@vibe-agent-toolkit/agent-skills 0.1.42 → 0.2.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/builder.d.ts.map +1 -1
- package/dist/builder.js +7 -17
- package/dist/builder.js.map +1 -1
- package/dist/content-type-routing.d.ts +32 -0
- package/dist/content-type-routing.d.ts.map +1 -1
- package/dist/content-type-routing.js +23 -0
- package/dist/content-type-routing.js.map +1 -1
- package/dist/files-config.d.ts +34 -1
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +277 -39
- package/dist/files-config.js.map +1 -1
- package/dist/fs-attribution.d.ts +45 -0
- package/dist/fs-attribution.d.ts.map +1 -0
- package/dist/fs-attribution.js +57 -0
- package/dist/fs-attribution.js.map +1 -0
- package/dist/import.d.ts.map +1 -1
- package/dist/import.js +0 -4
- package/dist/import.js.map +1 -1
- package/dist/index.d.ts +13 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -7
- package/dist/index.js.map +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
- package/dist/inventory/index.d.ts +1 -1
- package/dist/inventory/index.d.ts.map +1 -1
- package/dist/inventory/index.js +1 -1
- package/dist/inventory/index.js.map +1 -1
- package/dist/inventory/serialize.d.ts +4 -4
- package/dist/inventory/serialize.d.ts.map +1 -1
- package/dist/inventory/serialize.js +6 -6
- package/dist/inventory/serialize.js.map +1 -1
- package/dist/inventory/types.d.ts +8 -0
- package/dist/inventory/types.d.ts.map +1 -1
- package/dist/inventory/types.js.map +1 -1
- package/dist/link-graph-members.d.ts +18 -0
- package/dist/link-graph-members.d.ts.map +1 -0
- package/dist/link-graph-members.js +18 -0
- package/dist/link-graph-members.js.map +1 -0
- package/dist/parsers/frontmatter-parser.d.ts.map +1 -1
- package/dist/parsers/frontmatter-parser.js +30 -6
- package/dist/parsers/frontmatter-parser.js.map +1 -1
- package/dist/plugin-distribution-layout.d.ts +70 -6
- package/dist/plugin-distribution-layout.d.ts.map +1 -1
- package/dist/plugin-distribution-layout.js +84 -11
- package/dist/plugin-distribution-layout.js.map +1 -1
- package/dist/post-build-checks.d.ts +38 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js +68 -17
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +247 -0
- package/dist/projection/skill-extent.d.ts.map +1 -0
- package/dist/projection/skill-extent.js +560 -0
- package/dist/projection/skill-extent.js.map +1 -0
- package/dist/schemas/agent-skill-frontmatter.js +1 -1
- package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
- package/dist/schemas/allowed-tools.d.ts +31 -0
- package/dist/schemas/allowed-tools.d.ts.map +1 -0
- package/dist/schemas/allowed-tools.js +88 -0
- package/dist/schemas/allowed-tools.js.map +1 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/index.js.map +1 -1
- package/dist/schemas/installed-plugins-registry.d.ts.map +1 -1
- package/dist/schemas/installed-plugins-registry.js +7 -1
- package/dist/schemas/installed-plugins-registry.js.map +1 -1
- package/dist/schemas/marketplace-manifest.d.ts.map +1 -1
- package/dist/schemas/marketplace-manifest.js +17 -6
- package/dist/schemas/marketplace-manifest.js.map +1 -1
- package/dist/skill-identity.d.ts +18 -3
- package/dist/skill-identity.d.ts.map +1 -1
- package/dist/skill-identity.js +25 -6
- package/dist/skill-identity.js.map +1 -1
- package/dist/skill-packager.d.ts +144 -20
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +447 -105
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/content-hash.d.ts.map +1 -1
- package/dist/skill-source/content-hash.js +21 -10
- package/dist/skill-source/content-hash.js.map +1 -1
- package/dist/skill-source/fetch-cache.d.ts.map +1 -1
- package/dist/skill-source/fetch-cache.js +0 -5
- package/dist/skill-source/fetch-cache.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +10 -10
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-source/sources/url-source.d.ts.map +1 -1
- package/dist/skill-source/sources/url-source.js +2 -2
- package/dist/skill-source/sources/url-source.js.map +1 -1
- package/dist/skill-source/stage.d.ts.map +1 -1
- package/dist/skill-source/stage.js +0 -6
- package/dist/skill-source/stage.js.map +1 -1
- package/dist/skill-test/baseline-delta.d.ts +1575 -0
- package/dist/skill-test/baseline-delta.d.ts.map +1 -0
- package/dist/skill-test/baseline-delta.js +272 -0
- package/dist/skill-test/baseline-delta.js.map +1 -0
- package/dist/skill-test/baseline-integrity.d.ts +944 -0
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
- package/dist/skill-test/baseline-integrity.js +3090 -0
- package/dist/skill-test/baseline-integrity.js.map +1 -0
- package/dist/skill-test/build-hook.d.ts +3 -2
- package/dist/skill-test/build-hook.d.ts.map +1 -1
- package/dist/skill-test/build-hook.js +4 -5
- package/dist/skill-test/build-hook.js.map +1 -1
- package/dist/skill-test/configure-writer.js +1 -1
- package/dist/skill-test/configure-writer.js.map +1 -1
- package/dist/skill-test/declared-env.d.ts +5 -4
- package/dist/skill-test/declared-env.d.ts.map +1 -1
- package/dist/skill-test/declared-env.js +8 -9
- package/dist/skill-test/declared-env.js.map +1 -1
- package/dist/skill-test/eval-executor.d.ts +21 -5
- package/dist/skill-test/eval-executor.d.ts.map +1 -1
- package/dist/skill-test/eval-executor.js +6 -6
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +106 -28
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +78 -16
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts +65 -7
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +263 -64
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +110 -8
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +177 -22
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +17 -4
- package/dist/skill-test/eval-lint.d.ts.map +1 -1
- package/dist/skill-test/eval-lint.js +22 -9
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/eval-suite-isolation.d.ts.map +1 -1
- package/dist/skill-test/eval-suite-isolation.js +8 -5
- package/dist/skill-test/eval-suite-isolation.js.map +1 -1
- package/dist/skill-test/evals-template.d.ts +1 -1
- package/dist/skill-test/evals-template.d.ts.map +1 -1
- package/dist/skill-test/evals-template.js +1 -3
- package/dist/skill-test/evals-template.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +15 -7
- package/dist/skill-test/executor-prompt.d.ts.map +1 -1
- package/dist/skill-test/executor-prompt.js +10 -7
- package/dist/skill-test/executor-prompt.js.map +1 -1
- package/dist/skill-test/failure-reason.d.ts +113 -0
- package/dist/skill-test/failure-reason.d.ts.map +1 -0
- package/dist/skill-test/failure-reason.js +124 -0
- package/dist/skill-test/failure-reason.js.map +1 -0
- package/dist/skill-test/fragment-merge.d.ts +3 -20
- package/dist/skill-test/fragment-merge.d.ts.map +1 -1
- package/dist/skill-test/fragment-merge.js +67 -8
- package/dist/skill-test/fragment-merge.js.map +1 -1
- package/dist/skill-test/grader-prompt.d.ts +29 -9
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +264 -68
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grader-text.d.ts +145 -0
- package/dist/skill-test/grader-text.d.ts.map +1 -0
- package/dist/skill-test/grader-text.js +410 -0
- package/dist/skill-test/grader-text.js.map +1 -0
- package/dist/skill-test/grading-adapter.d.ts +38 -2
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +57 -9
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +160 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +66 -7
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/harness-location.d.ts +3 -2
- package/dist/skill-test/harness-location.d.ts.map +1 -1
- package/dist/skill-test/harness-location.js +5 -16
- package/dist/skill-test/harness-location.js.map +1 -1
- package/dist/skill-test/index.d.ts +2 -2
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +2 -2
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +58 -1
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +78 -6
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +19 -1
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +21 -10
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +59 -16
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/prompt-invariants.d.ts +4 -2
- package/dist/skill-test/prompt-invariants.d.ts.map +1 -1
- package/dist/skill-test/prompt-invariants.js +5 -4
- package/dist/skill-test/prompt-invariants.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +665 -85
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +1522 -237
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +20 -8
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +25 -18
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tier-plan.d.ts +1 -1
- package/dist/skill-test/tier-plan.js +1 -1
- package/dist/skill-test/tool-eval-schema.d.ts +6 -6
- package/dist/skill-test/tool-eval-schema.js +2 -2
- package/dist/skill-test/vendor-manifest.d.ts +2 -1
- package/dist/skill-test/vendor-manifest.d.ts.map +1 -1
- package/dist/skill-test/vendor-manifest.js +10 -10
- package/dist/skill-test/vendor-manifest.js.map +1 -1
- package/dist/test-input.d.ts +59 -8
- package/dist/test-input.d.ts.map +1 -1
- package/dist/test-input.js +64 -19
- package/dist/test-input.js.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.js +45 -2
- package/dist/validators/agent-instruction-presence.js.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +6 -4
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +39 -39
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/compat-detectors.d.ts +18 -1
- package/dist/validators/compat-detectors.d.ts.map +1 -1
- package/dist/validators/compat-detectors.js +47 -20
- package/dist/validators/compat-detectors.js.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.js +1 -1
- package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
- package/dist/validators/description-style-detection.d.ts +1 -1
- package/dist/validators/description-style-detection.d.ts.map +1 -1
- package/dist/validators/description-style-detection.js +1 -1
- package/dist/validators/description-style-detection.js.map +1 -1
- package/dist/validators/format-detection.d.ts +3 -0
- package/dist/validators/format-detection.d.ts.map +1 -1
- package/dist/validators/format-detection.js +41 -26
- package/dist/validators/format-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.d.ts +1 -1
- package/dist/validators/frontmatter-validation.d.ts.map +1 -1
- package/dist/validators/frontmatter-validation.js +2 -2
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/imperative-body-detection.d.ts +1 -1
- package/dist/validators/imperative-body-detection.d.ts.map +1 -1
- package/dist/validators/imperative-body-detection.js +1 -1
- package/dist/validators/imperative-body-detection.js.map +1 -1
- package/dist/validators/index.d.ts +1 -1
- package/dist/validators/index.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.d.ts +1 -1
- package/dist/validators/kebab-case-detection.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.js +1 -1
- package/dist/validators/kebab-case-detection.js.map +1 -1
- package/dist/validators/marketplace-validator.d.ts.map +1 -1
- package/dist/validators/marketplace-validator.js +7 -2
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/mcp-tool-qualification.d.ts +158 -0
- package/dist/validators/mcp-tool-qualification.d.ts.map +1 -0
- package/dist/validators/mcp-tool-qualification.js +467 -0
- package/dist/validators/mcp-tool-qualification.js.map +1 -0
- package/dist/validators/packaged-size-limit.d.ts +296 -0
- package/dist/validators/packaged-size-limit.d.ts.map +1 -0
- package/dist/validators/packaged-size-limit.js +492 -0
- package/dist/validators/packaged-size-limit.js.map +1 -0
- package/dist/validators/packaging-validator.d.ts +149 -10
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +371 -55
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +4 -2
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +6 -7
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
- package/dist/validators/plugin-recommended-fields.js +1 -1
- package/dist/validators/plugin-recommended-fields.js.map +1 -1
- package/dist/validators/referenced-path-missing.d.ts +211 -0
- package/dist/validators/referenced-path-missing.d.ts.map +1 -0
- package/dist/validators/referenced-path-missing.js +378 -0
- package/dist/validators/referenced-path-missing.js.map +1 -0
- package/dist/validators/registry-validator.d.ts.map +1 -1
- package/dist/validators/registry-validator.js +1 -2
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/index.d.ts +1 -1
- package/dist/validators/rule-engine/rule-context.d.ts +20 -3
- package/dist/validators/rule-engine/rule-context.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-context.js +3 -1
- package/dist/validators/rule-engine/rule-context.js.map +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +2 -2
- package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-engine.js +52 -13
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +91 -54
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/source-only-codes.js +1 -1
- package/dist/validators/source-only-codes.js.map +1 -1
- package/dist/validators/types.d.ts +21 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +4 -5
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +4 -4
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/validation-rules.js +9 -38
- package/dist/validators/validation-rules.js.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +9 -3
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +64 -19
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts +64 -5
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +376 -92
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +14 -22
- package/dist/skill-test/exit-codes.d.ts +0 -97
- package/dist/skill-test/exit-codes.d.ts.map +0 -1
- package/dist/skill-test/exit-codes.js +0 -141
- package/dist/skill-test/exit-codes.js.map +0 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { VatError } from '@vibe-agent-toolkit/utils';
|
|
1
2
|
import { z } from 'zod';
|
|
2
3
|
/** One graded expectation inside a per-eval grader fragment. */
|
|
3
4
|
export declare const EvalFragmentExpectationSchema: z.ZodObject<{
|
|
@@ -5,17 +6,17 @@ export declare const EvalFragmentExpectationSchema: z.ZodObject<{
|
|
|
5
6
|
passed: z.ZodBoolean;
|
|
6
7
|
evidence: z.ZodOptional<z.ZodString>;
|
|
7
8
|
}, "strict", z.ZodTypeAny, {
|
|
8
|
-
passed: boolean;
|
|
9
9
|
text: string;
|
|
10
|
+
passed: boolean;
|
|
10
11
|
evidence?: string | undefined;
|
|
11
12
|
}, {
|
|
12
|
-
passed: boolean;
|
|
13
13
|
text: string;
|
|
14
|
+
passed: boolean;
|
|
14
15
|
evidence?: string | undefined;
|
|
15
16
|
}>;
|
|
16
17
|
export type EvalFragmentExpectation = z.infer<typeof EvalFragmentExpectationSchema>;
|
|
17
18
|
/**
|
|
18
|
-
* Strict shape of a single grader subagent's output
|
|
19
|
+
* Strict shape of a single grader subagent's output. One
|
|
19
20
|
* GRADER spawn grades ONE eval's expectations from that eval's captured
|
|
20
21
|
* transcript and writes exactly one fragment matching this schema; vat reads
|
|
21
22
|
* every eval's fragment back and merges them into the run's aggregate grading
|
|
@@ -28,7 +29,7 @@ export type EvalFragmentExpectation = z.infer<typeof EvalFragmentExpectationSche
|
|
|
28
29
|
* `arm` distinguishes a WITH/WITHOUT baseline run's fragment; absent means the
|
|
29
30
|
* default 'with' (skill present) arm.
|
|
30
31
|
*
|
|
31
|
-
* `tool`
|
|
32
|
+
* `tool` carries this eval's tool-expectation verdict —
|
|
32
33
|
* a SEPARATE channel from `expectations[]` and from `friction` (see
|
|
33
34
|
* tool-eval-schema.ts). It is {@link ToolVerdictBodySchema}, i.e. a
|
|
34
35
|
* {@link import('./tool-eval-schema.js').ToolVerdict} WITHOUT `evalId`: the
|
|
@@ -38,19 +39,19 @@ export type EvalFragmentExpectation = z.infer<typeof EvalFragmentExpectationSche
|
|
|
38
39
|
*/
|
|
39
40
|
export declare const EvalFragmentSchema: z.ZodObject<{
|
|
40
41
|
runNonce: z.ZodString;
|
|
41
|
-
evalId: z.ZodString
|
|
42
|
+
evalId: z.ZodEffects<z.ZodUnion<[z.ZodString, z.ZodNumber]>, string, string | number>;
|
|
42
43
|
arm: z.ZodOptional<z.ZodEnum<["with", "without"]>>;
|
|
43
44
|
expectations: z.ZodArray<z.ZodObject<{
|
|
44
45
|
text: z.ZodString;
|
|
45
46
|
passed: z.ZodBoolean;
|
|
46
47
|
evidence: z.ZodOptional<z.ZodString>;
|
|
47
48
|
}, "strict", z.ZodTypeAny, {
|
|
48
|
-
passed: boolean;
|
|
49
49
|
text: string;
|
|
50
|
+
passed: boolean;
|
|
50
51
|
evidence?: string | undefined;
|
|
51
52
|
}, {
|
|
52
|
-
passed: boolean;
|
|
53
53
|
text: string;
|
|
54
|
+
passed: boolean;
|
|
54
55
|
evidence?: string | undefined;
|
|
55
56
|
}>, "many">;
|
|
56
57
|
friction: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -171,22 +172,61 @@ export declare const EvalFragmentSchema: z.ZodObject<{
|
|
|
171
172
|
evidence?: string | undefined;
|
|
172
173
|
}[] | undefined;
|
|
173
174
|
}>>;
|
|
175
|
+
/**
|
|
176
|
+
* WITHOUT-arm baseline-integrity hits (see baseline-integrity.ts). Like `arm`,
|
|
177
|
+
* this is attached by VAT after the strict parse — it is derived from the
|
|
178
|
+
* executor transcript, which the grader never sees, so a value arriving from a
|
|
179
|
+
* grader is meaningless and is overwritten rather than trusted.
|
|
180
|
+
*/
|
|
181
|
+
contamination: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
182
|
+
kind: z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>;
|
|
183
|
+
match: z.ZodString;
|
|
184
|
+
excerpt: z.ZodString;
|
|
185
|
+
}, "strict", z.ZodTypeAny, {
|
|
186
|
+
match: string;
|
|
187
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
188
|
+
excerpt: string;
|
|
189
|
+
}, {
|
|
190
|
+
match: string;
|
|
191
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
192
|
+
excerpt: string;
|
|
193
|
+
}>, "many">>;
|
|
194
|
+
/**
|
|
195
|
+
* Why THIS eval's contamination scan could not run at full strength, when it
|
|
196
|
+
* could not (see `BaselineScanDegradation`). Attached by VAT after the strict
|
|
197
|
+
* parse for exactly the reasons `contamination` is, and stripped from grader
|
|
198
|
+
* output for exactly the same reason: it is derived from the executor
|
|
199
|
+
* transcript's structure, which the grader never inspects, so a value arriving
|
|
200
|
+
* from a grader is meaningless — and this one is worse than meaningless,
|
|
201
|
+
* because a grader talked into omitting it turns a blind scan back into a
|
|
202
|
+
* confident clean one.
|
|
203
|
+
*
|
|
204
|
+
* This is the per-eval half of the run-level `baselineIntegrity.degraded` list.
|
|
205
|
+
* It exists because the degradation is detected per eval, deep inside a
|
|
206
|
+
* pipeline worker, while the block that has to report it is assembled once at
|
|
207
|
+
* the end from fragments — the fragment is the only channel between the two.
|
|
208
|
+
*/
|
|
209
|
+
degraded: z.ZodOptional<z.ZodObject<{
|
|
210
|
+
reason: z.ZodEnum<["transcript-unparsed", "cwd-unknown", "cwd-untracked", "transcript-malformed", "glob-unexpanded"]>;
|
|
211
|
+
detail: z.ZodString;
|
|
212
|
+
evalId: z.ZodOptional<z.ZodString>;
|
|
213
|
+
}, "strict", z.ZodTypeAny, {
|
|
214
|
+
detail: string;
|
|
215
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
216
|
+
evalId?: string | undefined;
|
|
217
|
+
}, {
|
|
218
|
+
detail: string;
|
|
219
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
220
|
+
evalId?: string | undefined;
|
|
221
|
+
}>>;
|
|
174
222
|
}, "strict", z.ZodTypeAny, {
|
|
175
223
|
expectations: {
|
|
176
|
-
passed: boolean;
|
|
177
224
|
text: string;
|
|
225
|
+
passed: boolean;
|
|
178
226
|
evidence?: string | undefined;
|
|
179
227
|
}[];
|
|
180
228
|
evalId: string;
|
|
181
229
|
runNonce: string;
|
|
182
|
-
arm?: "with" | "without" | undefined;
|
|
183
|
-
friction?: {
|
|
184
|
-
message: string;
|
|
185
|
-
severity: "high" | "medium" | "low";
|
|
186
|
-
category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
|
|
187
|
-
evidence?: string | undefined;
|
|
188
|
-
subjectFile?: string | undefined;
|
|
189
|
-
}[] | undefined;
|
|
190
230
|
tool?: {
|
|
191
231
|
passed: boolean;
|
|
192
232
|
mustRun?: {
|
|
@@ -210,14 +250,11 @@ export declare const EvalFragmentSchema: z.ZodObject<{
|
|
|
210
250
|
evidence?: string | undefined;
|
|
211
251
|
}[] | undefined;
|
|
212
252
|
} | undefined;
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
}[];
|
|
219
|
-
evalId: string;
|
|
220
|
-
runNonce: string;
|
|
253
|
+
degraded?: {
|
|
254
|
+
detail: string;
|
|
255
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
256
|
+
evalId?: string | undefined;
|
|
257
|
+
} | undefined;
|
|
221
258
|
arm?: "with" | "without" | undefined;
|
|
222
259
|
friction?: {
|
|
223
260
|
message: string;
|
|
@@ -226,6 +263,19 @@ export declare const EvalFragmentSchema: z.ZodObject<{
|
|
|
226
263
|
evidence?: string | undefined;
|
|
227
264
|
subjectFile?: string | undefined;
|
|
228
265
|
}[] | undefined;
|
|
266
|
+
contamination?: {
|
|
267
|
+
match: string;
|
|
268
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
269
|
+
excerpt: string;
|
|
270
|
+
}[] | undefined;
|
|
271
|
+
}, {
|
|
272
|
+
expectations: {
|
|
273
|
+
text: string;
|
|
274
|
+
passed: boolean;
|
|
275
|
+
evidence?: string | undefined;
|
|
276
|
+
}[];
|
|
277
|
+
evalId: string | number;
|
|
278
|
+
runNonce: string;
|
|
229
279
|
tool?: {
|
|
230
280
|
passed: boolean;
|
|
231
281
|
mustRun?: {
|
|
@@ -249,6 +299,24 @@ export declare const EvalFragmentSchema: z.ZodObject<{
|
|
|
249
299
|
evidence?: string | undefined;
|
|
250
300
|
}[] | undefined;
|
|
251
301
|
} | undefined;
|
|
302
|
+
degraded?: {
|
|
303
|
+
detail: string;
|
|
304
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
305
|
+
evalId?: string | undefined;
|
|
306
|
+
} | undefined;
|
|
307
|
+
arm?: "with" | "without" | undefined;
|
|
308
|
+
friction?: {
|
|
309
|
+
message: string;
|
|
310
|
+
severity: "high" | "medium" | "low";
|
|
311
|
+
category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
|
|
312
|
+
evidence?: string | undefined;
|
|
313
|
+
subjectFile?: string | undefined;
|
|
314
|
+
}[] | undefined;
|
|
315
|
+
contamination?: {
|
|
316
|
+
match: string;
|
|
317
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
318
|
+
excerpt: string;
|
|
319
|
+
}[] | undefined;
|
|
252
320
|
}>;
|
|
253
321
|
export type EvalFragment = z.infer<typeof EvalFragmentSchema>;
|
|
254
322
|
/**
|
|
@@ -261,7 +329,8 @@ export type EvalFragment = z.infer<typeof EvalFragmentSchema>;
|
|
|
261
329
|
* should point at the offending eval, not at "Re-sync the vendored
|
|
262
330
|
* skill-creator".
|
|
263
331
|
*/
|
|
264
|
-
export declare class EvalFragmentError extends
|
|
332
|
+
export declare class EvalFragmentError extends VatError {
|
|
333
|
+
readonly reason: "internal";
|
|
265
334
|
constructor(message: string);
|
|
266
335
|
}
|
|
267
336
|
/**
|
|
@@ -271,8 +340,17 @@ export declare class EvalFragmentError extends Error {
|
|
|
271
340
|
*
|
|
272
341
|
* The verdict channels are strict. The auxiliary `friction` field is sanitized
|
|
273
342
|
* leniently first (see {@link sanitizeFrictionField}); when items are dropped,
|
|
274
|
-
* `onWarn`
|
|
275
|
-
*
|
|
343
|
+
* `onWarn` is called so the operator sees that friction was partial — grading
|
|
344
|
+
* itself is never affected. `onWarn` is REQUIRED: a caller that could omit it
|
|
345
|
+
* received a fragment with friction silently missing, and "friction is
|
|
346
|
+
* advisory" is only true when the operator is told it was cut.
|
|
347
|
+
*
|
|
348
|
+
* THIS IS THE TEXT BOUNDARY. Every string in the fragment is run through
|
|
349
|
+
* {@link sanitizeGraderTextDeep} BEFORE anything else looks at it, so no
|
|
350
|
+
* grader-supplied newline or ANSI sequence can reach an artifact or an operator
|
|
351
|
+
* line downstream (see grader-text.ts for the threat). It runs first, ahead of
|
|
352
|
+
* the friction sanitizer and the strict parse, because both of those quote
|
|
353
|
+
* grader text into messages of their own.
|
|
276
354
|
*/
|
|
277
|
-
export declare function parseEvalFragment(raw: unknown, onWarn
|
|
355
|
+
export declare function parseEvalFragment(raw: unknown, onWarn: (message: string) => void): EvalFragment;
|
|
278
356
|
//# sourceMappingURL=eval-fragment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-fragment.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-fragment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-fragment.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-fragment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,gEAAgE;AAChE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;EAI/B,CAAC;AAEZ,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAU7B;;;;;OAKG;;;;;;;;;;;;;;IAEH;;;;;;;;;;;;;;OAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAEM,CAAC;AAEZ,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;;;GASG;AACH,qBAAa,iBAAkB,SAAQ,QAAQ;IAC7C,QAAQ,CAAC,MAAM,EAAG,UAAU,CAAU;gBAC1B,OAAO,EAAE,MAAM;CAG5B;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,YAAY,CAyB/F"}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { VatError } from '@vibe-agent-toolkit/utils';
|
|
1
2
|
import { z } from 'zod';
|
|
3
|
+
import { BaselineContaminationHitSchema, BaselineScanDegradationSchema } from './baseline-integrity.js';
|
|
2
4
|
import { FrictionItemSchema } from './friction-schema.js';
|
|
5
|
+
import { sanitizeGraderText, sanitizeGraderTextDeep } from './grader-text.js';
|
|
3
6
|
import { ToolVerdictBodySchema } from './tool-eval-schema.js';
|
|
4
7
|
/** One graded expectation inside a per-eval grader fragment. */
|
|
5
8
|
export const EvalFragmentExpectationSchema = z.object({
|
|
@@ -8,7 +11,7 @@ export const EvalFragmentExpectationSchema = z.object({
|
|
|
8
11
|
evidence: z.string().optional(),
|
|
9
12
|
}).strict();
|
|
10
13
|
/**
|
|
11
|
-
* Strict shape of a single grader subagent's output
|
|
14
|
+
* Strict shape of a single grader subagent's output. One
|
|
12
15
|
* GRADER spawn grades ONE eval's expectations from that eval's captured
|
|
13
16
|
* transcript and writes exactly one fragment matching this schema; vat reads
|
|
14
17
|
* every eval's fragment back and merges them into the run's aggregate grading
|
|
@@ -21,7 +24,7 @@ export const EvalFragmentExpectationSchema = z.object({
|
|
|
21
24
|
* `arm` distinguishes a WITH/WITHOUT baseline run's fragment; absent means the
|
|
22
25
|
* default 'with' (skill present) arm.
|
|
23
26
|
*
|
|
24
|
-
* `tool`
|
|
27
|
+
* `tool` carries this eval's tool-expectation verdict —
|
|
25
28
|
* a SEPARATE channel from `expectations[]` and from `friction` (see
|
|
26
29
|
* tool-eval-schema.ts). It is {@link ToolVerdictBodySchema}, i.e. a
|
|
27
30
|
* {@link import('./tool-eval-schema.js').ToolVerdict} WITHOUT `evalId`: the
|
|
@@ -31,11 +34,37 @@ export const EvalFragmentExpectationSchema = z.object({
|
|
|
31
34
|
*/
|
|
32
35
|
export const EvalFragmentSchema = z.object({
|
|
33
36
|
runNonce: z.string().min(1),
|
|
34
|
-
|
|
37
|
+
// evals.json allows an integer id and every other layer carries it as a
|
|
38
|
+
// string; a grader that echoes the number names the same eval, so it is
|
|
39
|
+
// normalised here rather than refused as a forged fragment.
|
|
40
|
+
evalId: z.union([z.string().min(1), z.number().int()]).transform(String),
|
|
35
41
|
arm: z.enum(['with', 'without']).optional(),
|
|
36
42
|
expectations: z.array(EvalFragmentExpectationSchema).min(1),
|
|
37
43
|
friction: z.array(FrictionItemSchema).optional(),
|
|
38
44
|
tool: ToolVerdictBodySchema.optional(),
|
|
45
|
+
/**
|
|
46
|
+
* WITHOUT-arm baseline-integrity hits (see baseline-integrity.ts). Like `arm`,
|
|
47
|
+
* this is attached by VAT after the strict parse — it is derived from the
|
|
48
|
+
* executor transcript, which the grader never sees, so a value arriving from a
|
|
49
|
+
* grader is meaningless and is overwritten rather than trusted.
|
|
50
|
+
*/
|
|
51
|
+
contamination: z.array(BaselineContaminationHitSchema).optional(),
|
|
52
|
+
/**
|
|
53
|
+
* Why THIS eval's contamination scan could not run at full strength, when it
|
|
54
|
+
* could not (see `BaselineScanDegradation`). Attached by VAT after the strict
|
|
55
|
+
* parse for exactly the reasons `contamination` is, and stripped from grader
|
|
56
|
+
* output for exactly the same reason: it is derived from the executor
|
|
57
|
+
* transcript's structure, which the grader never inspects, so a value arriving
|
|
58
|
+
* from a grader is meaningless — and this one is worse than meaningless,
|
|
59
|
+
* because a grader talked into omitting it turns a blind scan back into a
|
|
60
|
+
* confident clean one.
|
|
61
|
+
*
|
|
62
|
+
* This is the per-eval half of the run-level `baselineIntegrity.degraded` list.
|
|
63
|
+
* It exists because the degradation is detected per eval, deep inside a
|
|
64
|
+
* pipeline worker, while the block that has to report it is assembled once at
|
|
65
|
+
* the end from fragments — the fragment is the only channel between the two.
|
|
66
|
+
*/
|
|
67
|
+
degraded: BaselineScanDegradationSchema.optional(),
|
|
39
68
|
}).strict();
|
|
40
69
|
/**
|
|
41
70
|
* Thrown when a single grader subagent's fragment output does not match
|
|
@@ -47,26 +76,43 @@ export const EvalFragmentSchema = z.object({
|
|
|
47
76
|
* should point at the offending eval, not at "Re-sync the vendored
|
|
48
77
|
* skill-creator".
|
|
49
78
|
*/
|
|
50
|
-
export class EvalFragmentError extends
|
|
79
|
+
export class EvalFragmentError extends VatError {
|
|
80
|
+
reason = 'internal';
|
|
51
81
|
constructor(message) {
|
|
52
|
-
super(message);
|
|
53
|
-
this.name = 'EvalFragmentError';
|
|
82
|
+
super('EVAL_FRAGMENT', message);
|
|
54
83
|
}
|
|
55
84
|
}
|
|
56
|
-
/**
|
|
85
|
+
/**
|
|
86
|
+
* Best-effort extraction of `evalId` from raw input for error messages,
|
|
87
|
+
* tolerating any shape. Sanitized: this value lands in a thrown message that a
|
|
88
|
+
* human reads on stderr, and at the point it is read the fragment has NOT been
|
|
89
|
+
* validated — so it is raw grader text with no schema between it and the
|
|
90
|
+
* terminal.
|
|
91
|
+
*/
|
|
57
92
|
function extractRawEvalId(raw) {
|
|
58
93
|
if (typeof raw === 'object' && raw !== null && 'evalId' in raw) {
|
|
59
94
|
const value = raw['evalId'];
|
|
95
|
+
if (typeof value === 'number')
|
|
96
|
+
return `"${String(value)}"`;
|
|
60
97
|
if (typeof value === 'string' && value.length > 0)
|
|
61
|
-
return `"${value}"`;
|
|
98
|
+
return `"${sanitizeGraderText(value)}"`;
|
|
62
99
|
}
|
|
63
100
|
return '(unknown)';
|
|
64
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* Fragment keys whose value must NOT be normalized.
|
|
104
|
+
*
|
|
105
|
+
* `runNonce` is compared byte-for-byte against the run's secret by the
|
|
106
|
+
* integrity gate in eval-grader.ts. Normalizing it would mean the gate no
|
|
107
|
+
* longer compares what the grader actually wrote — the one field in this
|
|
108
|
+
* fragment that exists to be compared exactly.
|
|
109
|
+
*/
|
|
110
|
+
const UNSANITIZED_FRAGMENT_KEYS = new Set(['runNonce']);
|
|
65
111
|
/**
|
|
66
112
|
* Sanitize ONLY the (non-verdict-bearing) `friction` field before the strict
|
|
67
113
|
* fragment parse. Friction is auxiliary advisory data, not a verdict channel —
|
|
68
114
|
* so a grader that wobbles on the friction shape (e.g. emits bare strings, the
|
|
69
|
-
* common failure mode
|
|
115
|
+
* common failure mode) must NOT be allowed to discard the
|
|
70
116
|
* verdict-bearing grading for the whole run. We drop friction items that don't
|
|
71
117
|
* match {@link FrictionItemSchema} (and a `friction` that isn't an array at
|
|
72
118
|
* all), then hand the rest to the STRICT fragment parse — so the verdict
|
|
@@ -107,21 +153,37 @@ function sanitizeFrictionField(raw) {
|
|
|
107
153
|
*
|
|
108
154
|
* The verdict channels are strict. The auxiliary `friction` field is sanitized
|
|
109
155
|
* leniently first (see {@link sanitizeFrictionField}); when items are dropped,
|
|
110
|
-
* `onWarn`
|
|
111
|
-
*
|
|
156
|
+
* `onWarn` is called so the operator sees that friction was partial — grading
|
|
157
|
+
* itself is never affected. `onWarn` is REQUIRED: a caller that could omit it
|
|
158
|
+
* received a fragment with friction silently missing, and "friction is
|
|
159
|
+
* advisory" is only true when the operator is told it was cut.
|
|
160
|
+
*
|
|
161
|
+
* THIS IS THE TEXT BOUNDARY. Every string in the fragment is run through
|
|
162
|
+
* {@link sanitizeGraderTextDeep} BEFORE anything else looks at it, so no
|
|
163
|
+
* grader-supplied newline or ANSI sequence can reach an artifact or an operator
|
|
164
|
+
* line downstream (see grader-text.ts for the threat). It runs first, ahead of
|
|
165
|
+
* the friction sanitizer and the strict parse, because both of those quote
|
|
166
|
+
* grader text into messages of their own.
|
|
112
167
|
*/
|
|
113
168
|
export function parseEvalFragment(raw, onWarn) {
|
|
114
|
-
const
|
|
169
|
+
const sanitized = sanitizeGraderTextDeep(raw, UNSANITIZED_FRAGMENT_KEYS);
|
|
170
|
+
const { value, dropped } = sanitizeFrictionField(sanitized);
|
|
115
171
|
if (dropped > 0) {
|
|
116
|
-
onWarn
|
|
172
|
+
onWarn(`grader fragment for eval ${extractRawEvalId(sanitized)} had ${dropped} malformed friction item(s) — ` +
|
|
117
173
|
`dropped them (friction is advisory; grading is unaffected).`);
|
|
118
174
|
}
|
|
119
175
|
const result = EvalFragmentSchema.safeParse(value);
|
|
120
176
|
if (!result.success) {
|
|
121
177
|
const firstIssue = result.error.issues[0];
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
178
|
+
// Both halves quote grader-chosen text. The path names the offending FIELD,
|
|
179
|
+
// and a `.strict()` failure puts the offending KEY inside zod's own issue
|
|
180
|
+
// MESSAGE ("Unrecognized key(s) in object: '<key>'") — the key never becomes
|
|
181
|
+
// fragment data, so the deep walk above cannot reach it, and it is the only
|
|
182
|
+
// route by which an unparseable fragment can still paint the terminal.
|
|
183
|
+
const path = sanitizeGraderText(firstIssue?.path.join('.') ?? '(root)');
|
|
184
|
+
const issue = sanitizeGraderText(firstIssue?.message ?? 'unknown');
|
|
185
|
+
const evalId = extractRawEvalId(sanitized);
|
|
186
|
+
throw new EvalFragmentError(`grader fragment for eval ${evalId} has an invalid shape: invalid field at "${path}" (${issue})`);
|
|
125
187
|
}
|
|
126
188
|
return result.data;
|
|
127
189
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-fragment.js","sourceRoot":"","sources":["../../src/skill-test/eval-fragment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,gEAAgE;AAChE,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC,MAAM,EAAE,CAAC;AAIZ;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-fragment.js","sourceRoot":"","sources":["../../src/skill-test/eval-fragment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,8BAA8B,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAC;AACxG,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9D,gEAAgE;AAChE,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC,MAAM,EAAE,CAAC;AAIZ;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,wEAAwE;IACxE,wEAAwE;IACxE,4DAA4D;IAC5D,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;IACxE,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC3C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE;IAChD,IAAI,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACtC;;;;;OAKG;IACH,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,QAAQ,EAAE;IACjE;;;;;;;;;;;;;;OAcG;IACH,QAAQ,EAAE,6BAA6B,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC,MAAM,EAAE,CAAC;AAIZ;;;;;;;;;GASG;AACH,MAAM,OAAO,iBAAkB,SAAQ,QAAQ;IACpC,MAAM,GAAG,UAAmB,CAAC;IACtC,YAAY,OAAe;QACzB,KAAK,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IAClC,CAAC;CACF;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAI,GAA+B,CAAC,QAAQ,CAAC,CAAC;QACzD,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;QAC3D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC;IAC7F,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,yBAAyB,GAAwB,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;AAE7E;;;;;;;;;;;;;;GAcG;AACH,SAAS,qBAAqB,CAAC,GAAY;IACzC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IACjC,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC9D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,4EAA4E;QAC5E,yDAAyD;QACzD,MAAM,IAAI,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,GAAG,KAAK,UAAU;gBAAE,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QAC1C,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACrC,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;IACnF,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACvE,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;AACvF,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAY,EAAE,MAAiC;IAC/E,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACzE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC;IAC5D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,CACJ,4BAA4B,gBAAgB,CAAC,SAAS,CAAC,QAAQ,OAAO,gCAAgC;YACpG,6DAA6D,CAChE,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,4EAA4E;QAC5E,0EAA0E;QAC1E,6EAA6E;QAC7E,4EAA4E;QAC5E,uEAAuE;QACvE,MAAM,IAAI,GAAG,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC;QACxE,MAAM,KAAK,GAAG,kBAAkB,CAAC,UAAU,EAAE,OAAO,IAAI,SAAS,CAAC,CAAC;QACnE,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC3C,MAAM,IAAI,iBAAiB,CACzB,4BAA4B,MAAM,4CAA4C,IAAI,MAAM,KAAK,GAAG,CACjG,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { VatError } from '@vibe-agent-toolkit/utils';
|
|
2
|
+
import { spawnHeadlessClaude } from '@vibe-agent-toolkit/utils/skill-test';
|
|
2
3
|
import { type EvalFragment } from './eval-fragment.js';
|
|
3
4
|
import type { ToolExpectations } from './eval-inputs.js';
|
|
4
5
|
export interface RunGraderInput {
|
|
@@ -10,7 +11,7 @@ export interface RunGraderInput {
|
|
|
10
11
|
/** Path to the vendored skill-creator grader rubric (references/grader.md). */
|
|
11
12
|
rubricPath: string;
|
|
12
13
|
/**
|
|
13
|
-
* The eval's declared tool expectations
|
|
14
|
+
* The eval's declared tool expectations. Optional — when
|
|
14
15
|
* absent, `buildGraderPrompt` emits no tool-verdict instruction and the
|
|
15
16
|
* returned fragment omits `tool`. See `grader-prompt.ts`'s
|
|
16
17
|
* `BuildGraderPromptOptions.toolExpectations`.
|
|
@@ -40,15 +41,72 @@ export interface RunGraderInput {
|
|
|
40
41
|
env: NodeJS.ProcessEnv;
|
|
41
42
|
/** Injectable seam for tests; defaults to the real {@link spawnHeadlessClaude}. */
|
|
42
43
|
spawn?: typeof spawnHeadlessClaude;
|
|
43
|
-
/** Called with each stdout chunk as it streams (for progress echo). */
|
|
44
|
-
onProgress?: (chunk: string) => void;
|
|
45
44
|
/**
|
|
46
|
-
*
|
|
45
|
+
* Called with each stdout chunk as it streams (for progress echo), and with
|
|
46
|
+
* the `[skill-test]` warning line when the grader's friction was cut — see
|
|
47
|
+
* {@link fragmentWarnRouter}. Required: it is the only channel on which a
|
|
48
|
+
* dropped friction item is ever said, and an optional sink here was the
|
|
49
|
+
* optional `onWarn` one layer down wearing a different name.
|
|
50
|
+
*/
|
|
51
|
+
onProgress: (chunk: string) => void;
|
|
52
|
+
/**
|
|
53
|
+
* Reports each grader session's `total_cost_usd` (parsed from its stream-json
|
|
47
54
|
* transcript's terminal result; `undefined` when the transcript carried none,
|
|
48
|
-
* e.g. a mock spawn). Called once
|
|
49
|
-
*
|
|
55
|
+
* e.g. a mock spawn). Called once per successful grader spawn — twice when the
|
|
56
|
+
* unparseable-fragment re-grade fires — so the run can aggregate total spend
|
|
57
|
+
* across all executor+grader sessions (adopter follow-up).
|
|
50
58
|
*/
|
|
51
59
|
costSink?: (totalCostUsd: number | undefined) => void;
|
|
52
60
|
}
|
|
61
|
+
/** The reason named in {@link GraderFragmentUnparseableError}'s message and in every `evidence` of the failing fragment vat synthesizes from it. */
|
|
62
|
+
export declare const GRADER_FRAGMENT_UNPARSEABLE = "grader-fragment-unparseable";
|
|
63
|
+
/**
|
|
64
|
+
* Thrown when the grader's fragment was not valid JSON on BOTH attempts (see
|
|
65
|
+
* {@link runGraderForEval}). A per-eval GRADING failure, deliberately neither
|
|
66
|
+
* {@link InternalHarnessError} nor `EvalFragmentError`: the harness did its
|
|
67
|
+
* job twice and the model could not write strict JSON, which is a verdict about
|
|
68
|
+
* this eval's grading, not about the suite — so it ends on the verdict path
|
|
69
|
+
* (`FINDINGS`), never on `ERROR` with `Reason: internal`. `runEvalWorker` (run-harness.ts)
|
|
70
|
+
* turns it into that eval's failing fragment (reason
|
|
71
|
+
* {@link GRADER_FRAGMENT_UNPARSEABLE}) on the treatment arm, and into a recorded
|
|
72
|
+
* control-arm failure on the other — where a fabricated `fail` would read as
|
|
73
|
+
* skill lift. `detail` is already sanitized. Carries no `reason` on purpose:
|
|
74
|
+
* `skillTestFailureReason` would read it only if this escaped, and escaping IS
|
|
75
|
+
* the bug — the worker catches it on both arms.
|
|
76
|
+
*/
|
|
77
|
+
export declare class GraderFragmentUnparseableError extends VatError {
|
|
78
|
+
readonly evalId: string;
|
|
79
|
+
readonly detail: string;
|
|
80
|
+
constructor(evalId: string, detail: string);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Run ONE eval's grader: a blind `claude -p` spawn (grader model, NO
|
|
84
|
+
* skill/plugin loaded) that reads the executor's captured transcript via its
|
|
85
|
+
* prompt and writes a nonce'd fragment JSON to `graderOutDir`.
|
|
86
|
+
* `graderOutDir` is vat-only and separate from the skill sandbox the
|
|
87
|
+
* executor ran in — the grader's `sandboxDir` --add-dir is
|
|
88
|
+
* `graderOutDir` itself, and `pluginDirs: []` means no skill code ever runs
|
|
89
|
+
* inside the grader's process, so skill code cannot forge or tamper with the
|
|
90
|
+
* fragment this run trusts.
|
|
91
|
+
*
|
|
92
|
+
* The grader is vat infrastructure, not the thing under test: unlike the
|
|
93
|
+
* executor (see eval-executor.ts), there is NO clean-failure path here. A
|
|
94
|
+
* spawn rejection, watchdog kill (`timedOut`/`stalled`), non-zero exit, or a
|
|
95
|
+
* missing fragment file all mean the grader failed to do its ONE job and are
|
|
96
|
+
* thrown as {@link InternalHarnessError} (`ERROR`, `Reason: internal`) — never laundered into a
|
|
97
|
+
* passing (or even a valid failing) verdict.
|
|
98
|
+
*
|
|
99
|
+
* A fragment that exists but is not valid JSON is the one exception, because
|
|
100
|
+
* the grader MODEL writes it: one bad escape in a free-text `evidence` took a
|
|
101
|
+
* six-eval suite down as harness-broke after five evals had graded fine. That
|
|
102
|
+
* attempt is re-run ONCE — same transcript, a FRESH nonce (so the stale file, or
|
|
103
|
+
* anything left behind, cannot satisfy the retry), the bad file consumed first,
|
|
104
|
+
* and the prompt told what went wrong. Only a second unparseable fragment
|
|
105
|
+
* surfaces, as {@link GraderFragmentUnparseableError}. Once a fragment parses,
|
|
106
|
+
* shape problems throw `EvalFragmentError` (via {@link parseEvalFragment})
|
|
107
|
+
* and a nonce that doesn't match the nonce THAT attempt was prompted with throws
|
|
108
|
+
* {@link GradingNonceError} — both fail-closed with no retry, because the grader
|
|
109
|
+
* DID produce JSON, just not JSON we can trust.
|
|
110
|
+
*/
|
|
53
111
|
export declare function runGraderForEval(input: RunGraderInput): Promise<EvalFragment>;
|
|
54
112
|
//# sourceMappingURL=eval-grader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-grader.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"eval-grader.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAGA,OAAO,EAA2B,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAEL,mBAAmB,EACpB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAOzD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,uGAAuG;IACvG,mBAAmB,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChF;;;;;OAKG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,mFAAmF;IACnF,KAAK,CAAC,EAAE,OAAO,mBAAmB,CAAC;IACnC;;;;;;OAMG;IACH,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC;CACvD;AAED,oJAAoJ;AACpJ,eAAO,MAAM,2BAA2B,gCAAgC,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,qBAAa,8BAA+B,SAAQ,QAAQ;aAExC,MAAM,EAAE,MAAM;aACd,MAAM,EAAE,MAAM;gBADd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM;CAQjC;AAWD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC,CAwDnF"}
|