@vibe-agent-toolkit/agent-skills 0.1.40 → 0.1.41-rc.2

package/dist/skill-test/eval-inputs.d.ts

@@ -3,36 +3,129 @@ import { z } from 'zod';
 export declare class EvalInputError extends Error {
     constructor(message: string);
 }
+/**
+ * True when `key` is exactly one edit (insert/delete/substitute one char) from
+ * `target`. Used to catch typos of recognized fields (here) and of declared
+ * executable names ({@link ./eval-lint.ts}) — deliberately a tiny single-edit
+ * check, not a general edit-distance routine, so it stays cheap and never fires
+ * on legitimately distinct adopter strings (`name`, `category`, `notes`).
+ */
+export declare function isSingleEditAway(key: string, target: string): boolean;
 export declare const EvalEntrySchema: z.ZodEffects<z.ZodObject<{
     id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
     prompt: z.ZodString;
     expected_output: z.ZodOptional<z.ZodString>;
     files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     expectations: z.ZodArray<z.ZodString, "many">;
+    tier: z.ZodOptional<z.ZodNumber>;
+    toolExpectations: z.ZodOptional<z.ZodObject<{
+        mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strict", z.ZodTypeAny, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }>>;
 }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
     id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
     prompt: z.ZodString;
     expected_output: z.ZodOptional<z.ZodString>;
     files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     expectations: z.ZodArray<z.ZodString, "many">;
+    tier: z.ZodOptional<z.ZodNumber>;
+    toolExpectations: z.ZodOptional<z.ZodObject<{
+        mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strict", z.ZodTypeAny, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }>>;
 }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
     id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
     prompt: z.ZodString;
     expected_output: z.ZodOptional<z.ZodString>;
     files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     expectations: z.ZodArray<z.ZodString, "many">;
+    tier: z.ZodOptional<z.ZodNumber>;
+    toolExpectations: z.ZodOptional<z.ZodObject<{
+        mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strict", z.ZodTypeAny, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }>>;
 }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
     id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
     prompt: z.ZodString;
     expected_output: z.ZodOptional<z.ZodString>;
     files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     expectations: z.ZodArray<z.ZodString, "many">;
+    tier: z.ZodOptional<z.ZodNumber>;
+    toolExpectations: z.ZodOptional<z.ZodObject<{
+        mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strict", z.ZodTypeAny, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }>>;
 }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
     id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
     prompt: z.ZodString;
     expected_output: z.ZodOptional<z.ZodString>;
     files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     expectations: z.ZodArray<z.ZodString, "many">;
+    tier: z.ZodOptional<z.ZodNumber>;
+    toolExpectations: z.ZodOptional<z.ZodObject<{
+        mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strict", z.ZodTypeAny, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }, {
+        mustRun?: string[] | undefined;
+        mustNotRun?: string[] | undefined;
+        mustSucceed?: string[] | undefined;
+        sequence?: string[] | undefined;
+    }>>;
 }, z.ZodTypeAny, "passthrough">>;
 export declare const EvalSuiteSchema: z.ZodObject<{
     _comment: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
@@ -43,30 +136,115 @@ export declare const EvalSuiteSchema: z.ZodObject<{
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
 }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
     _comment: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
@@ -77,30 +255,115 @@ export declare const EvalSuiteSchema: z.ZodObject<{
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
 }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
     _comment: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
@@ -111,34 +374,126 @@ export declare const EvalSuiteSchema: z.ZodObject<{
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
         id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
         prompt: z.ZodString;
         expected_output: z.ZodOptional<z.ZodString>;
         files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         expectations: z.ZodArray<z.ZodString, "many">;
+        tier: z.ZodOptional<z.ZodNumber>;
+        toolExpectations: z.ZodOptional<z.ZodObject<{
+            mustRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustNotRun: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            mustSucceed: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+            sequence: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strict", z.ZodTypeAny, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }, {
+            mustRun?: string[] | undefined;
+            mustNotRun?: string[] | undefined;
+            mustSucceed?: string[] | undefined;
+            sequence?: string[] | undefined;
+        }>>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
 }, z.ZodTypeAny, "passthrough">>;
 export type EvalEntry = z.infer<typeof EvalEntrySchema>;
 export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
+/**
+ * The declared tool expectations of ONE eval (issue #145 Phase T). Derived from
+ * {@link EvalEntry} so the grader input + prompt builder share the EXACT shape the
+ * parser produces — no drift, and no duplicated inline `{ mustRun?; mustNotRun?;
+ * sequence? }` literal across those consumers.
+ */
+export type ToolExpectations = NonNullable<EvalEntry['toolExpectations']>;
 /** Parse + validate a skill-test eval suite. Throws {@link EvalInputError} on any problem. */
 export declare function parseEvalSuite(jsonText: string): EvalSuite;
 export interface StageEvalWorkspacesInput {

package/dist/skill-test/eval-inputs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AA4CD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAmCxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAmB1D;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAkC3E"}
+{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AAyBD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAWD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAsDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAmB1D;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAkC3E"}

package/dist/skill-test/eval-inputs.js

@@ -20,14 +20,23 @@ export class EvalInputError extends Error {
 const EVAL_ID_PATTERN = /^[A-Za-z0-9_-]+$/;
 /** Fields VAT recognizes per eval. Unknown fields are allowed (passthrough); a
  *  near-miss of one of these is flagged as a likely typo (see superRefine). */
-const RECOGNIZED_EVAL_FIELDS = ['id', 'prompt', 'expected_output', 'files', 'expectations'];
+const RECOGNIZED_EVAL_FIELDS = [
+    'id',
+    'prompt',
+    'expected_output',
+    'files',
+    'expectations',
+    'tier',
+    'toolExpectations',
+];
 /**
  * True when `key` is exactly one edit (insert/delete/substitute one char) from
- * `target`. Used only to catch typos of recognized fields — deliberately a tiny
- * single-edit check, not a general edit-distance routine, so it stays cheap and
- * never fires on legitimately distinct adopter keys (`name`, `category`, `notes`).
+ * `target`. Used to catch typos of recognized fields (here) and of declared
+ * executable names ({@link ./eval-lint.ts}) — deliberately a tiny single-edit
+ * check, not a general edit-distance routine, so it stays cheap and never fires
+ * on legitimately distinct adopter strings (`name`, `category`, `notes`).
  */
-function isSingleEditAway(key, target) {
+export function isSingleEditAway(key, target) {
     if (key === target)
         return false;
     const lk = key.length;
@@ -65,18 +74,37 @@ export const EvalEntrySchema = z
     // Optional: a human-readable success description. The pass/fail verdict is
     // always decided per `expectations` entry, so this is not load-bearing and
     // (per Postel's Law) is not required — real adopter suites (e.g. dxa-consumption)
-    // grade with `expectations` alone. When present, the experimenter prompt feeds it
-    // to the grader as prose CONTEXT informing judgment (see experimenter-prompt.ts).
+    // grade with `expectations` alone. When present, the grader prompt feeds it
+    // to the grader as prose CONTEXT informing judgment (see grader-prompt.ts).
     expected_output: z.string().min(1).optional(),
     files: z.array(z.string().min(1)).optional(),
     expectations: z.array(z.string().min(1)).min(1),
+    // Cost/foundational tier for fail-fast gating (Phase G): ascending, 0 = cheapest/first.
+    // Optional — adopters who don't opt into tiered execution omit it.
+    tier: z.number().int().nonnegative().optional(),
+    // Which tools SHOULD/SHOULD-NOT run, judged later by the grader against the
+    // transcript (Phase T). This sub-object's keys are VAT-defined (not adopter-owned
+    // like the entry itself), so it is `.strict()` — a typo here is a hard schema error,
+    // not silently passed through.
+    toolExpectations: z
+        .object({
+        mustRun: z.array(z.string().min(1)).optional(),
+        mustNotRun: z.array(z.string().min(1)).optional(),
+        // Each named executable must have RUN and its invoking tool_result must
+        // NOT be an error (feature #148). Judged from the transcript by the
+        // grader — see tool-eval-schema.ts's ToolSucceedCheckSchema.
+        mustSucceed: z.array(z.string().min(1)).optional(),
+        sequence: z.array(z.string().min(1)).optional(),
+    })
+        .strict()
+        .optional(),
 })
     .passthrough()
     .superRefine((entry, ctx) => {
     for (const key of Object.keys(entry)) {
         if (RECOGNIZED_EVAL_FIELDS.includes(key))
             continue;
-        const near = RECOGNIZED_EVAL_FIELDS.find((field) => isSingleEditAway(key.toLowerCase(), field));
+        const near = RECOGNIZED_EVAL_FIELDS.find((field) => isSingleEditAway(key.toLowerCase(), field.toLowerCase()));
         if (near !== undefined) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,

package/dist/skill-test/eval-inputs.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,CAAU,CAAC;AAErG;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,GAAW,EAAE,MAAc;IACnD,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,kFAAkF;AAClF,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,kFAAkF;IAClF,kFAAkF;IAClF,kFAAkF;IAClF,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAChD,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QAChG,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,OAAO,EAAE,uBAAuB,GAAG,qBAAqB,IAAI,kDAAkD;gBAC9G,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAKjB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,cAAc,CAAC,iCAAkC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAWD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpE,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC9B,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,sCAAuC,GAAa,CAAC,OAAO,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CAAC,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,yBAAyB,GAAG,EAAE,CAAC,CAAC;YACvG,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC9D,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,GAAG,yBAA0B,GAAa,CAAC,OAAO,EAAE,CACrG,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC"}
+{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,kFAAkF;AAClF,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,kFAAkF;IAClF,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,oEAAoE;QACpE,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,OAAO,EAAE,uBAAuB,GAAG,qBAAqB,IAAI,kDAAkD;gBAC9G,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,cAAc,CAAC,iCAAkC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAWD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpE,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC9B,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,sCAAuC,GAAa,CAAC,OAAO,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CAAC,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,yBAAyB,GAAG,EAAE,CAAC,CAAC;YACvG,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC9D,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,GAAG,yBAA0B,GAAa,CAAC,OAAO,EAAE,CACrG,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC"}

package/dist/skill-test/eval-lint.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Advisory (never-fatal) lint over an eval suite's `expectations` prose (issue #145
+ * follow-up). An adopter reported that WEAK expectations — pure "mentions X" /
+ * "includes Y" presence checks with no discriminating or negative counterpart —
+ * pass even for a hallucinated or right-for-the-wrong-reason answer, giving false
+ * confidence in a green run. This module flags that pattern so authors can
+ * strengthen the eval; it never blocks a run or changes an exit code.
+ *
+ * Deliberately conservative: this is a heuristic over free-text prose, not a
+ * grammar. When in doubt, do NOT warn — a false "your eval is weak" nag is worse
+ * than a missed one, since the grader (an LLM judging the transcript) is the real
+ * backstop for eval quality.
+ */
+import { type EvalEntry } from './eval-inputs.js';
+/** One advisory finding: `evalId` names the offending eval, `message` is the full advisory text. */
+export interface EvalLintWarning {
+    evalId: string;
+    message: string;
+}
+/**
+ * Lint a parsed eval suite's `expectations` for the weak-presence-only pattern.
+ * Flags an eval only when ALL of these hold (conservative — minimize false
+ * positives):
+ *   (a) it has at least one expectation;
+ *   (b) EVERY expectation string is presence-only (see {@link isPresenceOnlyExpectation});
+ *   (c) the eval declares no `toolExpectations` — a `mustRun`/`mustNotRun`/`sequence`
+ *       assertion is itself discriminating (it can fail a transcript that never
+ *       invoked the right tool), so its presence already covers the concern this
+ *       lint exists to catch.
+ *
+ * Pure and side-effect free — callers decide how/where to surface the warnings
+ * (this never throws and never affects exit code).
+ */
+export declare function lintEvalExpectations(evals: EvalEntry[]): EvalLintWarning[];
+/**
+ * Advisory (never-fatal) lint: flag a `toolExpectations` entry that names an
+ * executable which looks like a TYPO of one of the skill's `declaredExecutables`
+ * (issue #145 adopter follow-up). A name that never matches a real tool is a quiet
+ * footgun — `mustRun`/`mustSucceed`/`sequence` then fail for the wrong reason, and
+ * `mustNotRun` passes vacuously (the tool "never ran" only because the name is
+ * wrong). This is zero-token static plumbing, run before any spend.
+ *
+ * Conservative, like {@link lintEvalExpectations}: it only fires when there is a
+ * SPECIFIC declared name the reference is probably a typo of (so it can suggest
+ * "did you mean X?"). A reference that matches a declared name exactly, or that is
+ * not close to any declared name (a deliberate built-in/system tool reference), is
+ * never flagged. When the skill declares no executables there is nothing to compare
+ * against, so it returns no warnings. Pure + side-effect free.
+ */
+export declare function lintToolExpectationExecutables(evals: EvalEntry[], declaredExecutableNames: string[]): EvalLintWarning[];
+//# sourceMappingURL=eval-lint.d.ts.map

package/dist/skill-test/eval-lint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-lint.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAoB,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEpE,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAsBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,eAAe,EAAE,CAgB1E;AA0DD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,EAAE,EAClB,uBAAuB,EAAE,MAAM,EAAE,GAChC,eAAe,EAAE,CAuBnB"}