@lesliechan721/agent-skill-eval 0.1.0-beta.5 → 0.1.0-beta.50

package/dist/schemas/eval-spec.schema.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-skill-eval.invalid/schemas/eval-spec.schema.json",
+  "title": "EvalSpec",
+  "type": "object",
+  "required": ["version", "source", "workspace", "target", "cases", "provider", "runs", "timeoutMs", "baseline", "outputDir"],
+  "properties": {
+    "version": { "const": 1 },
+    "source": {
+      "type": "object",
+      "required": ["plugin", "skill", "suite", "profile"],
+      "properties": {
+        "plugin": { "type": "string", "minLength": 1 },
+        "skill": { "type": "string", "minLength": 1 },
+        "suite": { "type": "string", "minLength": 1 },
+        "profile": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    },
+    "workspace": { "$ref": "#/$defs/pathDigest" },
+    "target": { "$ref": "#/$defs/pathDigest" },
+    "cases": { "$ref": "#/$defs/pathDigest" },
+    "provider": {
+      "oneOf": [
+        { "enum": ["codex-trigger", "codex-conversation"] },
+        {
+          "type": "object",
+          "required": ["type", "command"],
+          "properties": {
+            "type": { "const": "custom-process" },
+            "command": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "type": "string", "minLength": 1 }
+            }
+          },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": ["type", "command", "allowedModels"],
+          "properties": {
+            "type": { "const": "codex-workflow" },
+            "command": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "type": "string", "minLength": 1 }
+            },
+            "allowedModels": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 },
+              "uniqueItems": true
+            }
+          },
+          "additionalProperties": false
+        }
+      ]
+    },
+    "runs": { "type": "integer", "minimum": 1 },
+    "timeoutMs": { "type": "integer", "minimum": 1 },
+    "baseline": {
+      "oneOf": [
+        {
+          "type": "object",
+          "required": ["type"],
+          "properties": { "type": { "const": "none" } },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": ["type"],
+          "properties": { "type": { "const": "without-skill" } },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": ["type", "target"],
+          "properties": {
+            "type": { "const": "previous-version" },
+            "target": { "$ref": "#/$defs/pathDigest" }
+          },
+          "additionalProperties": false
+        }
+      ]
+    },
+    "outputDir": { "$ref": "#/$defs/relativePath" }
+  },
+  "$defs": {
+    "digest": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "relativePath": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^(?![\\\\/])(?![A-Za-z]:[\\\\/])(?!.*(?:^|[\\\\/])\\.\\.(?:[\\\\/]|$)).+"
+    },
+    "pathDigest": {
+      "type": "object",
+      "required": ["path", "digest"],
+      "properties": {
+        "path": { "$ref": "#/$defs/relativePath" },
+        "digest": { "$ref": "#/$defs/digest" }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/dist/schemas/instruction-v2-receipt.schema.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://aw.dev/schemas/instruction-v2-receipt.schema.json",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["protocolVersion", "outcome", "instructionChannel", "instructionsRevision"],
+  "properties": {
+    "protocolVersion": { "const": 2 },
+    "outcome": { "enum": ["completed", "failed", "cancelled", "unknown"] },
+    "instructionChannel": { "enum": ["system", "developer", "equivalent-config", "user-prefix"] },
+    "instructionsRevision": { "type": "string", "minLength": 1, "maxLength": 256 },
+    "providerSessionId": { "type": "string", "minLength": 1, "maxLength": 4096 },
+    "actualModel": { "type": "string", "minLength": 1, "maxLength": 4096 },
+    "providerTerminal": { "type": "string", "minLength": 1, "maxLength": 4096 },
+    "traceFile": { "type": "string", "minLength": 1, "maxLength": 4096 },
+    "adapter": { "type": "object" },
+    "usage": {},
+    "generationStarted": { "type": "boolean" },
+    "localExitCode": { "type": ["integer", "null"], "minimum": 0 },
+    "code": { "type": "string", "minLength": 1, "maxLength": 256 },
+    "scopeViolation": { "type": "boolean" },
+    "remoteStopConfirmed": { "const": false }
+  }
+}

package/dist/schemas/instruction-v2-request.schema.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://aw.dev/schemas/instruction-v2-request.schema.json",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["protocolVersion", "run", "instructions", "task", "materials"],
+  "properties": {
+    "protocolVersion": { "const": 2 },
+    "run": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "mode", "access", "configRevision"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1, "maxLength": 256 },
+        "mode": { "enum": ["workspace", "bundle"] },
+        "access": { "enum": ["read-only", "workspace-write"] },
+        "configRevision": { "type": "string", "minLength": 1, "maxLength": 256 },
+        "model": { "type": "string", "minLength": 1, "maxLength": 256 },
+        "reasoningEffort": { "enum": ["minimal", "low", "medium", "high", "xhigh", "max"] },
+        "requiredIsolation": { "type": "boolean" },
+        "workspace": { "type": "string", "minLength": 1, "maxLength": 4096 }
+      },
+      "allOf": [{ "if": { "properties": { "mode": { "const": "bundle" } } }, "then": { "properties": { "access": { "const": "read-only" } } } }]
+    },
+    "instructions": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["workerKey", "revision", "text"],
+      "properties": {
+        "workerKey": { "type": "string", "minLength": 1, "maxLength": 256 },
+        "revision": { "type": "string", "pattern": "^(sha256:)?[a-f0-9]{64}$" },
+        "text": { "type": "string", "minLength": 1, "maxLength": 200000 }
+      }
+    },
+    "task": { "type": "string", "minLength": 1, "maxLength": 200000 },
+    "materials": {
+      "type": "array", "maxItems": 128,
+      "items": {
+        "type": "object", "additionalProperties": false,
+        "required": ["source", "path", "sha256", "startLine", "endLine"],
+        "properties": {
+          "source": { "type": "string", "minLength": 1, "maxLength": 4096 },
+          "path": { "type": "string", "minLength": 1, "maxLength": 4096 },
+          "sha256": { "type": "string", "pattern": "^(sha256:)?[a-f0-9]{64}$" },
+          "startLine": { "type": "integer", "minimum": 1 },
+          "endLine": { "type": "integer", "minimum": 1 }
+        }
+      }
+    }
+  }
+}

package/dist/schemas/workflow-evaluation.schema.json

@@ -0,0 +1,38 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://aw.dev/schemas/workflow-evaluation.schema.json",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["artifacts", "checks", "interactions"],
+  "properties": {
+    "artifacts": { "type": "array", "maxItems": 64, "uniqueItems": true, "items": { "$ref": "#/$defs/path" } },
+    "checks": { "type": "array", "minItems": 1, "maxItems": 64, "items": {
+      "type": "object", "additionalProperties": false, "required": ["name", "criterion"],
+      "properties": { "name": { "type": "string", "minLength": 1, "maxLength": 200 }, "criterion": { "type": "string", "minLength": 1, "maxLength": 20000 } }
+    } },
+    "interactions": { "type": "array", "maxItems": 8, "items": {
+      "type": "object", "additionalProperties": false, "required": ["id", "when", "answer", "protectedPaths"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1, "maxLength": 200 },
+        "when": { "type": "string", "minLength": 1, "maxLength": 20000 },
+        "answer": { "type": "string", "minLength": 1, "maxLength": 20000 },
+        "protectedPaths": { "type": "array", "maxItems": 64, "items": { "$ref": "#/$defs/path" } },
+        "action": { "$ref": "#/$defs/action" }
+      }
+    } },
+    "setupPrompt": { "type": "string", "minLength": 1, "maxLength": 20000 },
+    "setupActions": { "type": "array", "maxItems": 8, "items": { "$ref": "#/$defs/action" } },
+    "finalActions": { "type": "array", "maxItems": 8, "items": { "$ref": "#/$defs/action" } }
+  },
+  "$defs": {
+    "path": { "type": "string", "minLength": 1, "maxLength": 1024, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*\\\\)(?!.*//).+$" },
+    "action": {
+      "type": "object", "additionalProperties": false, "required": ["command", "input"],
+      "properties": {
+        "command": { "type": "array", "minItems": 2, "maxItems": 2, "prefixItems": [{ "const": "node" }, { "$ref": "#/$defs/path" }] },
+        "input": {},
+        "pathPrepend": { "type": "array", "maxItems": 8, "uniqueItems": true, "items": { "$ref": "#/$defs/path" } }
+      }
+    }
+  }
+}

package/dist/schemas/workflow-fixture-action.schema.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://aw.dev/schemas/workflow-fixture-action.schema.json",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["command", "input"],
+  "properties": {
+    "command": {
+      "type": "array", "minItems": 2, "maxItems": 2,
+      "prefixItems": [{ "const": "node" }, { "type": "string", "minLength": 1, "maxLength": 1024, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*\\\\)(?!.*//).+$" }]
+    },
+    "input": {},
+    "pathPrepend": { "type": "array", "maxItems": 8, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 1024, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*\\\\)(?!.*//).+$" } }
+  }
+}

package/dist/schemas.d.ts

@@ -1,5 +1,3 @@
-import type { Digest, RuntimeCapabilities, SchemaName, ValidationResult } from './types.js';
-export declare function isSchemaName(value: string): value is SchemaName;
+import type { Digest, SchemaName } from './types.js';
 export declare function getSchema(name: SchemaName): Promise<Record<string, unknown>>;
 export declare function getSchemaDigest(name: SchemaName): Promise<Digest>;
-export declare function validateRuntimeCapabilities(value: unknown): Promise<ValidationResult<RuntimeCapabilities>>;

package/dist/schemas.js

@@ -1,27 +1,29 @@
+import { createHash } from 'node:crypto';
 import { readFile } from 'node:fs/promises';
-import { validateJsonSchema } from './json-schema.js';
-const schemaNames = new Set([
-    'aw-skill-eval-execution-plan',
-    'eval-run-bundle',
-    'eval-runtime-capabilities',
-]);
-export function isSchemaName(value) {
-    return schemaNames.has(value);
-}
 export async function getSchema(name) {
-    const source = new URL(`./schemas/${name}.schema.json`, import.meta.url);
-    return JSON.parse(await readFile(source, 'utf8'));
+    return JSON.parse((await readSchema(name)).toString('utf8'));
 }
 export async function getSchemaDigest(name) {
-    const digests = JSON.parse(await readFile(new URL('./schemas/digests.json', import.meta.url), 'utf8'));
-    return digests[name];
+    try {
+        const digests = JSON.parse(await readFile(new URL('./schemas/digests.json', import.meta.url), 'utf8'));
+        return digests[name];
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+    }
+    const schema = JSON.parse((await readSchema(name)).toString('utf8'));
+    return `sha256:${createHash('sha256').update(JSON.stringify(schema)).digest('hex')}`;
 }
-export async function validateRuntimeCapabilities(value) {
-    const errors = validateJsonSchema(await getSchema('eval-runtime-capabilities'), value);
-    return {
-        valid: errors.length === 0,
-        errors,
-        calculated: errors.length === 0 ? value : null,
-    };
+async function readSchema(name) {
+    const embedded = new URL(`./schemas/${name}.schema.json`, import.meta.url);
+    try {
+        return await readFile(embedded);
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+        return readFile(new URL(`../schemas/${name}.schema.json`, import.meta.url));
+    }
 }
 //# sourceMappingURL=schemas.js.map

package/dist/schemas.js.map

@@ -1 +1 @@
-{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAQrD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAa;IACtC,8BAA8B;IAC9B,iBAAiB;IACjB,2BAA2B;CAC5B,CAAC,CAAA;AAEF,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,WAAW,CAAC,GAAG,CAAC,KAAmB,CAAC,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,aAAa,IAAI,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAA4B,CAAA;AAC9E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAgB;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,MAAM,QAAQ,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAC7C,CAAA;IAC/B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,KAAc;IAEd,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,SAAS,CAAC,2BAA2B,CAAC,EAAE,KAAK,CAAC,CAAA;IACtF,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;QACN,UAAU,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAA4B,CAAC,CAAC,CAAC,IAAI;KACtE,CAAA;AACH,CAAC"}
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAM3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAA4B,CAAA;AACzF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAgB;IACpD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,MAAM,QAAQ,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAC7C,CAAA;QAC/B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAA;IACrE,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IACpE,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AACtF,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAgB;IACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,aAAa,IAAI,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC1E,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,KAAK,CAAA;QACnE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,cAAc,IAAI,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7E,CAAC;AACH,CAAC"}

package/dist/selected-cases.d.ts

@@ -0,0 +1,30 @@
+import { type AwSkillEvalTriggerCase, type AwSkillEvalTriggerSuiteCaseContract } from './codex-eval-cases.js';
+import type { ValidationResult } from './types.js';
+export type SelectedCaseSuiteKind = 'trigger' | 'conversation' | 'custom-process' | 'codex-workflow';
+export interface SelectedCaseSuite {
+    kind: SelectedCaseSuiteKind;
+    unitUnderTest: string;
+    participants?: string[];
+    policy?: {
+        optimization?: string;
+        falseNegativeCost?: string;
+        falsePositiveCost?: string;
+        rationale?: string;
+    };
+    runtime?: Record<string, unknown>;
+    readonly providerSuite?: AwSkillEvalTriggerSuiteCaseContract;
+}
+export interface SelectedEvalCase extends Record<string, unknown> {
+    id: string;
+    split: string;
+    tags: string[];
+    readonly providerCase?: AwSkillEvalTriggerCase;
+}
+export interface SelectedCasesDocument {
+    version: 1;
+    suite: SelectedCaseSuite;
+    cases: SelectedEvalCase[];
+}
+export declare function parseSelectedCases(value: unknown): SelectedCasesDocument;
+export declare function validateSelectedCases(value: unknown): Promise<ValidationResult<SelectedCasesDocument>>;
+export declare function validateSelectedCasesFile(casesPath: string): Promise<ValidationResult<SelectedCasesDocument>>;