@bjoernrochel/sevro 1.0.0 → 1.1.0

package/dist/src/evaluation/run.js

@@ -0,0 +1,148 @@
+import { prepareEvaluation } from "./plan.js";
+import { canonicalJson, hashJson } from "../identity.js";
+import { exitCodeFor, summarizeCases } from "../results.js";
+import { runtimePolicySnapshot } from "../runtime/config.js";
+import { cleanupExpiredWorkspaces } from "../fixtures/workspace-state.js";
+import { assertCliResult, assertRunEvidence } from "../schema.js";
+import { atomicWriteJson } from "../storage.js";
+import { allocateRunResources, retainPreparationArtifacts, } from "./resources.js";
+import { ResultCollector } from "./collection.js";
+import { executeEvaluationTrials } from "./trial.js";
+import { evaluationProgress } from "./progress.js";
+function evaluationSignal(options) {
+    const reason = options.signal?.aborted
+        ? options.signal.reason
+        : null;
+    return reason === "SIGINT" || reason === "SIGTERM" ? reason : undefined;
+}
+function finalCliResult(options, resources, collected) {
+    const caseResult = collected.caseSummary;
+    const assessment = summarizeCases([caseResult]);
+    const diagnostic = collected.diagnostic;
+    return {
+        format: "sevro.cli-result.v1",
+        runId: resources.runId,
+        ...assessment,
+        exitCode: exitCodeFor(assessment, {
+            dryRun: options.dry,
+            signal: evaluationSignal(options),
+        }),
+        evidencePath: resources.evidencePath,
+        cases: [caseResult],
+        ...(diagnostic ? { diagnostic } : {}),
+    };
+}
+function finalConditionEvidence(options, requestedInstrumentation, trials) {
+    const actualConditions = [
+        ...new Set(trials.map((entry) => entry.condition.actual)),
+    ];
+    const commonAppliedInstrumentation = trials.length === options.trialCount &&
+        trials.every((entry) => canonicalJson(entry.condition.appliedInstrumentation) ===
+            canonicalJson(requestedInstrumentation))
+        ? requestedInstrumentation
+        : [];
+    return {
+        requested: options.condition,
+        actual: actualConditions.length === 1 ? actualConditions[0] : "unknown",
+        requestedInstrumentation,
+        appliedInstrumentation: commonAppliedInstrumentation,
+    };
+}
+function finalExtensionEvidence(options, replacedBuiltinGraders) {
+    const extension = options.extension;
+    if (!extension)
+        return null;
+    const identity = extension.session.identity;
+    return {
+        id: identity.id,
+        version: identity.version,
+        sourceDigest: identity.sourceDigest,
+        configurationDigest: identity.configurationDigest,
+        protocol: identity.protocol,
+        capabilities: identity.capabilities,
+        replacements: {
+            graders: replacedBuiltinGraders,
+            taskVerdictPolicy: identity.selectedTaskVerdictPolicy,
+        },
+    };
+}
+function finalRunEvidence(options, prepared, resources, collected, result) {
+    const condition = finalConditionEvidence(options, prepared.requestedInstrumentation, collected.trials);
+    const diagnostic = collected.diagnostic;
+    return {
+        format: "sevro.run-evidence.v1",
+        runId: resources.runId,
+        evaluationIdentity: prepared.evaluationIdentity,
+        configuration: {
+            digest: hashJson(prepared.redactedConfig),
+            redacted: prepared.redactedConfig,
+            ...(options.resolvedRunSettings
+                ? { resolved: options.resolvedRunSettings }
+                : {}),
+        },
+        runner: prepared.runner,
+        project: prepared.project,
+        extension: finalExtensionEvidence(options, prepared.checks.replacedBuiltinGraders),
+        condition,
+        graders: {
+            active: prepared.activeGraders,
+            replacedDefaults: prepared.checks.replacedBuiltinGraders,
+        },
+        routes: prepared.routes,
+        result,
+        trials: collected.trials,
+        ...(diagnostic ? { diagnostic } : {}),
+    };
+}
+async function finishEvaluationRun(options, context) {
+    const collected = context.results.snapshot({
+        caseId: options.case.id,
+        passThreshold: options.passThreshold,
+        cancelled: Boolean(options.signal?.aborted),
+    });
+    const result = finalCliResult(options, context.resources, collected);
+    const evidence = finalRunEvidence(options, context.prepared, context.resources, collected, result);
+    assertCliResult(result);
+    assertRunEvidence(evidence);
+    evaluationProgress(options, "evidence", "started");
+    await atomicWriteJson(context.resources.evidencePath, evidence);
+    context.results.checkpoint(options.signal?.aborted ? "interrupted" : "complete");
+    evaluationProgress(options, "evidence", "completed");
+    return { result };
+}
+/** Execute resolved evaluator data through an injected host and retain every trial. */
+export async function runEvaluation(options) {
+    options = {
+        ...options,
+        runtimePolicy: runtimePolicySnapshot(options.runtimePolicy),
+    };
+    evaluationProgress(options, "preparation", "started");
+    await cleanupExpiredWorkspaces();
+    const prepared = await prepareEvaluation(options);
+    const resources = await allocateRunResources(options, prepared.evaluationIdentity.digest);
+    const results = new ResultCollector(resources.owner);
+    try {
+        const preparationArtifacts = await retainPreparationArtifacts(prepared.fixture.inlineArtifacts, resources.runDir);
+        await resources.workspaces.reserve({
+            count: options.trialCount,
+            semantic: prepared.checks.preparedSemantic.length > 0,
+            signal: options.signal,
+        });
+        const context = {
+            prepared,
+            resources,
+            results,
+            preparationArtifacts,
+        };
+        await executeEvaluationTrials(options, context);
+        return await finishEvaluationRun(options, context);
+    }
+    catch (error) {
+        results.diagnoseFailure();
+        throw error;
+    }
+    finally {
+        await resources.workspaces.cleanup();
+    }
+}
+//# sourceMappingURL=run.js.map

package/dist/src/evaluation/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAA2B,kBAAe;AAEpE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,uBAAoB;AACtD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,sBAAmB;AACzD,OAAO,EAAE,qBAAqB,EAAE,6BAA0B;AAC1D,OAAO,EAAE,wBAAwB,EAAE,uCAAoC;AACvE,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,qBAAkB;AAC/D,OAAO,EAAE,eAAe,EAAE,sBAAmB;AAC7C,OAAO,EACL,oBAAoB,EACpB,0BAA0B,GAE3B,uBAAoB;AAErB,OAAO,EAAE,eAAe,EAAyB,wBAAqB;AACtE,OAAO,EAAE,uBAAuB,EAAE,mBAAgB;AAClD,OAAO,EAAE,kBAAkB,EAAE,sBAAmB;AAEhD,SAAS,gBAAgB,CACvB,OAA0B;IAE1B,MAAM,MAAM,GAAY,OAAO,CAAC,MAAM,EAAE,OAAO;QAC7C,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM;QACvB,CAAC,CAAC,IAAI,CAAC;IACT,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,CAAC;AAED,SAAS,cAAc,CACrB,OAA0B,EAC1B,SAAuB,EACvB,SAA2B;IAE3B,MAAM,UAAU,GAAG,SAAS,CAAC,WAAW,CAAC;IACzC,MAAM,UAAU,GAAG,cAAc,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;IACxC,OAAO;QACL,MAAM,EAAE,qBAAqB;QAC7B,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,GAAG,UAAU;QACb,QAAQ,EAAE,WAAW,CAAC,UAAU,EAAE;YAChC,MAAM,EAAE,OAAO,CAAC,GAAG;YACnB,MAAM,EAAE,gBAAgB,CAAC,OAAO,CAAC;SAClC,CAAC;QACF,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,KAAK,EAAE,CAAC,UAAU,CAAC;QACnB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA4D,EAC5D,wBAAwE,EACxE,MAAkC;IAElC,MAAM,gBAAgB,GAAG;QACvB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;KAC1D,CAAC;IACF,MAAM,4BAA4B,GAChC,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,UAAU;QACpC,MAAM,CAAC,KAAK,CACV,CAAC,KAAK,EAAE,EAAE,CACR,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,sBAAsB,CAAC;YACrD,aAAa,CAAC,wBAAwB,CAAC,CAC1C;QACC,CAAC,CAAC,wBAAwB;QAC1B,CAAC,CAAC,EAAE,CAAC;IACT,OAAO;QACL,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,gBAAgB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACvE,wBAAwB;QACxB,sBAAsB,EAAE,4BAA4B;KACrD,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA0B,EAC1B,sBAAgC;IAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC5C,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;QACjD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,YAAY,EAAE;YACZ,OAAO,EAAE,sBAAsB;YAC/B,iBAAiB,EAAE,QAAQ,CAAC,yBAAyB;SACtD;KACF,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,OAA0B,EAC1B,QAA4B,EAC5B,SAAuB,EACvB,SAA2B,EAC3B,MAAiB;IAEjB,MAAM,SAAS,GAAG,sBAAsB,CACtC,OAAO,EACP,QAAQ,CAAC,wBAAwB,EACjC,SAAS,CAAC,MAAM,CACjB,CAAC;IACF,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC;IACxC,OAAO;QACL,MAAM,EAAE,uBAAuB;QAC/B,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;QAC/C,aAAa,EAAE;YACb,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC;YACzC,QAAQ,EAAE,QAAQ,CAAC,cAAc;YACjC,GAAG,CAAC,OAAO,CAAC,mBAAmB;gBAC7B,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,mBAAmB,EAAE;gBAC3C,CAAC,CAAC,EAAE,CAAC;SACR;QACD,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,SAAS,EAAE,sBAAsB,CAC/B,OAAO,EACP,QAAQ,CAAC,MAAM,CAAC,sBAAsB,CACvC;QACD,SAAS;QACT,OAAO,EAAE;YACP,MAAM,EAAE,QAAQ,CAAC,aAAa;YAC9B,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,sBAAsB;SACzD;QACD,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,MAAM;QACN,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,OAA0B,EAC1B,OAA0B;IAE1B,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;QACzC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE;QACvB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;KAC5C,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,gBAAgB,CAC/B,OAAO,EACP,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,SAAS,EACjB,SAAS,EACT,MAAM,CACP,CAAC;IACF,eAAe,CAAC,MAAM,CAAC,CAAC;IACxB,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC5B,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAChE,OAAO,CAAC,OAAO,CAAC,UAAU,CACxB,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CACrD,CAAC;IACF,kBAAkB,CAAC,OAAO,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IACrD,OAAO,EAAE,MAAM,EAAE,CAAC;AACpB,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAA0B;IAE1B,OAAO,GAAG;QACR,GAAG,OAAO;QACV,aAAa,EAAE,qBAAqB,CAAC,OAAO,CAAC,aAAa,CAAC;KAC5D,CAAC;IACF,kBAAkB,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;IACtD,MAAM,wBAAwB,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAC1C,OAAO,EACP,QAAQ,CAAC,kBAAkB,CAAC,MAAM,CACnC,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,oBAAoB,GAAG,MAAM,0BAA0B,CAC3D,QAAQ,CAAC,OAAO,CAAC,eAAe,EAChC,SAAS,CAAC,MAAM,CACjB,CAAC;QACF,MAAM,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC;YACjC,KAAK,EAAE,OAAO,CAAC,UAAU;YACzB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;YACrD,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,MAAM,OAAO,GAAsB;YACjC,QAAQ;YACR,SAAS;YACT,OAAO;YACP,oBAAoB;SACrB,CAAC;QACF,MAAM,uBAAuB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,MAAM,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,eAAe,EAAE,CAAC;QAC1B,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IACvC,CAAC;AACH,CAAC","sourcesContent":["import { prepareEvaluation, type PreparedEvaluation } from \"./plan\";\nimport { type CliResult, type EvaluationOptions } from \"./types\";\nimport { canonicalJson, hashJson } from \"../identity\";\nimport { exitCodeFor, summarizeCases } from \"../results\";\nimport { runtimePolicySnapshot } from \"../runtime/config\";\nimport { cleanupExpiredWorkspaces } from \"../fixtures/workspace-state\";\nimport { assertCliResult, assertRunEvidence } from \"../schema\";\nimport { atomicWriteJson } from \"../storage\";\nimport {\n  allocateRunResources,\n  retainPreparationArtifacts,\n  type RunResources,\n} from \"./resources\";\nimport type { EvaluationContext } from \"./context\";\nimport { ResultCollector, type CollectedResults } from \"./collection\";\nimport { executeEvaluationTrials } from \"./trial\";\nimport { evaluationProgress } from \"./progress\";\n\nfunction evaluationSignal(\n  options: EvaluationOptions,\n): \"SIGINT\" | \"SIGTERM\" | undefined {\n  const reason: unknown = options.signal?.aborted\n    ? options.signal.reason\n    : null;\n  return reason === \"SIGINT\" || reason === \"SIGTERM\" ? reason : undefined;\n}\n\nfunction finalCliResult(\n  options: EvaluationOptions,\n  resources: RunResources,\n  collected: CollectedResults,\n): CliResult {\n  const caseResult = collected.caseSummary;\n  const assessment = summarizeCases([caseResult]);\n  const diagnostic = collected.diagnostic;\n  return {\n    format: \"sevro.cli-result.v1\",\n    runId: resources.runId,\n    ...assessment,\n    exitCode: exitCodeFor(assessment, {\n      dryRun: options.dry,\n      signal: evaluationSignal(options),\n    }),\n    evidencePath: resources.evidencePath,\n    cases: [caseResult],\n    ...(diagnostic ? { diagnostic } : {}),\n  };\n}\n\nfunction finalConditionEvidence(\n  options: Pick<EvaluationOptions, \"condition\" | \"trialCount\">,\n  requestedInstrumentation: PreparedEvaluation[\"requestedInstrumentation\"],\n  trials: CollectedResults[\"trials\"],\n) {\n  const actualConditions = [\n    ...new Set(trials.map((entry) => entry.condition.actual)),\n  ];\n  const commonAppliedInstrumentation =\n    trials.length === options.trialCount &&\n    trials.every(\n      (entry) =>\n        canonicalJson(entry.condition.appliedInstrumentation) ===\n        canonicalJson(requestedInstrumentation),\n    )\n      ? requestedInstrumentation\n      : [];\n  return {\n    requested: options.condition,\n    actual: actualConditions.length === 1 ? actualConditions[0] : \"unknown\",\n    requestedInstrumentation,\n    appliedInstrumentation: commonAppliedInstrumentation,\n  };\n}\n\nfunction finalExtensionEvidence(\n  options: EvaluationOptions,\n  replacedBuiltinGraders: string[],\n) {\n  const extension = options.extension;\n  if (!extension) return null;\n  const identity = extension.session.identity;\n  return {\n    id: identity.id,\n    version: identity.version,\n    sourceDigest: identity.sourceDigest,\n    configurationDigest: identity.configurationDigest,\n    protocol: identity.protocol,\n    capabilities: identity.capabilities,\n    replacements: {\n      graders: replacedBuiltinGraders,\n      taskVerdictPolicy: identity.selectedTaskVerdictPolicy,\n    },\n  };\n}\n\nfunction finalRunEvidence(\n  options: EvaluationOptions,\n  prepared: PreparedEvaluation,\n  resources: RunResources,\n  collected: CollectedResults,\n  result: CliResult,\n) {\n  const condition = finalConditionEvidence(\n    options,\n    prepared.requestedInstrumentation,\n    collected.trials,\n  );\n  const diagnostic = collected.diagnostic;\n  return {\n    format: \"sevro.run-evidence.v1\",\n    runId: resources.runId,\n    evaluationIdentity: prepared.evaluationIdentity,\n    configuration: {\n      digest: hashJson(prepared.redactedConfig),\n      redacted: prepared.redactedConfig,\n      ...(options.resolvedRunSettings\n        ? { resolved: options.resolvedRunSettings }\n        : {}),\n    },\n    runner: prepared.runner,\n    project: prepared.project,\n    extension: finalExtensionEvidence(\n      options,\n      prepared.checks.replacedBuiltinGraders,\n    ),\n    condition,\n    graders: {\n      active: prepared.activeGraders,\n      replacedDefaults: prepared.checks.replacedBuiltinGraders,\n    },\n    routes: prepared.routes,\n    result,\n    trials: collected.trials,\n    ...(diagnostic ? { diagnostic } : {}),\n  };\n}\n\nasync function finishEvaluationRun(\n  options: EvaluationOptions,\n  context: EvaluationContext,\n) {\n  const collected = context.results.snapshot({\n    caseId: options.case.id,\n    passThreshold: options.passThreshold,\n    cancelled: Boolean(options.signal?.aborted),\n  });\n  const result = finalCliResult(options, context.resources, collected);\n  const evidence = finalRunEvidence(\n    options,\n    context.prepared,\n    context.resources,\n    collected,\n    result,\n  );\n  assertCliResult(result);\n  assertRunEvidence(evidence);\n  evaluationProgress(options, \"evidence\", \"started\");\n  await atomicWriteJson(context.resources.evidencePath, evidence);\n  context.results.checkpoint(\n    options.signal?.aborted ? \"interrupted\" : \"complete\",\n  );\n  evaluationProgress(options, \"evidence\", \"completed\");\n  return { result };\n}\n\n/** Execute resolved evaluator data through an injected host and retain every trial. */\nexport async function runEvaluation(\n  options: EvaluationOptions,\n): Promise<{ result: CliResult }> {\n  options = {\n    ...options,\n    runtimePolicy: runtimePolicySnapshot(options.runtimePolicy),\n  };\n  evaluationProgress(options, \"preparation\", \"started\");\n  await cleanupExpiredWorkspaces();\n  const prepared = await prepareEvaluation(options);\n  const resources = await allocateRunResources(\n    options,\n    prepared.evaluationIdentity.digest,\n  );\n  const results = new ResultCollector(resources.owner);\n  try {\n    const preparationArtifacts = await retainPreparationArtifacts(\n      prepared.fixture.inlineArtifacts,\n      resources.runDir,\n    );\n    await resources.workspaces.reserve({\n      count: options.trialCount,\n      semantic: prepared.checks.preparedSemantic.length > 0,\n      signal: options.signal,\n    });\n    const context: EvaluationContext = {\n      prepared,\n      resources,\n      results,\n      preparationArtifacts,\n    };\n    await executeEvaluationTrials(options, context);\n    return await finishEvaluationRun(options, context);\n  } catch (error) {\n    results.diagnoseFailure();\n    throw error;\n  } finally {\n    await resources.workspaces.cleanup();\n  }\n}\n"]}

package/dist/src/evaluation/scheduler.js

@@ -0,0 +1,27 @@
+/** Bound trial admission, stop on errors/cancellation, and drain active work. */
+export async function scheduleTrials(options, run) {
+    let nextTrial = 1;
+    let stopped = false;
+    const stopAdmission = () => {
+        stopped = true;
+    };
+    let failure;
+    const admitting = () => !stopped && !options.signal?.aborted && nextTrial <= options.count;
+    const worker = async () => {
+        while (admitting()) {
+            const trial = nextTrial++;
+            try {
+                if (!(await run(trial, stopAdmission)))
+                    stopAdmission();
+            }
+            catch (error) {
+                stopped = true;
+                failure ??= { error };
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(options.jobs, options.count) }, worker));
+    if (failure)
+        throw failure.error;
+}
+//# sourceMappingURL=scheduler.js.map

package/dist/src/evaluation/scheduler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["scheduler.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA8D,EAC9D,GAAmE;IAEnE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,aAAa,GAAG,GAAG,EAAE;QACzB,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC,CAAC;IACF,IAAI,OAAuC,CAAC;IAC5C,MAAM,SAAS,GAAG,GAAG,EAAE,CACrB,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC;IACrE,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,SAAS,EAAE,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;oBAAE,aAAa,EAAE,CAAC;YAC1D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,KAAK,EAAE,KAAK,EAAE,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CACtE,CAAC;IACF,IAAI,OAAO;QAAE,MAAM,OAAO,CAAC,KAAK,CAAC;AACnC,CAAC","sourcesContent":["/** Bound trial admission, stop on errors/cancellation, and drain active work. */\nexport async function scheduleTrials(\n  options: { count: number; jobs: number; signal?: AbortSignal },\n  run: (trial: number, stopAdmission: () => void) => Promise<boolean>,\n): Promise<void> {\n  let nextTrial = 1;\n  let stopped = false;\n  const stopAdmission = () => {\n    stopped = true;\n  };\n  let failure: { error: unknown } | undefined;\n  const admitting = () =>\n    !stopped && !options.signal?.aborted && nextTrial <= options.count;\n  const worker = async () => {\n    while (admitting()) {\n      const trial = nextTrial++;\n      try {\n        if (!(await run(trial, stopAdmission))) stopAdmission();\n      } catch (error) {\n        stopped = true;\n        failure ??= { error };\n      }\n    }\n  };\n  await Promise.all(\n    Array.from({ length: Math.min(options.jobs, options.count) }, worker),\n  );\n  if (failure) throw failure.error;\n}\n"]}

package/dist/src/evaluation/semantic.js

@@ -0,0 +1,183 @@
+import { clearFixtureContents } from "../fixtures/cleanup.js";
+import { createFixture } from "../fixtures/create.js";
+import { usage } from "../evidence/usage.js";
+import { gradingRuntimePolicy, scopedHostArtifacts } from "./grading-host.js";
+import { HostExecutionFailure, } from "./types.js";
+import { readSemanticArtifact } from "../graders/artifact.js";
+import { parseSemanticVerdicts, semanticCheckGroups, semanticPrompt, } from "../graders/semantic.js";
+import { sha256 } from "../identity.js";
+import { HostDiagnosticPersistenceError } from "../host-diagnostics.js";
+function semanticGroupSuffix(pattern) {
+    return pattern === undefined ? "" : "." + sha256(pattern);
+}
+function semanticSource(artifact) {
+    return artifact
+        ? {
+            kind: "artifact",
+            path: artifact.path,
+            sha256: sha256(artifact.content),
+        }
+        : { kind: "response" };
+}
+function semanticHostPrompt(message, checks, artifact) {
+    return semanticPrompt(artifact?.content ?? message, checks, artifact ? "document" : "response");
+}
+class SemanticGrading {
+    input;
+    effects;
+    observations = [];
+    checks = [];
+    failure = null;
+    constructor(input, effects) {
+        this.input = input;
+        this.effects = effects;
+    }
+    async run() {
+        for (const group of semanticCheckGroups([...this.input.checks]))
+            await this.gradeSemanticGroup(group);
+        return {
+            checks: this.checks,
+            observations: this.observations,
+            failure: this.failure,
+        };
+    }
+    failGrading(message, cancelable = true) {
+        this.failure = { message, cancelable };
+        this.effects.onFailure(this.failure);
+    }
+    async semanticInput(pattern) {
+        return pattern === undefined
+            ? null
+            : readSemanticArtifact(this.input.candidateWorkspace, pattern);
+    }
+    async semanticWorkspace() {
+        return createFixture({ kind: "inline", files: {} }, {
+            projectRoot: this.input.projectRoot,
+            signal: this.input.signal,
+            reservedWorkspace: this.input.reservedWorkspace,
+        });
+    }
+    async runSemanticHost(group, pattern, suffix, workspace, message) {
+        let source = { kind: "response" };
+        try {
+            const artifact = await this.semanticInput(pattern);
+            source = semanticSource(artifact);
+            const host = this.input.host;
+            if (!host)
+                throw new Error("semantic host is unavailable");
+            const response = await host
+                .run({
+                retainHostDiagnostics: (bytes) => this.effects.evidence.retainArtifacts([
+                    {
+                        id: `sevro.semantic${suffix}.sevro.host.diagnostics`,
+                        bytes: Buffer.from(bytes),
+                    },
+                ]),
+                candidateTranscriptRoot: this.input.candidateTranscriptRoot,
+                candidateArtifactRoots: this.input.candidateArtifactRoots,
+                prompt: semanticHostPrompt(message, group, artifact),
+                runtimePolicy: gradingRuntimePolicy(this.input.runtimePolicy, pattern === undefined),
+                runtimeRole: "semantic",
+                workspace,
+                condition: "passive",
+                signal: this.input.signal,
+            })
+                .catch((cause) => {
+                if (cause instanceof HostExecutionFailure)
+                    return cause.result;
+                throw cause;
+            });
+            const artifacts = scopedHostArtifacts(response, `sevro.semantic${suffix}.`, this.effects.evidence.artifactRefs, this.input.declaredChecks, `sevro.semantic.verdicts${suffix}`);
+            return { result: response, artifacts, source, hostId: host.id };
+        }
+        catch (cause) {
+            if (cause instanceof HostDiagnosticPersistenceError)
+                throw cause;
+            this.failGrading("semantic grading did not complete");
+            return { result: null, artifacts: [], source, hostId: null };
+        }
+    }
+    recordSemanticVerdict(verdict, pattern, source, hostId) {
+        const id = `sevro.observation.semantic.${sha256(verdict.id)}`;
+        this.observations.push({
+            id,
+            source: hostId,
+            completeness: "complete",
+            data: {
+                verdict: verdict.verdict,
+                reason: verdict.reason,
+                ...(pattern === undefined ? {} : { source }),
+            },
+        });
+        this.checks.push({
+            id: verdict.id,
+            grader: "sevro.semantic",
+            status: verdict.verdict === "pass" ? "passed" : "failed",
+            detail: verdict.reason,
+            evidenceRefs: [id],
+        });
+    }
+    parseSemanticResults(result, group, pattern, source, hostId) {
+        try {
+            const message = this.completeSemanticGraderMessage(result);
+            const verdicts = parseSemanticVerdicts(message, group);
+            for (const verdict of verdicts)
+                this.recordSemanticVerdict(verdict, pattern, source, hostId);
+            return true;
+        }
+        catch {
+            this.failGrading(result.failureMessage ?? "semantic grading did not complete", result.processFailure?.reason === "cancelled");
+            return false;
+        }
+    }
+    completeSemanticGraderMessage(result) {
+        if (result.executionFailed || !result.complete || !result.finalMessage)
+            throw new Error("semantic grader result did not complete");
+        return result.finalMessage;
+    }
+    async gradeSemanticGroup(group) {
+        const pattern = group[0].artifactPath;
+        const suffix = semanticGroupSuffix(pattern);
+        if (this.failure)
+            return;
+        const message = this.input.message;
+        if (message === null) {
+            this.checks.push(...group.map((check) => ({
+                id: check.id,
+                grader: "sevro.semantic",
+                status: "unavailable",
+                evidenceRefs: [],
+            })));
+            return;
+        }
+        const workspace = await this.semanticWorkspace();
+        let successful = false;
+        try {
+            const outcome = await this.runSemanticHost(group, pattern, suffix, workspace, message);
+            if (outcome.result === null)
+                return;
+            await this.effects.evidence.retainArtifacts(outcome.artifacts);
+            this.observations.push({
+                id: `sevro.observation.semantic.usage${suffix}`,
+                source: outcome.hostId,
+                completeness: outcome.result.usageComplete ? "complete" : "partial",
+                data: usage(outcome.result),
+            });
+            await this.effects.evidence.retainSemanticVerdicts(outcome.result.finalMessage, suffix);
+            successful = this.parseSemanticResults(outcome.result, group, pattern, outcome.source, outcome.hostId);
+        }
+        finally {
+            await this.finishWorkspace(workspace, successful);
+        }
+    }
+    async finishWorkspace(workspace, successful) {
+        if (successful)
+            await clearFixtureContents(workspace);
+        else
+            await this.effects.retainWorkspace(workspace);
+    }
+}
+export async function gradeSemanticChecks(input, effects) {
+    return new SemanticGrading(input, effects).run();
+}
+//# sourceMappingURL=semantic.js.map

package/dist/src/evaluation/semantic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic.js","sourceRoot":"","sources":["semantic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,+BAA4B;AAE3D,OAAO,EAAE,aAAa,EAAE,8BAA2B;AAEnD,OAAO,EAAE,KAAK,EAAE,6BAA0B;AAC1C,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,0BAAuB;AAC3E,OAAO,EACL,oBAAoB,GAGrB,mBAAgB;AACjB,OAAO,EAAE,oBAAoB,EAAE,+BAA4B;AAC3D,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,cAAc,GAGf,+BAA4B;AAC7B,OAAO,EAAE,MAAM,EAAE,uBAAoB;AAQrC,OAAO,EAAE,8BAA8B,EAAE,+BAA4B;AA4CrE,SAAS,mBAAmB,CAAC,OAA2B;IACtD,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CACrB,QAA+B;IAE/B,OAAO,QAAQ;QACb,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;SACjC;QACH,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAe,EACf,MAAqB,EACrB,QAA+B;IAE/B,OAAO,cAAc,CACnB,QAAQ,EAAE,OAAO,IAAI,OAAO,EAC5B,MAAM,EACN,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CACnC,CAAC;AACJ,CAAC;AACD,MAAM,eAAe;IAKA;IACA;IALX,YAAY,GAAuB,EAAE,CAAC;IACtC,MAAM,GAAmB,EAAE,CAAC;IAC5B,OAAO,GAA0B,IAAI,CAAC;IAC9C,YACmB,KAA2B,EAC3B,OAA+B;QAD/B,UAAK,GAAL,KAAK,CAAsB;QAC3B,YAAO,GAAP,OAAO,CAAwB;IAC/C,CAAC;IACJ,KAAK,CAAC,GAAG;QACP,KAAK,MAAM,KAAK,IAAI,mBAAmB,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC7D,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACvC,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IACO,WAAW,CAAC,OAAe,EAAE,UAAU,GAAG,IAAI;QACpD,IAAI,CAAC,OAAO,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IACO,KAAK,CAAC,aAAa,CACzB,OAA2B;QAE3B,OAAO,OAAO,KAAK,SAAS;YAC1B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,OAAO,aAAa,CAClB,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,EAC7B;YACE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW;YACnC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACzB,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;SAChD,CACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,KAAoB,EACpB,OAA2B,EAC3B,MAAc,EACd,SAAiB,EACjB,OAAe;QAEf,IAAI,MAAM,GAA4B,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACnD,MAAM,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,MAAM,IAAI;iBACxB,GAAG,CAAC;gBACH,qBAAqB,EAAE,CAAC,KAAK,EAAE,EAAE,CAC/B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;oBACpC;wBACE,EAAE,EAAE,iBAAiB,MAAM,yBAAyB;wBACpD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;qBAC1B;iBACF,CAAC;gBACJ,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,uBAAuB;gBAC3D,sBAAsB,EAAE,IAAI,CAAC,KAAK,CAAC,sBAAsB;gBACzD,MAAM,EAAE,kBAAkB,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC;gBACpD,aAAa,EAAE,oBAAoB,CACjC,IAAI,CAAC,KAAK,CAAC,aAAa,EACxB,OAAO,KAAK,SAAS,CACtB;gBACD,WAAW,EAAE,UAAU;gBACvB,SAAS;gBACT,SAAS,EAAE,SAAS;gBACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;aAC1B,CAAC;iBACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBACxB,IAAI,KAAK,YAAY,oBAAoB;oBAAE,OAAO,KAAK,CAAC,MAAM,CAAC;gBAC/D,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;YACL,MAAM,SAAS,GAAG,mBAAmB,CACnC,QAAQ,EACR,iBAAiB,MAAM,GAAG,EAC1B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,EAClC,IAAI,CAAC,KAAK,CAAC,cAAc,EACzB,0BAA0B,MAAM,EAAE,CACnC,CAAC;YACF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,8BAA8B;gBAAE,MAAM,KAAK,CAAC;YACjE,IAAI,CAAC,WAAW,CAAC,mCAAmC,CAAC,CAAC;YACtD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;IAEO,qBAAqB,CAC3B,OAAwB,EACxB,OAA2B,EAC3B,MAA+B,EAC/B,MAAc;QAEd,MAAM,EAAE,GAAG,8BAA8B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACrB,EAAE;YACF,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;aAC7C;SACF,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACf,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,MAAM,EAAE,gBAAgB;YACxB,MAAM,EAAE,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACxD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,YAAY,EAAE,CAAC,EAAE,CAAC;SACnB,CAAC,CAAC;IACL,CAAC;IAEO,oBAAoB,CAC1B,MAAkB,EAClB,KAAoB,EACpB,OAA2B,EAC3B,MAA+B,EAC/B,MAAc;QAEd,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,6BAA6B,CAAC,MAAM,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACvD,KAAK,MAAM,OAAO,IAAI,QAAQ;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,WAAW,CACd,MAAM,CAAC,cAAc,IAAI,mCAAmC,EAC5D,MAAM,CAAC,cAAc,EAAE,MAAM,KAAK,WAAW,CAC9C,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,6BAA6B,CAAC,MAAkB;QACtD,IAAI,MAAM,CAAC,eAAe,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;YACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC,YAAY,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAoB;QACnD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;QACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QACnC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACvB,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,MAAM,EAAE,gBAAgB;gBACxB,MAAM,EAAE,aAAsB;gBAC9B,YAAY,EAAE,EAAE;aACjB,CAAC,CAAC,CACJ,CAAC;YACF,OAAO;QACT,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACjD,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CACxC,KAAK,EACL,OAAO,EACP,MAAM,EACN,SAAS,EACT,OAAO,CACR,CAAC;YACF,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI;gBAAE,OAAO;YACpC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;gBACrB,EAAE,EAAE,mCAAmC,MAAM,EAAE;gBAC/C,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBACnE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;aAC5B,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAChD,OAAO,CAAC,MAAM,CAAC,YAAY,EAC3B,MAAM,CACP,CAAC;YACF,UAAU,GAAG,IAAI,CAAC,oBAAoB,CACpC,OAAO,CAAC,MAAM,EACd,KAAK,EACL,OAAO,EACP,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,MAAM,CACf,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACO,KAAK,CAAC,eAAe,CAC3B,SAAiB,EACjB,UAAmB;QAEnB,IAAI,UAAU;YAAE,MAAM,oBAAoB,CAAC,SAAS,CAAC,CAAC;;YACjD,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;CACF;AACD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAA2B,EAC3B,OAA+B;IAE/B,OAAO,IAAI,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;AACnD,CAAC","sourcesContent":["import { clearFixtureContents } from \"../fixtures/cleanup\";\nimport type { CandidateArtifactRoots } from \"../candidate-artifacts\";\nimport { createFixture } from \"../fixtures/create\";\nimport { hostArtifacts } from \"../evidence/host-results\";\nimport { usage } from \"../evidence/usage\";\nimport { gradingRuntimePolicy, scopedHostArtifacts } from \"./grading-host\";\nimport {\n  HostExecutionFailure,\n  type HostAdapter,\n  type HostResult,\n} from \"./types\";\nimport { readSemanticArtifact } from \"../graders/artifact\";\nimport {\n  parseSemanticVerdicts,\n  semanticCheckGroups,\n  semanticPrompt,\n  type PreparedSemanticCheck,\n  type SemanticVerdict,\n} from \"../graders/semantic\";\nimport { sha256 } from \"../identity\";\nimport type { CheckOutcome } from \"../results\";\nimport type {\n  TrialEvidenceWriter,\n  TrialObservation,\n} from \"../evidence/trial-writer\";\nimport type { GradingFailure, GradingResult } from \"./grading\";\nimport type { RuntimePolicy } from \"../runtime/config\";\nimport { HostDiagnosticPersistenceError } from \"../host-diagnostics\";\n\ninterface SemanticGradingInput {\n  readonly checks: readonly PreparedSemanticCheck[];\n  readonly declaredChecks: readonly { id: string }[];\n  readonly host: HostAdapter | undefined;\n  readonly message: string | null;\n  readonly candidateWorkspace: string;\n  readonly candidateTranscriptRoot?: string;\n  readonly candidateArtifactRoots?: CandidateArtifactRoots;\n  readonly projectRoot: string;\n  readonly reservedWorkspace?: string;\n  readonly runtimePolicy?: RuntimePolicy;\n  readonly signal?: AbortSignal;\n}\ninterface SemanticGradingEffects {\n  readonly evidence: Pick<\n    TrialEvidenceWriter,\n    \"artifactRefs\" | \"retainArtifacts\" | \"retainSemanticVerdicts\"\n  >;\n  /** Close admission at detection, before retaining or cleaning auxiliary state. */\n  onFailure(failure: GradingFailure): void;\n  retainWorkspace(workspace: string): Promise<void>;\n}\ntype SemanticGroup = ReturnType<typeof semanticCheckGroups>[number];\n\ntype SemanticInputArtifact = Awaited<\n  ReturnType<typeof readSemanticArtifact>\n> | null;\n\ntype SemanticHostOutcome =\n  | {\n      result: HostResult;\n      artifacts: ReturnType<typeof hostArtifacts>;\n      source: Record<string, unknown>;\n      hostId: string;\n    }\n  | {\n      result: null;\n      artifacts: ReturnType<typeof hostArtifacts>;\n      source: Record<string, unknown>;\n      hostId: null;\n    };\n\nfunction semanticGroupSuffix(pattern: string | undefined): string {\n  return pattern === undefined ? \"\" : \".\" + sha256(pattern);\n}\n\nfunction semanticSource(\n  artifact: SemanticInputArtifact,\n): Record<string, unknown> {\n  return artifact\n    ? {\n        kind: \"artifact\",\n        path: artifact.path,\n        sha256: sha256(artifact.content),\n      }\n    : { kind: \"response\" };\n}\n\nfunction semanticHostPrompt(\n  message: string,\n  checks: SemanticGroup,\n  artifact: SemanticInputArtifact,\n): string {\n  return semanticPrompt(\n    artifact?.content ?? message,\n    checks,\n    artifact ? \"document\" : \"response\",\n  );\n}\nclass SemanticGrading {\n  private observations: TrialObservation[] = [];\n  private checks: CheckOutcome[] = [];\n  private failure: GradingFailure | null = null;\n  constructor(\n    private readonly input: SemanticGradingInput,\n    private readonly effects: SemanticGradingEffects,\n  ) {}\n  async run(): Promise<GradingResult> {\n    for (const group of semanticCheckGroups([...this.input.checks]))\n      await this.gradeSemanticGroup(group);\n    return {\n      checks: this.checks,\n      observations: this.observations,\n      failure: this.failure,\n    };\n  }\n  private failGrading(message: string, cancelable = true): void {\n    this.failure = { message, cancelable };\n    this.effects.onFailure(this.failure);\n  }\n  private async semanticInput(\n    pattern: string | undefined,\n  ): Promise<SemanticInputArtifact> {\n    return pattern === undefined\n      ? null\n      : readSemanticArtifact(this.input.candidateWorkspace, pattern);\n  }\n\n  private async semanticWorkspace(): Promise<string> {\n    return createFixture(\n      { kind: \"inline\", files: {} },\n      {\n        projectRoot: this.input.projectRoot,\n        signal: this.input.signal,\n        reservedWorkspace: this.input.reservedWorkspace,\n      },\n    );\n  }\n\n  private async runSemanticHost(\n    group: SemanticGroup,\n    pattern: string | undefined,\n    suffix: string,\n    workspace: string,\n    message: string,\n  ): Promise<SemanticHostOutcome> {\n    let source: Record<string, unknown> = { kind: \"response\" };\n    try {\n      const artifact = await this.semanticInput(pattern);\n      source = semanticSource(artifact);\n      const host = this.input.host;\n      if (!host) throw new Error(\"semantic host is unavailable\");\n      const response = await host\n        .run({\n          retainHostDiagnostics: (bytes) =>\n            this.effects.evidence.retainArtifacts([\n              {\n                id: `sevro.semantic${suffix}.sevro.host.diagnostics`,\n                bytes: Buffer.from(bytes),\n              },\n            ]),\n          candidateTranscriptRoot: this.input.candidateTranscriptRoot,\n          candidateArtifactRoots: this.input.candidateArtifactRoots,\n          prompt: semanticHostPrompt(message, group, artifact),\n          runtimePolicy: gradingRuntimePolicy(\n            this.input.runtimePolicy,\n            pattern === undefined,\n          ),\n          runtimeRole: \"semantic\",\n          workspace,\n          condition: \"passive\",\n          signal: this.input.signal,\n        })\n        .catch((cause: unknown) => {\n          if (cause instanceof HostExecutionFailure) return cause.result;\n          throw cause;\n        });\n      const artifacts = scopedHostArtifacts(\n        response,\n        `sevro.semantic${suffix}.`,\n        this.effects.evidence.artifactRefs,\n        this.input.declaredChecks,\n        `sevro.semantic.verdicts${suffix}`,\n      );\n      return { result: response, artifacts, source, hostId: host.id };\n    } catch (cause) {\n      if (cause instanceof HostDiagnosticPersistenceError) throw cause;\n      this.failGrading(\"semantic grading did not complete\");\n      return { result: null, artifacts: [], source, hostId: null };\n    }\n  }\n\n  private recordSemanticVerdict(\n    verdict: SemanticVerdict,\n    pattern: string | undefined,\n    source: Record<string, unknown>,\n    hostId: string,\n  ): void {\n    const id = `sevro.observation.semantic.${sha256(verdict.id)}`;\n    this.observations.push({\n      id,\n      source: hostId,\n      completeness: \"complete\",\n      data: {\n        verdict: verdict.verdict,\n        reason: verdict.reason,\n        ...(pattern === undefined ? {} : { source }),\n      },\n    });\n    this.checks.push({\n      id: verdict.id,\n      grader: \"sevro.semantic\",\n      status: verdict.verdict === \"pass\" ? \"passed\" : \"failed\",\n      detail: verdict.reason,\n      evidenceRefs: [id],\n    });\n  }\n\n  private parseSemanticResults(\n    result: HostResult,\n    group: SemanticGroup,\n    pattern: string | undefined,\n    source: Record<string, unknown>,\n    hostId: string,\n  ): boolean {\n    try {\n      const message = this.completeSemanticGraderMessage(result);\n      const verdicts = parseSemanticVerdicts(message, group);\n      for (const verdict of verdicts)\n        this.recordSemanticVerdict(verdict, pattern, source, hostId);\n      return true;\n    } catch {\n      this.failGrading(\n        result.failureMessage ?? \"semantic grading did not complete\",\n        result.processFailure?.reason === \"cancelled\",\n      );\n      return false;\n    }\n  }\n\n  private completeSemanticGraderMessage(result: HostResult): string {\n    if (result.executionFailed || !result.complete || !result.finalMessage)\n      throw new Error(\"semantic grader result did not complete\");\n    return result.finalMessage;\n  }\n\n  private async gradeSemanticGroup(group: SemanticGroup): Promise<void> {\n    const pattern = group[0].artifactPath;\n    const suffix = semanticGroupSuffix(pattern);\n    if (this.failure) return;\n    const message = this.input.message;\n    if (message === null) {\n      this.checks.push(\n        ...group.map((check) => ({\n          id: check.id,\n          grader: \"sevro.semantic\",\n          status: \"unavailable\" as const,\n          evidenceRefs: [],\n        })),\n      );\n      return;\n    }\n    const workspace = await this.semanticWorkspace();\n    let successful = false;\n    try {\n      const outcome = await this.runSemanticHost(\n        group,\n        pattern,\n        suffix,\n        workspace,\n        message,\n      );\n      if (outcome.result === null) return;\n      await this.effects.evidence.retainArtifacts(outcome.artifacts);\n      this.observations.push({\n        id: `sevro.observation.semantic.usage${suffix}`,\n        source: outcome.hostId,\n        completeness: outcome.result.usageComplete ? \"complete\" : \"partial\",\n        data: usage(outcome.result),\n      });\n      await this.effects.evidence.retainSemanticVerdicts(\n        outcome.result.finalMessage,\n        suffix,\n      );\n      successful = this.parseSemanticResults(\n        outcome.result,\n        group,\n        pattern,\n        outcome.source,\n        outcome.hostId,\n      );\n    } finally {\n      await this.finishWorkspace(workspace, successful);\n    }\n  }\n  private async finishWorkspace(\n    workspace: string,\n    successful: boolean,\n  ): Promise<void> {\n    if (successful) await clearFixtureContents(workspace);\n    else await this.effects.retainWorkspace(workspace);\n  }\n}\nexport async function gradeSemanticChecks(\n  input: SemanticGradingInput,\n  effects: SemanticGradingEffects,\n) {\n  return new SemanticGrading(input, effects).run();\n}\n"]}