@smartmemory/stratum 0.3.3 → 0.3.4

package/dist/engine/engine.js

@@ -4,6 +4,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { promisify } from "node:util";
+import { z } from "zod";
 import { runAgent } from "../connectors/runner.js";
 import { extractReferences } from "../ir/refs.js";
 import { validateSpec } from "../ir/validate.js";
@@ -11,6 +12,37 @@ import { BudgetLedger, validUsage } from "./ledger.js";
 import { commitCheckpoint, revertCheckpoint } from "./checkpoint.js";
 import { StateStore } from "./state.js";
 const execFileAsync = promisify(execFile);
+/**
+ * The fixed shape every S1 `evaluate:` step must return. Engine-owned and
+ * strict — an author's `out` contract governs what is *referenceable*, this
+ * schema governs what the data must *be*. It is the trust anchor: a transport
+ * failure can never be laundered into a `closed` verdict, and the cross-field
+ * invariants (`closed` ⇒ no children, `open` ⇒ ≥1 child) are enforced here.
+ */
+export const evaluatorResultSchema = z.object({
+    status: z.enum(["closed", "open", "failed"]),
+    children: z.array(z.unknown()),
+    reason: z.string(),
+    score: z.number().optional(),
+    route: z.enum(["claude", "codex"]).optional(),
+}).strict().superRefine((result, ctx) => {
+    if (result.status === "closed" && result.children.length > 0) {
+        ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["children"], message: "a closed verdict must carry no children" });
+    }
+    if (result.status === "open" && result.children.length === 0) {
+        ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["children"], message: "an open verdict must carry at least one child" });
+    }
+});
+/**
+ * The engine validates the runner's ENVELOPE at runtime, not just the verdict
+ * inside it — the `ok` discriminant is the runner's word for whether it even
+ * succeeded, and a malformed envelope must never let a `{status:"closed"}`
+ * payload reach the success path. Same trust posture as the judge verdict.
+ */
+const evaluateRunResultSchema = z.discriminatedUnion("ok", [
+    z.object({ ok: z.literal(true), result: z.unknown() }),
+    z.object({ ok: z.literal(false), kind: z.enum(["exit", "timeout", "parse"]), reason: z.string() }),
+]);
 export class CheckpointOperationError extends Error {
     errorType;
     available;
@@ -33,6 +65,7 @@ export class StratumEngine {
     store;
     evaluator;
     judge;
+    evaluateRunner;
     connector;
     // Serializes load-modify-save per run: plan may hand out several ready steps, so
     // stepDone/resume can race in-process. The state root is owned by one engine process in v1.
@@ -51,6 +84,8 @@ export class StratumEngine {
         this.evaluator = options.evaluator;
         if (options.judge)
             this.judge = options.judge;
+        if (options.evaluateRunner)
+            this.evaluateRunner = options.evaluateRunner;
         this.connector = options.connector ?? defaultConnector;
     }
     async loadRun(runId) {
@@ -895,6 +930,74 @@ export class StratumEngine {
                         break;
                     }
                 }
+                if (step.evaluate !== undefined) {
+                    const attempt = state.attempts.length + 1;
+                    const evaluate = step.evaluate;
+                    // Deterministic, single-shot, and atomic: no intermediate `running`
+                    // is persisted, so a crash mid-evaluate leaves the step `pending` and
+                    // it re-runs on resume. `forceExhausted` terminalizes every failure —
+                    // retrying a deterministic evaluator is pointless (backtrack is S3).
+                    const evalFail = (reason) => this.failAttempt(run, spec, contracts, scope, step, state, attempt, reason, {}, undefined, undefined, true);
+                    if (!this.evaluateRunner) {
+                        await evalFail("evaluate: no evaluate runner configured");
+                        changed = true;
+                        break;
+                    }
+                    let input;
+                    try {
+                        input = evaluate.in === undefined ? undefined : this.renderValue(evaluate.in, scope);
+                    }
+                    catch (error) {
+                        await evalFail(`evaluate: input render failed: ${message(error)}`);
+                        changed = true;
+                        break;
+                    }
+                    // Sandboxing (workspaceRoot jail) is deferred — see design open question 3.
+                    let rawOutcome;
+                    try {
+                        rawOutcome = await this.evaluateRunner({ command: evaluate.command, input, timeoutMs: evaluate.timeout_ms }, {});
+                    }
+                    catch (error) {
+                        await evalFail(`evaluate: runner threw: ${message(error)}`);
+                        changed = true;
+                        break;
+                    }
+                    const envelope = evaluateRunResultSchema.safeParse(rawOutcome);
+                    if (!envelope.success) {
+                        await evalFail(`evaluate: runner returned a malformed result envelope: ${envelope.error.message}`);
+                        changed = true;
+                        break;
+                    }
+                    const outcome = envelope.data;
+                    if (!outcome.ok) {
+                        const detail = outcome.reason;
+                        const reason = outcome.kind === "exit" ? `evaluate: command exited with a non-zero status (${detail})`
+                            : outcome.kind === "timeout" ? `evaluate: command timed out (${detail})`
+                                : `evaluate: output was not valid JSON (${detail})`;
+                        await evalFail(reason);
+                        changed = true;
+                        break;
+                    }
+                    const parsed = evaluatorResultSchema.safeParse(outcome.result);
+                    if (!parsed.success) {
+                        await evalFail(`evaluate: output failed the evaluator-result contract: ${parsed.error.message}`);
+                        changed = true;
+                        break;
+                    }
+                    const outError = this.contractError(step, parsed.data, contracts);
+                    if (outError) {
+                        await evalFail(`evaluate: output failed the ${step.out} contract: ${outError}`);
+                        changed = true;
+                        break;
+                    }
+                    state.status = "succeeded";
+                    state.output = parsed.data;
+                    state.attempts.push({ attempt, at: now(), result: parsed.data });
+                    this.event(run, "result", this.scopedId(scope, step.id), { attempt, result: parsed.data });
+                    await this.persist(run);
+                    changed = true;
+                    continue;
+                }
                 if (step.do === undefined) {
                     await this.failScope(run, spec, contracts, scope, "construct is outside P1 engine scope");
                     break;
@@ -2231,6 +2334,8 @@ function stringLeaves(step) {
     // over "${prep.output.items}" must wait for prep, not fail at resolve time.
     if (step.with !== undefined)
         collect(step.with);
+    if (step.evaluate?.in !== undefined)
+        collect(step.evaluate.in);
     if (step.fanout !== undefined) {
         collect(step.fanout.over);
         for (const stage of step.fanout.steps) {