@eir-labs/coltrane 0.5.1 → 0.7.2

package/dist/src/runtime.js

@@ -11,9 +11,15 @@ import { resolveModel } from "./claude_invoker.js";
 // core type → the process primitive that produces it (reverse of PRIMITIVE_OUTPUT_TYPE).
 // A skill-backed chair seals its output as this primitive/core when its output_contract is
 // a core type.
-const CORE_TO_PRIMITIVE = Object.fromEntries(Object.entries(PRIMITIVE_OUTPUT_TYPE).map(([prim, core]) => [String(core), prim]));
-import { sha256Hex, canonJson, runFingerprint, outputContentHash, CANONICAL_FORM_VERSION } from "./canonical_form.js";
-import { producersSha, reuseCacheKey, checkReuseEntry, runIdentityMismatch, CHECKPOINT_SCHEMA_VERSION, REUSE_SCHEMA_VERSION, } from "./reuse.js";
+//
+// EXPORTED because `primitive` is folded into `content_sha`. Anything that has to re-derive
+// what a chair WOULD have sealed — the drain reconstruction in src/worker.ts — has to arrive
+// at the same primitive this seal boundary does, and a second copy of the mapping is exactly
+// the drift that makes two gates on one concern answer differently.
+export const CORE_TO_PRIMITIVE = Object.fromEntries(Object.entries(PRIMITIVE_OUTPUT_TYPE).map(([prim, core]) => [String(core), prim]));
+import { sha256Hex, canonJson, canonStructuralJson, runFingerprint, outputContentHash, CANONICAL_FORM_VERSION } from "./canonical_form.js";
+import { producersSha, reuseCacheKey, checkReuseEntry, runIdentityMismatch, checkpointRoleKey, CHECKPOINT_SCHEMA_VERSION, REUSE_SCHEMA_VERSION, } from "./reuse.js";
+import { drainGigHeader } from "./output_mirror.js";
 import { LEDGER_SCHEMA_VERSION } from "./ledger.js";
 /**
  * #236 — settled spend used to be discarded on every failed gig: `usage` was written only on
@@ -141,9 +147,16 @@ export function computeAppendCost(ctx, base_cost, k) {
 // in a canonical (sorted, JCS) form. This is the reproducibility key — same defs,
 // same genome_hash, regardless of model or run.
 /**
- * Resolve a list of skill slugs against the genome's skills map. REPORTS, never decides:
- * it returns both what resolved and what did not, and `prepareChair` decides what a miss
- * means (fatal when the chair declared the skill REQUIRED, reported otherwise).
+ * Resolve an agent's skills: the ones it CARRIES on its record, unioned with the repertoire
+ * packages its `skill_slugs` name. REPORTS, never decides: it returns both what resolved and what
+ * did not, and `prepareChair` decides what a miss means (fatal when the chair declared the skill
+ * REQUIRED, reported otherwise).
+ *
+ * CARRIED-FIRST, and carried wins the slug. An agent's own definition needs no genome lookup (it
+ * travels with the player into any institution), and where both a carried definition and a
+ * repertoire package answer to one slug the carried one SHADOWS it — the player's own technique is
+ * the one that plays, and the same slug never resolves to two skills in one prompt. A slug covered
+ * by a carried definition is therefore not missing.
  *
  * The boundary (#241): a skill package that LOADS is a legitimate degradation candidate —
  * it has an identity, a version, a code_hash, and its degradation is already surfaced and
@@ -155,12 +168,15 @@ export function computeAppendCost(ctx, base_cost, k) {
  * An ABSENT map means resolution was never configured (the documented v0 back-compat path)
  * — that is not evidence of a dangling binding, so `missing` stays empty.
  */
-function resolveSkills(slugs, map) {
+function resolveSkills(slugs, map, carried) {
+    const skills = [...(carried ?? [])];
+    const carriedSlugs = new Set(skills.map((s) => s.slug));
     if (!slugs || slugs.length === 0 || !map)
-        return { skills: [], missing: [] };
-    const skills = [];
+        return { skills, missing: [] };
     const missing = [];
     for (const slug of slugs) {
+        if (carriedSlugs.has(slug))
+            continue; // the carried definition already answered this slug
         const rec = map.get(slug);
         if (rec)
             skills.push(rec);
@@ -169,7 +185,16 @@ function resolveSkills(slugs, map) {
     }
     return { skills, missing };
 }
-function genomeHash(standard) {
+/**
+ * The structural identity of a pipeline: the standard's phase graph plus each bound agent's
+ * type surface. Deterministic across machines for a given structure.
+ *
+ * EXPORTED because it is the one identity a DRAINED gig carries. The sink's gig header records
+ * `genome_hash` and nothing else about the run's producers, so a worker reconstructing a resume
+ * from the sink (src/worker.ts) has to be able to ask "is the standard I just loaded the one
+ * those outputs were sealed under" — and it must ask with this function, not a lookalike.
+ */
+export function genomeHash(standard) {
     const agents = [...standard.agents]
         .map((a) => ({
         slug: a.slug,
@@ -179,7 +204,13 @@ function genomeHash(standard) {
         domain: a.domain,
     }))
         .sort((x, y) => (x.slug < y.slug ? -1 : 1));
-    return sha256Hex(canonJson({ standard: { slug: standard.slug, domain: standard.domain, phases: standard.phases }, agents }));
+    // canonStructuralJson, not canonJson: a field whose value states NOTHING (an empty array, an
+    // absent optional, a null domain) must not be able to move a STRUCTURAL hash. 0.6.6 added two
+    // `.default([])` chair fields, no standard's structure changed, and genome_hash moved for the
+    // entire genome — re-keying the ledger and refusing resumes for a drift that did not exist.
+    // Reaching this canonicalization moved the hash ONE final time; after it, a new schema default
+    // is hash-neutral. Pinned by tests/genome_hash_stability.test.ts, which states the bump loudly.
+    return sha256Hex(canonStructuralJson({ standard: { slug: standard.slug, domain: standard.domain, phases: standard.phases }, agents }));
 }
 /**
  * Execute one gig: walk phases in order, each phase's agent consumes the prior
@@ -193,7 +224,10 @@ function genomeHash(standard) {
 // declarations are polymorphic, so a base player written against `Interpretation`
 // consumes any downstream subtype while domain contracts keep their precision.
 const CORE_TYPE_SET = new Set(CORE_TYPES);
-function outputSatisfiesType(output, declared) {
+/** EXPORTED because the chart layer asks the same question at the movement boundary — which of a
+ *  source movement's sealed records does an edge of type T carry — and two layers answering "does
+ *  this record satisfy this declared type" differently is the #263 defect wearing a new hat. */
+export function outputSatisfiesType(output, declared) {
     if (output.domain_type === declared)
         return true;
     if (CORE_TYPE_SET.has(declared) && output.core_type === declared)
@@ -303,6 +337,10 @@ export async function runGig(standard, gigInput, deps) {
     // any chair"). The payload is validated BEFORE any chair fires — a missing gig input is a
     // hard stop, so no model tokens are spent on bad input.
     const standardInputs = new Set(standard.input_types ?? []);
+    // Sealed records an earlier MOVEMENT handed to this one over a chart edge (RunDeps.seed_outputs).
+    // They are inputs, not products: available to entry chairs, never folded into `produced`.
+    const seedRecords = deps.seed_outputs ?? [];
+    const seedsConsumed = new Map(); // output_id → record, for the manifest
     // Keys are the HYPHENATED type slug. `grant_requirements` vs `grant-requirements` is the
     // single most common dispatch mistake, and the caller's own keys are in scope here.
     const normalizeKey = (k) => k.toLowerCase().replace(/[_\-\s]/g, "");
@@ -337,6 +375,10 @@ export async function runGig(standard, gigInput, deps) {
                 sealedByRole.set(ch.role, [ch.output_contract[0] ?? "Signal"]);
                 continue;
             }
+            if (ch.human === true && (ch.agent_slug ?? "") === "") {
+                sealedByRole.set(ch.role, [ch.output_contract[0] ?? "Judgment"]);
+                continue;
+            }
             const ag = standard.agents.find((a) => a.slug === ch.agent_slug);
             if (!ag)
                 continue; // prepareChair reports an unknown agent_slug precisely; don't pre-empt it
@@ -375,6 +417,10 @@ export async function runGig(standard, gigInput, deps) {
                         continue; // supplied
                     if (reachable.some((t) => mightSatisfy(t, need)))
                         continue; // an upstream can cover it
+                    // A chart edge satisfies a declared gig input with a SEALED RECORD rather than a payload
+                    // key. Without this the pre-flight would refuse a correctly-arranged movement at t=0.
+                    if (seedRecords.some((s) => outputSatisfiesType(s, need)))
+                        continue;
                     throw missingGigInput(need, ch.role);
                 }
                 sealedThisPhase.push(...(sealedByRole.get(ch.role) ?? []));
@@ -475,7 +521,10 @@ export async function runGig(standard, gigInput, deps) {
     };
     const identity = () => ({
         standard_slug: standard.slug,
-        genome_hash,
+        // A movement's resume gate carries the ARRANGEMENT's identity when it has one: chairs from
+        // chart B consuming a movement's sealed outputs from chart A is the same splice as a moved
+        // genome, one level up. Byte-identical for a degenerate chart, where chart_hash IS genome_hash.
+        genome_hash: deps.chart?.chart_hash ?? genome_hash,
         // #278 review — genome_hash does NOT see an agent's identity/method/constraints/tools,
         // nor a skill's code. Those are the producer, and editing one under a stable slug is the
         // ordinary response to a bad run. Without this the resume gate accepted exactly that.
@@ -523,6 +572,15 @@ export async function runGig(standard, gigInput, deps) {
             if (!rolesInStandard.has(r.role)) {
                 throw new ResumeRefused(gig_id, `its checkpoint names role "${r.role}", which this standard does not define`);
             }
+            // THE SEAT'S FULL IDENTITY: (chart_slug, movement_id, role). Two movements of one chart may
+            // each declare a chair named "reviewer", and restoring one movement's sealed output into the
+            // other's seat would be a splice with nothing in the manifest recording it. A legacy row
+            // carries no movement_id and defaults to the standard's own slug, so it still restores.
+            const seatNow = checkpointRoleKey(deps.chart?.chart_slug ?? standard.slug, deps.chart?.movement_id, r.role);
+            const seatThen = checkpointRoleKey(deps.chart?.chart_slug ?? standard.slug, r.movement_id, r.role);
+            if (seatNow !== seatThen) {
+                throw new ResumeRefused(gig_id, `its checkpoint names seat "${seatThen}" and this run is seat "${seatNow}" — a movement does not restore another movement's chair`);
+            }
             const records = [];
             for (let i = 0; i < r.output_ids.length; i++) {
                 const id = r.output_ids[i];
@@ -568,6 +626,9 @@ export async function runGig(standard, gigInput, deps) {
             return;
         checkpointRoles.set(role, {
             role, phase: phaseName,
+            // WHICH movement's seat this is. Two movements may both declare a chair named "reviewer";
+            // the composite (chart_slug, movement_id, role) is what keeps their checkpoints apart.
+            ...(deps.chart ? { movement_id: deps.chart.movement_id } : {}),
             output_ids: records.map((r) => r.id),
             content_shas: records.map((r) => r.content_sha),
             domain_types: records.map((r) => r.domain_type),
@@ -596,6 +657,9 @@ export async function runGig(standard, gigInput, deps) {
                 updated_at: new Date().toISOString(),
                 roles: [...checkpointRoles.values()],
                 ...(prior ? { prior_usage: JSON.parse(JSON.stringify(prior)) } : {}),
+                // The chart's cumulative spend AT THIS MOVEMENT'S BOUNDARY, so a resumed performance can
+                // compare it to the envelope before spawning anything (src/chart.ts, edge case B).
+                ...(deps.chart?.prior_budget_state ? { prior_budget_state: deps.chart.prior_budget_state } : {}),
             });
         }
         catch (e) {
@@ -653,6 +717,80 @@ export async function runGig(standard, gigInput, deps) {
                     const stuck = [...remaining.values()].map((c) => c.role).join(", ");
                     throw new RuntimeError(`phase "${phase.name}" cannot advance — chairs [${stuck}] have unresolved depends_on`);
                 }
+                // ── THE HUMAN SEAT ─────────────────────────────────────────────────────────────
+                // A human chair in the frontier is handled before any model dispatch. With its
+                // approval supplied (deps.approvals[role]) the incumbent's verdict seals through the
+                // SAME output gate as every record — schema-validated, under the approving
+                // principal's name, carrying the input_shas of exactly what was approved. Without
+                // it, the gig PARKS: checkpointed, honestly drained as awaiting_approval, nothing
+                // hollow sealed. The sketching happens before dispatch; this gate is light.
+                const humanReady = ready.filter((c) => c.human === true && (c.agent_slug ?? "") === "");
+                for (const hc of humanReady) {
+                    const approval = deps.approvals?.[hc.role];
+                    if (!approval) {
+                        checkpoint();
+                        emit({ type: "gig_awaiting_approval", phase: phase.name, role: hc.role });
+                        // AWAITED, unlike the fire-and-forget completion drain: parking is the runtime's
+                        // last act before the caller (often a CLI) exits, and an in-flight fetch dies with
+                        // the process — which left the sink's row saying "running" about a gig that was
+                        // waiting on a person. Parking is not latency-critical; the truth is.
+                        await drainGigHeader({
+                            gig_id,
+                            standard_slug: standard.slug,
+                            status: "awaiting_approval",
+                            genome_hash,
+                            started_at,
+                            finished_at: new Date().toISOString(),
+                            outputs_count: produced.length,
+                            error: `awaiting approval at human chair "${hc.role}" (phase "${phase.name}")`,
+                        }).catch((de) => {
+                            if (process.env["COLTRANE_DRAIN_DEBUG"])
+                                console.error(`[drain] awaiting header ${gig_id}: ${String(de)}`);
+                        });
+                        return {
+                            gig_id,
+                            standard_slug: standard.slug,
+                            ...(deps.chart ? { chart_slug: deps.chart.chart_slug, movement_id: deps.chart.movement_id } : {}),
+                            genome_hash,
+                            run_fingerprint: "",
+                            outputs: produced,
+                            eval_scores: {},
+                            status: "awaiting_approval",
+                            awaiting: { phase: phase.name, role: hc.role },
+                        };
+                    }
+                    remaining.delete(hc.role);
+                    const domain_type = hc.output_contract[0] ?? "Judgment";
+                    const core = deps.outputs.coreTypeOf(domain_type) ?? domain_type;
+                    const primitive = CORE_TO_PRIMITIVE[core] ?? "JUDGE";
+                    const approvalInputs = hc.depends_on.flatMap((d) => producedByRole.get(d) ?? []);
+                    const t0 = Date.now();
+                    emit({ type: "chair_start", phase: phase.name, role: hc.role, producer: deps.approved_by ?? "human" });
+                    const rec = deps.outputs.write({
+                        core_type: core,
+                        domain_type,
+                        domain: standard.domain,
+                        gig_id,
+                        agent_slug: deps.approved_by ?? "human",
+                        from_role: hc.role,
+                        phase: phase.name,
+                        primitive,
+                        data: approval,
+                        input_refs: approvalInputs.map((i) => i.id),
+                        input_shas: approvalInputs.map((i) => i.content_sha),
+                    });
+                    for (const i of approvalInputs)
+                        deps.outputs.addRef(rec.id, i.id, "derived_from", primitive);
+                    producedByRole.set(hc.role, [rec]);
+                    produced.push(rec);
+                    emit({
+                        type: "chair_complete", phase: phase.name, role: hc.role, producer: deps.approved_by ?? "human",
+                        output_types: [domain_type], duration_ms: Date.now() - t0,
+                    });
+                }
+                ready = ready.filter((c) => !(c.human === true && (c.agent_slug ?? "") === ""));
+                if (ready.length === 0)
+                    continue;
                 // Routing policy: when a selector is injected, it narrows the frontier to the
                 // chairs to dispatch THIS iteration; the rest stay in `remaining` and re-enter
                 // the next frontier. The return is validated strictly — empty or containing a
@@ -746,6 +884,9 @@ export async function runGig(standard, gigInput, deps) {
                 return undefined;
             const key = reuseCacheKey({
                 standard_slug: standard.slug,
+                // A chart may name one standard TWICE. Keyed on movement_id, the two instances occupy
+                // separate namespaces even with byte-identical inputs — isolation by default.
+                ...(deps.chart ? { chart_slug: deps.chart.chart_slug, movement_id: deps.chart.movement_id } : {}),
                 phase: a.phaseName,
                 chair: a.chair,
                 agent: a.agent ?? null,
@@ -816,6 +957,25 @@ export async function runGig(standard, gigInput, deps) {
             }
             return { key, hit: { cache_key: key, source_gig_id: entry.source_gig_id, outputs: entry.outputs } };
         }
+        /**
+         * Offer an ENTRY chair the seeds a chart edge carried in.
+         *
+         * Scoped to a chair with no `depends_on`: a chair that named its upstream roles asked for those
+         * specific seats, and a movement's seed is not one of them. Records enter `inputs` — so what the
+         * chair seals carries their content_shas — and are recorded as consumed for the manifest.
+         */
+        function pullSeeds(chair, inputs, wanted) {
+            if (seedRecords.length === 0 || chair.depends_on.length > 0)
+                return;
+            for (const s of seedRecords) {
+                if (inputs.includes(s))
+                    continue;
+                if (!wanted.some((t) => outputSatisfiesType(s, t)))
+                    continue;
+                inputs.push(s);
+                seedsConsumed.set(s.id, s);
+            }
+        }
         function prepareChair(chair, phaseName) {
             // A skill-backed chair runs the skill's deterministic code half — no agent, no model.
             if (chair.skill_slug && (chair.agent_slug ?? "") === "") {
@@ -834,6 +994,7 @@ export async function runGig(standard, gigInput, deps) {
                         throw new RuntimeError(`chair "${chair.role}" depends_on "${dep}" which has not been produced`);
                     inputs.push(...recs);
                 }
+                pullSeeds(chair, inputs, chair.input_contract);
                 if (chair.input_contract.length > 0) {
                     for (const need of chair.input_contract) {
                         // #156: a type satisfied by an upstream record OR by the gig payload (entry-chair seed).
@@ -900,6 +1061,9 @@ export async function runGig(standard, gigInput, deps) {
             }
             else {
                 inputs = produced.filter((o) => agent.input_types.some((t) => outputSatisfiesType(o, t)));
+                // A chart edge's carriers are offered to the same chair on the same terms as an in-gig
+                // upstream record — by type, as records, so provenance survives the movement boundary.
+                pullSeeds(chair, inputs, [...chair.input_contract, ...agent.input_types]);
             }
             // Runtime input_contract check: every type the chair declares it expects
             // on input must be satisfied by its actual upstream inputs. Subtype-aware
@@ -955,7 +1119,7 @@ export async function runGig(standard, gigInput, deps) {
             // Resolve this agent's skill bindings (slugs) against the genome's skills map.
             // resolveSkills REPORTS; this is where the engine DECIDES — and it decides BEFORE the
             // budget deduction below, so a dangling binding costs nothing.
-            const { skills, missing } = resolveSkills(agent.skill_slugs, deps.skills);
+            const { skills, missing } = resolveSkills(agent.skill_slugs, deps.skills, agent.skills);
             if (missing.length > 0) {
                 // #242 — `Chair.required_skills` was validated exactly once, at compose time, as a
                 // string-subset check against the agent's own declaration. A chair could declare a
@@ -1456,7 +1620,11 @@ export async function runGig(standard, gigInput, deps) {
             eval_scores[slug] = scoreEval(slug, produced, deps.evals);
         }
         const run_fingerprint = runFingerprint({
-            genome_hash,
+            // THE CHART SLOT. When this run is a movement of an arrangement, the arrangement's identity
+            // folds in exactly where `genome_hash` folded — because the reproducible thing is no longer
+            // "this standard's structure" but "this movement of this chart". For a degenerate chart
+            // `chart_hash === genomeHash(standard)`, so a single-standard gig's fingerprint is unmoved.
+            genome_hash: deps.chart?.chart_hash ?? genome_hash,
             model_version: deps.model_version ?? "unknown",
             canonical_form_version: CANONICAL_FORM_VERSION,
             eval_scores,
@@ -1464,22 +1632,43 @@ export async function runGig(standard, gigInput, deps) {
             ...(unresolved_evals.length > 0 ? { unresolved_evals } : {}),
         });
         const settledUsage = finalizeUsage();
+        const gig_finished_at = new Date().toISOString();
         deps.ledger.append({
             kind: "gig",
             schema_version: LEDGER_SCHEMA_VERSION,
+            // One ROW per movement, because a movement runs under its own gig id (src/chart.ts
+            // `movementGigId`) — and for the degenerate chart that id IS the chart's, which is what an
+            // existing single-standard reader looks up.
             entry_id: gig_id,
             gig_id,
             standard_slug: standard.slug,
+            ...(deps.chart ? { chart_slug: deps.chart.chart_slug, movement_id: deps.chart.movement_id } : {}),
             genome_hash,
             run_fingerprint,
             output_hashes,
             started_at,
-            finished_at: new Date().toISOString(),
+            finished_at: gig_finished_at,
             // settled model spend (#195) — omitted when nothing was CAPTURED (skill-only gigs, stubbed
             // invokers, or a run whose every invocation reported no usage payload). #235: an absent
             // usage block means "not captured", never "$0.00".
             ...(settledUsage ? { usage: settledUsage } : {}),
         });
+        // Drain the gig HEADER to the sink (fire-and-forget, like every output before it) — the
+        // stub row the drain service fabricated for FK integrity is replaced by the run's own record.
+        void drainGigHeader({
+            gig_id,
+            standard_slug: standard.slug,
+            status: "complete",
+            genome_hash,
+            run_fingerprint,
+            started_at,
+            finished_at: gig_finished_at,
+            outputs_count: produced.length,
+            ...(settledUsage ? { usage: { total_cost_usd: settledUsage.total_cost_usd, input_tokens: settledUsage.input_tokens, output_tokens: settledUsage.output_tokens } } : {}),
+        }).catch((e) => {
+            if (process.env["COLTRANE_DRAIN_DEBUG"])
+                console.error(`[drain] gig header ${gig_id}: ${String(e)}`);
+        });
         // Cycle complete — when a budget was supplied, mark it `settled` and
         // surface the final state in the manifest. `settled` mirrors the
         // budget-state.json cycle terminal-state semantics for a closed cycle.
@@ -1495,7 +1684,18 @@ export async function runGig(standard, gigInput, deps) {
             deps.checkpoints?.remove(gig_id);
         }
         catch { /* reclaiming disk must not fail a run that succeeded */ }
-        const result = { gig_id, standard_slug: standard.slug, genome_hash, run_fingerprint, outputs: produced, eval_scores, status: "complete" };
+        const result = {
+            gig_id, standard_slug: standard.slug,
+            ...(deps.chart ? { chart_slug: deps.chart.chart_slug, movement_id: deps.chart.movement_id } : {}),
+            genome_hash, run_fingerprint, outputs: produced, eval_scores, status: "complete",
+            ...(seedsConsumed.size > 0
+                ? {
+                    seeded_from: [...seedsConsumed.values()].map((s) => ({
+                        gig_id: s.gig_id, output_id: s.id, domain_type: s.domain_type, content_sha: s.content_sha,
+                    })),
+                }
+                : {}),
+        };
         if (settledUsage)
             result.usage = settledUsage;
         if (budget)
@@ -1528,13 +1728,31 @@ export async function runGig(standard, gigInput, deps) {
         // not. Attaching the partial accounting to the error is what lets gig_monitor and a
         // synchronous caller report it. This does NOT write a ledger row — absence-of-row remains
         // the honest "un-sealed gig" signal (recorder_durability_mid_crash.spec.ts).
+        let partial;
         if (e && typeof e === "object") {
-            const partial = finalizeUsage();
+            partial = finalizeUsage();
             if (partial)
                 e["usage"] = partial;
             if (budget)
                 e["budget_state"] = budget;
         }
+        // The sink learns the truth either way: a failed run drains a FAILED header (same
+        // fire-and-forget seam as the success path), so the queue row never sits stale on a
+        // local failure. Found live: the worker's first day exposed the success-only drain.
+        void drainGigHeader({
+            gig_id,
+            standard_slug: standard.slug,
+            status: "failed",
+            genome_hash,
+            started_at,
+            finished_at: new Date().toISOString(),
+            outputs_count: produced.length,
+            error: e instanceof Error ? e.message : String(e),
+            ...(partial ? { usage: { total_cost_usd: partial.total_cost_usd, input_tokens: partial.input_tokens, output_tokens: partial.output_tokens } } : {}),
+        }).catch((de) => {
+            if (process.env["COLTRANE_DRAIN_DEBUG"])
+                console.error(`[drain] failed-gig header ${gig_id}: ${String(de)}`);
+        });
         throw e;
     }
 }