@eir-labs/coltrane 0.5.1 → 0.6.2

package/dist/src/server.js

@@ -9,12 +9,14 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot
 import { MCP_TOOLS, requiresApproval, AGENT_STATUS_ORDER, STANDARD_STATUS_ORDER, SKILL_STATUS_ORDER, checkPromotion, PromotionError, } from "./mcp.js";
 import { loadRegistry, domainTypeDefect } from "./registry.js";
 import { resolveGenome } from "./loader.js";
-import { SkillSchema, AgentSchema, StandardSchema } from "./genome_schema.js";
+import { SkillSchema, AgentSchema, StandardSchema, DomainTypeSchema } from "./genome_schema.js";
 import { runSkillFixtures, executeSkill, loadFixtures } from "./skill_subprocess.js";
 import { evolveSkill } from "./skills.js";
 import { sealAgentDefinition, sealDefinition, sealSkillPackage, recordIdentity } from "./genome_writer.js";
 import { createOutputStore, defaultOutputsPersistDir } from "./outputs.js";
+import { createOutputMirror, defaultMirrorDir, outputPreview, mirrorStorageRef } from "./output_mirror.js";
 import { FileLedger, LedgerError, LEDGER_SCHEMA_VERSION, defaultLedgerPath, } from "./ledger.js";
+import { sealDrill } from "./seal_drill.js";
 import { standardSimulate } from "./simulate.js";
 import { runGig, BudgetExhausted, GigAborted, ResumeRefused, partialGigUsage, partialBudgetState } from "./runtime.js";
 import { createCheckpointStore, createReuseStore } from "./reuse.js";
@@ -172,6 +174,46 @@ function syncMap(target, source, before) {
     }
     return { added, modified, removed };
 }
+// A queryable output row: the Tier-1 metadata projection (+ preview + storage_ref), with the
+// full `data` payload only when the caller wants it. Keeps output_query's default shape
+// backward-compatible (data present) while making the compact traversal and the deep second
+// pass first-class.
+function outputRow(rec, deps, withData) {
+    const row = {
+        id: rec.id,
+        gig_id: rec.gig_id,
+        agent_slug: rec.agent_slug,
+        from_role: rec.from_role,
+        phase: rec.phase,
+        primitive: rec.primitive,
+        core_type: rec.core_type,
+        domain_type: rec.domain_type,
+        domain_type_version: rec.domain_type_version,
+        domain: rec.domain,
+        content_sha: rec.content_sha,
+        input_refs: rec.input_refs,
+        input_shas: rec.input_shas,
+        cost_usd: rec.cost_usd,
+        tokens_used: rec.tokens_used,
+        duration_ms: rec.duration_ms,
+        model: rec.model,
+        model_tier: rec.model_tier,
+        created_at: rec.created_at,
+        preview: outputPreview(rec.data),
+        storage_ref: deps.output_mirror ? mirrorStorageRef(rec.content_sha) : "",
+    };
+    if (withData)
+        row["data"] = rec.data;
+    if (rec.skill_provenance !== undefined)
+        row["skill_provenance"] = rec.skill_provenance;
+    if (rec.reused_from !== undefined)
+        row["reused_from"] = rec.reused_from;
+    return row;
+}
+// A Tier-1 metadata row (from the mirror) shaped as an output_query row.
+function metaToRow(meta) {
+    return { ...meta };
+}
 /**
  * Pure tool dispatcher. Routes a tool call to its implementation. No transport,
  * no I/O beyond the injected deps — fully unit-testable.
@@ -347,28 +389,70 @@ async function runImpl(slug, args, deps, approval) {
                     ...(std ? { standard: { slug: std.slug, phases: std.phases.map((p) => ({ name: p.name, chairs: p.chairs.length })) } } : {}),
                     ...(observed.length > 0 ? { observed_costs_usd: observed } : {}),
                 });
-                return { ok: true, requires_approval: approval, data: res };
+                // WU-0008 — the seal drill: before quoting a price, prove every chair contract
+                // can seal AT ALL. A cost estimate for a run whose terminal chair is doomed is
+                // worse than useless (it prices a failure as if it were work). Failures ride in
+                // the SUCCESS payload loudly; dispatch stays unchanged — the operator decides.
+                const drill = sealDrill({ phases: std.phases.map((p) => ({ name: p.name, chairs: p.chairs.map((c) => ({ role: c.role, output_contract: c.output_contract })) })) }, deps.registry);
+                return { ok: true, requires_approval: approval, data: { ...res, seal_drill: drill } };
             }
             case "output_query": {
-                let outs = deps.outputs.all();
+                const mirror = deps.output_mirror;
+                // SECOND PASS — a single output's FULL payload by id or content_sha. This is the deeper
+                // Tier-2 read: prefer the content-addressed artifact tier (local mirror, or remote when
+                // drained), falling back to the store. Returns exactly the one output, with its data.
+                const wantId = args["output_id"] ? String(args["output_id"]) : undefined;
+                const wantSha = args["content_sha"] ? String(args["content_sha"]) : undefined;
+                if (wantId || wantSha) {
+                    const rec = wantId
+                        ? deps.outputs.get(wantId)
+                        : deps.outputs.all().find((o) => o.content_sha === wantSha);
+                    let row = rec ? outputRow(rec, deps, true) : undefined;
+                    // Store miss (or a mirror-only payload): read the artifact tier directly.
+                    if ((!row || row["data"] === undefined) && mirror) {
+                        const sel = {};
+                        if (wantId)
+                            sel.id = wantId;
+                        if (wantSha)
+                            sel.content_sha = wantSha;
+                        const hit = mirror.readPayload(sel);
+                        if (hit?.meta)
+                            row = { ...metaToRow(hit.meta), data: hit.data };
+                        else if (row && hit?.data !== undefined)
+                            row["data"] = hit.data;
+                    }
+                    const outputs = row ? [row] : [];
+                    return { ok: true, requires_approval: approval, data: { outputs, total_count: outputs.length } };
+                }
+                // LIST — Tier-1 traversal. The store is now cross-process-fresh (the `fullyHydrated`
+                // latch is gone), so a gig sealed by a separate CLI process is visible here. `data` is
+                // carried by default (backward-compat) and on `data_filter`; pass `include_data:false`
+                // for the compact Tier-1 rows (metadata + preview + storage_ref, no payload).
+                const dataFilter = args["data_filter"];
+                const hasDataFilter = Boolean(dataFilter && typeof dataFilter === "object" && !Array.isArray(dataFilter));
+                const includeData = args["include_data"] === false ? false : true;
+                let recs = deps.outputs.all();
                 if (args["domain_type"])
-                    outs = outs.filter((o) => o.domain_type === args["domain_type"]);
+                    recs = recs.filter((o) => o.domain_type === args["domain_type"]);
                 if (args["gig_id"])
-                    outs = outs.filter((o) => o.gig_id === args["gig_id"]);
+                    recs = recs.filter((o) => o.gig_id === args["gig_id"]);
                 if (args["agent_slug"])
-                    outs = outs.filter((o) => o.agent_slug === args["agent_slug"]);
+                    recs = recs.filter((o) => o.agent_slug === args["agent_slug"]);
                 // #234 — `data_filter` was advertised and ignored, so a caller narrowing a query by
                 // payload got the UNFILTERED set back and a `total_count` describing it. Every key must
                 // match (AND), compared structurally so an object or array value filters as written.
-                const dataFilter = args["data_filter"];
-                if (dataFilter && typeof dataFilter === "object" && !Array.isArray(dataFilter)) {
+                if (hasDataFilter) {
                     const entries = Object.entries(dataFilter);
-                    outs = outs.filter((o) => {
+                    recs = recs.filter((o) => {
                         const data = (o.data ?? {});
                         return entries.every(([k, v]) => canonJson(data[k]) === canonJson(v));
                     });
                 }
-                return { ok: true, requires_approval: approval, data: { outputs: outs, total_count: outs.length } };
+                // Payload is dropped only when the caller explicitly asks for the compact rows AND is not
+                // filtering by payload (a data_filter needs the payload to have been read).
+                const withData = includeData || hasDataFilter;
+                const outputs = recs.map((o) => outputRow(o, deps, withData));
+                return { ok: true, requires_approval: approval, data: { outputs, total_count: outputs.length } };
             }
             case "output_trace": {
                 const id = String(args["output_id"] ?? "");
@@ -501,6 +585,19 @@ async function runImpl(slug, args, deps, approval) {
                 const warnings = stdStatus === "deprecated"
                     ? [`standard "${slug2}" is deprecated — it still runs, but should not be built on.`]
                     : [];
+                // WU-0008 preflight: run the same sealDrill used by standard_simulate BEFORE spending
+                // on any chair. A structurally-unsealable standard is refused here (pennies) instead of
+                // after a chair runs and aborts. Gate is placed once, above the wait/async split, so a
+                // single check covers both runGig call-sites below.
+                const drill = sealDrill({ phases: standard.phases.map((p) => ({ name: p.name, chairs: p.chairs.map((c) => ({ role: c.role, output_contract: c.output_contract })) })) }, deps.registry);
+                if (!drill.ok) {
+                    return {
+                        ok: false, requires_approval: approval,
+                        error: `standard "${slug2}" cannot seal: ` +
+                            drill.failures.map((f) => `${f.phase}/${f.role} → ${f.domain_type} (${f.errors.join("; ")})`).join(", "),
+                        data: { seal_drill: drill },
+                    };
+                }
                 // Optional budget arg — when present, runtime enforces per-gig cost-budget
                 // and raises BudgetExhausted on depletion (PR for T10 gap, see runtime.ts).
                 const budgetArg = args["budget"];
@@ -935,6 +1032,9 @@ async function runImpl(slug, args, deps, approval) {
                     slug: baseDef.slug,
                     extends: baseDef.extends,
                     schema: { properties: nextProps },
+                    // 2026-08-08 — the undeclared-required check needs the extension's required
+                    // set, or an extend could version-in a requirement no producer can see.
+                    required_fields: nextRequired,
                 });
                 if (extendDefect) {
                     return { ok: false, requires_approval: approval, error: `type_extend rejected: ${extendDefect}` };
@@ -1620,6 +1720,46 @@ async function runImpl(slug, args, deps, approval) {
             // production status, and never RUN, TESTED, LISTED or REVISED through the engine. The
             // fixture gate on promotion made that gap sharper — you could be refused for failing
             // fixtures with no way to run them and see why.
+            // ── discoverability parity — a dispatcher must be able to FIND a slug over MCP ──────
+            // (tests/genome_browse_parity.test.ts). Backed by the deps maps, so the same handler
+            // serves a working-tree load and a hosted store load identically; no filesystem.
+            case "standard_browse": {
+                if (!deps.standards)
+                    return { ok: false, not_implemented: true, requires_approval: approval, error: "standard_browse needs a standards map (bootstrap from a genome)" };
+                let list = [...deps.standards.values()];
+                if (args["domain"])
+                    list = list.filter((s) => s.domain === args["domain"]);
+                if (args["status"])
+                    list = list.filter((s) => (s.status ?? "active") === args["status"]);
+                const standards = list
+                    .map((s) => ({
+                    slug: s.slug, domain: s.domain, status: s.status ?? "active",
+                    phases: s.phases.map((p) => p.name), phase_count: s.phases.length,
+                    chair_count: s.phases.reduce((n, p) => n + p.chairs.length, 0),
+                    input_types: s.input_types ?? [], output_types: s.output_types ?? [],
+                    eval_slugs: s.eval_slugs ?? [], description: s.description ?? null,
+                }))
+                    .sort((a, b) => (a.slug < b.slug ? -1 : 1));
+                return { ok: true, requires_approval: approval, data: { standards, count: standards.length } };
+            }
+            case "agent_browse": {
+                if (!deps.agents)
+                    return { ok: false, not_implemented: true, requires_approval: approval, error: "agent_browse needs an agents map (bootstrap from a genome)" };
+                let list = [...deps.agents.values()];
+                if (args["domain"])
+                    list = list.filter((a) => a.domain === args["domain"]);
+                if (args["primitive"])
+                    list = list.filter((a) => a.primitives.includes(args["primitive"]));
+                const agents = list
+                    .map((a) => ({
+                    slug: a.slug, primitives: a.primitives, domain: a.domain,
+                    input_types: a.input_types, output_types: a.output_types,
+                    behavioral_primitives: a.behavioral_primitives,
+                    skill_slugs: a.skill_slugs ?? [], model_tier: a.model_tier ?? null,
+                }))
+                    .sort((x, y) => (x.slug < y.slug ? -1 : 1));
+                return { ok: true, requires_approval: approval, data: { agents, count: agents.length } };
+            }
             case "skill_browse": {
                 if (!deps.skills)
                     return { ok: false, not_implemented: true, requires_approval: approval, error: "skill_browse needs a skills map (bootstrap from a genome)" };
@@ -2153,18 +2293,99 @@ async function runImpl(slug, args, deps, approval) {
         return { ok: false, requires_approval: approval, error: e instanceof Error ? e.message : String(e) };
     }
 }
-/** Build the low-level MCP Server with ListTools + CallTool wired to the dispatcher. */
+// Tools whose implementation is inherently local-process. Each entry names WHY, so the
+// hosted refusal teaches instead of stonewalling.
+const HOSTED_BLOCKED = {
+    server_restart: "server_restart is a local-process concern (the stdio relay restarts its child); a hosted, per-request surface has no server process to restart",
+    skill_execute: "skill_execute runs a skill's code half in a local subprocess against its on-disk package; hosted skills carry no local package — run code skills where the genome files live",
+    skill_evolve: "skill_evolve runs candidate code against local fixtures in a subprocess; hosted skills carry no local package to evolve",
+    charter_read: "charter_read reads a charter file from a local path; a hosted surface has no filesystem",
+    gig_logs: "gig_logs tails per-chair log files under the local outputs dir; hosted runs are executed by the drain worker and their record lives in the store",
+    genome_reload: "genome_reload re-reads genome files from disk; a hosted surface loads its genome from the store per-request, so there is nothing to reload",
+};
+// The genome-mutation tools whose success must ALSO land in the hosted store, and the class
+// each persists as. Payload is filtered by the class's own Zod schema key list (the same
+// single source the handlers copy from), so the store payload can't drift from the schema.
+const HOSTED_UPSERT = {
+    agent_define: { cls: "agent", keys: Object.keys(AgentSchema.shape) },
+    standard_compose: { cls: "standard", keys: Object.keys(StandardSchema.shape) },
+    type_register: { cls: "domain_type", keys: Object.keys(DomainTypeSchema.shape) },
+    skill_define: { cls: "skill", keys: Object.keys(SkillSchema.shape) },
+};
+async function callSurfaceTool(slug, args, deps) {
+    if (deps.hosted) {
+        const blocked = HOSTED_BLOCKED[slug];
+        if (blocked)
+            return { ok: false, hosted_unsupported: true, error: blocked };
+        if (slug === "gig_dispatch") {
+            // Hosted dispatch NEVER spawns. With a queue seam it queues (the gig table is the
+            // queue; a drain worker claims and runs); without one it says so, typed.
+            if (deps.queueGig) {
+                try {
+                    const data = await deps.queueGig(args);
+                    return { ok: true, data };
+                }
+                catch (e) {
+                    return { ok: false, error: e instanceof Error ? e.message : String(e) };
+                }
+            }
+            return {
+                ok: false,
+                hosted_unsupported: true,
+                error: "hosted dispatch goes through the queue RPC (coltrane_gig_dispatch) — nothing spawns in a hosted " +
+                    "surface. Wire deps.queueGig (e.g. postgrestQueueGig(ctx) from ./genome_store) to queue the gig " +
+                    "for the drain worker.",
+            };
+        }
+    }
+    const result = await dispatchTool(slug, args, deps);
+    // Hosted persistence: without a genome_dir the handlers compute + validate + seal identity
+    // but write no file (the validation path). The store upsert is the hosted write half; a
+    // refused upsert fails the mutation loudly — a definition that only ever lived in this
+    // request's memory must not report success.
+    const up = HOSTED_UPSERT[slug];
+    if (deps.hosted && deps.store && result.ok && up) {
+        const payload = {};
+        for (const k of up.keys)
+            if (args[k] !== undefined)
+                payload[k] = args[k];
+        try {
+            await deps.store.upsert(up.cls, payload);
+        }
+        catch (e) {
+            return { ok: false, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    return result;
+}
+/** The engine's FULL MCP tool surface as a transport-agnostic registry. The stdio server
+ *  below consumes this internally; a hosted host mounts it per-request with hosted deps. */
+export function createToolSurface(deps) {
+    return MCP_TOOLS.map((t) => ({
+        name: t.slug,
+        description: `${t.category} tool`,
+        input_schema: t.input_schema,
+        call: (args) => callSurfaceTool(t.slug, args, deps),
+    }));
+}
+/** Build the low-level MCP Server with ListTools + CallTool wired to the tool surface. */
 export function createColtraneServer(deps, recorder) {
     const server = new Server({ name: "coltrane", version: COLTRANE_VERSION }, { capabilities: { tools: {} } });
+    const surface = createToolSurface(deps);
+    const byName = new Map(surface.map((t) => [t.name, t]));
     server.setRequestHandler(ListToolsRequestSchema, async () => ({
-        tools: MCP_TOOLS.map((t) => ({
-            name: t.slug,
-            description: `${t.category} tool`,
+        tools: surface.map((t) => ({
+            name: t.name,
+            description: t.description,
             inputSchema: t.input_schema,
         })),
     }));
     server.setRequestHandler(CallToolRequestSchema, async (req) => {
-        const result = await dispatchTool(req.params.name, (req.params.arguments ?? {}), deps);
+        const args = (req.params.arguments ?? {});
+        const tool = byName.get(req.params.name);
+        // A name outside the surface falls through to the dispatcher's own unknown-tool answer,
+        // keeping the reply byte-identical to the pre-surface path.
+        const result = tool ? await tool.call(args) : await dispatchTool(req.params.name, args, deps);
         if (recorder) {
             recorder.recordToolCall(req.params.name);
             recorder.recordObservability(`call:${req.params.name}`, { ok: result.ok });
@@ -2234,13 +2455,19 @@ export function bootstrapServerDeps(genomeRoot) {
     // the tool as mcp__coltrane__<slug> — the name the server exposes — so a bare-slug grant (the form
     // agent_define accepts) is actually callable instead of a silent dead name that seals nothing.
     const toolProviders = new Map([...REGISTERED_TOOL_SLUGS].map((slug) => [slug, { tool: slug, kind: "in_house", server: ENGINE_MCP_SERVER }]));
+    // The two-tier local mirror, rooted at `<root>/.coltrane` (gitignored, beside the ledger).
+    // The SAME instance is handed to the OutputStore and exposed as `output_mirror`, so every
+    // sealed output — whether this process ran the gig or a separate CLI process did — lands in
+    // one content-addressed store MCP retrieval reads. COLTRANE_MIRROR_DIR overrides (tests).
+    const output_mirror = createOutputMirror(defaultMirrorDir(root));
     return {
         registry,
         toolProviders,
+        output_mirror,
         // PR #78 follow-up: persist outputs to disk so the audit chain survives an
         // MCP session close (Rob cold-trial requirement). COLTRANE_OUTPUTS_DIR
         // overrides the default ~/.eir/coltrane_outputs path (tests + sandboxes).
-        outputs: createOutputStore(registry, { persistDir: defaultOutputsPersistDir() }),
+        outputs: createOutputStore(registry, { persistDir: defaultOutputsPersistDir(), mirror: output_mirror }),
         // #209 — the audit spine is durable by default. The line above gives OUTPUTS a persistDir
         // under an explicit "the audit chain must survive an MCP session close" requirement
         // (PR #78); the ledger sat in RAM directly beneath it, which made absence-of-row mean