onto-mcp 0.4.11 → 0.4.13

package/dist/core-runtime/reconstruct/materialize-preparation.js

@@ -1,9 +1,11 @@
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
-import { stringify as stringifyYaml } from "yaml";
+import { atomicWriteYamlDocument as writeYamlDocument } from "../artifact-io.js";
 import { aggregateTargetMaterialDetections, detectTargetMaterialRefs, } from "../target-material-kind.js";
+import { observeSpreadsheetSource, projectInventoryForAdmission, SPREADSHEET_CAPTURE_TRUNCATED_PHRASE, SPREADSHEET_MACRO_PRESENT_PHRASE, SPREADSHEET_OBSERVER_ADAPTER_ID, } from "../spreadsheet-structure-observer.js";
 import { validateSourceObservationBoundary, } from "./source-observations.js";
+import { buildDeterministicComprehensionArtifact } from "./comprehension-artifact.js";
 import { loadReconstructSourceProfiles, } from "./source-profiles.js";
 const CONCRETE_TARGET_MATERIAL_KINDS = new Set([
     "code",
@@ -17,10 +19,6 @@ function isConcreteTargetMaterialKind(kind) {
 function isoNow() {
     return new Date().toISOString();
 }
-async function writeYamlDocument(filePath, value) {
-    await fs.mkdir(path.dirname(filePath), { recursive: true });
-    await fs.writeFile(filePath, stringifyYaml(value), "utf8");
-}
 function stableObservationId(args) {
     const digest = crypto
         .createHash("sha256")
@@ -114,10 +112,199 @@ function defaultProfileForKind(profiles, kind) {
     return matchingProfiles.find((candidate) => candidate.is_default_for_kind) ??
         matchingProfiles[0];
 }
-async function textStats(ref) {
+/**
+ * Structural `content_excerpt` capture budgets (chars). Code source and text-readable
+ * document prose are captured whole so the file tail (later definitions, document
+ * goals/milestones/decisions) reaches seed authoring instead of being lost to a
+ * leading slice — both are projected whole at the seed stage, so a small capture
+ * would silently author the seed from a partial file. Other kinds (binary documents,
+ * spreadsheet/database structural inventories) keep a small leading sample — the
+ * observation is a structural sample there, not the whole file.
+ *
+ * Capture is MODEL-AGNOSTIC: it runs before the seed-stage (provider, model) is
+ * resolved, so it captures up to a fixed `DOCUMENT_CAPTURE_CEILING_CHARS` ceiling
+ * (a disk/pathological-input safety bound, NOT a window bound). The seed-stage
+ * prompt PROJECTION then slices that captured prose to a dynamic, model-aware
+ * budget (see deriveDocumentExcerptProjectionBudget + run.ts). The ceiling is set
+ * comfortably above any projection budget so projection only ever narrows, never
+ * needs more than was captured. A document longer than the ceiling is captured-
+ * truncated (`excerpt_truncated`); one longer than the projection budget is
+ * projection-truncated at the seed stage (recorded durably there). True
+ * window-overflow recovery (selecting the relevant tail) is a later stage (see
+ * development-records/design/20260616-large-input-observation).
+ *
+ * Material kind is detected by extension (target-material-kind.ts `DOCUMENT_EXTENSIONS`),
+ * and `document` includes binary formats (.pdf/.docx/.ppt/.rtf) that `textStats` still
+ * reads as UTF-8. Only text-readable document formats earn the whole-document budget —
+ * a binary document keeps the small sample so we never capture the ceiling in decoded
+ * binary bytes and spend the prompt on garbage; binary documents need an extraction step
+ * before reconstruct. This set must be the text-prose subset of `DOCUMENT_EXTENSIONS`: an
+ * extension that the classifier does not map to `document` (e.g. `.html`) would never
+ * reach this budget, so listing it here would be dead and misleading.
+ */
+const DEFAULT_STRUCTURAL_EXCERPT_CHAR_LIMIT = 6000;
+/**
+ * Capture ceiling (chars) for whole-document prose. Static and model-agnostic —
+ * the upper bound for both capture AND the dynamic projection clamp, so the
+ * captured excerpt is always a superset of any projection. Set well above the
+ * largest projection budget (opus/gpt-5.5 ~1M-token windows derive ~0.5M-char
+ * budgets) to leave headroom; the absolute bound is disk, not a model window.
+ */
+export const DOCUMENT_CAPTURE_CEILING_CHARS = 5_000_000;
+/**
+ * Floor (chars) for the seed-stage document projection budget. The static value
+ * used when the model window is unknown (mock realization, provider-only seat, an
+ * unregistered model, or a registry entry without context_window_tokens). Equal to
+ * the prior fixed document excerpt budget, so a model-unaware run is unchanged.
+ */
+export const DOCUMENT_EXCERPT_PROJECTION_FLOOR = 200_000;
+/**
+ * Fraction of the model context window the projection budget may consume, leaving
+ * margin for prompt instructions, non-excerpt payload, output tokens, and
+ * tokenization variance. Window-proportional (not a fixed char margin) so the
+ * margin scales with the window (C1).
+ */
+const WINDOW_BUDGET_FRACTION = 0.5;
+/**
+ * Conservative LOWER bound on chars-per-token used to convert a token window into
+ * a char budget. NOT the average (~4 for English): CJK-dense prose tokenizes to
+ * far fewer chars per token (≈1 or below), so a low bound keeps the char budget
+ * from implying more tokens than the window holds (C1). This is an approximation
+ * pending a real tokenizer; the CJK-dense live fixture (P6) validates/calibrates
+ * it, and the re-design trigger lowers it if even this conservative budget
+ * overflows.
+ */
+const CHARS_PER_TOKEN_LB = 1;
+/**
+ * Chars reserved from the derived budget for the prompt's instructions, the
+ * non-excerpt structural payload, and the model's output allowance — all of which
+ * also consume the window alongside the document excerpt (C1).
+ */
+const PROMPT_OVERHEAD_RESERVE_CHARS = 50_000;
+// Clamp integrity: the capture ceiling must be >= the projection floor so the
+// derived budget's clamp(raw, FLOOR, CEILING) range is well-formed and the
+// captured excerpt is always a superset of any projection (C4). A module-load
+// assert fails fast on a future mis-edit of these constants.
+if (DOCUMENT_EXCERPT_PROJECTION_FLOOR > DOCUMENT_CAPTURE_CEILING_CHARS) {
+    throw new Error("Invalid document excerpt budgets: DOCUMENT_EXCERPT_PROJECTION_FLOOR " +
+        `(${DOCUMENT_EXCERPT_PROJECTION_FLOOR}) must be <= ` +
+        `DOCUMENT_CAPTURE_CEILING_CHARS (${DOCUMENT_CAPTURE_CEILING_CHARS}).`);
+}
+const TEXT_READABLE_DOCUMENT_EXTENSIONS = new Set([".md", ".txt", ".adoc"]);
+// M3a — explicit positive allowlist of real source-language extensions that earn whole-file
+// capture/projection. The classifier also maps config/data extensions to `code`
+// (.json/.yaml/.yml/.toml/.xml/.env/.cfg/.conf/.lock); those are deliberately NOT here and
+// default to the bounded structural sample (a quality + cost volume reduction — NOT a
+// secret-exposure closure, which is Track 2). Enumerated, no wildcard, so adding a language is
+// a deliberate edit; fail-safe is bounded for any code extension not listed.
+const CODE_WHOLE_CAPTURE_EXTENSIONS = new Set([
+    ".c",
+    ".cc",
+    ".cpp",
+    ".cs",
+    ".css",
+    ".dockerfile",
+    ".go",
+    ".graphql",
+    ".java",
+    ".js",
+    ".jsx",
+    ".kt",
+    ".mjs",
+    ".php",
+    ".prisma",
+    ".proto",
+    ".py",
+    ".rb",
+    ".rs",
+    ".scss",
+    ".sh",
+    ".swift",
+    ".ts",
+    ".tsx",
+]);
+// Extensionless build-language source basenames the classifier maps to `code` by name
+// (CODE_BASENAMES). These are real source/build scripts → whole capture. The config/data
+// basenames (package.json/tsconfig.json/cargo.toml/go.mod/pom.xml) are deliberately NOT here
+// — they default to the bounded sample like other config/data (M3a-default), and are small
+// enough that the sample is the whole file anyway.
+const CODE_WHOLE_CAPTURE_BASENAMES = new Set([
+    "dockerfile",
+    "makefile",
+    "rakefile",
+    "gemfile",
+]);
+/**
+ * Single whole-capture / full-excerpt eligibility predicate (M3a). Consumed by BOTH the
+ * capture owner (`structuralExcerptCharLimit` here) and the seed-stage prompt projection
+ * (`isFullExcerptProjectionEligible` in run.ts), so capture and projection can never disagree
+ * — a bounded capture under a whole-projection budget would silently author the seed from a
+ * partial file. Takes the source REF (path/filename) so it is basename-aware: source-language
+ * code (allowlisted extension OR build-language basename) and text-readable documents
+ * (.md/.txt/.adoc) earn whole capture; binary documents, config/data code files, and
+ * structural-inventory kinds (spreadsheet/database) stay bounded.
+ */
+export function isFullExcerptCaptureEligible(kind, ref) {
+    const lower = typeof ref === "string" ? ref.toLowerCase() : "";
+    const ext = path.extname(lower);
+    const base = path.basename(lower);
+    if (kind === "code") {
+        return CODE_WHOLE_CAPTURE_EXTENSIONS.has(ext) ||
+            CODE_WHOLE_CAPTURE_BASENAMES.has(base);
+    }
+    return kind === "document" && isTextReadableDocumentExtension(ext);
+}
+/**
+ * Derives the seed-stage document projection budget (chars) for the active
+ * (provider, model) seat from its registered context window. Pure and total
+ * (never throws): an unresolved provider/model, an unregistered pair, or an entry
+ * without a window all fall back to the static FLOOR, so mock realization and
+ * provider-only seats are unchanged.
+ *
+ * The route key is the MODEL provider (the registry key, e.g. "openai"), not the
+ * runtime adapter provider (openai OAuth dispatches as "codex"; anthropic OAuth
+ * stays "anthropic"). The caller passes the model provider so the default
+ * gpt-5.5 OAuth seat resolves against `openai/gpt-5.5` (see reconstruct-api).
+ *
+ * This is the SINGLE model→budget conversion point: model literals never reach
+ * the char-budget tuning constants, so G2/INV-CFG-1 stay satisfied.
+ */
+export function deriveDocumentExcerptProjectionBudget(route, registry) {
+    if (!route.provider || !route.modelId) {
+        return DOCUMENT_EXCERPT_PROJECTION_FLOOR;
+    }
+    const window = registry.supported_models.find((entry) => entry.provider === route.provider && entry.model === route.modelId)?.context_window_tokens;
+    if (!window)
+        return DOCUMENT_EXCERPT_PROJECTION_FLOOR;
+    const raw = Math.floor(window * WINDOW_BUDGET_FRACTION * CHARS_PER_TOKEN_LB) -
+        PROMPT_OVERHEAD_RESERVE_CHARS;
+    return Math.min(Math.max(raw, DOCUMENT_EXCERPT_PROJECTION_FLOOR), DOCUMENT_CAPTURE_CEILING_CHARS);
+}
+/**
+ * A text-readable document earns the whole-document excerpt budget. Both the capture
+ * (here) and the seed-stage prompt projection (run.ts) consult this single predicate so
+ * a binary document (.pdf/.docx/.ppt/.rtf) is never expanded — neither the captured
+ * artifact nor the prompt carries decoded binary bytes.
+ */
+export function isTextReadableDocumentExtension(extension) {
+    return (typeof extension === "string" &&
+        TEXT_READABLE_DOCUMENT_EXTENSIONS.has(extension.toLowerCase()));
+}
+function structuralExcerptCharLimit(kind, ref) {
+    // Whole-capture eligibility is the single shared predicate (M3a): a source-language file
+    // or a text-readable document is projected whole into seed authoring, so it must be
+    // CAPTURED whole too — a small leading sample would make seed authoring read only the
+    // file's head while the prompt-projection truncation never fires (capture < projection
+    // budget), silently authoring the seed from a partial file. Config/data code files,
+    // binary documents, and structural-inventory kinds (spreadsheet/database) stay bounded.
+    if (isFullExcerptCaptureEligible(kind, ref)) {
+        return DOCUMENT_CAPTURE_CEILING_CHARS;
+    }
+    return DEFAULT_STRUCTURAL_EXCERPT_CHAR_LIMIT;
+}
+async function textStats(ref, excerptLimit) {
     try {
         const text = await fs.readFile(ref, "utf8");
-        const excerptLimit = 6000;
         return {
             line_count: text.length === 0 ? 0 : text.split(/\r?\n/).length,
             char_count: text.length,
@@ -136,20 +323,57 @@ async function textStats(ref) {
         };
     }
 }
-export async function buildReconstructSourceObservation(detection, lineage) {
+export async function buildReconstructSourceObservation(detection, lineage, 
+// Defect-3 basis A: the caller asserts whether this ref is the user-provided
+// reconstruct runtime target. Set true ONLY by the initial-target
+// materialization; frontier-discovered / maturation-closure re-entry callers
+// omit it (conservative default false). Every observation literal below sets
+// is_runtime_target_source from this so source-safety can grant the
+// material_claim/public_output tiers by provenance without leaking to
+// non-target sources.
+options) {
     if (!detection.exists || !isConcreteTargetMaterialKind(detection.kind)) {
         return null;
     }
     const extension = path.extname(detection.ref).toLowerCase();
     const location = detection.ref;
-    const stat = await fs.stat(detection.ref);
-    const stats = stat.isFile() ? await textStats(detection.ref) : {
-        line_count: null,
-        char_count: null,
-        content_sha256: null,
-        content_excerpt: null,
-        excerpt_truncated: false,
-    };
+    // Re-observation runs after an earlier detection: the ref may have vanished
+    // in between (TOCTOU). Treat a missing ref as nothing-to-observe (degrade to
+    // null, like the !detection.exists guard above) instead of crashing the run
+    // with an uncontextualized ENOENT. Other stat failures still propagate.
+    let stat;
+    try {
+        stat = await fs.stat(detection.ref);
+    }
+    catch (error) {
+        const code = error.code;
+        // ENOENT: the ref was deleted; ENOTDIR: a parent path component became a
+        // file. Both mean the ref is no longer an observable source (TOCTOU), so
+        // degrade to null. Other stat failures still propagate.
+        if (code === "ENOENT" || code === "ENOTDIR")
+            return null;
+        throw error;
+    }
+    // Spreadsheet sources route through the shared deterministic structure
+    // observer instead of the generic raw-text path (design S1 §2.2 / §11).
+    if (detection.kind === "spreadsheet") {
+        return buildSpreadsheetSourceObservation({
+            detection,
+            stat,
+            location,
+            lineage,
+            isRuntimeTargetSource: Boolean(options?.isRuntimeTargetSource),
+        });
+    }
+    const stats = stat.isFile()
+        ? await textStats(detection.ref, structuralExcerptCharLimit(detection.kind, detection.ref))
+        : {
+            line_count: null,
+            char_count: null,
+            content_sha256: null,
+            content_excerpt: null,
+            excerpt_truncated: false,
+        };
     const observation = {
         observation_id: stableObservationId({
             sourceRef: detection.ref,
@@ -158,6 +382,7 @@ export async function buildReconstructSourceObservation(detection, lineage) {
         round_id: lineage?.roundId ?? "initial_source_frontier",
         observation_batch_id: lineage?.observationBatchId ?? "source-observation-batch:initial",
         triggering_frontier_validation_ref: lineage?.triggeringFrontierValidationRef ?? null,
+        is_runtime_target_source: Boolean(options?.isRuntimeTargetSource),
         target_material_kind: detection.kind,
         adapter_id: `minimal-${detection.kind}-structure-observer`,
         source_ref: detection.ref,
@@ -181,6 +406,96 @@ export async function buildReconstructSourceObservation(detection, lineage) {
     }
     return observation;
 }
+/**
+ * Observe a spreadsheet source through the shared structure observer (design S1
+ * §2.2) — a deterministic, LLM-free structural inventory. The observation carries
+ * NO raw cell values: it never emits the generic path's `content_excerpt` (which for
+ * a workbook would be raw data values) and the inventory's aggregate-only vocab
+ * leaves `top_values` absent. The one NARROWED exception (design-C, not raw data) is
+ * `data_validations[].members` — the bounded, DECLARED type=list enum labels parsed from a
+ * formula1 literal (never observed values). `content_sha256` is the RAW-byte hash (§11 HASH-1)
+ * surfaced at the structural_data top level because the downstream source-scout-pack
+ * reads it there; the full inventory is nested under `workbook_inventory` as the
+ * structural substrate the seed-authoring prompt observes. xlsx-family kinds are
+ * not yet extractable (P4) and arrive here carrying an `unsupported_reason`.
+ */
+/**
+ * The honest observation summary for a workbook inventory (P6): the base disclosure
+ * plus the fixed phrases the boundary gate asserts when the capture/macro flags are
+ * set. This is the single emit path; it and the gate bind to the SAME exported phrase
+ * consts, so assert + emit cannot drift. Exported for the gate-binding regression test.
+ */
+export function buildSpreadsheetObservationSummary(basename, inventory) {
+    const parts = [
+        inventory.unsupported_reason
+            ? `spreadsheet workbook observed at ${basename} — extraction unsupported (${inventory.unsupported_reason}), structure_inspected_only`
+            : `spreadsheet workbook observed at ${basename} — ${inventory.sheets.length} sheet(s), structure_inspected_only`,
+    ];
+    if (inventory.capture_truncated)
+        parts.push(SPREADSHEET_CAPTURE_TRUNCATED_PHRASE);
+    if (inventory.macro_present)
+        parts.push(SPREADSHEET_MACRO_PRESENT_PHRASE);
+    return parts.join("; ");
+}
+async function buildSpreadsheetSourceObservation(args) {
+    const { detection, stat, location, lineage, isRuntimeTargetSource } = args;
+    const basename = path.basename(detection.ref);
+    const extension = path.extname(detection.ref).toLowerCase();
+    // Route through the single shared projection: the structural_data inventory
+    // carries aggregate counts / structure only, never raw cell values (the sole narrowed
+    // exception is data_validations[].members — bounded declared type=list enum labels).
+    const inventory = projectInventoryForAdmission(await observeSpreadsheetSource(detection.ref));
+    const summary = buildSpreadsheetObservationSummary(basename, inventory);
+    const observationId = stableObservationId({ sourceRef: detection.ref, location });
+    // P1-C1 §5.7: the ComprehensionArtifact COMPANIONS the inventory (joined by observation_id) — a
+    // deterministic-only edition in this cut (LLM-touch fields are explicit not_applicable). It is
+    // validated by validateSourceObservationBoundary below (construction throws on an invalid contract).
+    const comprehensionArtifact = buildDeterministicComprehensionArtifact({
+        observationId,
+        inventory,
+    });
+    const observation = {
+        observation_id: observationId,
+        round_id: lineage?.roundId ?? "initial_source_frontier",
+        observation_batch_id: lineage?.observationBatchId ?? "source-observation-batch:initial",
+        triggering_frontier_validation_ref: lineage?.triggeringFrontierValidationRef ?? null,
+        is_runtime_target_source: isRuntimeTargetSource,
+        target_material_kind: "spreadsheet",
+        adapter_id: SPREADSHEET_OBSERVER_ADAPTER_ID,
+        source_ref: detection.ref,
+        location,
+        summary,
+        structural_data: {
+            basename,
+            extension: extension || null,
+            path_kind: stat.isDirectory() ? "directory" : "file",
+            size_bytes: stat.isFile() ? stat.size : null,
+            // Raw-byte hash (§11 HASH-1) surfaced top-level for source-scout-pack
+            // admission, which reads structural_data.content_sha256.
+            content_sha256: inventory.content_sha256,
+            workbook_inventory: inventory,
+            comprehension_artifact: comprehensionArtifact,
+        },
+    };
+    const validation = validateSourceObservationBoundary(observation);
+    if (!validation.valid) {
+        throw new Error(`Invalid source observation boundary for ${detection.ref}: ${validation.violations.join("; ")}`);
+    }
+    return observation;
+}
+/**
+ * The `unsupported_reason` of a spreadsheet observation's workbook inventory, or
+ * null when the observation is not an unsupported workbook. An unsupported workbook
+ * (.xls/.xlsb/.ods, corrupt, oversized) carries no structural evidence, so the
+ * materialize loop demotes it to a skip rather than admitting an empty observation.
+ */
+export function spreadsheetUnsupportedReason(observation) {
+    if (observation.target_material_kind !== "spreadsheet")
+        return null;
+    const inventory = observation.structural_data.workbook_inventory;
+    const reason = inventory?.unsupported_reason;
+    return typeof reason === "string" && reason.length > 0 ? reason : null;
+}
 function stableFrontierRefId(unit) {
     const digest = crypto
         .createHash("sha256")
@@ -291,10 +606,6 @@ export async function materializeReconstructPreparationArtifacts(params) {
             source: "binding",
         },
     };
-    const initialSourceFrontier = buildInitialSourceFrontier({
-        sessionId,
-        inventory,
-    });
     const observations = [];
     const skippedRefs = [];
     for (const unit of inventory.inventory_units) {
@@ -309,10 +620,56 @@ export async function materializeReconstructPreparationArtifacts(params) {
         const refDetection = detection.per_ref.find((candidate) => candidate.ref === unit.ref);
         if (!refDetection)
             continue;
-        const observation = await buildReconstructSourceObservation(refDetection);
-        if (observation)
-            observations.push(observation);
+        // Defect-3 basis A: these are the user-provided reconstruct runtime-target
+        // refs (the initial target inventory), so they are runtime-target sources.
+        const observation = await buildReconstructSourceObservation(refDetection, undefined, {
+            isRuntimeTargetSource: true,
+        });
+        if (observation) {
+            const unsupportedReason = spreadsheetUnsupportedReason(observation);
+            if (unsupportedReason) {
+                // A workbook whose format the observer cannot extract (.xls/.xlsb/.ods,
+                // corrupt, oversized) yields an inventory carrying only `unsupported_reason`
+                // — no structural evidence. After the gate flip such a ref is runnable and
+                // produces an observation, so without this demotion a sole legacy-format
+                // target would pass the zero-observation halt and reach LLM authoring with
+                // empty evidence. Demote it to a skip so the run fails loud as unsupported
+                // (the evidence gate stays honest about un-observed material).
+                unit.scan_status = "skipped";
+                unit.skip_reason = `spreadsheet extraction unsupported: ${unsupportedReason}`;
+                skippedRefs.push({
+                    ref: unit.ref,
+                    target_material_kind: unit.target_material_kind,
+                    reason: unit.skip_reason,
+                });
+            }
+            else {
+                observations.push(observation);
+            }
+        }
+        else {
+            // buildReconstructSourceObservation returns null when the ref is no longer
+            // a concrete, existing source at observation time (e.g. deleted between
+            // detection and re-observation). Mark the inventory unit skipped — the
+            // single source of truth that the initial frontier (built below), the
+            // zero-observation halt, and later frontier admission all derive from — so
+            // the vanished ref is excluded everywhere instead of being silently dropped
+            // or re-queued by the deterministic first-frontier scout.
+            unit.scan_status = "skipped";
+            unit.skip_reason = "source ref unavailable at observation time";
+            skippedRefs.push({
+                ref: unit.ref,
+                target_material_kind: unit.target_material_kind,
+                reason: unit.skip_reason,
+            });
+        }
     }
+    // Built after observation so refs marked skipped above (vanished mid-run) are
+    // excluded from frontier source_refs rather than re-admitted later.
+    const initialSourceFrontier = buildInitialSourceFrontier({
+        sessionId,
+        inventory,
+    });
     const sourceObservations = {
         schema_version: "1",
         session_id: sessionId,