onto-mcp 0.4.12 → 0.4.13

package/dist/core-runtime/reconstruct/materialize-preparation.js

@@ -3,7 +3,9 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { atomicWriteYamlDocument as writeYamlDocument } from "../artifact-io.js";
 import { aggregateTargetMaterialDetections, detectTargetMaterialRefs, } from "../target-material-kind.js";
+import { observeSpreadsheetSource, projectInventoryForAdmission, SPREADSHEET_CAPTURE_TRUNCATED_PHRASE, SPREADSHEET_MACRO_PRESENT_PHRASE, SPREADSHEET_OBSERVER_ADAPTER_ID, } from "../spreadsheet-structure-observer.js";
 import { validateSourceObservationBoundary, } from "./source-observations.js";
+import { buildDeterministicComprehensionArtifact } from "./comprehension-artifact.js";
 import { loadReconstructSourceProfiles, } from "./source-profiles.js";
 const CONCRETE_TARGET_MATERIAL_KINDS = new Set([
     "code",
@@ -111,28 +113,173 @@ function defaultProfileForKind(profiles, kind) {
         matchingProfiles[0];
 }
 /**
- * Structural `content_excerpt` capture budgets (chars). Code and other kinds keep
- * a small leading-sample budget — the observation is a structural sample, not the
- * whole file. Text-readable document prose is captured whole so the document tail
- * (goals, milestones, decisions) reaches seed authoring instead of being lost to a
- * leading slice: a typical business/policy document is well within any modern model
- * window (e.g. ~12.5K chars ≈ ~3K tokens). The 200K cap is a safety ceiling against
- * pathological inputs; documents that genuinely exceed it are decomposed by a later
- * stage (see development-records/design/20260616-large-input-observation), not
- * silently truncated here.
+ * Structural `content_excerpt` capture budgets (chars). Code source and text-readable
+ * document prose are captured whole so the file tail (later definitions, document
+ * goals/milestones/decisions) reaches seed authoring instead of being lost to a
+ * leading slice — both are projected whole at the seed stage, so a small capture
+ * would silently author the seed from a partial file. Other kinds (binary documents,
+ * spreadsheet/database structural inventories) keep a small leading sample — the
+ * observation is a structural sample there, not the whole file.
+ *
+ * Capture is MODEL-AGNOSTIC: it runs before the seed-stage (provider, model) is
+ * resolved, so it captures up to a fixed `DOCUMENT_CAPTURE_CEILING_CHARS` ceiling
+ * (a disk/pathological-input safety bound, NOT a window bound). The seed-stage
+ * prompt PROJECTION then slices that captured prose to a dynamic, model-aware
+ * budget (see deriveDocumentExcerptProjectionBudget + run.ts). The ceiling is set
+ * comfortably above any projection budget so projection only ever narrows, never
+ * needs more than was captured. A document longer than the ceiling is captured-
+ * truncated (`excerpt_truncated`); one longer than the projection budget is
+ * projection-truncated at the seed stage (recorded durably there). True
+ * window-overflow recovery (selecting the relevant tail) is a later stage (see
+ * development-records/design/20260616-large-input-observation).
  *
  * Material kind is detected by extension (target-material-kind.ts `DOCUMENT_EXTENSIONS`),
  * and `document` includes binary formats (.pdf/.docx/.ppt/.rtf) that `textStats` still
  * reads as UTF-8. Only text-readable document formats earn the whole-document budget —
- * a binary document keeps the small sample so we never capture 200K of decoded binary
- * bytes and spend the prompt on garbage; binary documents need an extraction step before
- * reconstruct. This set must be the text-prose subset of `DOCUMENT_EXTENSIONS`: an
+ * a binary document keeps the small sample so we never capture the ceiling in decoded
+ * binary bytes and spend the prompt on garbage; binary documents need an extraction step
+ * before reconstruct. This set must be the text-prose subset of `DOCUMENT_EXTENSIONS`: an
  * extension that the classifier does not map to `document` (e.g. `.html`) would never
  * reach this budget, so listing it here would be dead and misleading.
  */
 const DEFAULT_STRUCTURAL_EXCERPT_CHAR_LIMIT = 6000;
-export const DOCUMENT_EXCERPT_CHAR_LIMIT = 200_000;
+/**
+ * Capture ceiling (chars) for whole-document prose. Static and model-agnostic —
+ * the upper bound for both capture AND the dynamic projection clamp, so the
+ * captured excerpt is always a superset of any projection. Set well above the
+ * largest projection budget (opus/gpt-5.5 ~1M-token windows derive ~0.5M-char
+ * budgets) to leave headroom; the absolute bound is disk, not a model window.
+ */
+export const DOCUMENT_CAPTURE_CEILING_CHARS = 5_000_000;
+/**
+ * Floor (chars) for the seed-stage document projection budget. The static value
+ * used when the model window is unknown (mock realization, provider-only seat, an
+ * unregistered model, or a registry entry without context_window_tokens). Equal to
+ * the prior fixed document excerpt budget, so a model-unaware run is unchanged.
+ */
+export const DOCUMENT_EXCERPT_PROJECTION_FLOOR = 200_000;
+/**
+ * Fraction of the model context window the projection budget may consume, leaving
+ * margin for prompt instructions, non-excerpt payload, output tokens, and
+ * tokenization variance. Window-proportional (not a fixed char margin) so the
+ * margin scales with the window (C1).
+ */
+const WINDOW_BUDGET_FRACTION = 0.5;
+/**
+ * Conservative LOWER bound on chars-per-token used to convert a token window into
+ * a char budget. NOT the average (~4 for English): CJK-dense prose tokenizes to
+ * far fewer chars per token (≈1 or below), so a low bound keeps the char budget
+ * from implying more tokens than the window holds (C1). This is an approximation
+ * pending a real tokenizer; the CJK-dense live fixture (P6) validates/calibrates
+ * it, and the re-design trigger lowers it if even this conservative budget
+ * overflows.
+ */
+const CHARS_PER_TOKEN_LB = 1;
+/**
+ * Chars reserved from the derived budget for the prompt's instructions, the
+ * non-excerpt structural payload, and the model's output allowance — all of which
+ * also consume the window alongside the document excerpt (C1).
+ */
+const PROMPT_OVERHEAD_RESERVE_CHARS = 50_000;
+// Clamp integrity: the capture ceiling must be >= the projection floor so the
+// derived budget's clamp(raw, FLOOR, CEILING) range is well-formed and the
+// captured excerpt is always a superset of any projection (C4). A module-load
+// assert fails fast on a future mis-edit of these constants.
+if (DOCUMENT_EXCERPT_PROJECTION_FLOOR > DOCUMENT_CAPTURE_CEILING_CHARS) {
+    throw new Error("Invalid document excerpt budgets: DOCUMENT_EXCERPT_PROJECTION_FLOOR " +
+        `(${DOCUMENT_EXCERPT_PROJECTION_FLOOR}) must be <= ` +
+        `DOCUMENT_CAPTURE_CEILING_CHARS (${DOCUMENT_CAPTURE_CEILING_CHARS}).`);
+}
 const TEXT_READABLE_DOCUMENT_EXTENSIONS = new Set([".md", ".txt", ".adoc"]);
+// M3a — explicit positive allowlist of real source-language extensions that earn whole-file
+// capture/projection. The classifier also maps config/data extensions to `code`
+// (.json/.yaml/.yml/.toml/.xml/.env/.cfg/.conf/.lock); those are deliberately NOT here and
+// default to the bounded structural sample (a quality + cost volume reduction — NOT a
+// secret-exposure closure, which is Track 2). Enumerated, no wildcard, so adding a language is
+// a deliberate edit; fail-safe is bounded for any code extension not listed.
+const CODE_WHOLE_CAPTURE_EXTENSIONS = new Set([
+    ".c",
+    ".cc",
+    ".cpp",
+    ".cs",
+    ".css",
+    ".dockerfile",
+    ".go",
+    ".graphql",
+    ".java",
+    ".js",
+    ".jsx",
+    ".kt",
+    ".mjs",
+    ".php",
+    ".prisma",
+    ".proto",
+    ".py",
+    ".rb",
+    ".rs",
+    ".scss",
+    ".sh",
+    ".swift",
+    ".ts",
+    ".tsx",
+]);
+// Extensionless build-language source basenames the classifier maps to `code` by name
+// (CODE_BASENAMES). These are real source/build scripts → whole capture. The config/data
+// basenames (package.json/tsconfig.json/cargo.toml/go.mod/pom.xml) are deliberately NOT here
+// — they default to the bounded sample like other config/data (M3a-default), and are small
+// enough that the sample is the whole file anyway.
+const CODE_WHOLE_CAPTURE_BASENAMES = new Set([
+    "dockerfile",
+    "makefile",
+    "rakefile",
+    "gemfile",
+]);
+/**
+ * Single whole-capture / full-excerpt eligibility predicate (M3a). Consumed by BOTH the
+ * capture owner (`structuralExcerptCharLimit` here) and the seed-stage prompt projection
+ * (`isFullExcerptProjectionEligible` in run.ts), so capture and projection can never disagree
+ * — a bounded capture under a whole-projection budget would silently author the seed from a
+ * partial file. Takes the source REF (path/filename) so it is basename-aware: source-language
+ * code (allowlisted extension OR build-language basename) and text-readable documents
+ * (.md/.txt/.adoc) earn whole capture; binary documents, config/data code files, and
+ * structural-inventory kinds (spreadsheet/database) stay bounded.
+ */
+export function isFullExcerptCaptureEligible(kind, ref) {
+    const lower = typeof ref === "string" ? ref.toLowerCase() : "";
+    const ext = path.extname(lower);
+    const base = path.basename(lower);
+    if (kind === "code") {
+        return CODE_WHOLE_CAPTURE_EXTENSIONS.has(ext) ||
+            CODE_WHOLE_CAPTURE_BASENAMES.has(base);
+    }
+    return kind === "document" && isTextReadableDocumentExtension(ext);
+}
+/**
+ * Derives the seed-stage document projection budget (chars) for the active
+ * (provider, model) seat from its registered context window. Pure and total
+ * (never throws): an unresolved provider/model, an unregistered pair, or an entry
+ * without a window all fall back to the static FLOOR, so mock realization and
+ * provider-only seats are unchanged.
+ *
+ * The route key is the MODEL provider (the registry key, e.g. "openai"), not the
+ * runtime adapter provider (openai OAuth dispatches as "codex"; anthropic OAuth
+ * stays "anthropic"). The caller passes the model provider so the default
+ * gpt-5.5 OAuth seat resolves against `openai/gpt-5.5` (see reconstruct-api).
+ *
+ * This is the SINGLE model→budget conversion point: model literals never reach
+ * the char-budget tuning constants, so G2/INV-CFG-1 stay satisfied.
+ */
+export function deriveDocumentExcerptProjectionBudget(route, registry) {
+    if (!route.provider || !route.modelId) {
+        return DOCUMENT_EXCERPT_PROJECTION_FLOOR;
+    }
+    const window = registry.supported_models.find((entry) => entry.provider === route.provider && entry.model === route.modelId)?.context_window_tokens;
+    if (!window)
+        return DOCUMENT_EXCERPT_PROJECTION_FLOOR;
+    const raw = Math.floor(window * WINDOW_BUDGET_FRACTION * CHARS_PER_TOKEN_LB) -
+        PROMPT_OVERHEAD_RESERVE_CHARS;
+    return Math.min(Math.max(raw, DOCUMENT_EXCERPT_PROJECTION_FLOOR), DOCUMENT_CAPTURE_CEILING_CHARS);
+}
 /**
  * A text-readable document earns the whole-document excerpt budget. Both the capture
  * (here) and the seed-stage prompt projection (run.ts) consult this single predicate so
@@ -144,8 +291,14 @@ export function isTextReadableDocumentExtension(extension) {
         TEXT_READABLE_DOCUMENT_EXTENSIONS.has(extension.toLowerCase()));
 }
 function structuralExcerptCharLimit(kind, ref) {
-    if (kind === "document" && isTextReadableDocumentExtension(path.extname(ref))) {
-        return DOCUMENT_EXCERPT_CHAR_LIMIT;
+    // Whole-capture eligibility is the single shared predicate (M3a): a source-language file
+    // or a text-readable document is projected whole into seed authoring, so it must be
+    // CAPTURED whole too — a small leading sample would make seed authoring read only the
+    // file's head while the prompt-projection truncation never fires (capture < projection
+    // budget), silently authoring the seed from a partial file. Config/data code files,
+    // binary documents, and structural-inventory kinds (spreadsheet/database) stay bounded.
+    if (isFullExcerptCaptureEligible(kind, ref)) {
+        return DOCUMENT_CAPTURE_CEILING_CHARS;
     }
     return DEFAULT_STRUCTURAL_EXCERPT_CHAR_LIMIT;
 }
@@ -170,7 +323,15 @@ async function textStats(ref, excerptLimit) {
         };
     }
 }
-export async function buildReconstructSourceObservation(detection, lineage) {
+export async function buildReconstructSourceObservation(detection, lineage, 
+// Defect-3 basis A: the caller asserts whether this ref is the user-provided
+// reconstruct runtime target. Set true ONLY by the initial-target
+// materialization; frontier-discovered / maturation-closure re-entry callers
+// omit it (conservative default false). Every observation literal below sets
+// is_runtime_target_source from this so source-safety can grant the
+// material_claim/public_output tiers by provenance without leaking to
+// non-target sources.
+options) {
     if (!detection.exists || !isConcreteTargetMaterialKind(detection.kind)) {
         return null;
     }
@@ -193,6 +354,17 @@ export async function buildReconstructSourceObservation(detection, lineage) {
             return null;
         throw error;
     }
+    // Spreadsheet sources route through the shared deterministic structure
+    // observer instead of the generic raw-text path (design S1 §2.2 / §11).
+    if (detection.kind === "spreadsheet") {
+        return buildSpreadsheetSourceObservation({
+            detection,
+            stat,
+            location,
+            lineage,
+            isRuntimeTargetSource: Boolean(options?.isRuntimeTargetSource),
+        });
+    }
     const stats = stat.isFile()
         ? await textStats(detection.ref, structuralExcerptCharLimit(detection.kind, detection.ref))
         : {
@@ -210,6 +382,7 @@ export async function buildReconstructSourceObservation(detection, lineage) {
         round_id: lineage?.roundId ?? "initial_source_frontier",
         observation_batch_id: lineage?.observationBatchId ?? "source-observation-batch:initial",
         triggering_frontier_validation_ref: lineage?.triggeringFrontierValidationRef ?? null,
+        is_runtime_target_source: Boolean(options?.isRuntimeTargetSource),
         target_material_kind: detection.kind,
         adapter_id: `minimal-${detection.kind}-structure-observer`,
         source_ref: detection.ref,
@@ -233,6 +406,96 @@ export async function buildReconstructSourceObservation(detection, lineage) {
     }
     return observation;
 }
+/**
+ * Observe a spreadsheet source through the shared structure observer (design S1
+ * §2.2) — a deterministic, LLM-free structural inventory. The observation carries
+ * NO raw cell values: it never emits the generic path's `content_excerpt` (which for
+ * a workbook would be raw data values) and the inventory's aggregate-only vocab
+ * leaves `top_values` absent. The one NARROWED exception (design-C, not raw data) is
+ * `data_validations[].members` — the bounded, DECLARED type=list enum labels parsed from a
+ * formula1 literal (never observed values). `content_sha256` is the RAW-byte hash (§11 HASH-1)
+ * surfaced at the structural_data top level because the downstream source-scout-pack
+ * reads it there; the full inventory is nested under `workbook_inventory` as the
+ * structural substrate the seed-authoring prompt observes. xlsx-family kinds are
+ * not yet extractable (P4) and arrive here carrying an `unsupported_reason`.
+ */
+/**
+ * The honest observation summary for a workbook inventory (P6): the base disclosure
+ * plus the fixed phrases the boundary gate asserts when the capture/macro flags are
+ * set. This is the single emit path; it and the gate bind to the SAME exported phrase
+ * consts, so assert + emit cannot drift. Exported for the gate-binding regression test.
+ */
+export function buildSpreadsheetObservationSummary(basename, inventory) {
+    const parts = [
+        inventory.unsupported_reason
+            ? `spreadsheet workbook observed at ${basename} — extraction unsupported (${inventory.unsupported_reason}), structure_inspected_only`
+            : `spreadsheet workbook observed at ${basename} — ${inventory.sheets.length} sheet(s), structure_inspected_only`,
+    ];
+    if (inventory.capture_truncated)
+        parts.push(SPREADSHEET_CAPTURE_TRUNCATED_PHRASE);
+    if (inventory.macro_present)
+        parts.push(SPREADSHEET_MACRO_PRESENT_PHRASE);
+    return parts.join("; ");
+}
+async function buildSpreadsheetSourceObservation(args) {
+    const { detection, stat, location, lineage, isRuntimeTargetSource } = args;
+    const basename = path.basename(detection.ref);
+    const extension = path.extname(detection.ref).toLowerCase();
+    // Route through the single shared projection: the structural_data inventory
+    // carries aggregate counts / structure only, never raw cell values (the sole narrowed
+    // exception is data_validations[].members — bounded declared type=list enum labels).
+    const inventory = projectInventoryForAdmission(await observeSpreadsheetSource(detection.ref));
+    const summary = buildSpreadsheetObservationSummary(basename, inventory);
+    const observationId = stableObservationId({ sourceRef: detection.ref, location });
+    // P1-C1 §5.7: the ComprehensionArtifact COMPANIONS the inventory (joined by observation_id) — a
+    // deterministic-only edition in this cut (LLM-touch fields are explicit not_applicable). It is
+    // validated by validateSourceObservationBoundary below (construction throws on an invalid contract).
+    const comprehensionArtifact = buildDeterministicComprehensionArtifact({
+        observationId,
+        inventory,
+    });
+    const observation = {
+        observation_id: observationId,
+        round_id: lineage?.roundId ?? "initial_source_frontier",
+        observation_batch_id: lineage?.observationBatchId ?? "source-observation-batch:initial",
+        triggering_frontier_validation_ref: lineage?.triggeringFrontierValidationRef ?? null,
+        is_runtime_target_source: isRuntimeTargetSource,
+        target_material_kind: "spreadsheet",
+        adapter_id: SPREADSHEET_OBSERVER_ADAPTER_ID,
+        source_ref: detection.ref,
+        location,
+        summary,
+        structural_data: {
+            basename,
+            extension: extension || null,
+            path_kind: stat.isDirectory() ? "directory" : "file",
+            size_bytes: stat.isFile() ? stat.size : null,
+            // Raw-byte hash (§11 HASH-1) surfaced top-level for source-scout-pack
+            // admission, which reads structural_data.content_sha256.
+            content_sha256: inventory.content_sha256,
+            workbook_inventory: inventory,
+            comprehension_artifact: comprehensionArtifact,
+        },
+    };
+    const validation = validateSourceObservationBoundary(observation);
+    if (!validation.valid) {
+        throw new Error(`Invalid source observation boundary for ${detection.ref}: ${validation.violations.join("; ")}`);
+    }
+    return observation;
+}
+/**
+ * The `unsupported_reason` of a spreadsheet observation's workbook inventory, or
+ * null when the observation is not an unsupported workbook. An unsupported workbook
+ * (.xls/.xlsb/.ods, corrupt, oversized) carries no structural evidence, so the
+ * materialize loop demotes it to a skip rather than admitting an empty observation.
+ */
+export function spreadsheetUnsupportedReason(observation) {
+    if (observation.target_material_kind !== "spreadsheet")
+        return null;
+    const inventory = observation.structural_data.workbook_inventory;
+    const reason = inventory?.unsupported_reason;
+    return typeof reason === "string" && reason.length > 0 ? reason : null;
+}
 function stableFrontierRefId(unit) {
     const digest = crypto
         .createHash("sha256")
@@ -357,9 +620,32 @@ export async function materializeReconstructPreparationArtifacts(params) {
         const refDetection = detection.per_ref.find((candidate) => candidate.ref === unit.ref);
         if (!refDetection)
             continue;
-        const observation = await buildReconstructSourceObservation(refDetection);
+        // Defect-3 basis A: these are the user-provided reconstruct runtime-target
+        // refs (the initial target inventory), so they are runtime-target sources.
+        const observation = await buildReconstructSourceObservation(refDetection, undefined, {
+            isRuntimeTargetSource: true,
+        });
         if (observation) {
-            observations.push(observation);
+            const unsupportedReason = spreadsheetUnsupportedReason(observation);
+            if (unsupportedReason) {
+                // A workbook whose format the observer cannot extract (.xls/.xlsb/.ods,
+                // corrupt, oversized) yields an inventory carrying only `unsupported_reason`
+                // — no structural evidence. After the gate flip such a ref is runnable and
+                // produces an observation, so without this demotion a sole legacy-format
+                // target would pass the zero-observation halt and reach LLM authoring with
+                // empty evidence. Demote it to a skip so the run fails loud as unsupported
+                // (the evidence gate stays honest about un-observed material).
+                unit.scan_status = "skipped";
+                unit.skip_reason = `spreadsheet extraction unsupported: ${unsupportedReason}`;
+                skippedRefs.push({
+                    ref: unit.ref,
+                    target_material_kind: unit.target_material_kind,
+                    reason: unit.skip_reason,
+                });
+            }
+            else {
+                observations.push(observation);
+            }
         }
         else {
             // buildReconstructSourceObservation returns null when the ref is no longer