@oscharko-dev/keiko-server 0.2.0

package/dist/grounded-document-evidence.js

@@ -0,0 +1,430 @@
+// Bounded small-document evidence for grounded Repository Search (Issue #1285).
+//
+// Repository Search stays code-first, but a document that the user has EXPLICITLY connected to the
+// chat (a `files` scope with `explicitConnection === true`) should no longer be silently dropped
+// just because it is a `.docx`, `.xlsx`, or text-layer `.pdf`. This module reads those documents
+// through the existing workspace path-safety primitives, extracts a bounded text projection via
+// the reused Local Knowledge parsers (keiko-local-knowledge `extractBoundedDocumentText`), redacts
+// the text, and turns the result into request-local `document-extract` evidence atoms + excerpt
+// windows. Documents that cannot be used produce a stable skipped-document diagnostic instead of a
+// crash or a silent omission.
+//
+// Trust-boundary notes:
+//   - The 2 MiB input ceiling is enforced via `fs.stat` BEFORE any bytes are read.
+//   - Bytes are read through `resolveWithinWorkspace` + `containedRealPathInfo` + the deny-list,
+//     identical to every other workspace read; denied/escaping paths never reach the parser.
+//   - Extracted text is `redact()`-ed here (the Local Knowledge extractor is pure and does not
+//     redact) before it enters any excerpt, so credential-shaped strings inside a document never
+//     reach the prompt, the citation surface, or the evidence manifest.
+//   - Nothing is persisted: no Local Knowledge capsule, vector, index, or cache is written.
+import { extractBoundedDocumentText, } from "@oscharko-dev/keiko-local-knowledge";
+import { isValidScopePath, } from "@oscharko-dev/keiko-contracts/connected-context";
+import { stripUnsafeFormatChars } from "@oscharko-dev/keiko-contracts/text-safety";
+import { redact } from "@oscharko-dev/keiko-security";
+import { containedRealPathInfo, evidenceAtomStableId, isDenied, resolveWithinWorkspace, } from "@oscharko-dev/keiko-workspace";
+import { createHash } from "node:crypto";
+// ─── Bounds ──────────────────────────────────────────────────────────────────
+export const MAX_DOCUMENT_INPUT_BYTES = 2 * 1024 * 1024; // 2 MiB, enforced before parser execution
+export const MAX_DOCUMENT_EXTRACTED_BYTES = 32 * 1024; // per-document extracted-text ceiling
+export const MAX_TOTAL_DOCUMENT_EXTRACTED_BYTES = 64 * 1024; // per grounded request, across documents
+export const MAX_DOCUMENT_WINDOW_BYTES = 8 * 1024; // per excerpt window (matches the pack assembler)
+export const MAX_DOCUMENT_WINDOWS = 8; // per document
+export const DOCUMENT_PARSE_TIMEOUT_MS = 5_000; // per-document wall-clock budget
+export const MAX_DOCUMENT_PARSE_UNITS = 5_000; // bounds paragraphs/rows/pages parsed
+const TEXT_ENCODER = new TextEncoder();
+const EXTENSION_BINDINGS = new Map([
+    [
+        "docx",
+        {
+            format: "docx",
+            mediaType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        },
+    ],
+    [
+        "xlsx",
+        {
+            format: "xlsx",
+            mediaType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        },
+    ],
+    ["pdf", { format: "pdf", mediaType: "application/pdf" }],
+]);
+// Document container formats that are deliberately out of scope for bounded extraction (legacy
+// binary Office, presentations, OpenDocument, rich text, Apple iWork). A connected file with one of
+// these extensions is reported with a stable `unsupported-format` diagnostic instead of being
+// silently dropped on the binary excerpt path — Repository Search does not parse these (Issue
+// #1285 Out of Scope), and Local Knowledge remains the path for them.
+const UNSUPPORTED_DOCUMENT_EXTENSIONS = new Set([
+    "doc",
+    "dot",
+    "ppt",
+    "pptx",
+    "odt",
+    "ods",
+    "odp",
+    "rtf",
+    "pages",
+    "key",
+    "numbers",
+]);
+function extensionOf(scopePath) {
+    const name = scopePath.slice(scopePath.lastIndexOf("/") + 1);
+    const dot = name.lastIndexOf(".");
+    return dot < 0 ? "" : name.slice(dot + 1).toLowerCase();
+}
+function documentBindingForPath(scopePath) {
+    return EXTENSION_BINDINGS.get(extensionOf(scopePath));
+}
+function normalizeScopePath(scopePath) {
+    return scopePath.replace(/\\/gu, "/");
+}
+function isWithinSelectedScope(searchScope, realRelativePath) {
+    if (searchScope.relativePaths.length === 0) {
+        return true;
+    }
+    return searchScope.relativePaths.some((selectedPath) => {
+        const normalizedSelectedPath = normalizeScopePath(selectedPath);
+        return (realRelativePath === normalizedSelectedPath ||
+            realRelativePath.startsWith(`${normalizedSelectedPath}/`));
+    });
+}
+// True when `scopePath` is any document the bounded-extraction path owns — either a supported small
+// document (DOCX/XLSX/PDF) or a known-but-unsupported document format. The orchestrator uses this
+// to keep every such file off the code-first excerpt-read path so document handling (extraction or
+// a stable unsupported diagnostic) is the single source of truth (Issue #1285).
+export function isConnectedDocumentPath(scopePath) {
+    const extension = extensionOf(scopePath);
+    return EXTENSION_BINDINGS.has(extension) || UNSUPPORTED_DOCUMENT_EXTENSIONS.has(extension);
+}
+const EMPTY_RESULT = {
+    atoms: [],
+    candidates: [],
+    excerpts: new Map(),
+    omitted: [],
+    uncertainty: [],
+};
+// ─── Outcome → omission reason ───────────────────────────────────────────────────
+function omissionReasonForOutcome(outcome) {
+    switch (outcome) {
+        case "oversized":
+            return "size-exceeded";
+        case "unsupported-format":
+            return "unsupported-format";
+        // A scanned PDF and an empty-but-valid container are both "supported container, no extractable
+        // text"; they share the no-text-layer diagnostic so the user sees one stable message.
+        case "no-text-layer":
+        case "empty":
+            return "no-text-layer";
+        case "malformed":
+            return "malformed-document";
+        case "encrypted":
+            return "encrypted-document";
+        case "timed-out":
+            return "budget-exhausted";
+    }
+}
+// ─── Excerpt windowing over extracted text ───────────────────────────────────────
+function utf8ByteLength(value) {
+    return TEXT_ENCODER.encode(value).length;
+}
+// Splits a single line that exceeds the per-window byte cap into UTF-8-safe chunks, each at most
+// MAX_DOCUMENT_WINDOW_BYTES. This guarantees no window ever exceeds the cap (which the pack
+// assembler would otherwise clamp, dropping bounded-but-extracted text), even for a document whose
+// extracted projection has a very long unbroken line (e.g. a PDF page with no line breaks).
+function splitToWindowChunks(line) {
+    if (utf8ByteLength(line) <= MAX_DOCUMENT_WINDOW_BYTES) {
+        return [line];
+    }
+    const encoded = TEXT_ENCODER.encode(line);
+    const decoder = new TextDecoder("utf-8", { fatal: false });
+    const chunks = [];
+    let offset = 0;
+    while (offset < encoded.length) {
+        const end = Math.min(offset + MAX_DOCUMENT_WINDOW_BYTES, encoded.length);
+        const chunk = decoder.decode(encoded.subarray(offset, end)).replace(/�+$/u, "");
+        const consumed = utf8ByteLength(chunk) || end - offset;
+        chunks.push(chunk);
+        offset += consumed;
+    }
+    return chunks;
+}
+// Split already-redacted extracted text into non-overlapping windows, each at most
+// MAX_DOCUMENT_WINDOW_BYTES and capped at MAX_DOCUMENT_WINDOWS. Line numbers index the extracted
+// text projection itself (DOCX paragraph / XLSX row / PDF page lines), so a citation such as
+// `[report.docx:1-40]` references a reproducible span of the extracted text rather than an
+// original on-screen line.
+function windowExtractedText(text) {
+    const lines = text.split("\n").flatMap((line) => splitToWindowChunks(line));
+    const windows = [];
+    let startIndex = 0;
+    let buffer = [];
+    let bufferBytes = 0;
+    const flush = (endIndexExclusive) => {
+        if (buffer.length === 0) {
+            return;
+        }
+        windows.push({
+            startLine: startIndex + 1,
+            endLine: endIndexExclusive,
+            content: buffer.join("\n"),
+        });
+        buffer = [];
+        bufferBytes = 0;
+        startIndex = endIndexExclusive;
+    };
+    for (let i = 0; i < lines.length; i += 1) {
+        if (windows.length >= MAX_DOCUMENT_WINDOWS) {
+            break;
+        }
+        const line = lines[i] ?? "";
+        const lineBytes = utf8ByteLength(line) + 1;
+        if (buffer.length > 0 && bufferBytes + lineBytes > MAX_DOCUMENT_WINDOW_BYTES) {
+            flush(i);
+            if (windows.length >= MAX_DOCUMENT_WINDOWS) {
+                break;
+            }
+        }
+        buffer.push(line);
+        bufferBytes += lineBytes;
+    }
+    if (windows.length < MAX_DOCUMENT_WINDOWS) {
+        flush(startIndex + buffer.length);
+    }
+    return windows;
+}
+// ─── Atom construction ───────────────────────────────────────────────────────────
+function documentQueryFingerprint(query) {
+    return createHash("sha256")
+        .update(JSON.stringify({
+        kind: "document-extract",
+        queryKind: query.kind,
+        text: query.text,
+        caseSensitive: query.caseSensitive,
+    }))
+        .digest("hex")
+        .slice(0, 16);
+}
+function documentAtom(scope, scopePath, window, queryFingerprint, nowMs) {
+    const lineRange = { startLine: window.startLine, endLine: window.endLine };
+    return {
+        schemaVersion: scope.schemaVersion,
+        stableId: evidenceAtomStableId({
+            scopeId: scope.scopeId,
+            scopePath,
+            lineRange,
+            provenanceKind: "document-extract",
+            provenanceTool: "repo.documentExtract",
+            queryFingerprint,
+        }),
+        scopePath,
+        lineRange,
+        score: 1,
+        provenance: {
+            kind: "document-extract",
+            tool: "repo.documentExtract",
+            queryFingerprint,
+        },
+        redactionState: "redacted",
+        emittedAtMs: nowMs,
+        ledgerRef: undefined,
+    };
+}
+function documentCandidate(scopePath) {
+    return {
+        scopePath,
+        score: 1,
+        signals: [{ name: "document-extract", value: 1 }],
+        omitted: undefined,
+    };
+}
+function omit(acc, scopePath, reason, nowMs) {
+    acc.omitted.push({ scopePath, reason, omittedAtMs: nowMs });
+}
+function resolveDocument(inputs, scopePath) {
+    const normalizedScopePath = normalizeScopePath(scopePath);
+    if (isDenied(normalizedScopePath)) {
+        return "denied";
+    }
+    let contained;
+    try {
+        const absolute = resolveWithinWorkspace(inputs.searchScope.workspace.root, normalizedScopePath);
+        contained = containedRealPathInfo(inputs.fs, inputs.searchScope.workspace.root, absolute);
+    }
+    catch {
+        return "denied";
+    }
+    const realRelative = normalizeScopePath(contained.realRelative);
+    if (isDenied(realRelative) || !isWithinSelectedScope(inputs.searchScope, realRelative)) {
+        return "denied";
+    }
+    try {
+        const stat = inputs.fs.stat(contained.path);
+        if (!stat.isFile) {
+            return "unreadable";
+        }
+        if (stat.hardLinkCount !== undefined && stat.hardLinkCount > 1) {
+            return "denied";
+        }
+        return {
+            absolutePath: contained.path,
+            realRelative,
+            sizeBytes: stat.size,
+        };
+    }
+    catch {
+        return "unreadable";
+    }
+}
+async function readDocumentBytes(inputs, absolutePath, sizeBytes) {
+    if (inputs.fs.readFileBytes === undefined) {
+        return "unreadable";
+    }
+    try {
+        return await inputs.fs.readFileBytes(absolutePath, Math.min(sizeBytes, MAX_DOCUMENT_INPUT_BYTES));
+    }
+    catch {
+        return "unreadable";
+    }
+}
+// Resolves and reads the document bytes through the workspace path-safety primitives, returning the
+// bytes or the precise omission reason for an unusable path.
+async function loadDocumentBytes(inputs, scopePath) {
+    const resolved = resolveDocument(inputs, scopePath);
+    if (resolved === "denied") {
+        return "outside-scope";
+    }
+    if (resolved === "unreadable") {
+        return "tool-unavailable";
+    }
+    if (resolved.sizeBytes > MAX_DOCUMENT_INPUT_BYTES) {
+        return "size-exceeded";
+    }
+    const bytes = await readDocumentBytes(inputs, resolved.absolutePath, resolved.sizeBytes);
+    return bytes === "unreadable" ? "tool-unavailable" : bytes;
+}
+// Redacts the extracted text and records the document atoms + excerpt windows, or an omission when
+// redaction left no usable text.
+function recordExtractedDocument(inputs, scopePath, extraction, queryFingerprint, acc, nowMs) {
+    // The Local Knowledge extractor is pure and does NOT redact; the document text is redacted here,
+    // before it enters any excerpt window, prompt, or citation surface (Issue #1285 trust boundary).
+    const windows = windowExtractedText(redact(stripUnsafeFormatChars(extraction.text)));
+    if (windows.length === 0) {
+        // All extracted text was redaction-stripped — disclose the document was reachable but unusable.
+        omit(acc, scopePath, "redacted-only", nowMs);
+        return;
+    }
+    for (const window of windows) {
+        acc.atoms.push(documentAtom(inputs.scope, scopePath, window, queryFingerprint, nowMs));
+    }
+    acc.candidates.push(documentCandidate(scopePath));
+    acc.excerpts.set(scopePath, windows);
+    acc.totalExtractedBytes += windows.reduce((sum, window) => sum + utf8ByteLength(window.content), 0);
+    if (extraction.truncated) {
+        // The document was clipped to the extraction byte budget; disclose that the evidence is partial.
+        acc.uncertainty.push({
+            kind: "scope-incomplete",
+            claim: `Document ${scopePath} was truncated to the extraction budget; its evidence is partial.`,
+            impactedAtomIds: [],
+            emittedAtMs: nowMs,
+        });
+    }
+}
+async function processDocument(inputs, scopePath, binding, queryFingerprint, acc) {
+    const nowMs = inputs.nowMs();
+    const remainingBudget = MAX_TOTAL_DOCUMENT_EXTRACTED_BYTES - acc.totalExtractedBytes;
+    if (remainingBudget <= 0) {
+        omit(acc, scopePath, "budget-exhausted", nowMs);
+        return;
+    }
+    const loaded = await loadDocumentBytes(inputs, scopePath);
+    if (typeof loaded === "string") {
+        omit(acc, scopePath, loaded, nowMs);
+        return;
+    }
+    const extraction = await extractBoundedDocumentText({ bytes: loaded, extension: binding.format, mediaType: binding.mediaType }, {
+        maxInputBytes: MAX_DOCUMENT_INPUT_BYTES,
+        maxOutputBytes: Math.min(MAX_DOCUMENT_EXTRACTED_BYTES, remainingBudget),
+        maxUnits: MAX_DOCUMENT_PARSE_UNITS,
+        timeoutMs: DOCUMENT_PARSE_TIMEOUT_MS,
+        now: inputs.nowMs,
+        ...(inputs.signal === undefined ? {} : { signal: inputs.signal }),
+    });
+    if (extraction.outcome !== "extracted") {
+        omit(acc, scopePath, omissionReasonForOutcome(extraction.outcome), nowMs);
+        return;
+    }
+    recordExtractedDocument(inputs, scopePath, { text: extraction.text, truncated: extraction.truncated }, queryFingerprint, acc, nowMs);
+}
+// ─── Public entry ────────────────────────────────────────────────────────────────
+function disclosureMarker(omittedCount, nowMs) {
+    return {
+        kind: "scope-incomplete",
+        claim: `${String(omittedCount)} connected document(s) were skipped: Repository Search reads text, code, and small DOCX/XLSX/text-layer-PDF files only.`,
+        impactedAtomIds: [],
+        emittedAtMs: nowMs,
+    };
+}
+// Processes one connected scope entry: a supported document is extracted, a known-unsupported
+// document format is reported, and anything else (code/text) is left to the code-first path.
+async function processConnectedEntry(inputs, entry, seen, queryFingerprint, acc) {
+    if (!isValidScopePath(entry, { mustBeRelative: true })) {
+        return;
+    }
+    const scopePath = entry.replace(/\\/gu, "/");
+    if (seen.has(scopePath)) {
+        return;
+    }
+    seen.add(scopePath);
+    const binding = documentBindingForPath(scopePath);
+    const knownUnsupportedDocument = UNSUPPORTED_DOCUMENT_EXTENSIONS.has(extensionOf(scopePath));
+    if ((binding !== undefined || knownUnsupportedDocument) && isDenied(scopePath)) {
+        omit(acc, scopePath, "outside-scope", inputs.nowMs());
+        return;
+    }
+    if (binding !== undefined) {
+        await processDocument(inputs, scopePath, binding, queryFingerprint, acc);
+        return;
+    }
+    if (knownUnsupportedDocument) {
+        // A known-but-unsupported document container (e.g. legacy .doc): emit a stable diagnostic.
+        omit(acc, scopePath, "unsupported-format", inputs.nowMs());
+    }
+}
+// Enumerates the explicitly-connected document files in a `files` scope, extracts bounded redacted
+// text from each, and returns request-local document evidence ready to merge into the grounded
+// context-pack assembly. Returns empty evidence for any scope that is not an explicit `files`
+// connection, keeping whole-workspace and directory scopes on the unchanged code-first path.
+export async function collectConnectedDocumentEvidence(inputs) {
+    if (inputs.scope.kind !== "files" || inputs.scope.explicitConnection !== true) {
+        return EMPTY_RESULT;
+    }
+    const queryFingerprint = documentQueryFingerprint(inputs.query);
+    const acc = {
+        atoms: [],
+        candidates: [],
+        excerpts: new Map(),
+        omitted: [],
+        uncertainty: [],
+        totalExtractedBytes: 0,
+    };
+    const seen = new Set();
+    for (const entry of inputs.scope.relativePaths) {
+        if (inputs.signal?.aborted === true) {
+            break;
+        }
+        await processConnectedEntry(inputs, entry, seen, queryFingerprint, acc);
+    }
+    // The skipped-document disclosure (one marker summarizing all omissions) plus any per-document
+    // truncation markers accumulated during extraction.
+    const uncertainty = [
+        ...(acc.omitted.length > 0 ? [disclosureMarker(acc.omitted.length, inputs.nowMs())] : []),
+        ...acc.uncertainty,
+    ];
+    return {
+        atoms: acc.atoms,
+        candidates: acc.candidates,
+        excerpts: acc.excerpts,
+        omitted: acc.omitted,
+        uncertainty,
+    };
+}

package/dist/grounded-handoff.d.ts

@@ -0,0 +1,4 @@
+import { type UiHandlerDeps } from "./deps.js";
+import type { RouteContext, RouteResult } from "./routes.js";
+export declare function handleGroundedWorkflowHandoff(ctx: RouteContext, deps: UiHandlerDeps): Promise<RouteResult>;
+//# sourceMappingURL=grounded-handoff.d.ts.map

package/dist/grounded-handoff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grounded-handoff.d.ts","sourceRoot":"","sources":["../src/grounded-handoff.ts"],"names":[],"mappings":"AAmBA,OAAO,EAA2B,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAYxE,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AA8hB7D,wBAAsB,6BAA6B,CACjD,GAAG,EAAE,YAAY,EACjB,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,WAAW,CAAC,CAOtB"}