@oscharko-dev/keiko 0.2.0-beta.4 → 0.2.0-beta.6

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/embedding-batcher.js

@@ -113,6 +113,113 @@ async function embedChunkWithRetry(options, chunk) {
     }
     return outcome;
 }
+// ─── Array-batch embedding (#189 GRD-004) ────────────────────────────────────
+// When the adapter exposes `requestBatch`, embed many unique chunks per HTTP round-trip.
+// Items per request are bounded so a single response stays well under the gateway's 10 MB
+// JSON cap (96 × 3072 float32 ≈ 4.4 MB) and inside provider per-request token limits.
+const BATCH_ITEM_CAP = 96;
+const BATCH_CHAR_CAP = 120_000;
+function groupIntoBatches(requests) {
+    const batches = [];
+    let current = [];
+    let currentChars = 0;
+    for (const request of requests) {
+        const len = request.representative.text.length;
+        if (current.length > 0 &&
+            (current.length >= BATCH_ITEM_CAP || currentChars + len > BATCH_CHAR_CAP)) {
+            batches.push(current);
+            current = [];
+            currentChars = 0;
+        }
+        current.push(request);
+        currentChars += len;
+    }
+    if (current.length > 0)
+        batches.push(current);
+    return batches;
+}
+function isTransientBatchOutcome(outcome) {
+    return !outcome.ok && TRANSIENT_EMBED_KINDS.has(outcome.kind);
+}
+function errorFromKind(kind) {
+    return { code: "EMBEDDING_ADAPTER_FAILED", message: `embedding adapter returned ${kind}` };
+}
+async function embedArrayBatchWithRetry(options, inputs) {
+    const adapter = options.adapter;
+    const requestBatch = adapter.requestBatch;
+    if (requestBatch === undefined) {
+        return { ok: false, kind: "transport" };
+    }
+    const base = {
+        endpoint: adapter.endpoint,
+        apiKey: adapter.apiKey,
+        ...(adapter.apiKeyHeaderName !== undefined
+            ? { apiKeyHeaderName: adapter.apiKeyHeaderName }
+            : {}),
+        modelId: options.pinnedIdentity.modelId,
+        inputs,
+        ...(options.signal !== undefined ? { signal: options.signal } : {}),
+    };
+    const retry = resolveRetry(options.retry);
+    let outcome = await requestBatch(base);
+    for (let attempt = 1; attempt <= retry.maxRetries; attempt += 1) {
+        if (!isTransientBatchOutcome(outcome) || options.signal?.aborted === true) {
+            return outcome;
+        }
+        try {
+            await retry.sleep(backoffMs(attempt, retry.baseDelayMs), options.signal);
+        }
+        catch {
+            return outcome; // aborted mid-backoff; the abort gate converts this to CANCELLED
+        }
+        outcome = await requestBatch(base);
+    }
+    return outcome;
+}
+// Apply the per-vector identity gate exactly as the scalar path does. Order-independent:
+// once `state.identityFailure` is set, embedChunkBatch persists nothing, so which concurrent
+// batch first observes the drift is irrelevant to the outcome.
+function gateVectorOutcome(representative, success, options, state) {
+    if (state.identityFailure !== undefined) {
+        return { ok: false, chunk: representative, error: state.identityFailure };
+    }
+    const observed = identityFromAdapter(options.pinnedIdentity, success);
+    const compat = assertCompatibleEmbeddingIdentity(options.pinnedIdentity, observed);
+    if (!compat.ok) {
+        state.identityFailure = {
+            code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
+            message: compat.safeMessage,
+        };
+        return { ok: false, chunk: representative, error: state.identityFailure };
+    }
+    return { ok: true, chunk: representative, success };
+}
+async function embedUniqueBatch(batch, options, state) {
+    if (state.identityFailure !== undefined) {
+        const failure = state.identityFailure;
+        return batch.map((r) => ({ ok: false, chunk: r.representative, error: failure }));
+    }
+    const abortError = checkAbort(options.signal);
+    if (abortError !== undefined) {
+        return batch.map((r) => ({ ok: false, chunk: r.representative, error: abortError }));
+    }
+    const outcome = await embedArrayBatchWithRetry(options, batch.map((r) => r.representative.text));
+    if (!outcome.ok) {
+        const error = errorFromKind(outcome.kind);
+        return batch.map((r) => ({ ok: false, chunk: r.representative, error }));
+    }
+    return batch.map((request, i) => {
+        const success = outcome.value[i];
+        if (success === undefined) {
+            return {
+                ok: false,
+                chunk: request.representative,
+                error: errorFromKind("invalid-response"),
+            };
+        }
+        return gateVectorOutcome(request.representative, success, options, state);
+    });
+}
 // ─── Identity verification ───────────────────────────────────────────────────
 function identityFromAdapter(pinned, success) {
     // `provider` and `vectorMetric` are not echoed by the OpenAI API response — they come from
@@ -208,9 +315,20 @@ function expandUniqueOutcomes(uniqueRequests, uniqueOutcomes) {
 async function buildChunkOutcomes(chunks, options) {
     const state = { identityFailure: undefined };
     const uniqueRequests = dedupeEmbeddingRequests(chunks);
-    const uniqueOutcomes = await runBounded(uniqueRequests, options.concurrency, async (request) => {
-        return buildUniqueChunkOutcome(request, options, state);
-    });
+    let uniqueOutcomes;
+    if (typeof options.adapter.requestBatch === "function") {
+        // Array-batch path: collapse the unique requests into ceil(N / itemCap) HTTP calls,
+        // run those calls with bounded concurrency, then flatten back into request order.
+        const batches = groupIntoBatches(uniqueRequests);
+        const batchOutcomes = await runBounded(batches, options.concurrency, async (batch) => embedUniqueBatch(batch, options, state));
+        uniqueOutcomes = batchOutcomes.flat();
+    }
+    else {
+        // Scalar fallback (adapters/stubs without `requestBatch`): one HTTP call per unique chunk.
+        uniqueOutcomes = await runBounded(uniqueRequests, options.concurrency, async (request) => {
+            return buildUniqueChunkOutcome(request, options, state);
+        });
+    }
     const outcomes = expandUniqueOutcomes(uniqueRequests, uniqueOutcomes);
     return state.identityFailure === undefined
         ? { outcomes }

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AA0EA,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,eAAe,EAErB,MAAM,YAAY,CAAC;AA4lCpB,wBAAuB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,aAAa,CAAC,aAAa,CAAC,CA2B5F"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AA0EA,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,eAAe,EAErB,MAAM,YAAY,CAAC;AA6nCpB,wBAAuB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,aAAa,CAAC,aAAa,CAAC,CA2B5F"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.js

@@ -280,14 +280,45 @@ function handleExtractionSkipped(state, result) {
         reason: "unchanged",
     };
 }
+// GRD-010: transient IO failure codes that must NOT destroy a previously-good index on an
+// incremental refresh. Mirrors the gate in discovery/extract.ts buildFailureResult.
+const TRANSIENT_DISCOVERY_CODES = new Set(["READ_FAILED", "STAT_FAILED"]);
 function handleExtractionFailed(state, result) {
-    state.failedDocuments += 1;
     const errMessage = result.outcome.kind === "failed" ? result.outcome.error.message : "extraction failed";
     const errCode = result.outcome.kind === "failed" ? result.outcome.error.code : "READ_FAILED";
     if (result.outcome.kind === "failed") {
-        clearDocumentArtifacts(state, result.outcome.document.id, { deleteChunks: true });
-        markDocumentFailed(state, result.outcome.document.id);
+        const documentId = result.outcome.document.id;
+        // GRD-010: a transient re-read failure on a document that still has a prior good index
+        // (extract.ts preserved its chunks/vectors) is reported as a non-destructive skip, NOT a
+        // failure — the retrievable content survives until a successful re-extraction.
+        if (TRANSIENT_DISCOVERY_CODES.has(errCode) &&
+            countChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId) > 0) {
+            state.skippedDocuments += 1;
+            return {
+                kind: "document-skipped",
+                jobId: state.jobId,
+                capsuleId: state.capsule.id,
+                sourceId: result.sourceId,
+                documentId,
+                reason: "unchanged",
+            };
+        }
+        state.failedDocuments += 1;
+        clearDocumentArtifacts(state, documentId, { deleteChunks: true });
+        markDocumentFailed(state, documentId);
+        const error = { code: `DISCOVERY_FAILED:${errCode}`, message: errMessage };
+        state.lastError = error;
+        return {
+            kind: "document-failed",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: result.sourceId,
+            documentId,
+            relativePath: result.relativePath,
+            error,
+        };
     }
+    state.failedDocuments += 1;
     const error = { code: `DISCOVERY_FAILED:${errCode}`, message: errMessage };
     state.lastError = error;
     return {
@@ -295,7 +326,6 @@ function handleExtractionFailed(state, result) {
         jobId: state.jobId,
         capsuleId: state.capsule.id,
         sourceId: result.sourceId,
-        ...(result.outcome.kind === "failed" ? { documentId: result.outcome.document.id } : {}),
         relativePath: result.relativePath,
         error,
     };

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/_internal.d.ts

@@ -15,5 +15,6 @@ export interface DecodedText {
     readonly text: string;
     readonly bomBytes: number;
 }
+export declare function decodeXmlEntities(value: string): string;
 export declare function decodeUtf8(bytes: Uint8Array): DecodedText;
 //# sourceMappingURL=_internal.d.ts.map

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/_internal.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"_internal.d.ts","sourceRoot":"","sources":["../../src/parsers/_internal.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,YAAY,EACb,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEnF,wBAAgB,WAAW,CACzB,UAAU,EAAE,gBAAgB,EAC5B,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,aAAa,EACtB,WAAW,GAAE,SAAS,gBAAgB,EAAO,EAC7C,KAAK,GAAE,SAAS,UAAU,EAAO,GAChC,YAAY,CAUd;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,cAAc,CAQ3E;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,UAAU,EACtB,QAAQ,GAAE,gBAAgB,CAAC,UAAU,CAAU,GAC9C,gBAAgB,CAElB;AAID,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,UAAU,CACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,GACnB,UAAU,CAmBZ;AAED,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,gBAAgB,CAOlB;AAED,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,qBAAqB,EAAE,MAAM,GAC5B,gBAAgB,CAOlB;AAMD,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,CAOzD"}
+{"version":3,"file":"_internal.d.ts","sourceRoot":"","sources":["../../src/parsers/_internal.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,YAAY,EACb,MAAM,+BAA+B,CAAC;AAEvC,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEnF,wBAAgB,WAAW,CACzB,UAAU,EAAE,gBAAgB,EAC5B,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,aAAa,EACtB,WAAW,GAAE,SAAS,gBAAgB,EAAO,EAC7C,KAAK,GAAE,SAAS,UAAU,EAAO,GAChC,YAAY,CAUd;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,cAAc,CAQ3E;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,UAAU,EACtB,QAAQ,GAAE,gBAAgB,CAAC,UAAU,CAAU,GAC9C,gBAAgB,CAElB;AAID,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,UAAU,CACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,GACnB,UAAU,CAmBZ;AAED,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,gBAAgB,CAOlB;AAED,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,UAAU,EACtB,qBAAqB,EAAE,MAAM,GAC5B,gBAAgB,CAOlB;AAMD,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAkDD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAWvD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,CASzD"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/_internal.js

@@ -49,7 +49,70 @@ export function oversizeDiagnostic(documentId, byteLength, maxBytes) {
 export function objectLimitDiagnostic(documentId, maxObjectsPerDocument) {
     return diagnostic("OBJECT_LIMIT_REACHED", `reached maxObjectsPerDocument=${String(maxObjectsPerDocument)}`, documentId, "error");
 }
+// GRD-012: detect UTF-16 LE/BE by BOM (common Windows .txt/.csv/.json exports) and re-decode
+// with the matching codec. Without this they decode as UTF-8 mojibake (every other byte a NUL /
+// replacement char) and are silently chunked/embedded as garbage. UTF-32 LE (FF FE 00 00) is
+// explicitly excluded so it is not mis-read as UTF-16 LE.
+function utf16CodecForBom(bytes) {
+    if (bytes.byteLength < 2)
+        return undefined;
+    const b0 = bytes[0];
+    const b1 = bytes[1];
+    if (b0 === 0xfe && b1 === 0xff)
+        return "utf-16be";
+    if (b0 !== 0xff || b1 !== 0xfe)
+        return undefined;
+    // FF FE is UTF-16 LE — unless it is the UTF-32 LE BOM (FF FE 00 00), which is not handled here.
+    const isUtf32Le = bytes.byteLength >= 4 && bytes[2] === 0x00 && bytes[3] === 0x00;
+    return isUtf32Le ? undefined : "utf-16le";
+}
+function decodeUtf16(bytes) {
+    const codec = utf16CodecForBom(bytes);
+    if (codec === undefined)
+        return undefined;
+    const text = new TextDecoder(codec, { fatal: false }).decode(bytes);
+    // The 2-byte UTF-16 BOM is normally consumed by TextDecoder; strip defensively so a leading
+    // U+FEFF never survives into offsets. bomBytes is the consumed BOM byte length (2).
+    const stripped = text.length > 0 && text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    return { text: stripped, bomBytes: 2 };
+}
+// GRD-027: decode an XML numeric character reference body (the part between `&#` and `;`),
+// e.g. "8217" (decimal) or "xE9" / "x2019" (hex). Returns undefined for malformed or
+// out-of-range references (incl. surrogates) so the caller leaves the literal text intact —
+// String.fromCodePoint throws on those, which must never crash the parser.
+// Valid Unicode scalar value: in range and not a lone surrogate (String.fromCodePoint throws
+// on surrogates / out-of-range).
+function isValidScalarCodePoint(cp) {
+    if (!Number.isInteger(cp) || cp < 0 || cp > 0x10ffff)
+        return false;
+    return cp < 0xd800 || cp > 0xdfff;
+}
+function decodeNumericCharacterReference(body) {
+    const isHex = body.startsWith("x") || body.startsWith("X");
+    const digits = isHex ? body.slice(1) : body;
+    if (digits.length === 0)
+        return undefined;
+    if (!(isHex ? /^[0-9a-fA-F]+$/ : /^[0-9]+$/).test(digits))
+        return undefined;
+    const codePoint = Number.parseInt(digits, isHex ? 16 : 10);
+    return isValidScalarCodePoint(codePoint) ? String.fromCodePoint(codePoint) : undefined;
+}
+// Shared OOXML/HTML entity decoder for docx/xlsx text runs. Decodes numeric references first
+// (decimal `&#8217;` and hex `&#xE9;` — smart quotes, accents), then the five named refs, with
+// `&amp;` LAST so an escaped ampersand (`&amp;#65;`) is not re-interpreted as a numeric ref.
+export function decodeXmlEntities(value) {
+    const withNumeric = value.replace(/&#(x?[0-9a-fA-F]+);/g, (match, body) => decodeNumericCharacterReference(body) ?? match);
+    return withNumeric
+        .replaceAll("&lt;", "<")
+        .replaceAll("&gt;", ">")
+        .replaceAll("&quot;", '"')
+        .replaceAll("&apos;", "'")
+        .replaceAll("&amp;", "&");
+}
 export function decodeUtf8(bytes) {
+    const utf16 = decodeUtf16(bytes);
+    if (utf16 !== undefined)
+        return utf16;
     const decoder = new TextDecoder("utf-8", { fatal: false });
     const raw = decoder.decode(bytes);
     if (raw.length > 0 && raw.charCodeAt(0) === 0xfeff) {

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/docx-parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"docx-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/docx-parser.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,YAAY,CAAC;AAmoBpB,eAAO,MAAM,UAAU,EAAE,kBAKvB,CAAC"}
+{"version":3,"file":"docx-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/docx-parser.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,YAAY,CAAC;AAgoBpB,eAAO,MAAM,UAAU,EAAE,kBAKvB,CAAC"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/docx-parser.js

@@ -1,6 +1,6 @@
 import { Buffer } from "node:buffer";
 import yauzl from "yauzl";
-import { diagnostic, emptyResult, objectLimitDiagnostic, oversizeDiagnostic, parserIdentity, shouldStop, } from "./_internal.js";
+import { decodeXmlEntities as decodeXmlEntitiesShared, diagnostic, emptyResult, objectLimitDiagnostic, oversizeDiagnostic, parserIdentity, shouldStop, } from "./_internal.js";
 const PARSER_ID = "docx";
 const PARSER_VERSION = "1";
 const DEPENDENCY_VERSIONS = Object.freeze([
@@ -176,13 +176,10 @@ async function readDocumentXml(bytes, maxInputBytes) {
         closeZipQuietly(zip);
     }
 }
+// GRD-027: delegate to the shared decoder so docx text runs also resolve numeric character
+// references (smart quotes, accented letters) instead of surfacing literal `’`.
 function decodeXmlEntities(value) {
-    return value
-        .replaceAll("&lt;", "<")
-        .replaceAll("&gt;", ">")
-        .replaceAll("&quot;", '"')
-        .replaceAll("&apos;", "'")
-        .replaceAll("&amp;", "&");
+    return decodeXmlEntitiesShared(value);
 }
 function limitDiagnostic(input, limit) {
     if (!limit.stop || limit.code === undefined || limit.message === undefined) {

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/html-parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/html-parser.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,aAAa,EAAuC,MAAM,YAAY,CAAC;AA+QrF,eAAO,MAAM,UAAU,EAAE,aAsBvB,CAAC"}
+{"version":3,"file":"html-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/html-parser.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAEV,aAAa,EAGd,MAAM,YAAY,CAAC;AAyUpB,eAAO,MAAM,UAAU,EAAE,aA4BvB,CAAC"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/html-parser.js

@@ -16,6 +16,26 @@
 // `headingPath` is set from the most recent `<h1>`-`<h6>` stack at the moment the block
 // opens. Inline text outside any heading produces a `html-block` with `headingPath: []`.
 import { decodeUtf8, diagnostic, emptyResult, oversizeDiagnostic, shouldStop, } from "./_internal.js";
+// Collapse internal whitespace runs to single spaces and trim. Applied to each block's
+// visible text so the cleaned projection reads as flowing prose (the raw HTML had source
+// indentation / newlines between inline tags).
+function collapseWhitespace(value) {
+    let out = "";
+    let inWs = false;
+    for (let i = 0; i < value.length; i += 1) {
+        const code = value.charCodeAt(i);
+        const ws = code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0d;
+        if (ws) {
+            inWs = true;
+            continue;
+        }
+        if (inWs && out.length > 0)
+            out += " ";
+        inWs = false;
+        out += value.charAt(i);
+    }
+    return out;
+}
 const PARSER_ID = "html";
 const PARSER_VERSION = "1";
 const HTML_EXTENSIONS = new Set(["html", "htm", "xhtml"]);
@@ -128,34 +148,54 @@ function isWhitespaceOnly(text, start, end) {
     }
     return true;
 }
+function resetBlock(state) {
+    state.pendingBlockStart = null;
+    state.pendingBlockHasText = false;
+    state.blockText = "";
+}
 function flushBlock(state, end) {
     if (state.pendingBlockStart === null)
         return;
     if (end <= state.pendingBlockStart || !state.pendingBlockHasText) {
-        state.pendingBlockStart = null;
-        state.pendingBlockHasText = false;
+        resetBlock(state);
         return;
     }
     const limit = shouldStop(state.startedAt, state.options, state.units.length);
     if (limit.stop && limit.code !== undefined && limit.message !== undefined) {
         state.diagnostics.push(diagnostic(limit.code, limit.message, state.input.documentId, "info"));
         state.stopped = true;
-        state.pendingBlockStart = null;
-        state.pendingBlockHasText = false;
+        resetBlock(state);
         return;
     }
+    // Offsets index the CLEANED projection, not the raw HTML — so citation slices and chunk
+    // spans never expose tags, <script>/<style> bodies, or embedded secrets.
+    const cleaned = collapseWhitespace(state.blockText);
+    if (cleaned.length === 0) {
+        resetBlock(state);
+        return;
+    }
+    if (state.cleanedParts.length > 0) {
+        // Single newline separator between blocks (a gap not covered by any unit range).
+        state.cleanedParts.push("\n");
+        state.cleanedOffset += 1;
+    }
+    const start = state.cleanedOffset;
+    state.cleanedParts.push(cleaned);
+    state.cleanedOffset += cleaned.length;
     state.units.push({
         kind: "html-block",
         documentId: state.input.documentId,
         headingPath: [...state.heading.stack],
-        characterStart: state.pendingBlockStart,
-        characterEnd: end,
+        characterStart: start,
+        characterEnd: state.cleanedOffset,
     });
-    state.pendingBlockStart = null;
-    state.pendingBlockHasText = false;
+    resetBlock(state);
 }
 function openBlock(state, at, hasText) {
-    state.pendingBlockStart ??= at;
+    if (state.pendingBlockStart === null) {
+        state.pendingBlockStart = at;
+        state.blockText = "";
+    }
     if (hasText)
         state.pendingBlockHasText = true;
 }
@@ -183,6 +223,9 @@ function appendTextRun(state, from, to) {
         return;
     }
     openBlock(state, from, !isWhitespaceOnly(state.text, from, to));
+    // Accumulate only inter-tag text runs — inline tag literals (<b>, <a …>) are never part of
+    // a run, so the cleaned projection is tag-free by construction.
+    state.blockText += state.text.slice(from, to);
 }
 function handleTag(state, tag) {
     if (RAW_TEXT_TAGS.has(tag.name) && tag.kind === "open") {
@@ -227,13 +270,20 @@ function emitHtml(text, input, options) {
         pendingHeadingLabel: null,
         pendingHeadingLevel: 0,
         stopped: false,
+        blockText: "",
+        cleanedParts: [],
+        cleanedOffset: 0,
     };
     let cursor = 0;
     while (cursor < text.length && !state.stopped) {
         cursor = step(state, cursor);
     }
     flushBlock(state, text.length);
-    return { units: state.units, diagnostics: state.diagnostics };
+    return {
+        units: state.units,
+        diagnostics: state.diagnostics,
+        normalizedText: state.cleanedParts.join(""),
+    };
 }
 export const htmlParser = Object.freeze({
     capability: Object.freeze({
@@ -249,6 +299,12 @@ export const htmlParser = Object.freeze({
         }
         const decoded = decodeUtf8(input.bytes);
         const emission = emitHtml(decoded.text, input, options);
-        return emptyResult(htmlParser.capability, input.documentId, options, emission.diagnostics, emission.units);
+        const result = {
+            ...emptyResult(htmlParser.capability, input.documentId, options, emission.diagnostics, emission.units),
+            // GRD-003: persist the cleaned, tag/script/style-free projection so extract.ts NEVER
+            // falls back to raw bytes (which carried <script> bodies + embedded secrets).
+            normalizedText: emission.normalizedText,
+        };
+        return result;
     },
 });

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/index.d.ts

@@ -7,6 +7,7 @@ export { csvParser } from "./csv-parser.js";
 export { htmlParser } from "./html-parser.js";
 export { pdfParser } from "./pdf-parser.js";
 export { docxParser } from "./docx-parser.js";
+export { xlsxParser } from "./xlsx-parser.js";
 import type { ParserRegistry } from "./types.js";
 export declare function createDefaultParserRegistry(): ParserRegistry;
 export { nullOcrAdapter, createOcrPipelineParser, type OcrAdapter, type OcrPageResult, type OcrPipelineAdapter, } from "./ocr/index.js";

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parsers/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,iBAAiB,EACjB,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,kBAAkB,EAClB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,GAC1B,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7E,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,GAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAY9C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,wBAAgB,2BAA2B,IAAI,cAAc,CAY5D;AAGD,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,kBAAkB,GACxB,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parsers/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,iBAAiB,EACjB,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,kBAAkB,EAClB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,oBAAoB,GAC1B,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7E,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,cAAc,EACd,aAAa,EACb,iBAAiB,GAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAY9C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGjD,wBAAgB,2BAA2B,IAAI,cAAc,CAa5D;AAGD,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,kBAAkB,GACxB,MAAM,gBAAgB,CAAC"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/index.js

@@ -9,6 +9,7 @@ export { csvParser } from "./csv-parser.js";
 export { htmlParser } from "./html-parser.js";
 export { pdfParser } from "./pdf-parser.js";
 export { docxParser } from "./docx-parser.js";
+export { xlsxParser } from "./xlsx-parser.js";
 // Convenience: a registry pre-populated with every shipped adapter. Resolution order is
 // JSON → CSV/TSV → HTML → text. Text registers last because its `matches` predicate is
 // the most permissive (accepts any `text/*`); registering it first would shadow CSV and HTML.
@@ -19,6 +20,7 @@ import { jsonParser } from "./json-parser.js";
 import { pdfParser } from "./pdf-parser.js";
 import { createParserRegistry, registerParser } from "./registry.js";
 import { textParser } from "./text-parser.js";
+import { xlsxParser } from "./xlsx-parser.js";
 export function createDefaultParserRegistry() {
     let registry = createParserRegistry();
     registry = registerParser(registry, jsonParser);
@@ -26,6 +28,7 @@ export function createDefaultParserRegistry() {
     registry = registerParser(registry, htmlParser);
     registry = registerParser(registry, pdfParser);
     registry = registerParser(registry, docxParser);
+    registry = registerParser(registry, xlsxParser);
     // Text parser is registered last among the real adapters because its `matches` predicate
     // is the most permissive (it accepts any `text/*` media type), so it must not shadow the
     // structured adapters.

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/json-parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/json-parser.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAEV,aAAa,EAGd,MAAM,YAAY,CAAC;AAiJpB,eAAO,MAAM,UAAU,EAAE,aA4CvB,CAAC"}
+{"version":3,"file":"json-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/json-parser.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAEV,aAAa,EAGd,MAAM,YAAY,CAAC;AAoLpB,eAAO,MAAM,UAAU,EAAE,aAkDvB,CAAC"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/json-parser.js

@@ -114,6 +114,35 @@ function parseJsonValue(text) {
         return { ok: false, error: error instanceof Error ? error.name : "SyntaxError" };
     }
 }
+// GRD-011: JSON Lines / NDJSON is a sequence of independent JSON values, one per line — NOT a
+// single JSON document. Whole-document `JSON.parse` always throws on a multi-record file, so
+// `.jsonl`/`.ndjson` (advertised as supported) were 100% unparseable. Detect and parse per line.
+const JSONL_EXTENSIONS = new Set(["jsonl", "ndjson"]);
+function isJsonLines(input) {
+    if (JSONL_EXTENSIONS.has(input.extension.toLowerCase()))
+        return true;
+    return input.mediaType.toLowerCase() === "application/x-ndjson";
+}
+// Parse each non-empty line independently and walk it under an `/<lineIndex>` pointer prefix
+// (RFC 6901 array-index style), so a record on file line N cites as `/N/...`. A malformed line
+// is surfaced as a non-fatal `warning` diagnostic and skipped — one bad line never discards the
+// whole file's good records.
+function walkJsonLines(ctx) {
+    const lines = ctx.text.split("\n");
+    for (let i = 0; i < lines.length; i += 1) {
+        if (ctx.stopped)
+            return;
+        const raw = lines[i];
+        if (raw === undefined || raw.trim().length === 0)
+            continue;
+        const parsed = parseJsonValue(raw);
+        if (!parsed.ok) {
+            ctx.diagnostics.push(diagnostic("MALFORMED_INPUT", `JSONL line ${String(i)} parse failed: ${parsed.error}`, ctx.input.documentId, "warning"));
+            continue;
+        }
+        walk(ctx, parsed.value, joinPointer("", String(i)), 0);
+    }
+}
 export const jsonParser = Object.freeze({
     capability: Object.freeze({
         parserId: PARSER_ID,
@@ -127,8 +156,10 @@ export const jsonParser = Object.freeze({
             ]);
         }
         const decoded = decodeUtf8(input.bytes);
-        const parsed = parseJsonValue(decoded.text);
-        if (!parsed.ok) {
+        const jsonLines = isJsonLines(input);
+        // Whole-document parse only for true JSON; JSONL is parsed line-by-line below.
+        const parsed = jsonLines ? undefined : parseJsonValue(decoded.text);
+        if (parsed !== undefined && !parsed.ok) {
             return emptyResult(jsonParser.capability, input.documentId, options, [
                 diagnostic("MALFORMED_INPUT", `JSON parse failed: ${parsed.error}`, input.documentId, "error"),
             ]);
@@ -144,7 +175,12 @@ export const jsonParser = Object.freeze({
             normalizedLength: 0,
             stopped: false,
         };
-        walk(ctx, parsed.value, "", 0);
+        if (jsonLines) {
+            walkJsonLines(ctx);
+        }
+        else if (parsed?.ok === true) {
+            walk(ctx, parsed.value, "", 0);
+        }
         if (ctx.diagnostics.some((diagnostic) => diagnostic.code === "NESTING_LIMIT_REACHED")) {
             return emptyResult(jsonParser.capability, input.documentId, options, ctx.diagnostics);
         }

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/ocr/null-ocr-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"null-ocr-adapter.d.ts","sourceRoot":"","sources":["../../../src/parsers/ocr/null-ocr-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,YAAY,CAAC;AAO5D,eAAO,MAAM,cAAc,EAAE,UAI3B,CAAC"}
+{"version":3,"file":"null-ocr-adapter.d.ts","sourceRoot":"","sources":["../../../src/parsers/ocr/null-ocr-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,YAAY,CAAC;AAO5D,eAAO,MAAM,cAAc,EAAE,UAM3B,CAAC"}

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/parsers/ocr/ocr-pipeline-parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ocr-pipeline-parser.d.ts","sourceRoot":"","sources":["../../../src/parsers/ocr/ocr-pipeline-parser.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAG9E,OAAO,KAAK,EAAE,aAAa,EAAoB,aAAa,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACxG,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,YAAY,CAAC;AAsH5D,MAAM,WAAW,kBAAmB,SAAQ,aAAa;IACvD,QAAQ,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,aAAa,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CACrG;AAID,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,UAAU,GAAG,kBAAkB,CAG/E"}
+{"version":3,"file":"ocr-pipeline-parser.d.ts","sourceRoot":"","sources":["../../../src/parsers/ocr/ocr-pipeline-parser.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAG9E,OAAO,KAAK,EACV,aAAa,EAEb,aAAa,EACb,oBAAoB,EACrB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,YAAY,CAAC;AAwK5D,MAAM,WAAW,kBAAmB,SAAQ,aAAa;IACvD,QAAQ,CAAC,UAAU,EAAE,CACnB,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,aAAa,KACnB,OAAO,CAAC,YAAY,CAAC,CAAC;CAC5B;AAID,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,UAAU,GAAG,kBAAkB,CAW/E"}