@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/parsers/csv-parser.js

@@ -0,0 +1,202 @@
+// CSV / TSV parser adapter (Epic #189, Issue #266). Pure, hand-rolled RFC 4180 tokenizer.
+//
+// No `csv-parse` or other dependency — the rules are small enough that a single state
+// machine over the decoded string handles every case we need:
+//
+//   * Quoted fields preserve embedded delimiters (`,` or `\t`) and embedded newlines.
+//   * `""` inside a quoted field decodes to a single literal `"`.
+//   * CRLF, LF, and bare CR row terminators all work.
+//   * A trailing newline does NOT emit a synthetic empty row.
+//   * A row with only whitespace + empty fields is preserved verbatim (we do not lose data).
+//
+// Emits one ParsedUnit { kind: "csv-row" } per non-header row. The first row is treated as
+// the header and is NOT emitted as a unit — its values are kept only for the implicit table
+// schema. If the document has no header row (a single line), THAT line is emitted as the
+// header AND a single data row, so a one-line CSV stays observable.
+import { decodeUtf8, diagnostic, emptyResult, oversizeDiagnostic, shouldStop, } from "./_internal.js";
+const PARSER_ID = "csv";
+const PARSER_VERSION = "1";
+const CSV_EXTENSIONS = new Set(["csv"]);
+const TSV_EXTENSIONS = new Set(["tsv", "tab"]);
+function selectDelimiter(input) {
+    const ext = input.extension.toLowerCase();
+    if (CSV_EXTENSIONS.has(ext))
+        return ",";
+    if (TSV_EXTENSIONS.has(ext))
+        return "\t";
+    const media = input.mediaType.toLowerCase();
+    if (media === "text/csv")
+        return ",";
+    if (media === "text/tab-separated-values")
+        return "\t";
+    return null;
+}
+function readField(state) {
+    const { text } = state;
+    if (state.cursor >= text.length)
+        return { endOfRow: true };
+    const code = text.charCodeAt(state.cursor);
+    if (code === 0x22 /* " */)
+        return readQuotedField(state);
+    return readBareField(state);
+}
+function readBareField(state) {
+    const { text, delimiter } = state;
+    while (state.cursor < text.length) {
+        const code = text.charCodeAt(state.cursor);
+        if (code === delimiter) {
+            state.cursor += 1;
+            return { endOfRow: false };
+        }
+        if (code === 0x0a /* LF */) {
+            // Caller advances past LF; we just signal end-of-row here.
+            return { endOfRow: true };
+        }
+        if (code === 0x0d /* CR */) {
+            return { endOfRow: true };
+        }
+        state.cursor += 1;
+    }
+    return { endOfRow: true };
+}
+function readQuotedField(state) {
+    const { text } = state;
+    // Skip the opening quote.
+    state.cursor += 1;
+    while (state.cursor < text.length) {
+        const code = text.charCodeAt(state.cursor);
+        if (code === 0x22 /* " */) {
+            // Escaped quote? Peek ahead.
+            if (state.cursor + 1 < text.length && text.charCodeAt(state.cursor + 1) === 0x22) {
+                state.cursor += 2;
+                continue;
+            }
+            // Closing quote — consume it then expect delimiter / row terminator / EOF.
+            state.cursor += 1;
+            return consumeAfterQuote(state);
+        }
+        state.cursor += 1;
+    }
+    // Unterminated quoted field: treat the rest of the document as part of this field.
+    return { endOfRow: true };
+}
+function consumeAfterQuote(state) {
+    if (state.cursor >= state.text.length)
+        return { endOfRow: true };
+    const code = state.text.charCodeAt(state.cursor);
+    if (code === state.delimiter) {
+        state.cursor += 1;
+        return { endOfRow: false };
+    }
+    if (code === 0x0a || code === 0x0d)
+        return { endOfRow: true };
+    // Malformed: bytes after the closing quote that are neither delimiter nor newline. We
+    // tolerate by consuming until the next delimiter / newline rather than crashing.
+    while (state.cursor < state.text.length) {
+        const inner = state.text.charCodeAt(state.cursor);
+        if (inner === state.delimiter) {
+            state.cursor += 1;
+            return { endOfRow: false };
+        }
+        if (inner === 0x0a || inner === 0x0d)
+            return { endOfRow: true };
+        state.cursor += 1;
+    }
+    return { endOfRow: true };
+}
+function consumeRowTerminator(state) {
+    if (state.cursor >= state.text.length)
+        return;
+    const code = state.text.charCodeAt(state.cursor);
+    if (code === 0x0d) {
+        state.cursor += 1;
+        if (state.cursor < state.text.length && state.text.charCodeAt(state.cursor) === 0x0a) {
+            state.cursor += 1;
+        }
+        return;
+    }
+    if (code === 0x0a)
+        state.cursor += 1;
+}
+function readRow(state) {
+    if (state.cursor >= state.text.length) {
+        return { start: state.cursor, end: state.cursor, fieldCount: 0, done: true };
+    }
+    const start = state.cursor;
+    state.rowStart = start;
+    let fieldCount = 0;
+    for (;;) {
+        const field = readField(state);
+        fieldCount += 1;
+        if (field.endOfRow)
+            break;
+    }
+    const end = state.cursor;
+    consumeRowTerminator(state);
+    return { start, end, fieldCount, done: false };
+}
+function emitRows(text, delimiter, input, options) {
+    const tableName = input.extension.toLowerCase() === "tsv" ? "tsv" : "csv";
+    const state = { text, delimiter: delimiter.charCodeAt(0), cursor: 0, rowStart: 0 };
+    const startedAt = options.now();
+    const units = [];
+    const diagnostics = [];
+    // Read the header row first. If there are no further rows, emit the header itself as a
+    // single data row at index 0 so a one-line CSV remains observable.
+    const header = readRow(state);
+    if (header.done)
+        return { units, diagnostics };
+    if (state.cursor >= text.length) {
+        units.push(csvUnit(input, tableName, 0, header.start, header.end));
+        return { units, diagnostics };
+    }
+    let rowIndex = 0;
+    while (state.cursor < text.length) {
+        const limit = shouldStop(startedAt, options, units.length);
+        if (limit.stop && limit.code !== undefined && limit.message !== undefined) {
+            diagnostics.push(diagnostic(limit.code, limit.message, input.documentId, "info"));
+            break;
+        }
+        const row = readRow(state);
+        if (row.done)
+            break;
+        units.push(csvUnit(input, tableName, rowIndex, row.start, row.end));
+        rowIndex += 1;
+    }
+    return { units, diagnostics };
+}
+function csvUnit(input, tableName, rowIndex, start, end) {
+    return {
+        kind: "csv-row",
+        documentId: input.documentId,
+        tableName,
+        rowIndex,
+        characterStart: start,
+        characterEnd: end,
+    };
+}
+export const csvParser = Object.freeze({
+    capability: Object.freeze({
+        parserId: PARSER_ID,
+        parserVersion: PARSER_VERSION,
+        matches: (input) => selectDelimiter(input) !== null,
+    }),
+    parse: (input, options) => {
+        if (input.bytes.byteLength > options.maxBytes) {
+            return emptyResult(csvParser.capability, input.documentId, options, [
+                oversizeDiagnostic(input.documentId, input.bytes.byteLength, options.maxBytes),
+            ]);
+        }
+        const delimiter = selectDelimiter(input);
+        if (delimiter === null) {
+            // Defensive: registry never routes here without a delimiter, but we honour the
+            // contract by returning a typed diagnostic rather than throwing.
+            return emptyResult(csvParser.capability, input.documentId, options, [
+                diagnostic("UNSUPPORTED_FORMAT", "no delimiter selected", input.documentId, "error"),
+            ]);
+        }
+        const decoded = decodeUtf8(input.bytes);
+        const emission = emitRows(decoded.text, delimiter, input, options);
+        return emptyResult(csvParser.capability, input.documentId, options, emission.diagnostics, emission.units);
+    },
+});

package/dist/parsers/docx-parser.d.ts

@@ -0,0 +1,3 @@
+import type { AsyncParserAdapter } from "./types.js";
+export declare const docxParser: AsyncParserAdapter;
+//# sourceMappingURL=docx-parser.d.ts.map

package/dist/parsers/docx-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docx-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/docx-parser.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,YAAY,CAAC;AAgoBpB,eAAO,MAAM,UAAU,EAAE,kBAKvB,CAAC"}

package/dist/parsers/docx-parser.js

@@ -0,0 +1,390 @@
+import { Buffer } from "node:buffer";
+import yauzl from "yauzl";
+import { decodeXmlEntities as decodeXmlEntitiesShared, diagnostic, emptyResult, objectLimitDiagnostic, oversizeDiagnostic, parserIdentity, shouldStop, } from "./_internal.js";
+const PARSER_ID = "docx";
+const PARSER_VERSION = "1";
+const DEPENDENCY_VERSIONS = Object.freeze([
+    Object.freeze({ packageName: "yauzl", version: "3.4.0" }),
+]);
+const DOCX_MEDIA = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+const DOCUMENT_XML_ENTRY = "word/document.xml";
+const MAX_DOCUMENT_XML_INFLATED_BYTES = 16 * 1024 * 1024;
+const MAX_DOCUMENT_XML_INFLATE_RATIO = 100;
+const HEADING_STYLE_PATTERN = /<w:pStyle\b[^>]*w:val="Heading([1-6])"/i;
+const PARAGRAPH_PATTERN = /<w:p\b[\s\S]*?<\/w:p>/gi;
+const TEXT_RUN_PATTERN = /<w:t\b[^>]*>([\s\S]*?)<\/w:t>/gi;
+function isDocx(input) {
+    return input.extension.toLowerCase() === "docx" || input.mediaType.toLowerCase() === DOCX_MEDIA;
+}
+function cancelled(capability, input, options) {
+    return emptyResult(capability, input.documentId, options, [
+        diagnostic("PARSER_CANCELLED", "caller aborted parser", input.documentId, "info"),
+    ]);
+}
+function syncFallback(capability) {
+    return (input, options) => {
+        return emptyResult(capability, input.documentId, options, [
+            diagnostic("UNSUPPORTED_FORMAT", "docx parser requires async caller; use parseAsync via discovery", input.documentId, "info"),
+        ], [unsupportedMediaUnit(input.documentId, "docx-async-required")]);
+    };
+}
+function toError(error, fallback) {
+    return error instanceof Error ? error : new Error(fallback);
+}
+function closeZipQuietly(zip) {
+    try {
+        zip.close();
+    }
+    catch {
+        // Close failures are non-fatal during parser cleanup.
+    }
+}
+function openZip(bytes) {
+    return new Promise((resolve, reject) => {
+        yauzl.fromBuffer(Buffer.from(bytes), { lazyEntries: true, decodeStrings: true }, (error, zip) => {
+            if (error !== null) {
+                reject(toError(error, "failed to open docx zip"));
+                return;
+            }
+            resolve(zip);
+        });
+    });
+}
+function maxInflatedEntryBytes(maxInputBytes) {
+    const inputCap = Math.max(1, Math.floor(maxInputBytes));
+    return Math.min(MAX_DOCUMENT_XML_INFLATED_BYTES, inputCap * 10);
+}
+function assertEntryWithinLimits(entry, limits) {
+    if (entry.uncompressedSize > limits.maxInflatedEntryBytes) {
+        throw new Error(`docx document.xml inflated size ${String(entry.uncompressedSize)} exceeds limit ${String(limits.maxInflatedEntryBytes)}`);
+    }
+    if (entry.compressedSize > 0 &&
+        entry.uncompressedSize / entry.compressedSize > limits.maxInflateRatio) {
+        throw new Error("docx document.xml compression ratio exceeds limit");
+    }
+}
+function destroyStream(readStream, error) {
+    const destroy = readStream.destroy;
+    if (typeof destroy === "function") {
+        destroy.call(readStream, error);
+    }
+}
+function readEntryText(zip, entry, limits) {
+    assertEntryWithinLimits(entry, limits);
+    return new Promise((resolve, reject) => {
+        zip.openReadStream(entry, (error, stream) => {
+            if (error !== null) {
+                reject(toError(error, "failed to open docx entry stream"));
+                return;
+            }
+            const readStream = stream;
+            const chunks = [];
+            let inflatedBytes = 0;
+            let settled = false;
+            const rejectOnce = (streamError) => {
+                if (settled) {
+                    return;
+                }
+                settled = true;
+                reject(streamError);
+                destroyStream(readStream, streamError);
+            };
+            readStream.on("data", (chunk) => {
+                if (settled) {
+                    return;
+                }
+                inflatedBytes += chunk.byteLength;
+                if (inflatedBytes > limits.maxInflatedEntryBytes) {
+                    rejectOnce(new Error("docx document.xml inflated stream exceeds limit"));
+                    return;
+                }
+                chunks.push(chunk);
+            });
+            readStream.on("end", () => {
+                if (settled) {
+                    return;
+                }
+                settled = true;
+                resolve(Buffer.concat(chunks).toString("utf8"));
+            });
+            readStream.on("error", (streamError) => {
+                rejectOnce(streamError);
+            });
+        });
+    });
+}
+function readDocumentXmlFromZip(zip, limits) {
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        const resolveOnce = (value) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            zip.removeListener("entry", onEntry);
+            zip.removeListener("end", onEnd);
+            zip.removeListener("error", onError);
+            resolve(value);
+        };
+        const rejectOnce = (error) => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            zip.removeListener("entry", onEntry);
+            zip.removeListener("end", onEnd);
+            zip.removeListener("error", onError);
+            reject(error);
+        };
+        const onEnd = () => {
+            rejectOnce(new Error("docx missing word/document.xml"));
+        };
+        const onError = (error) => {
+            rejectOnce(toError(error, "failed to read docx zip"));
+        };
+        const handleEntry = async (entry) => {
+            if (entry.fileName !== DOCUMENT_XML_ENTRY) {
+                zip.readEntry();
+                return;
+            }
+            try {
+                const xml = await readEntryText(zip, entry, limits);
+                resolveOnce(xml);
+            }
+            catch (error) {
+                rejectOnce(toError(error, "failed to read docx entry"));
+            }
+        };
+        const onEntry = (entry) => {
+            void handleEntry(entry);
+        };
+        zip.on("entry", onEntry);
+        zip.once("end", onEnd);
+        zip.once("error", onError);
+        zip.readEntry();
+    });
+}
+async function readDocumentXml(bytes, maxInputBytes) {
+    const zip = (await openZip(bytes));
+    try {
+        return await readDocumentXmlFromZip(zip, {
+            maxInflatedEntryBytes: maxInflatedEntryBytes(maxInputBytes),
+            maxInflateRatio: MAX_DOCUMENT_XML_INFLATE_RATIO,
+        });
+    }
+    finally {
+        closeZipQuietly(zip);
+    }
+}
+// GRD-027: delegate to the shared decoder so docx text runs also resolve numeric character
+// references (smart quotes, accented letters) instead of surfacing literal `&#8217;`.
+function decodeXmlEntities(value) {
+    return decodeXmlEntitiesShared(value);
+}
+function limitDiagnostic(input, limit) {
+    if (!limit.stop || limit.code === undefined || limit.message === undefined) {
+        return undefined;
+    }
+    return diagnostic(limit.code, limit.message, input.documentId, "info");
+}
+function headingLevelOf(paragraphXml) {
+    const match = HEADING_STYLE_PATTERN.exec(paragraphXml);
+    return match?.[1] === undefined ? undefined : Number(match[1]);
+}
+function paragraphText(paragraphXml) {
+    const withBreaks = paragraphXml
+        .replaceAll(/<w:tab\s*\/>/gi, "\t")
+        .replaceAll(/<w:br\s*\/>/gi, "\n")
+        .replaceAll(/<w:cr\s*\/>/gi, "\n");
+    const parts = Array.from(withBreaks.matchAll(TEXT_RUN_PATTERN))
+        .map((match) => decodeXmlEntities(match[1] ?? ""))
+        .filter((part) => part.length > 0);
+    return parts.join("").trim();
+}
+function parseParagraphs(xml, input, options, startedAt) {
+    const out = [];
+    const diagnostics = [];
+    let scannedParagraphs = 0;
+    for (const match of xml.matchAll(PARAGRAPH_PATTERN)) {
+        if (scannedParagraphs >= options.maxObjectsPerDocument) {
+            diagnostics.push(objectLimitDiagnostic(input.documentId, options.maxObjectsPerDocument));
+            break;
+        }
+        const limit = shouldStop(startedAt, options, scannedParagraphs);
+        const stopped = limitDiagnostic(input, limit);
+        if (stopped !== undefined) {
+            diagnostics.push(stopped);
+            break;
+        }
+        scannedParagraphs += 1;
+        const paragraphXml = match[0];
+        const text = paragraphText(paragraphXml);
+        if (text.length === 0)
+            continue;
+        const headingLevel = headingLevelOf(paragraphXml);
+        out.push(headingLevel === undefined ? { text } : { text, headingLevel });
+    }
+    return { paragraphs: out, diagnostics };
+}
+function paragraphStarts(paragraphs) {
+    const starts = [];
+    let cursor = 0;
+    for (const paragraph of paragraphs) {
+        starts.push(cursor);
+        cursor += paragraph.text.length + 1;
+    }
+    return { starts, end: Math.max(0, cursor - 1) };
+}
+function isHeadingEntry(entry) {
+    return entry.paragraph.headingLevel !== undefined;
+}
+function collectHeadings(paragraphs) {
+    return paragraphs.map((paragraph, index) => ({ paragraph, index })).filter(isHeadingEntry);
+}
+function unsupportedMediaUnit(documentId, reason) {
+    return { kind: "unsupported-media", documentId, reason };
+}
+function sectionUnit(section) {
+    return {
+        kind: "section",
+        documentId: section.documentId,
+        sectionPath: section.sectionPath,
+        characterStart: section.characterStart,
+        characterEnd: section.characterEnd,
+    };
+}
+function appendSectionRecord(sections, units, input, sectionPath, start, end) {
+    const sectionRecord = {
+        documentId: input.documentId,
+        sectionPath,
+        characterStart: start,
+        characterEnd: end,
+    };
+    sections.push(sectionRecord);
+    units.push(sectionUnit(sectionRecord));
+}
+function appendLimitedSectionRecord(sections, units, diagnostics, input, options, startedAt, sectionPath, start, end) {
+    const limit = shouldStop(startedAt, options, units.length);
+    const stopped = limitDiagnostic(input, limit);
+    if (stopped !== undefined) {
+        diagnostics.push(stopped);
+        return false;
+    }
+    appendSectionRecord(sections, units, input, sectionPath, start, end);
+    return true;
+}
+function buildUnsectionedSections(input, options, startedAt, end) {
+    const sections = [];
+    const units = [];
+    const diagnostics = [];
+    appendLimitedSectionRecord(sections, units, diagnostics, input, options, startedAt, [], 0, end);
+    return { sections, units, diagnostics };
+}
+function appendLeadingPreambleSection(sections, units, diagnostics, input, headings, offsets, options, startedAt) {
+    const firstHeading = headings[0];
+    if (firstHeading === undefined) {
+        return true;
+    }
+    const firstHeadingStart = offsets.starts[firstHeading.index] ?? 0;
+    if (firstHeadingStart > 0) {
+        return appendLimitedSectionRecord(sections, units, diagnostics, input, options, startedAt, [], 0, firstHeadingStart);
+    }
+    return true;
+}
+function appendHeadingSection(state, current, next) {
+    const level = current.paragraph.headingLevel;
+    while (state.stack.length >= level)
+        state.stack.pop();
+    state.stack.push(current.paragraph.text);
+    const start = state.offsets.starts[current.index] ?? 0;
+    const end = next === undefined
+        ? state.offsets.end
+        : (state.offsets.starts[next.index] ?? state.offsets.end);
+    return appendLimitedSectionRecord(state.sections, state.units, state.diagnostics, state.input, state.options, state.startedAt, [...state.stack], start, end);
+}
+function appendHeadingSections(state, headings) {
+    for (const [i, current] of headings.entries()) {
+        if (!appendHeadingSection(state, current, headings[i + 1])) {
+            return;
+        }
+    }
+}
+function buildSections(paragraphs, input, options, startedAt) {
+    const stack = [];
+    const sections = [];
+    const units = [];
+    const diagnostics = [];
+    const offsets = paragraphStarts(paragraphs);
+    const headings = collectHeadings(paragraphs);
+    if (headings.length === 0) {
+        return buildUnsectionedSections(input, options, startedAt, offsets.end);
+    }
+    if (!appendLeadingPreambleSection(sections, units, diagnostics, input, headings, offsets, options, startedAt)) {
+        return { sections, units, diagnostics };
+    }
+    appendHeadingSections({ sections, units, diagnostics, input, options, startedAt, offsets, stack }, headings);
+    return { sections, units, diagnostics };
+}
+function docxNoTextResult(capability, input, options) {
+    return emptyResult(capability, input.documentId, options, [
+        diagnostic("UNSUPPORTED_FORMAT", "docx has no extractable text content", input.documentId, "info"),
+    ], [unsupportedMediaUnit(input.documentId, "docx-no-text")]);
+}
+function docxParseResult(capability, input, options, paragraphs, diagnostics, startedAt) {
+    const built = buildSections(paragraphs, input, options, startedAt);
+    const normalizedText = paragraphs.map((paragraph) => paragraph.text).join("\n");
+    return {
+        documentId: input.documentId,
+        parser: parserIdentity(capability),
+        pages: [],
+        sections: built.sections,
+        units: built.units,
+        diagnostics: [...diagnostics, ...built.diagnostics],
+        extractedAt: options.now(),
+        normalizedText,
+    };
+}
+async function asyncParse(capability, input, options) {
+    if (input.bytes.byteLength > options.maxBytes) {
+        return emptyResult(capability, input.documentId, options, [
+            oversizeDiagnostic(input.documentId, input.bytes.byteLength, options.maxBytes),
+        ]);
+    }
+    if (options.signal?.aborted === true) {
+        return cancelled(capability, input, options);
+    }
+    const startedAt = options.now();
+    try {
+        const xml = await readDocumentXml(input.bytes, options.maxBytes);
+        const limit = shouldStop(startedAt, options, 0);
+        if (limit.stop && limit.code !== undefined && limit.message !== undefined) {
+            return emptyResult(capability, input.documentId, options, [
+                diagnostic(limit.code, limit.message, input.documentId, "info"),
+            ]);
+        }
+        const parsed = parseParagraphs(xml, input, options, startedAt);
+        if (parsed.paragraphs.length === 0) {
+            if (parsed.diagnostics.length > 0) {
+                return emptyResult(capability, input.documentId, options, parsed.diagnostics);
+            }
+            return docxNoTextResult(capability, input, options);
+        }
+        return docxParseResult(capability, input, options, parsed.paragraphs, parsed.diagnostics, startedAt);
+    }
+    catch {
+        return emptyResult(capability, input.documentId, options, [
+            diagnostic("MALFORMED_INPUT", "docx parser rejected malformed or unsupported document", input.documentId, "error"),
+        ]);
+    }
+}
+const capability = Object.freeze({
+    parserId: PARSER_ID,
+    parserVersion: PARSER_VERSION,
+    dependencyVersions: DEPENDENCY_VERSIONS,
+    matches: isDocx,
+});
+export const docxParser = Object.freeze({
+    capability,
+    parse: syncFallback(capability),
+    parseAsync: (input, options) => asyncParse(capability, input, options),
+});

package/dist/parsers/html-parser.d.ts

@@ -0,0 +1,3 @@
+import type { ParserAdapter } from "./types.js";
+export declare const htmlParser: ParserAdapter;
+//# sourceMappingURL=html-parser.d.ts.map

package/dist/parsers/html-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/html-parser.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAEV,aAAa,EAGd,MAAM,YAAY,CAAC;AAyUpB,eAAO,MAAM,UAAU,EAAE,aA4BvB,CAAC"}