@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/parsers/text-parser.js

@@ -0,0 +1,214 @@
+// Plain-text + markdown adapter (Epic #189, Issue #266). Pure: takes bytes, returns a
+// ParserResult. Plain text emits a single section unit covering the whole document; markdown
+// is split on ATX-style headings (`^#{1,6}\\s`) into one section per heading run with the
+// hierarchical sectionPath populated from heading text.
+//
+// Source code (.ts/.js/.py/.yaml/etc) routes here too via `languageHint`. The lexical scan
+// runs once over the decoded string and stops emitting units the moment any limit (deadline
+// / cancellation / unit count) trips.
+import { decodeUtf8, diagnostic, emptyResult, oversizeDiagnostic, shouldStop, } from "./_internal.js";
+const PARSER_ID = "text";
+const PARSER_VERSION = "1";
+// Extensions accepted by the text adapter. The list is intentionally explicit so unknown
+// binary extensions fall through to the unsupported adapter rather than being mis-parsed as
+// "plain text". `languageHint` is emitted for source / config files so #195 (chunker) can
+// route them through code-aware splitters when those land.
+const TEXT_EXTENSIONS = new Set([
+    "txt",
+    "log",
+    "md",
+    "markdown",
+    "rst",
+    "adoc",
+    "asciidoc",
+    "ts",
+    "tsx",
+    "js",
+    "jsx",
+    "mjs",
+    "cjs",
+    "py",
+    "rb",
+    "go",
+    "rs",
+    "java",
+    "kt",
+    "swift",
+    "c",
+    "cc",
+    "cpp",
+    "h",
+    "hpp",
+    "cs",
+    "php",
+    "sh",
+    "bash",
+    "zsh",
+    "fish",
+    "ps1",
+    "yaml",
+    "yml",
+    "toml",
+    "ini",
+    "cfg",
+    "conf",
+    "env",
+    "properties",
+    "sql",
+    "graphql",
+    "gql",
+]);
+const TEXT_MEDIA_PREFIXES = ["text/"];
+function isMarkdown(input) {
+    const ext = input.extension.toLowerCase();
+    if (ext === "md" || ext === "markdown")
+        return true;
+    return input.mediaType.toLowerCase() === "text/markdown";
+}
+function isTextLike(input) {
+    const ext = input.extension.toLowerCase();
+    if (TEXT_EXTENSIONS.has(ext))
+        return true;
+    const media = input.mediaType.toLowerCase();
+    for (const prefix of TEXT_MEDIA_PREFIXES) {
+        if (media.startsWith(prefix))
+            return true;
+    }
+    return false;
+}
+function scanHeadings(text) {
+    const out = [];
+    let cursor = 0;
+    while (cursor < text.length) {
+        const newline = text.indexOf("\n", cursor);
+        const lineEnd = newline === -1 ? text.length : newline;
+        const line = text.slice(cursor, lineEnd);
+        const heading = matchAtxHeading(line);
+        if (heading !== null) {
+            out.push({ ...heading, characterStart: cursor });
+        }
+        cursor = lineEnd + 1;
+    }
+    return out;
+}
+const HASH = 0x23;
+const SPACE = 0x20;
+const TAB = 0x09;
+function isHSpace(code) {
+    return code === SPACE || code === TAB;
+}
+function countHashes(line) {
+    let i = 0;
+    while (i < line.length && line.charCodeAt(i) === HASH && i < 6)
+        i += 1;
+    return i;
+}
+function skipHSpace(line, from) {
+    let i = from;
+    while (i < line.length && isHSpace(line.charCodeAt(i)))
+        i += 1;
+    return i;
+}
+function trimAtxTrailing(line, start) {
+    let end = line.length;
+    while (end > start && isHSpace(line.charCodeAt(end - 1)))
+        end -= 1;
+    while (end > start && line.charCodeAt(end - 1) === HASH)
+        end -= 1;
+    while (end > start && isHSpace(line.charCodeAt(end - 1)))
+        end -= 1;
+    return end;
+}
+function matchAtxHeading(line) {
+    const level = countHashes(line);
+    if (level < 1 || level > 6)
+        return null;
+    if (level >= line.length || !isHSpace(line.charCodeAt(level)))
+        return null;
+    const textStart = skipHSpace(line, level + 1);
+    const textEnd = trimAtxTrailing(line, textStart);
+    if (textEnd <= textStart)
+        return null;
+    return { level, text: line.slice(textStart, textEnd) };
+}
+function sectionUnit(input, path, start, end) {
+    return {
+        kind: "section",
+        documentId: input.documentId,
+        sectionPath: path,
+        characterStart: start,
+        characterEnd: end,
+    };
+}
+function pushHeading(stack, current, next, text, input) {
+    while (stack.length >= current.level)
+        stack.pop();
+    stack.push(current.text);
+    const sectionEnd = next === undefined ? text.length : next.characterStart;
+    return sectionUnit(input, [...stack], current.characterStart, sectionEnd);
+}
+function emitMarkdownSections(text, input, options, startedAt) {
+    const headings = scanHeadings(text);
+    if (headings.length === 0) {
+        return emitPlainSection(text, input, options, startedAt);
+    }
+    const units = [];
+    const diagnostics = [];
+    const stack = [];
+    const firstStart = headings[0]?.characterStart ?? 0;
+    if (firstStart > 0)
+        units.push(sectionUnit(input, [], 0, firstStart));
+    for (let i = 0; i < headings.length; i += 1) {
+        const limit = shouldStop(startedAt, options, units.length);
+        if (limit.stop && limit.code !== undefined && limit.message !== undefined) {
+            diagnostics.push(diagnostic(limit.code, limit.message, input.documentId, "info"));
+            break;
+        }
+        const current = headings[i];
+        if (current === undefined)
+            break;
+        units.push(pushHeading(stack, current, headings[i + 1], text, input));
+    }
+    return { units, diagnostics };
+}
+function emitPlainSection(text, input, options, startedAt) {
+    const limit = shouldStop(startedAt, options, 0);
+    if (limit.stop && limit.code !== undefined && limit.message !== undefined) {
+        return {
+            units: [],
+            diagnostics: [diagnostic(limit.code, limit.message, input.documentId, "info")],
+        };
+    }
+    return {
+        units: [
+            {
+                kind: "section",
+                documentId: input.documentId,
+                sectionPath: [],
+                characterStart: 0,
+                characterEnd: text.length,
+            },
+        ],
+        diagnostics: [],
+    };
+}
+export const textParser = Object.freeze({
+    capability: Object.freeze({
+        parserId: PARSER_ID,
+        parserVersion: PARSER_VERSION,
+        matches: (input) => isTextLike(input),
+    }),
+    parse: (input, options) => {
+        if (input.bytes.byteLength > options.maxBytes) {
+            return emptyResult(textParser.capability, input.documentId, options, [
+                oversizeDiagnostic(input.documentId, input.bytes.byteLength, options.maxBytes),
+            ]);
+        }
+        const startedAt = options.now();
+        const decoded = decodeUtf8(input.bytes);
+        const emission = isMarkdown(input)
+            ? emitMarkdownSections(decoded.text, input, options, startedAt)
+            : emitPlainSection(decoded.text, input, options, startedAt);
+        return emptyResult(textParser.capability, input.documentId, options, emission.diagnostics, emission.units);
+    },
+});

package/dist/parsers/types.d.ts

@@ -0,0 +1,53 @@
+import type { DocumentId, ParsedUnit, ParserDiagnostic, ParserDependencyVersion, ParserResult } from "@oscharko-dev/keiko-contracts";
+export interface ParserOptions {
+    readonly maxBytes: number;
+    readonly maxUnitsPerDocument: number;
+    readonly maxNestingDepth: number;
+    readonly maxObjectsPerDocument: number;
+    readonly timeoutMs: number;
+    readonly signal?: AbortSignal;
+    readonly now: () => number;
+}
+export declare const DEFAULT_MAX_BYTES: number;
+export declare const DEFAULT_MAX_UNITS = 50000;
+export declare const DEFAULT_MAX_NESTING_DEPTH = 128;
+export declare const DEFAULT_MAX_OBJECTS = 25000000;
+export declare const DEFAULT_TIMEOUT_MS: number;
+export interface ParserSelectionInput {
+    readonly documentId: DocumentId;
+    readonly bytes: Uint8Array;
+    readonly extension: string;
+    readonly mediaType: string;
+    readonly languageHint?: string;
+}
+export interface ParserCapability {
+    readonly parserId: string;
+    readonly parserVersion: string;
+    readonly dependencyVersions?: readonly ParserDependencyVersion[];
+    readonly matches: (input: ParserSelectionInput) => boolean;
+}
+export interface ParserAdapter {
+    readonly capability: ParserCapability;
+    readonly parse: (input: ParserSelectionInput, options: ParserOptions) => ParserResult;
+}
+export interface AsyncParserAdapter extends ParserAdapter {
+    readonly parseAsync: (input: ParserSelectionInput, options: ParserOptions) => Promise<ParserResult>;
+}
+export interface InternalParserResult extends ParserResult {
+    readonly normalizedText?: string;
+}
+export interface ParserRegistry {
+    readonly list: () => readonly ParserAdapter[];
+    readonly resolve: (input: ParserSelectionInput) => ParserResolution;
+}
+export type ParserResolution = {
+    readonly kind: "matched";
+    readonly adapter: ParserAdapter;
+} | {
+    readonly kind: "unsupported";
+    readonly reason: string;
+};
+export type ParserErrorCode = "OVERSIZED_FILE" | "UNIT_LIMIT_REACHED" | "PARSER_TIMEOUT" | "PARSER_CANCELLED" | "NESTING_LIMIT_REACHED" | "OBJECT_LIMIT_REACHED" | "MALFORMED_INPUT" | "UNSUPPORTED_FORMAT";
+export declare const PARSER_ERROR_CODES: readonly ParserErrorCode[];
+export type { ParserResult, ParsedUnit, ParserDiagnostic };
+//# sourceMappingURL=types.d.ts.map

package/dist/parsers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/parsers/types.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,uBAAuB,EACvB,YAAY,EACb,MAAM,+BAA+B,CAAC;AAIvC,MAAM,WAAW,aAAa;IAG5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IAGrC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAGjC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;IAEvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAG3B,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAG9B,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;CAC5B;AAED,eAAO,MAAM,iBAAiB,QAAqB,CAAC;AACpD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AACxC,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAC7C,eAAO,MAAM,mBAAmB,WAAa,CAAC;AAC9C,eAAO,MAAM,kBAAkB,QAAiB,CAAC;AAIjD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAEhC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAG3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAG3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAG3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAE/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAG1B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAG/B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,SAAS,uBAAuB,EAAE,CAAC;IAGjE,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,OAAO,CAAC;CAC5D;AAID,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IAKtC,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,aAAa,KAAK,YAAY,CAAC;CACvF;AAED,MAAM,WAAW,kBAAmB,SAAQ,aAAa;IACvD,QAAQ,CAAC,UAAU,EAAE,CACnB,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,aAAa,KACnB,OAAO,CAAC,YAAY,CAAC,CAAC;CAC5B;AAOD,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAID,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,SAAS,aAAa,EAAE,CAAC;IAI9C,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,gBAAgB,CAAC;CACrE;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAA;CAAE,GAC7D;IAAE,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAI9D,MAAM,MAAM,eAAe,GACvB,gBAAgB,GAChB,oBAAoB,GACpB,gBAAgB,GAChB,kBAAkB,GAClB,uBAAuB,GACvB,sBAAsB,GACtB,iBAAiB,GACjB,oBAAoB,CAAC;AAEzB,eAAO,MAAM,kBAAkB,EAAE,SAAS,eAAe,EAS/C,CAAC;AAIX,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC"}

package/dist/parsers/types.js

@@ -0,0 +1,21 @@
+// Public types for the parser registry + format adapters (Epic #189, Issue #266).
+//
+// Adapters are pure: `(input, options) -> ParserResult`. No FS, no clock beyond a single
+// injected `now()`, no randomness, and no implicit runtime services. The runtime layer at
+// #194 reads bytes from disk and hands them to a parser; that keeps parsers trivially
+// testable with synthetic strings.
+export const DEFAULT_MAX_BYTES = 1024 * 1024 * 1024;
+export const DEFAULT_MAX_UNITS = 50_000;
+export const DEFAULT_MAX_NESTING_DEPTH = 128;
+export const DEFAULT_MAX_OBJECTS = 25_000_000;
+export const DEFAULT_TIMEOUT_MS = 60 * 60 * 1000;
+export const PARSER_ERROR_CODES = [
+    "OVERSIZED_FILE",
+    "UNIT_LIMIT_REACHED",
+    "PARSER_TIMEOUT",
+    "PARSER_CANCELLED",
+    "NESTING_LIMIT_REACHED",
+    "OBJECT_LIMIT_REACHED",
+    "MALFORMED_INPUT",
+    "UNSUPPORTED_FORMAT",
+];

package/dist/parsers/unsupported-parser.d.ts

@@ -0,0 +1,4 @@
+import type { ParserAdapter, ParserSelectionInput } from "./types.js";
+export declare function classifyUnsupported(input: ParserSelectionInput): string | undefined;
+export declare const unsupportedParser: ParserAdapter;
+//# sourceMappingURL=unsupported-parser.d.ts.map

package/dist/parsers/unsupported-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unsupported-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/unsupported-parser.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,aAAa,EAAiB,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAwErF,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,oBAAoB,GAAG,MAAM,GAAG,SAAS,CASnF;AAED,eAAO,MAAM,iBAAiB,EAAE,aAuB9B,CAAC"}

package/dist/parsers/unsupported-parser.js

@@ -0,0 +1,97 @@
+// Unsupported-format parser (Epic #189, Issue #266). Emits a single
+// `unsupported-media` ParsedUnit plus a typed `UNSUPPORTED_FORMAT` diagnostic so
+// downstream layers (chunker #195, indexer #196, UI #198) can render a stable signal
+// without sniffing the bytes themselves.
+//
+// CRITICAL: archives (.zip / .tar / .gz / .tar.gz) hit THIS adapter unchanged. We never
+// open them — zip-bomb protection lives in not-decompressing, not in size caps.
+import { diagnostic, emptyResult } from "./_internal.js";
+const PARSER_ID = "unsupported";
+const PARSER_VERSION = "1";
+// Extensions we recognise but cannot handle in this PR. Each maps to a stable reason string
+// so UI surfaces can render distinct guidance (e.g. "PDF — extract via OCR adapter").
+const UNSUPPORTED_EXTENSIONS = Object.freeze({
+    pdf: "pdf-not-implemented",
+    docx: "docx-not-implemented",
+    doc: "legacy-doc-not-implemented",
+    png: "image-not-supported",
+    jpg: "image-not-supported",
+    jpeg: "image-not-supported",
+    gif: "image-not-supported",
+    bmp: "image-not-supported",
+    tif: "image-not-supported",
+    tiff: "image-not-supported",
+    webp: "image-not-supported",
+    mp3: "audio-not-supported",
+    wav: "audio-not-supported",
+    flac: "audio-not-supported",
+    ogg: "audio-not-supported",
+    mp4: "video-not-supported",
+    mov: "video-not-supported",
+    mkv: "video-not-supported",
+    avi: "video-not-supported",
+    webm: "video-not-supported",
+    zip: "archive-not-decompressed",
+    tar: "archive-not-decompressed",
+    gz: "archive-not-decompressed",
+    tgz: "archive-not-decompressed",
+    bz2: "archive-not-decompressed",
+    "7z": "archive-not-decompressed",
+    rar: "archive-not-decompressed",
+    exe: "binary-not-supported",
+    dll: "binary-not-supported",
+    bin: "binary-not-supported",
+    so: "binary-not-supported",
+    dylib: "binary-not-supported",
+});
+// Magic-byte signatures for content sniffing. Each entry is `(bytes prefix) -> reason`.
+// We compare against the first few bytes so we still classify a `.txt`-named PDF as
+// "pdf-not-implemented" rather than misclassifying it as plain text.
+const MAGIC_BYTES = Object.freeze([
+    { prefix: [0x25, 0x50, 0x44, 0x46], reason: "pdf-not-implemented" }, // %PDF
+    { prefix: [0x50, 0x4b, 0x03, 0x04], reason: "archive-not-decompressed" }, // PK\3\4 (zip / docx / xlsx)
+    { prefix: [0x1f, 0x8b], reason: "archive-not-decompressed" }, // gzip
+    { prefix: [0x89, 0x50, 0x4e, 0x47], reason: "image-not-supported" }, // PNG
+    { prefix: [0xff, 0xd8, 0xff], reason: "image-not-supported" }, // JPEG
+    { prefix: [0x47, 0x49, 0x46, 0x38], reason: "image-not-supported" }, // GIF8
+    { prefix: [0x42, 0x4d], reason: "image-not-supported" }, // BMP
+]);
+function magicByteReason(bytes) {
+    for (const entry of MAGIC_BYTES) {
+        if (bytes.length < entry.prefix.length)
+            continue;
+        let match = true;
+        for (let i = 0; i < entry.prefix.length; i += 1) {
+            if (bytes[i] !== entry.prefix[i]) {
+                match = false;
+                break;
+            }
+        }
+        if (match)
+            return entry.reason;
+    }
+    return undefined;
+}
+export function classifyUnsupported(input) {
+    // Extension table is checked first so that known formats (e.g. .docx, which IS a ZIP) get
+    // the specific reason string rather than the generic magic-byte fallback.
+    const lower = input.extension.toLowerCase();
+    if (lower in UNSUPPORTED_EXTENSIONS) {
+        return UNSUPPORTED_EXTENSIONS[lower];
+    }
+    // Fall back to magic-byte sniffing for files with unknown/absent extensions.
+    return magicByteReason(input.bytes);
+}
+export const unsupportedParser = Object.freeze({
+    capability: Object.freeze({
+        parserId: PARSER_ID,
+        parserVersion: PARSER_VERSION,
+        matches: (input) => classifyUnsupported(input) !== undefined,
+    }),
+    parse: (input, options) => {
+        const reason = classifyUnsupported(input) ?? "unknown-format";
+        return emptyResult(unsupportedParser.capability, input.documentId, options, [
+            diagnostic("UNSUPPORTED_FORMAT", `format not parseable in this build (${reason})`, input.documentId, "info"),
+        ], [{ kind: "unsupported-media", documentId: input.documentId, reason }]);
+    },
+});

package/dist/parsers/xlsx-parser.d.ts

@@ -0,0 +1,3 @@
+import type { AsyncParserAdapter } from "./types.js";
+export declare const xlsxParser: AsyncParserAdapter;
+//# sourceMappingURL=xlsx-parser.d.ts.map

package/dist/parsers/xlsx-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"xlsx-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/xlsx-parser.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,YAAY,CAAC;AAugBpB,eAAO,MAAM,UAAU,EAAE,kBAyCvB,CAAC"}