@kolisachint/hoocode-agent 0.4.140 → 0.4.142

package/dist/core/embsearch/chunker.d.ts

@@ -0,0 +1,31 @@
+/**
+ * File chunking for semantic indexing.
+ *
+ * Splits a file into overlapping line windows, each capped by characters so a
+ * chunk stays within the embedding model's effective token window (MiniLM
+ * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;
+ * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)
+ * so search hits can be rendered as `path:start-end`.
+ *
+ * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the
+ * sidecar triggers a clean rebuild of the store.
+ */
+export declare const CHUNKER_VERSION = 1;
+export interface Chunk {
+    /** `relpath#index` — the id stored in the vector index. */
+    id: string;
+    /** Text sent to the embedder. */
+    text: string;
+    /** 1-based inclusive start line. */
+    startLine: number;
+    /** 1-based inclusive end line. */
+    endLine: number;
+}
+/** Heuristic binary sniff: NUL byte in the first 8KB. */
+export declare function looksBinary(content: string): boolean;
+/**
+ * Split `content` into chunks. `relPath` becomes the id prefix. Returns an
+ * empty array for empty or binary-looking content.
+ */
+export declare function chunkFile(relPath: string, content: string): Chunk[];
+//# sourceMappingURL=chunker.d.ts.map

package/dist/core/embsearch/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,eAAe,IAAI,CAAC;AASjC,MAAM,WAAW,KAAK;IACrB,6DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,yDAAyD;AACzD,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAGpD;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,CAoCnE","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 1;\n\n/** Target lines per chunk. */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/** Hard character cap per chunk (MiniLM truncates ~256 tokens ≈ 1000 chars). */\nconst CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nexport function looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n */\nexport function chunkFile(relPath: string, content: string): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > CHUNK_MAX_CHARS && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\tlet text = lines.slice(start, end).join(\"\\n\").trim();\n\t\tif (text.length > CHUNK_MAX_CHARS) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, CHUNK_MAX_CHARS);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: start + 1,\n\t\t\t\tendLine: end,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}

package/dist/core/embsearch/chunker.js

@@ -0,0 +1,68 @@
+/**
+ * File chunking for semantic indexing.
+ *
+ * Splits a file into overlapping line windows, each capped by characters so a
+ * chunk stays within the embedding model's effective token window (MiniLM
+ * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;
+ * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)
+ * so search hits can be rendered as `path:start-end`.
+ *
+ * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the
+ * sidecar triggers a clean rebuild of the store.
+ */
+export const CHUNKER_VERSION = 1;
+/** Target lines per chunk. */
+const CHUNK_LINES = 60;
+/** Overlapping lines between consecutive chunks, for context continuity. */
+const CHUNK_OVERLAP_LINES = 10;
+/** Hard character cap per chunk (MiniLM truncates ~256 tokens ≈ 1000 chars). */
+const CHUNK_MAX_CHARS = 1000;
+/** Heuristic binary sniff: NUL byte in the first 8KB. */
+export function looksBinary(content) {
+    const probe = content.slice(0, 8192);
+    return probe.includes("\u0000");
+}
+/**
+ * Split `content` into chunks. `relPath` becomes the id prefix. Returns an
+ * empty array for empty or binary-looking content.
+ */
+export function chunkFile(relPath, content) {
+    if (!content.trim() || looksBinary(content))
+        return [];
+    const lines = content.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n");
+    const chunks = [];
+    let start = 0; // 0-based
+    let index = 0;
+    while (start < lines.length) {
+        let end = start; // exclusive
+        let chars = 0;
+        while (end < lines.length && end - start < CHUNK_LINES) {
+            const lineLen = lines[end].length + 1;
+            if (chars + lineLen > CHUNK_MAX_CHARS && end > start)
+                break;
+            chars += lineLen;
+            end++;
+        }
+        let text = lines.slice(start, end).join("\n").trim();
+        if (text.length > CHUNK_MAX_CHARS) {
+            // Oversized chunk (e.g. long minified line): keep the prefix. The
+            // underlying model would truncate anyway, so this stays bounded.
+            text = text.slice(0, CHUNK_MAX_CHARS);
+        }
+        if (text) {
+            chunks.push({
+                id: `${relPath}#${index}`,
+                text,
+                startLine: start + 1,
+                endLine: end,
+            });
+            index++;
+        }
+        if (end >= lines.length)
+            break;
+        // Step forward with overlap, but always make progress.
+        start = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);
+    }
+    return chunks;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/core/embsearch/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAEjC,8BAA8B;AAC9B,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,4EAA4E;AAC5E,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,kFAAgF;AAChF,MAAM,eAAe,GAAG,IAAI,CAAC;AAa7B,yDAAyD;AACzD,MAAM,UAAU,WAAW,CAAC,OAAe,EAAW;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAAA,CAChC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,OAAe,EAAW;IACpE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,WAAW,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU;IACzB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,YAAY;QAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,GAAG,KAAK,GAAG,WAAW,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACtC,IAAI,KAAK,GAAG,OAAO,GAAG,eAAe,IAAI,GAAG,GAAG,KAAK;gBAAE,MAAM;YAC5D,KAAK,IAAI,OAAO,CAAC;YACjB,GAAG,EAAE,CAAC;QACP,CAAC;QACD,IAAI,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACrD,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACnC,kEAAkE;YAClE,iEAAiE;YACjE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,OAAO,IAAI,KAAK,EAAE;gBACzB,IAAI;gBACJ,SAAS,EAAE,KAAK,GAAG,CAAC;gBACpB,OAAO,EAAE,GAAG;aACZ,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACT,CAAC;QACD,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM;QAC/B,uDAAuD;QACvD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,mBAAmB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 1;\n\n/** Target lines per chunk. */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/** Hard character cap per chunk (MiniLM truncates ~256 tokens ≈ 1000 chars). */\nconst CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nexport function looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n */\nexport function chunkFile(relPath: string, content: string): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > CHUNK_MAX_CHARS && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\tlet text = lines.slice(start, end).join(\"\\n\").trim();\n\t\tif (text.length > CHUNK_MAX_CHARS) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, CHUNK_MAX_CHARS);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: start + 1,\n\t\t\t\tendLine: end,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}

package/dist/core/embsearch/client.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Client for the `embsearch` stdio daemon (vendored from
+ * github.com/kolisachint/embeddingsearchtools ts/client.ts).
+ *
+ * Spawns `embsearch serve` once and talks newline-delimited JSON over its
+ * stdin/stdout. The process stays alive so the model and index load a single
+ * time; every query after startup is hot.
+ */
+export interface EmbSearchResult {
+    id: string;
+    score: number;
+}
+export interface EmbSearchDaemonInfo {
+    modelId: string;
+    dim: number;
+    count: number;
+}
+export interface EmbSearchBulkResult {
+    inserted: number;
+    updated: number;
+}
+export interface EmbSearchClientOptions {
+    /** Path to the `embsearch` binary. */
+    binaryPath: string;
+    /** Store directory passed as `--path`. */
+    storePath: string;
+    /** Metric for a freshly created store. Default: "cosine". */
+    metric?: "cosine" | "dot" | "euclidean";
+}
+export declare class EmbSearchClient {
+    private proc;
+    private queue;
+    private buffer;
+    private closed;
+    private readyPromise;
+    constructor(opts: EmbSearchClientOptions);
+    /** Resolves once the daemon has loaded the model + index. */
+    ready(): Promise<void>;
+    get isClosed(): boolean;
+    private onStdout;
+    private send;
+    /** Search for the top-`k` matches for `text`. */
+    query(text: string, k?: number): Promise<EmbSearchResult[]>;
+    /**
+     * Batched insert-or-replace. One embedding inference for the whole batch —
+     * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a
+     * concurrent query is not stuck behind a huge inference.
+     */
+    bulk(items: Array<{
+        id: string;
+        text: string;
+    }>): Promise<EmbSearchBulkResult>;
+    /** Model id, dimensionality, and live vector count of the daemon. */
+    info(): Promise<EmbSearchDaemonInfo>;
+    /** Remove a record. Resolves to `true` if it existed. */
+    remove(id: string): Promise<boolean>;
+    /** Reclaim tombstoned rows left behind by `remove`. */
+    compact(): Promise<void>;
+    /** Persist the index to the store directory. */
+    save(): Promise<void>;
+    /** Shut the daemon down, closing stdin so it exits cleanly. */
+    close(): Promise<void>;
+}
+//# sourceMappingURL=client.d.ts.map

package/dist/core/embsearch/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACtC,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,CAAC;CACxC;AAoBD,qBAAa,eAAe;IAC3B,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAgB;IAEpC,YAAY,IAAI,EAAE,sBAAsB,EA2BvC;IAED,6DAA6D;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,OAAO,CAAC,QAAQ;IAqBhB,OAAO,CAAC,IAAI;IAQZ,iDAAiD;IAC3C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAK,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAG5D;IAED;;;;OAIG;IACG,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAGnF;IAED,qEAAqE;IAC/D,IAAI,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAGzC;IAED,yDAAyD;IACnD,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGzC;IAED,uDAAuD;IACjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;IAED,gDAAgD;IAC1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAED,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAI3B;CACD","sourcesContent":["/**\n * Client for the `embsearch` stdio daemon (vendored from\n * github.com/kolisachint/embeddingsearchtools ts/client.ts).\n *\n * Spawns `embsearch serve` once and talks newline-delimited JSON over its\n * stdin/stdout. The process stays alive so the model and index load a single\n * time; every query after startup is hot.\n */\n\nimport { type ChildProcessWithoutNullStreams, spawn } from \"child_process\";\n\nexport interface EmbSearchResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchDaemonInfo {\n\tmodelId: string;\n\tdim: number;\n\tcount: number;\n}\n\nexport interface EmbSearchBulkResult {\n\tinserted: number;\n\tupdated: number;\n}\n\nexport interface EmbSearchClientOptions {\n\t/** Path to the `embsearch` binary. */\n\tbinaryPath: string;\n\t/** Store directory passed as `--path`. */\n\tstorePath: string;\n\t/** Metric for a freshly created store. Default: \"cosine\". */\n\tmetric?: \"cosine\" | \"dot\" | \"euclidean\";\n}\n\ninterface Pending {\n\tresolve: (value: EmbSearchRawResponse) => void;\n\treject: (err: Error) => void;\n}\n\ninterface EmbSearchRawResponse {\n\tok: boolean;\n\terror?: string;\n\tresults?: EmbSearchResult[];\n\tinserted?: boolean;\n\tremoved?: boolean;\n\tcount?: number;\n\tinserted_count?: number;\n\tupdated_count?: number;\n\tmodel_id?: string;\n\tdim?: number;\n}\n\nexport class EmbSearchClient {\n\tprivate proc: ChildProcessWithoutNullStreams;\n\tprivate queue: Pending[] = [];\n\tprivate buffer = \"\";\n\tprivate closed = false;\n\tprivate readyPromise: Promise<void>;\n\n\tconstructor(opts: EmbSearchClientOptions) {\n\t\tconst args = [\"serve\", \"--path\", opts.storePath];\n\t\tif (opts.metric) args.push(\"--metric\", opts.metric);\n\n\t\tthis.proc = spawn(opts.binaryPath, args, { stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\n\t\tthis.proc.stdout.setEncoding(\"utf8\");\n\t\tthis.proc.stdout.on(\"data\", (chunk: string) => this.onStdout(chunk));\n\t\t// Swallow the informational stderr banner; errors surface via responses/exit.\n\t\tthis.proc.stderr.resume();\n\n\t\t// Readiness = the daemon answering a ping (probes the actual request loop).\n\t\tthis.readyPromise = this.send({ op: \"ping\" }).then(() => undefined);\n\t\t// Spawn failures reject the pending ping via the exit handler; mark the\n\t\t// promise handled so a failed spawn doesn't raise an unhandled rejection\n\t\t// before ready() is awaited.\n\t\tthis.readyPromise.catch(() => {});\n\n\t\tthis.proc.on(\"error\", (err) => {\n\t\t\tthis.closed = true;\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t\tthis.proc.on(\"exit\", (code) => {\n\t\t\tthis.closed = true;\n\t\t\tconst err = new Error(`embsearch daemon exited (code ${code})`);\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t}\n\n\t/** Resolves once the daemon has loaded the model + index. */\n\tready(): Promise<void> {\n\t\treturn this.readyPromise;\n\t}\n\n\tget isClosed(): boolean {\n\t\treturn this.closed;\n\t}\n\n\tprivate onStdout(chunk: string): void {\n\t\tthis.buffer += chunk;\n\t\t// Each response is one line; dispatch FIFO against the pending queue.\n\t\tfor (let nl = this.buffer.indexOf(\"\\n\"); nl !== -1; nl = this.buffer.indexOf(\"\\n\")) {\n\t\t\tconst line = this.buffer.slice(0, nl).trim();\n\t\t\tthis.buffer = this.buffer.slice(nl + 1);\n\t\t\tif (!line) continue;\n\t\t\tconst pending = this.queue.shift();\n\t\t\tif (!pending) continue;\n\t\t\tlet msg: EmbSearchRawResponse;\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line) as EmbSearchRawResponse;\n\t\t\t} catch {\n\t\t\t\tpending.reject(new Error(`bad response: ${line}`));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (msg.ok) pending.resolve(msg);\n\t\t\telse pending.reject(new Error(msg.error ?? \"unknown error\"));\n\t\t}\n\t}\n\n\tprivate send(req: Record<string, unknown>): Promise<EmbSearchRawResponse> {\n\t\tif (this.closed) return Promise.reject(new Error(\"embsearch client is closed\"));\n\t\treturn new Promise<EmbSearchRawResponse>((resolve, reject) => {\n\t\t\tthis.queue.push({ resolve, reject });\n\t\t\tthis.proc.stdin.write(`${JSON.stringify(req)}\\n`);\n\t\t});\n\t}\n\n\t/** Search for the top-`k` matches for `text`. */\n\tasync query(text: string, k = 10): Promise<EmbSearchResult[]> {\n\t\tconst res = await this.send({ op: \"query\", text, k });\n\t\treturn res.results ?? [];\n\t}\n\n\t/**\n\t * Batched insert-or-replace. One embedding inference for the whole batch —\n\t * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a\n\t * concurrent query is not stuck behind a huge inference.\n\t */\n\tasync bulk(items: Array<{ id: string; text: string }>): Promise<EmbSearchBulkResult> {\n\t\tconst res = await this.send({ op: \"bulk\", items });\n\t\treturn { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };\n\t}\n\n\t/** Model id, dimensionality, and live vector count of the daemon. */\n\tasync info(): Promise<EmbSearchDaemonInfo> {\n\t\tconst res = await this.send({ op: \"info\" });\n\t\treturn { modelId: res.model_id ?? \"\", dim: res.dim ?? 0, count: res.count ?? 0 };\n\t}\n\n\t/** Remove a record. Resolves to `true` if it existed. */\n\tasync remove(id: string): Promise<boolean> {\n\t\tconst res = await this.send({ op: \"remove\", id });\n\t\treturn res.removed === true;\n\t}\n\n\t/** Reclaim tombstoned rows left behind by `remove`. */\n\tasync compact(): Promise<void> {\n\t\tawait this.send({ op: \"compact\" });\n\t}\n\n\t/** Persist the index to the store directory. */\n\tasync save(): Promise<void> {\n\t\tawait this.send({ op: \"save\" });\n\t}\n\n\t/** Shut the daemon down, closing stdin so it exits cleanly. */\n\tasync close(): Promise<void> {\n\t\tif (this.closed) return;\n\t\tthis.proc.stdin.end();\n\t\tawait new Promise<void>((resolve) => this.proc.on(\"exit\", () => resolve()));\n\t}\n}\n"]}

package/dist/core/embsearch/client.js

@@ -0,0 +1,123 @@
+/**
+ * Client for the `embsearch` stdio daemon (vendored from
+ * github.com/kolisachint/embeddingsearchtools ts/client.ts).
+ *
+ * Spawns `embsearch serve` once and talks newline-delimited JSON over its
+ * stdin/stdout. The process stays alive so the model and index load a single
+ * time; every query after startup is hot.
+ */
+import { spawn } from "child_process";
+export class EmbSearchClient {
+    proc;
+    queue = [];
+    buffer = "";
+    closed = false;
+    readyPromise;
+    constructor(opts) {
+        const args = ["serve", "--path", opts.storePath];
+        if (opts.metric)
+            args.push("--metric", opts.metric);
+        this.proc = spawn(opts.binaryPath, args, { stdio: ["pipe", "pipe", "pipe"] });
+        this.proc.stdout.setEncoding("utf8");
+        this.proc.stdout.on("data", (chunk) => this.onStdout(chunk));
+        // Swallow the informational stderr banner; errors surface via responses/exit.
+        this.proc.stderr.resume();
+        // Readiness = the daemon answering a ping (probes the actual request loop).
+        this.readyPromise = this.send({ op: "ping" }).then(() => undefined);
+        // Spawn failures reject the pending ping via the exit handler; mark the
+        // promise handled so a failed spawn doesn't raise an unhandled rejection
+        // before ready() is awaited.
+        this.readyPromise.catch(() => { });
+        this.proc.on("error", (err) => {
+            this.closed = true;
+            for (const p of this.queue.splice(0))
+                p.reject(err);
+        });
+        this.proc.on("exit", (code) => {
+            this.closed = true;
+            const err = new Error(`embsearch daemon exited (code ${code})`);
+            for (const p of this.queue.splice(0))
+                p.reject(err);
+        });
+    }
+    /** Resolves once the daemon has loaded the model + index. */
+    ready() {
+        return this.readyPromise;
+    }
+    get isClosed() {
+        return this.closed;
+    }
+    onStdout(chunk) {
+        this.buffer += chunk;
+        // Each response is one line; dispatch FIFO against the pending queue.
+        for (let nl = this.buffer.indexOf("\n"); nl !== -1; nl = this.buffer.indexOf("\n")) {
+            const line = this.buffer.slice(0, nl).trim();
+            this.buffer = this.buffer.slice(nl + 1);
+            if (!line)
+                continue;
+            const pending = this.queue.shift();
+            if (!pending)
+                continue;
+            let msg;
+            try {
+                msg = JSON.parse(line);
+            }
+            catch {
+                pending.reject(new Error(`bad response: ${line}`));
+                continue;
+            }
+            if (msg.ok)
+                pending.resolve(msg);
+            else
+                pending.reject(new Error(msg.error ?? "unknown error"));
+        }
+    }
+    send(req) {
+        if (this.closed)
+            return Promise.reject(new Error("embsearch client is closed"));
+        return new Promise((resolve, reject) => {
+            this.queue.push({ resolve, reject });
+            this.proc.stdin.write(`${JSON.stringify(req)}\n`);
+        });
+    }
+    /** Search for the top-`k` matches for `text`. */
+    async query(text, k = 10) {
+        const res = await this.send({ op: "query", text, k });
+        return res.results ?? [];
+    }
+    /**
+     * Batched insert-or-replace. One embedding inference for the whole batch —
+     * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a
+     * concurrent query is not stuck behind a huge inference.
+     */
+    async bulk(items) {
+        const res = await this.send({ op: "bulk", items });
+        return { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };
+    }
+    /** Model id, dimensionality, and live vector count of the daemon. */
+    async info() {
+        const res = await this.send({ op: "info" });
+        return { modelId: res.model_id ?? "", dim: res.dim ?? 0, count: res.count ?? 0 };
+    }
+    /** Remove a record. Resolves to `true` if it existed. */
+    async remove(id) {
+        const res = await this.send({ op: "remove", id });
+        return res.removed === true;
+    }
+    /** Reclaim tombstoned rows left behind by `remove`. */
+    async compact() {
+        await this.send({ op: "compact" });
+    }
+    /** Persist the index to the store directory. */
+    async save() {
+        await this.send({ op: "save" });
+    }
+    /** Shut the daemon down, closing stdin so it exits cleanly. */
+    async close() {
+        if (this.closed)
+            return;
+        this.proc.stdin.end();
+        await new Promise((resolve) => this.proc.on("exit", () => resolve()));
+    }
+}
+//# sourceMappingURL=client.js.map

package/dist/core/embsearch/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../src/core/embsearch/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAuC,KAAK,EAAE,MAAM,eAAe,CAAC;AA6C3E,MAAM,OAAO,eAAe;IACnB,IAAI,CAAiC;IACrC,KAAK,GAAc,EAAE,CAAC;IACtB,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,GAAG,KAAK,CAAC;IACf,YAAY,CAAgB;IAEpC,YAAY,IAA4B,EAAE;QACzC,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpD,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,8EAA8E;QAC9E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAE1B,4EAA4E;QAC5E,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpE,wEAAwE;QACxE,yEAAyE;QACzE,6BAA6B;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAA,CACpD,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,iCAAiC,IAAI,GAAG,CAAC,CAAC;YAChE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAA,CACpD,CAAC,CAAC;IAAA,CACH;IAED,6DAA6D;IAC7D,KAAK,GAAkB;QACtB,OAAO,IAAI,CAAC,YAAY,CAAC;IAAA,CACzB;IAED,IAAI,QAAQ,GAAY;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC;IAAA,CACnB;IAEO,QAAQ,CAAC,KAAa,EAAQ;QACrC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACrB,sEAAsE;QACtE,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,IAAI,GAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;gBACnD,SAAS;YACV,CAAC;YACD,IAAI,GAAG,CAAC,EAAE;gBAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;;gBAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC,CAAC;QAC9D,CAAC;IAAA,CACD;IAEO,IAAI,CAAC,GAA4B,EAAiC;QACzE,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,OAAO,CAAuB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;YAC7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAAA,CAClD,CAAC,CAAC;IAAA,CACH;IAED,iDAAiD;IACjD,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,CAAC,GAAG,EAAE,EAA8B;QAC7D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAAA,CACzB;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CAAC,KAA0C,EAAgC;QACpF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,cAAc,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;IAAA,CAC9E;IAED,qEAAqE;IACrE,KAAK,CAAC,IAAI,GAAiC;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;IAAA,CACjF;IAED,yDAAyD;IACzD,KAAK,CAAC,MAAM,CAAC,EAAU,EAAoB;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC;IAAA,CAC5B;IAED,uDAAuD;IACvD,KAAK,CAAC,OAAO,GAAkB;QAC9B,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAAA,CACnC;IAED,gDAAgD;IAChD,KAAK,CAAC,IAAI,GAAkB;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAAA,CAChC;IAED,+DAA+D;IAC/D,KAAK,CAAC,KAAK,GAAkB;QAC5B,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAAA,CAC5E;CACD","sourcesContent":["/**\n * Client for the `embsearch` stdio daemon (vendored from\n * github.com/kolisachint/embeddingsearchtools ts/client.ts).\n *\n * Spawns `embsearch serve` once and talks newline-delimited JSON over its\n * stdin/stdout. The process stays alive so the model and index load a single\n * time; every query after startup is hot.\n */\n\nimport { type ChildProcessWithoutNullStreams, spawn } from \"child_process\";\n\nexport interface EmbSearchResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchDaemonInfo {\n\tmodelId: string;\n\tdim: number;\n\tcount: number;\n}\n\nexport interface EmbSearchBulkResult {\n\tinserted: number;\n\tupdated: number;\n}\n\nexport interface EmbSearchClientOptions {\n\t/** Path to the `embsearch` binary. */\n\tbinaryPath: string;\n\t/** Store directory passed as `--path`. */\n\tstorePath: string;\n\t/** Metric for a freshly created store. Default: \"cosine\". */\n\tmetric?: \"cosine\" | \"dot\" | \"euclidean\";\n}\n\ninterface Pending {\n\tresolve: (value: EmbSearchRawResponse) => void;\n\treject: (err: Error) => void;\n}\n\ninterface EmbSearchRawResponse {\n\tok: boolean;\n\terror?: string;\n\tresults?: EmbSearchResult[];\n\tinserted?: boolean;\n\tremoved?: boolean;\n\tcount?: number;\n\tinserted_count?: number;\n\tupdated_count?: number;\n\tmodel_id?: string;\n\tdim?: number;\n}\n\nexport class EmbSearchClient {\n\tprivate proc: ChildProcessWithoutNullStreams;\n\tprivate queue: Pending[] = [];\n\tprivate buffer = \"\";\n\tprivate closed = false;\n\tprivate readyPromise: Promise<void>;\n\n\tconstructor(opts: EmbSearchClientOptions) {\n\t\tconst args = [\"serve\", \"--path\", opts.storePath];\n\t\tif (opts.metric) args.push(\"--metric\", opts.metric);\n\n\t\tthis.proc = spawn(opts.binaryPath, args, { stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\n\t\tthis.proc.stdout.setEncoding(\"utf8\");\n\t\tthis.proc.stdout.on(\"data\", (chunk: string) => this.onStdout(chunk));\n\t\t// Swallow the informational stderr banner; errors surface via responses/exit.\n\t\tthis.proc.stderr.resume();\n\n\t\t// Readiness = the daemon answering a ping (probes the actual request loop).\n\t\tthis.readyPromise = this.send({ op: \"ping\" }).then(() => undefined);\n\t\t// Spawn failures reject the pending ping via the exit handler; mark the\n\t\t// promise handled so a failed spawn doesn't raise an unhandled rejection\n\t\t// before ready() is awaited.\n\t\tthis.readyPromise.catch(() => {});\n\n\t\tthis.proc.on(\"error\", (err) => {\n\t\t\tthis.closed = true;\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t\tthis.proc.on(\"exit\", (code) => {\n\t\t\tthis.closed = true;\n\t\t\tconst err = new Error(`embsearch daemon exited (code ${code})`);\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t}\n\n\t/** Resolves once the daemon has loaded the model + index. */\n\tready(): Promise<void> {\n\t\treturn this.readyPromise;\n\t}\n\n\tget isClosed(): boolean {\n\t\treturn this.closed;\n\t}\n\n\tprivate onStdout(chunk: string): void {\n\t\tthis.buffer += chunk;\n\t\t// Each response is one line; dispatch FIFO against the pending queue.\n\t\tfor (let nl = this.buffer.indexOf(\"\\n\"); nl !== -1; nl = this.buffer.indexOf(\"\\n\")) {\n\t\t\tconst line = this.buffer.slice(0, nl).trim();\n\t\t\tthis.buffer = this.buffer.slice(nl + 1);\n\t\t\tif (!line) continue;\n\t\t\tconst pending = this.queue.shift();\n\t\t\tif (!pending) continue;\n\t\t\tlet msg: EmbSearchRawResponse;\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line) as EmbSearchRawResponse;\n\t\t\t} catch {\n\t\t\t\tpending.reject(new Error(`bad response: ${line}`));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (msg.ok) pending.resolve(msg);\n\t\t\telse pending.reject(new Error(msg.error ?? \"unknown error\"));\n\t\t}\n\t}\n\n\tprivate send(req: Record<string, unknown>): Promise<EmbSearchRawResponse> {\n\t\tif (this.closed) return Promise.reject(new Error(\"embsearch client is closed\"));\n\t\treturn new Promise<EmbSearchRawResponse>((resolve, reject) => {\n\t\t\tthis.queue.push({ resolve, reject });\n\t\t\tthis.proc.stdin.write(`${JSON.stringify(req)}\\n`);\n\t\t});\n\t}\n\n\t/** Search for the top-`k` matches for `text`. */\n\tasync query(text: string, k = 10): Promise<EmbSearchResult[]> {\n\t\tconst res = await this.send({ op: \"query\", text, k });\n\t\treturn res.results ?? [];\n\t}\n\n\t/**\n\t * Batched insert-or-replace. One embedding inference for the whole batch —\n\t * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a\n\t * concurrent query is not stuck behind a huge inference.\n\t */\n\tasync bulk(items: Array<{ id: string; text: string }>): Promise<EmbSearchBulkResult> {\n\t\tconst res = await this.send({ op: \"bulk\", items });\n\t\treturn { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };\n\t}\n\n\t/** Model id, dimensionality, and live vector count of the daemon. */\n\tasync info(): Promise<EmbSearchDaemonInfo> {\n\t\tconst res = await this.send({ op: \"info\" });\n\t\treturn { modelId: res.model_id ?? \"\", dim: res.dim ?? 0, count: res.count ?? 0 };\n\t}\n\n\t/** Remove a record. Resolves to `true` if it existed. */\n\tasync remove(id: string): Promise<boolean> {\n\t\tconst res = await this.send({ op: \"remove\", id });\n\t\treturn res.removed === true;\n\t}\n\n\t/** Reclaim tombstoned rows left behind by `remove`. */\n\tasync compact(): Promise<void> {\n\t\tawait this.send({ op: \"compact\" });\n\t}\n\n\t/** Persist the index to the store directory. */\n\tasync save(): Promise<void> {\n\t\tawait this.send({ op: \"save\" });\n\t}\n\n\t/** Shut the daemon down, closing stdin so it exits cleanly. */\n\tasync close(): Promise<void> {\n\t\tif (this.closed) return;\n\t\tthis.proc.stdin.end();\n\t\tawait new Promise<void>((resolve) => this.proc.on(\"exit\", () => resolve()));\n\t}\n}\n"]}

package/dist/core/embsearch/embsearch-service.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Orchestrates semantic indexing and search for a repository.
+ *
+ * Lifecycle (all behind --enable-embsearchtools):
+ *  1. `start()` — resolve the embsearch binary, scan the repo (ignore-aware),
+ *     apply the byte threshold. Under threshold → dormant. Over → spawn the
+ *     daemon, verify the backend is not the mock embedder, then index changed
+ *     files in the background in small batches, reporting progress.
+ *  2. `search()` — top-k semantic query, mapping chunk ids back to
+ *     `path:start-end` via the sidecar metadata.
+ *  3. `dispose()` — save + close the daemon.
+ *
+ * Every failure degrades to `unavailable` with a reason; nothing here ever
+ * blocks session startup or affects grep/find.
+ */
+export type EmbsearchState = {
+    phase: "idle";
+} | {
+    phase: "skipped";
+    reason: string;
+} | {
+    phase: "indexing";
+    done: number;
+    total: number;
+} | {
+    phase: "ready";
+    chunkCount: number;
+} | {
+    phase: "unavailable";
+    reason: string;
+};
+export interface EmbsearchServiceOptions {
+    cwd: string;
+    /** Explicit binary path (settings override). Default: "embsearch" from PATH. */
+    binaryPath?: string;
+    /** Minimum indexable bytes before indexing kicks in. */
+    thresholdBytes: number;
+    /** Progress callback for UI (footer / stderr lines). */
+    onProgress?: (state: EmbsearchState) => void;
+}
+export interface SemanticHit {
+    path: string;
+    startLine: number;
+    endLine: number;
+    score: number;
+}
+export declare class EmbsearchService {
+    private readonly options;
+    private client;
+    private meta;
+    private state;
+    private disposed;
+    constructor(options: EmbsearchServiceOptions);
+    getState(): EmbsearchState;
+    /** Semantic search is usable (index ready, or still building with partial data). */
+    isAvailable(): boolean;
+    private setState;
+    private resolveBinary;
+    /**
+     * Scan, threshold-check, and (when needed) index in the background.
+     * Resolves when indexing completes or the feature settles dormant.
+     */
+    start(signal?: AbortSignal): Promise<void>;
+    private run;
+    private indexChangedFiles;
+    private countChunks;
+    /** Top-`k` semantic hits as `path` + line range + score. */
+    search(query: string, k?: number): Promise<SemanticHit[]>;
+    private closeClient;
+    /** Persist state and shut the daemon down. Safe to call twice. */
+    dispose(): Promise<void>;
+}
+export declare function registerEmbsearchService(cwd: string, service: EmbsearchService): void;
+export declare function getEmbsearchService(cwd: string): EmbsearchService | undefined;
+export declare function unregisterEmbsearchService(cwd: string): void;
+//# sourceMappingURL=embsearch-service.d.ts.map

package/dist/core/embsearch/embsearch-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embsearch-service.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/embsearch-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA0BH,MAAM,MAAM,cAAc,GACvB;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GACjB;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,KAAK,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5C,MAAM,WAAW,uBAAuB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,gFAAgF;IAChF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAClD,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,KAAK,CAAqC;IAClD,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,uBAAuB,EAE3C;IAED,QAAQ,IAAI,cAAc,CAEzB;IAED,oFAAoF;IACpF,WAAW,IAAI,OAAO,CAErB;IAED,OAAO,CAAC,QAAQ;YAKF,aAAa;IAO3B;;;OAGG;IACG,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ/C;YAEa,GAAG;YAmCH,iBAAiB;IAqF/B,OAAO,CAAC,WAAW;IAMnB,4DAA4D;IACtD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,SAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAgB1D;YAEa,WAAW;IAYzB,kEAAkE;IAC5D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAW7B;CACD;AAWD,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAMrF;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS,CAE7E;AAED,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE5D","sourcesContent":["/**\n * Orchestrates semantic indexing and search for a repository.\n *\n * Lifecycle (all behind --enable-embsearchtools):\n *  1. `start()` — resolve the embsearch binary, scan the repo (ignore-aware),\n *     apply the byte threshold. Under threshold → dormant. Over → spawn the\n *     daemon, verify the backend is not the mock embedder, then index changed\n *     files in the background in small batches, reporting progress.\n *  2. `search()` — top-k semantic query, mapping chunk ids back to\n *     `path:start-end` via the sidecar metadata.\n *  3. `dispose()` — save + close the daemon.\n *\n * Every failure degrades to `unavailable` with a reason; nothing here ever\n * blocks session startup or affects grep/find.\n */\n\nimport { readFileSync } from \"fs\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { chunkFile } from \"./chunker.js\";\nimport { EmbSearchClient } from \"./client.js\";\nimport {\n\temptyIndexMeta,\n\ttype FileMeta,\n\tgetEmbsearchStoreDir,\n\tgetVectorStoreDir,\n\thashContent,\n\ttype IndexMeta,\n\tloadIndexMeta,\n\tsaveIndexMeta,\n} from \"./index-meta.js\";\nimport { type RepoScanFile, scanRepo } from \"./repo-scan.js\";\n\n/** Chunks per bulk request. Small enough that a concurrent query is never\n *  stuck long behind one padded batch inference. */\nconst BULK_BATCH_SIZE = 48;\n/** Yield between batches so background indexing doesn't starve the session. */\nconst BATCH_YIELD_MS = 15;\n/** The Rust mock backend's model id — semantically meaningless, never index with it. */\nconst MOCK_MODEL_ID = \"mock-hash-v1\";\n\nexport type EmbsearchState =\n\t| { phase: \"idle\" }\n\t| { phase: \"skipped\"; reason: string }\n\t| { phase: \"indexing\"; done: number; total: number }\n\t| { phase: \"ready\"; chunkCount: number }\n\t| { phase: \"unavailable\"; reason: string };\n\nexport interface EmbsearchServiceOptions {\n\tcwd: string;\n\t/** Explicit binary path (settings override). Default: \"embsearch\" from PATH. */\n\tbinaryPath?: string;\n\t/** Minimum indexable bytes before indexing kicks in. */\n\tthresholdBytes: number;\n\t/** Progress callback for UI (footer / stderr lines). */\n\tonProgress?: (state: EmbsearchState) => void;\n}\n\nexport interface SemanticHit {\n\tpath: string;\n\tstartLine: number;\n\tendLine: number;\n\tscore: number;\n}\n\nexport class EmbsearchService {\n\tprivate readonly options: EmbsearchServiceOptions;\n\tprivate client: EmbSearchClient | undefined;\n\tprivate meta: IndexMeta | undefined;\n\tprivate state: EmbsearchState = { phase: \"idle\" };\n\tprivate disposed = false;\n\n\tconstructor(options: EmbsearchServiceOptions) {\n\t\tthis.options = options;\n\t}\n\n\tgetState(): EmbsearchState {\n\t\treturn this.state;\n\t}\n\n\t/** Semantic search is usable (index ready, or still building with partial data). */\n\tisAvailable(): boolean {\n\t\treturn this.state.phase === \"ready\" || this.state.phase === \"indexing\";\n\t}\n\n\tprivate setState(state: EmbsearchState): void {\n\t\tthis.state = state;\n\t\tthis.options.onProgress?.(state);\n\t}\n\n\tprivate async resolveBinary(): Promise<string | undefined> {\n\t\tif (this.options.binaryPath) {\n\t\t\treturn this.options.binaryPath;\n\t\t}\n\t\treturn await ensureTool(\"embsearch\", true);\n\t}\n\n\t/**\n\t * Scan, threshold-check, and (when needed) index in the background.\n\t * Resolves when indexing completes or the feature settles dormant.\n\t */\n\tasync start(signal?: AbortSignal): Promise<void> {\n\t\ttry {\n\t\t\tawait this.run(signal);\n\t\t} catch (e) {\n\t\t\tconst reason = e instanceof Error ? e.message : String(e);\n\t\t\tthis.setState({ phase: \"unavailable\", reason });\n\t\t\tawait this.closeClient();\n\t\t}\n\t}\n\n\tprivate async run(signal?: AbortSignal): Promise<void> {\n\t\tconst binary = await this.resolveBinary();\n\t\tif (!binary) {\n\t\t\tthis.setState({\n\t\t\t\tphase: \"unavailable\",\n\t\t\t\treason: \"embsearch binary not found (PATH or embsearchBinaryPath setting)\",\n\t\t\t});\n\t\t\treturn;\n\t\t}\n\n\t\tconst scan = scanRepo(this.options.cwd, signal);\n\t\tif (scan.totalBytes < this.options.thresholdBytes) {\n\t\t\tthis.setState({\n\t\t\t\tphase: \"skipped\",\n\t\t\t\treason: `repo under threshold (${scan.totalBytes} < ${this.options.thresholdBytes} bytes)`,\n\t\t\t});\n\t\t\treturn;\n\t\t}\n\n\t\tconst storeDir = getEmbsearchStoreDir(this.options.cwd);\n\t\tthis.client = new EmbSearchClient({ binaryPath: binary, storePath: getVectorStoreDir(storeDir) });\n\t\tawait this.client.ready();\n\n\t\tconst info = await this.client.info();\n\t\tif (info.modelId === MOCK_MODEL_ID) {\n\t\t\tthrow new Error(\"embsearch binary uses the mock embedder (not semantic); install an onnx build\");\n\t\t}\n\n\t\t// Missing/stale sidecar (format, chunker, or model changed) → clean rebuild.\n\t\tthis.meta = loadIndexMeta(storeDir, info.modelId) ?? emptyIndexMeta(this.options.cwd, info.modelId);\n\t\tthis.meta.lastUsedMs = Date.now();\n\n\t\tawait this.indexChangedFiles(scan.files, storeDir, signal);\n\t}\n\n\tprivate async indexChangedFiles(files: RepoScanFile[], storeDir: string, signal?: AbortSignal): Promise<void> {\n\t\tconst meta = this.meta!;\n\t\tconst client = this.client!;\n\n\t\t// Diff scan vs sidecar: cheap mtime+size check first, hash only on delta.\n\t\tconst toIndex: Array<{ file: RepoScanFile; content: string; hash: string }> = [];\n\t\tconst seen = new Set<string>();\n\t\tfor (const file of files) {\n\t\t\tseen.add(file.rel);\n\t\t\tconst known = meta.files[file.rel];\n\t\t\tif (known && known.mtimeMs === file.mtimeMs && known.size === file.size) continue;\n\t\t\tlet content: string;\n\t\t\ttry {\n\t\t\t\tcontent = readFileSync(file.abs, \"utf-8\");\n\t\t\t} catch {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst hash = hashContent(content);\n\t\t\tif (known && known.hash === hash) {\n\t\t\t\t// Touched but unchanged — refresh stat info only.\n\t\t\t\tknown.mtimeMs = file.mtimeMs;\n\t\t\t\tknown.size = file.size;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\ttoIndex.push({ file, content, hash });\n\t\t}\n\t\tconst toRemove = Object.keys(meta.files).filter((rel) => !seen.has(rel));\n\n\t\tif (toIndex.length === 0 && toRemove.length === 0) {\n\t\t\tsaveIndexMeta(storeDir, meta);\n\t\t\tthis.setState({ phase: \"ready\", chunkCount: this.countChunks(meta) });\n\t\t\treturn;\n\t\t}\n\n\t\t// Chunk changed files; count total upserts for exact progress.\n\t\tconst work: Array<{ rel: string; fileMeta: FileMeta; chunks: Array<{ id: string; text: string }> }> = [];\n\t\tlet totalChunks = 0;\n\t\tfor (const { file, content, hash } of toIndex) {\n\t\t\tconst chunks = chunkFile(file.rel, content);\n\t\t\twork.push({\n\t\t\t\trel: file.rel,\n\t\t\t\tfileMeta: {\n\t\t\t\t\tmtimeMs: file.mtimeMs,\n\t\t\t\t\tsize: file.size,\n\t\t\t\t\thash,\n\t\t\t\t\tchunks: chunks.map((c) => [c.startLine, c.endLine]),\n\t\t\t\t},\n\t\t\t\tchunks: chunks.map((c) => ({ id: c.id, text: c.text })),\n\t\t\t});\n\t\t\ttotalChunks += chunks.length;\n\t\t}\n\n\t\tthis.setState({ phase: \"indexing\", done: 0, total: totalChunks });\n\n\t\t// Drop vectors of deleted files and superseded chunk tails.\n\t\tfor (const rel of toRemove) {\n\t\t\tfor (let i = 0; i < meta.files[rel].chunks.length; i++) await client.remove(`${rel}#${i}`);\n\t\t\tdelete meta.files[rel];\n\t\t}\n\n\t\tlet done = 0;\n\t\tfor (const item of work) {\n\t\t\tif (signal?.aborted || this.disposed) return;\n\t\t\tconst oldChunkCount = meta.files[item.rel]?.chunks.length ?? 0;\n\t\t\t// Remove old chunks beyond the new count (upsert covers the rest).\n\t\t\tfor (let i = item.chunks.length; i < oldChunkCount; i++) await client.remove(`${item.rel}#${i}`);\n\n\t\t\tfor (let offset = 0; offset < item.chunks.length; offset += BULK_BATCH_SIZE) {\n\t\t\t\tif (signal?.aborted || this.disposed) return;\n\t\t\t\tconst batch = item.chunks.slice(offset, offset + BULK_BATCH_SIZE);\n\t\t\t\tawait client.bulk(batch);\n\t\t\t\tdone += batch.length;\n\t\t\t\tthis.setState({ phase: \"indexing\", done, total: totalChunks });\n\t\t\t\t// Yield so queries and the event loop stay responsive.\n\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, BATCH_YIELD_MS));\n\t\t\t}\n\t\t\tmeta.files[item.rel] = item.fileMeta;\n\t\t}\n\n\t\tawait client.compact();\n\t\tawait client.save();\n\t\tsaveIndexMeta(storeDir, meta);\n\t\tthis.setState({ phase: \"ready\", chunkCount: this.countChunks(meta) });\n\t}\n\n\tprivate countChunks(meta: IndexMeta): number {\n\t\tlet n = 0;\n\t\tfor (const rel of Object.keys(meta.files)) n += meta.files[rel].chunks.length;\n\t\treturn n;\n\t}\n\n\t/** Top-`k` semantic hits as `path` + line range + score. */\n\tasync search(query: string, k = 10): Promise<SemanticHit[]> {\n\t\tif (!this.client || this.client.isClosed || !this.meta) {\n\t\t\tthrow new Error(\"semantic index is not available\");\n\t\t}\n\t\tconst results = await this.client.query(query, k);\n\t\tconst hits: SemanticHit[] = [];\n\t\tfor (const result of results) {\n\t\t\tconst sep = result.id.lastIndexOf(\"#\");\n\t\t\tif (sep === -1) continue;\n\t\t\tconst rel = result.id.slice(0, sep);\n\t\t\tconst chunkIndex = Number.parseInt(result.id.slice(sep + 1), 10);\n\t\t\tconst range = this.meta.files[rel]?.chunks[chunkIndex];\n\t\t\tif (!range) continue;\n\t\t\thits.push({ path: rel, startLine: range[0], endLine: range[1], score: result.score });\n\t\t}\n\t\treturn hits;\n\t}\n\n\tprivate async closeClient(): Promise<void> {\n\t\tconst client = this.client;\n\t\tthis.client = undefined;\n\t\tif (client && !client.isClosed) {\n\t\t\ttry {\n\t\t\t\tawait client.close();\n\t\t\t} catch {\n\t\t\t\t// already dead\n\t\t\t}\n\t\t}\n\t}\n\n\t/** Persist state and shut the daemon down. Safe to call twice. */\n\tasync dispose(): Promise<void> {\n\t\tif (this.disposed) return;\n\t\tthis.disposed = true;\n\t\tif (this.client && !this.client.isClosed) {\n\t\t\ttry {\n\t\t\t\tawait this.client.save();\n\t\t\t} catch {\n\t\t\t\t// daemon may have exited; nothing to save\n\t\t\t}\n\t\t}\n\t\tawait this.closeClient();\n\t}\n}\n\n// --- Per-cwd service registry ---\n//\n// The semantic_search tool is constructed by the generic tool factory table and\n// only receives `cwd`; the service is created later during session init (flag\n// gated). This registry connects the two without threading a service instance\n// through every layer between main.ts and the tool factories.\n\nconst services = new Map<string, EmbsearchService>();\n\nexport function registerEmbsearchService(cwd: string, service: EmbsearchService): void {\n\tconst old = services.get(cwd);\n\tif (old && old !== service) {\n\t\told.dispose().catch(() => {});\n\t}\n\tservices.set(cwd, service);\n}\n\nexport function getEmbsearchService(cwd: string): EmbsearchService | undefined {\n\treturn services.get(cwd);\n}\n\nexport function unregisterEmbsearchService(cwd: string): void {\n\tservices.delete(cwd);\n}\n"]}