@kolisachint/hoocode-agent 0.5.59 → 0.5.61

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## [0.5.61] - 2026-09-06
+
+## [0.5.60] - 2026-09-05
+
+### Fixed
+
+- **A semantic index built by an older embedding model now rebuilds instead of
+  switching semantic search off for good.** The embsearch daemon refuses to open
+  a store whose recorded model disagrees with its own — rightly, since vectors
+  from two models are not comparable — but refusing was where it stopped: the
+  store stayed on disk, the daemon never came up, and the service parked itself
+  in `unavailable` with a rebuild one directory-removal away. The check meant to
+  catch this ran *after* the spawn and reset only the sidecar, never the store,
+  so on a model change it was unreachable anyway. Recovery now catches the
+  refusal and confirms, via embsearch's `store-info` — which reads a store's
+  manifest without loading a model, and so is the only thing that can still
+  identify a store nothing will open — that a readable store is what the daemon
+  choked on, then rebuilds. A store it cannot read is left alone and the original
+  error stands, because deleting an index is the wrong answer to an unknown
+  fault. This was about to bite: embsearch 0.3.2 changed its model id, and only
+  the fact that an already-installed binary is never upgraded in place kept
+  anyone from hitting it. `store-info` ships in **embsearch 0.3.3**; against an
+  older binary the probe simply fails and the previous behaviour stands, so no
+  version floor is needed and nothing regresses on an older daemon.
+- **A chunker change no longer leaves dead text in the index.** Bumping
+  `CHUNKER_VERSION` resets the sidecar, and the sidecar is the only record of how
+  many chunks each file produced — so it reported zero for every file, and the
+  loop that drops superseded chunks counts down from that number. Any file that
+  re-chunked into fewer pieces left its tail vectors behind holding text that no
+  longer existed anywhere, retrievable forever. Upserts hid it: chunk counts
+  looked right, the sidecar looked right, only the search results were wrong.
+
 ## [0.5.59] - 2026-09-05
 
 ### Fixed

package/dist/core/canvas/lifecycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../src/core/canvas/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,OAAO,KAAK,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAIlF,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC7B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED,8CAA8C;AAC9C,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,4EAA4E;IAC5E,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B;;;OAGG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC1C;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,MAAM,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5C;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC7B,SAAS,EAAE,yBAAyB,EACpC,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAChC,gBAAgB,GAAG,SAAS,CAG9B;AAwDD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACpC,SAAS,EAAE,yBAAyB,EACpC,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAChC,kBAAkB,GAAG,sBAAsB,CA4C7C;AAED,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;CACZ;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACpC,SAAS,EAAE,yBAAyB,EACpC,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAChC,kBAAkB,GAAG,sBAAsB,CAc7C;AAED,2DAA2D;AAC3D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,sBAAsB,CAE9E","sourcesContent":["/**\n * Renaming and removing a canvas extension.\n *\n * Design: `docs/canvas-extensions-design.md` §13.6. These exist because a canvas\n * has more identity than a file does, and it is spread across four places: the\n * directory name (which *is* the extension id, since discovery keys off\n * position), the `id` a canvas declares, its `displayName`, and whatever the\n * header comment tells the reader to type. Renaming by hand means getting all\n * four right, and getting the `id` wrong is not a typo — it drops the canvas the\n * person is looking at on the next reload, because the instance was opened\n * against a canvas that no longer exists.\n *\n * So the rewriting is deliberately narrow. It changes the three places the\n * scaffold puts the name and nothing else, then **reports every other line the\n * old name still appears on** rather than guessing at prose. A rename that\n * silently edited a description would be worse than one that admits what it left\n * behind.\n */\n\nimport { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CanvasSearchRoot, DiscoveredCanvasExtension } from \"./discovery.js\";\nimport { CANVAS_ENTRY_FILE } from \"./discovery.js\";\nimport { validateCanvasName } from \"./scaffold.js\";\n\n/** One line the rename changed. */\nexport interface CanvasRewrite {\n\t/** 1-based, so it can be read straight off an editor gutter. */\n\tline: number;\n\tbefore: string;\n\tafter: string;\n}\n\n/** What {@link renameCanvasExtension} did. */\nexport interface CanvasRenameResult {\n\tfrom: string;\n\tto: string;\n\t/** Workspace-relative where possible, for a message a person can act on. */\n\tdir: string;\n\trewrites: CanvasRewrite[];\n\t/**\n\t * Lines where the old name survives, because they are prose rather than\n\t * identity. Surfaced, never silently edited.\n\t */\n\tleftovers: number[];\n}\n\n/**\n * Why a canvas cannot be renamed or removed in place.\n *\n * `packaged` is the interesting one: a plugin's canvases are named by its\n * manifest rather than by where they sit, so moving the directory would either\n * do nothing or break the plugin. That is `/plugin`'s job, not this one's.\n */\nexport type CanvasLifecycleRefusal =\n\t| { reason: \"invalid-name\"; detail: string }\n\t| { reason: \"exists\"; detail: string }\n\t| { reason: \"packaged\"; detail: string }\n\t| { reason: \"unwritable\"; detail: string };\n\n/**\n * Whether this extension is one we may move or delete.\n *\n * The test is positional and deliberately so: an extension is ours to edit when\n * its directory sits *directly* inside one of the search roots, which is exactly\n * the layout `discoverCanvasExtensions` walks. Anything else arrived inside a\n * package — resolved through a manifest by `plugin-canvases.ts` — and its\n * location is that package's business.\n */\nexport function canvasHomeRoot(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasSearchRoot | undefined {\n\tconst parent = path.dirname(path.resolve(extension.dir));\n\treturn roots.find((root) => path.resolve(root.dir) === parent);\n}\n\n/**\n * Swap the old name for the new one where it is *identity* rather than prose.\n *\n * Two rules, and the first is the interesting one: **a string literal whose\n * entire content is the old name**. That covers `id`, `displayName`, `title` and\n * the scaffold's `const ID = \"…\"` without knowing any of their names, and it\n * does not touch a sentence that merely mentions the canvas — `\"the board is the\n * point of the board\"` is not the string `\"board\"`. A whole-word replacement\n * would have rewritten that sentence, which is worse than leaving it.\n *\n * The second rule keeps the instructions in a comment honest: `/canvas open\n * <old>` would otherwise tell the next reader to type a name that no longer\n * resolves.\n *\n * Anything else is left and reported. The scaffold names itself in exactly three\n * places, all covered here, so a canvas that has not been renamed by hand comes\n * through with nothing left over.\n */\nfunction rewriteEntrySource(source: string, from: string, to: string): { source: string; rewrites: CanvasRewrite[] } {\n\tconst name = escapeForRegExp(from);\n\tconst substitutions: [RegExp, string][] = [\n\t\t// A quoted string that is *only* the old name. Backticks included, but only\n\t\t// without interpolation — `${x}-${from}` is a computed value, not a literal.\n\t\t[new RegExp(`([\"'])${name}\\\\1`, \"g\"), `$1${to}$1`],\n\t\t[new RegExp(\"`\" + name + \"`\", \"g\"), `\\`${to}\\``],\n\t\t// Instructions in a comment: /canvas open <old>, /canvas reload <old>, …\n\t\t[new RegExp(`(/canvas\\\\s+(?:open|reload|rename|remove|close)\\\\s+)${name}\\\\b`, \"g\"), `$1${to}`],\n\t];\n\n\tconst before = source.split(\"\\n\");\n\tconst after = before.map((line) =>\n\t\tsubstitutions.reduce((text, [pattern, replacement]) => text.replace(pattern, replacement), line),\n\t);\n\n\tconst rewrites: CanvasRewrite[] = [];\n\tafter.forEach((line, index) => {\n\t\tif (line !== before[index]) rewrites.push({ line: index + 1, before: before[index] as string, after: line });\n\t});\n\treturn { source: after.join(\"\\n\"), rewrites };\n}\n\n/** Lines still mentioning `name` after the rewrite. */\nfunction remainingMentions(source: string, name: string): number[] {\n\tconst lines: number[] = [];\n\tsource.split(\"\\n\").forEach((line, index) => {\n\t\tif (line.includes(name)) lines.push(index + 1);\n\t});\n\treturn lines;\n}\n\nfunction escapeForRegExp(value: string): string {\n\treturn value.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\n/**\n * Rename a canvas extension: its directory, and the name inside its code.\n *\n * The caller is responsible for closing open instances first. This does not do\n * it, because closing is the *session's* concern and this module has no session —\n * but skipping it leaves an instance pointing at a directory that has moved.\n */\nexport function renameCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\tto: string,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRenameResult | CanvasLifecycleRefusal {\n\tconst invalid = validateCanvasName(to);\n\tif (invalid) return { reason: \"invalid-name\", detail: invalid };\n\tif (to === extension.id) return { reason: \"invalid-name\", detail: `it is already called \"${to}\"` };\n\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin, which names its canvases in its own manifest. Renaming it here would break the plugin; use /plugin instead.`,\n\t\t};\n\t}\n\n\tconst target = path.join(root.dir, to);\n\tif (existsSync(target)) return { reason: \"exists\", detail: `${target} already exists` };\n\n\tconst entry = path.join(extension.dir, CANVAS_ENTRY_FILE);\n\tlet source: string;\n\ttry {\n\t\tsource = readFileSync(entry, \"utf8\");\n\t} catch (cause) {\n\t\treturn {\n\t\t\treason: \"unwritable\",\n\t\t\tdetail: `could not read ${entry}: ${cause instanceof Error ? cause.message : String(cause)}`,\n\t\t};\n\t}\n\n\tconst rewritten = rewriteEntrySource(source, extension.id, to);\n\ttry {\n\t\t// Write before moving: if the write fails the directory has not moved and\n\t\t// nothing is half-renamed.\n\t\tif (rewritten.source !== source) writeFileSync(entry, rewritten.source, \"utf8\");\n\t\trenameSync(extension.dir, target);\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\n\treturn {\n\t\tfrom: extension.id,\n\t\tto,\n\t\tdir: target,\n\t\trewrites: rewritten.rewrites,\n\t\tleftovers: remainingMentions(rewritten.source, extension.id),\n\t};\n}\n\n/** What {@link removeCanvasExtension} deleted. */\nexport interface CanvasRemoveResult {\n\tid: string;\n\tdir: string;\n}\n\n/**\n * Delete a canvas extension's directory.\n *\n * As with rename, the caller closes open instances first — a deleted directory\n * whose child is still forked leaves a process serving code that no longer\n * exists on disk, which is the most confusing state of all.\n */\nexport function removeCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRemoveResult | CanvasLifecycleRefusal {\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin. Deleting its directory would leave the plugin broken rather than uninstalled; use /plugin instead.`,\n\t\t};\n\t}\n\ttry {\n\t\trmSync(extension.dir, { recursive: true, force: true });\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\treturn { id: extension.id, dir: extension.dir };\n}\n\n/** Narrow a lifecycle return value to its refusal case. */\nexport function isCanvasRefusal(value: object): value is CanvasLifecycleRefusal {\n\treturn \"reason\" in value;\n}\n"]}
+{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../src/core/canvas/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,OAAO,KAAK,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAIlF,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC7B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED,8CAA8C;AAC9C,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,4EAA4E;IAC5E,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B;;;OAGG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC1C;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,MAAM,EAAE,YAAY,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5C;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC7B,SAAS,EAAE,yBAAyB,EACpC,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAChC,gBAAgB,GAAG,SAAS,CAG9B;AAwDD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACpC,SAAS,EAAE,yBAAyB,EACpC,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAChC,kBAAkB,GAAG,sBAAsB,CA4C7C;AAED,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;CACZ;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACpC,SAAS,EAAE,yBAAyB,EACpC,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAChC,kBAAkB,GAAG,sBAAsB,CAc7C;AAED,2DAA2D;AAC3D,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,sBAAsB,CAE9E","sourcesContent":["/**\n * Renaming and removing a canvas extension.\n *\n * Design: `docs/canvas-extensions-design.md` §13.6. These exist because a canvas\n * has more identity than a file does, and it is spread across four places: the\n * directory name (which *is* the extension id, since discovery keys off\n * position), the `id` a canvas declares, its `displayName`, and whatever the\n * header comment tells the reader to type. Renaming by hand means getting all\n * four right, and getting the `id` wrong is not a typo — it drops the canvas the\n * person is looking at on the next reload, because the instance was opened\n * against a canvas that no longer exists.\n *\n * So the rewriting is deliberately narrow. It changes the three places the\n * scaffold puts the name and nothing else, then **reports every other line the\n * old name still appears on** rather than guessing at prose. A rename that\n * silently edited a description would be worse than one that admits what it left\n * behind.\n */\n\nimport { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CanvasSearchRoot, DiscoveredCanvasExtension } from \"./discovery.js\";\nimport { CANVAS_ENTRY_FILE } from \"./discovery.js\";\nimport { validateCanvasName } from \"./scaffold.js\";\n\n/** One line the rename changed. */\nexport interface CanvasRewrite {\n\t/** 1-based, so it can be read straight off an editor gutter. */\n\tline: number;\n\tbefore: string;\n\tafter: string;\n}\n\n/** What {@link renameCanvasExtension} did. */\nexport interface CanvasRenameResult {\n\tfrom: string;\n\tto: string;\n\t/** Workspace-relative where possible, for a message a person can act on. */\n\tdir: string;\n\trewrites: CanvasRewrite[];\n\t/**\n\t * Lines where the old name survives, because they are prose rather than\n\t * identity. Surfaced, never silently edited.\n\t */\n\tleftovers: number[];\n}\n\n/**\n * Why a canvas cannot be renamed or removed in place.\n *\n * `packaged` is the interesting one: a plugin's canvases are named by its\n * manifest rather than by where they sit, so moving the directory would either\n * do nothing or break the plugin. That is `/plugin`'s job, not this one's.\n */\nexport type CanvasLifecycleRefusal =\n\t| { reason: \"invalid-name\"; detail: string }\n\t| { reason: \"exists\"; detail: string }\n\t| { reason: \"packaged\"; detail: string }\n\t| { reason: \"unwritable\"; detail: string };\n\n/**\n * Whether this extension is one we may move or delete.\n *\n * The test is positional and deliberately so: an extension is ours to edit when\n * its directory sits *directly* inside one of the search roots, which is exactly\n * the layout `discoverCanvasExtensions` walks. Anything else arrived inside a\n * package — resolved through a manifest by `plugin-canvases.ts` — and its\n * location is that package's business.\n */\nexport function canvasHomeRoot(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasSearchRoot | undefined {\n\tconst parent = path.dirname(path.resolve(extension.dir));\n\treturn roots.find((root) => path.resolve(root.dir) === parent);\n}\n\n/**\n * Swap the old name for the new one where it is *identity* rather than prose.\n *\n * Two rules, and the first is the interesting one: **a string literal whose\n * entire content is the old name**. That covers `id`, `displayName`, `title` and\n * the scaffold's `const ID = \"…\"` without knowing any of their names, and it\n * does not touch a sentence that merely mentions the canvas — `\"the board is the\n * point of the board\"` is not the string `\"board\"`. A whole-word replacement\n * would have rewritten that sentence, which is worse than leaving it.\n *\n * The second rule keeps the instructions in a comment honest: `/canvas open\n * <old>` would otherwise tell the next reader to type a name that no longer\n * resolves.\n *\n * Anything else is left and reported. The scaffold names itself in exactly three\n * places, all covered here, so a canvas that has not been renamed by hand comes\n * through with nothing left over.\n */\nfunction rewriteEntrySource(source: string, from: string, to: string): { source: string; rewrites: CanvasRewrite[] } {\n\tconst name = escapeForRegExp(from);\n\tconst substitutions: [RegExp, string][] = [\n\t\t// A quoted string that is *only* the old name. Backticks included, but only\n\t\t// without interpolation — `${x}-${from}` is a computed value, not a literal.\n\t\t[new RegExp(`([\"'])${name}\\\\1`, \"g\"), `$1${to}$1`],\n\t\t[new RegExp(`\\`${name}\\``, \"g\"), `\\`${to}\\``],\n\t\t// Instructions in a comment: /canvas open <old>, /canvas reload <old>, …\n\t\t[new RegExp(`(/canvas\\\\s+(?:open|reload|rename|remove|close)\\\\s+)${name}\\\\b`, \"g\"), `$1${to}`],\n\t];\n\n\tconst before = source.split(\"\\n\");\n\tconst after = before.map((line) =>\n\t\tsubstitutions.reduce((text, [pattern, replacement]) => text.replace(pattern, replacement), line),\n\t);\n\n\tconst rewrites: CanvasRewrite[] = [];\n\tafter.forEach((line, index) => {\n\t\tif (line !== before[index]) rewrites.push({ line: index + 1, before: before[index] as string, after: line });\n\t});\n\treturn { source: after.join(\"\\n\"), rewrites };\n}\n\n/** Lines still mentioning `name` after the rewrite. */\nfunction remainingMentions(source: string, name: string): number[] {\n\tconst lines: number[] = [];\n\tsource.split(\"\\n\").forEach((line, index) => {\n\t\tif (line.includes(name)) lines.push(index + 1);\n\t});\n\treturn lines;\n}\n\nfunction escapeForRegExp(value: string): string {\n\treturn value.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\n/**\n * Rename a canvas extension: its directory, and the name inside its code.\n *\n * The caller is responsible for closing open instances first. This does not do\n * it, because closing is the *session's* concern and this module has no session —\n * but skipping it leaves an instance pointing at a directory that has moved.\n */\nexport function renameCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\tto: string,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRenameResult | CanvasLifecycleRefusal {\n\tconst invalid = validateCanvasName(to);\n\tif (invalid) return { reason: \"invalid-name\", detail: invalid };\n\tif (to === extension.id) return { reason: \"invalid-name\", detail: `it is already called \"${to}\"` };\n\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin, which names its canvases in its own manifest. Renaming it here would break the plugin; use /plugin instead.`,\n\t\t};\n\t}\n\n\tconst target = path.join(root.dir, to);\n\tif (existsSync(target)) return { reason: \"exists\", detail: `${target} already exists` };\n\n\tconst entry = path.join(extension.dir, CANVAS_ENTRY_FILE);\n\tlet source: string;\n\ttry {\n\t\tsource = readFileSync(entry, \"utf8\");\n\t} catch (cause) {\n\t\treturn {\n\t\t\treason: \"unwritable\",\n\t\t\tdetail: `could not read ${entry}: ${cause instanceof Error ? cause.message : String(cause)}`,\n\t\t};\n\t}\n\n\tconst rewritten = rewriteEntrySource(source, extension.id, to);\n\ttry {\n\t\t// Write before moving: if the write fails the directory has not moved and\n\t\t// nothing is half-renamed.\n\t\tif (rewritten.source !== source) writeFileSync(entry, rewritten.source, \"utf8\");\n\t\trenameSync(extension.dir, target);\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\n\treturn {\n\t\tfrom: extension.id,\n\t\tto,\n\t\tdir: target,\n\t\trewrites: rewritten.rewrites,\n\t\tleftovers: remainingMentions(rewritten.source, extension.id),\n\t};\n}\n\n/** What {@link removeCanvasExtension} deleted. */\nexport interface CanvasRemoveResult {\n\tid: string;\n\tdir: string;\n}\n\n/**\n * Delete a canvas extension's directory.\n *\n * As with rename, the caller closes open instances first — a deleted directory\n * whose child is still forked leaves a process serving code that no longer\n * exists on disk, which is the most confusing state of all.\n */\nexport function removeCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRemoveResult | CanvasLifecycleRefusal {\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin. Deleting its directory would leave the plugin broken rather than uninstalled; use /plugin instead.`,\n\t\t};\n\t}\n\ttry {\n\t\trmSync(extension.dir, { recursive: true, force: true });\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\treturn { id: extension.id, dir: extension.dir };\n}\n\n/** Narrow a lifecycle return value to its refusal case. */\nexport function isCanvasRefusal(value: object): value is CanvasLifecycleRefusal {\n\treturn \"reason\" in value;\n}\n"]}

package/dist/core/canvas/lifecycle.js

@@ -57,7 +57,7 @@ function rewriteEntrySource(source, from, to) {
         // A quoted string that is *only* the old name. Backticks included, but only
         // without interpolation — `${x}-${from}` is a computed value, not a literal.
         [new RegExp(`(["'])${name}\\1`, "g"), `$1${to}$1`],
-        [new RegExp("`" + name + "`", "g"), `\`${to}\``],
+        [new RegExp(`\`${name}\``, "g"), `\`${to}\``],
         // Instructions in a comment: /canvas open <old>, /canvas reload <old>, …
         [new RegExp(`(/canvas\\s+(?:open|reload|rename|remove|close)\\s+)${name}\\b`, "g"), `$1${to}`],
     ];

package/dist/core/canvas/lifecycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"lifecycle.js","sourceRoot":"","sources":["../../../src/core/canvas/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAqCnD;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAC7B,SAAoC,EACpC,KAAkC,EACH;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,CAAC;AAAA,CAC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAS,kBAAkB,CAAC,MAAc,EAAE,IAAY,EAAE,EAAU,EAAiD;IACpH,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAuB;QACzC,4EAA4E;QAC5E,+EAA6E;QAC7E,CAAC,IAAI,MAAM,CAAC,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC;QAClD,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,IAAI,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC;QAChD,2EAAyE;QACzE,CAAC,IAAI,MAAM,CAAC,uDAAuD,IAAI,KAAK,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;KAC9F,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,CAChG,CAAC;IAEF,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC9B,IAAI,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAW,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAAA,CAC7G,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC;AAAA,CAC9C;AAED,uDAAuD;AACvD,SAAS,iBAAiB,CAAC,MAAc,EAAE,IAAY,EAAY;IAClE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAAA,CAC/C,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,eAAe,CAAC,KAAa,EAAU;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AAAA,CACpD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACpC,SAAoC,EACpC,EAAU,EACV,KAAkC,EACY;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACvC,IAAI,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAChE,IAAI,EAAE,KAAK,SAAS,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,yBAAyB,EAAE,GAAG,EAAE,CAAC;IAEnG,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO;YACN,MAAM,EAAE,UAAU;YAClB,MAAM,EAAE,IAAI,SAAS,CAAC,EAAE,8IAA8I;SACtK,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAC;IAExF,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAC1D,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACJ,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO;YACN,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,kBAAkB,KAAK,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;SAC5F,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC;QACJ,0EAA0E;QAC1E,2BAA2B;QAC3B,IAAI,SAAS,CAAC,MAAM,KAAK,MAAM;YAAE,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChF,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACjG,CAAC;IAED,OAAO;QACN,IAAI,EAAE,SAAS,CAAC,EAAE;QAClB,EAAE;QACF,GAAG,EAAE,MAAM;QACX,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,SAAS,EAAE,iBAAiB,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;KAC5D,CAAC;AAAA,CACF;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACpC,SAAoC,EACpC,KAAkC,EACY;IAC9C,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO;YACN,MAAM,EAAE,UAAU;YAClB,MAAM,EAAE,IAAI,SAAS,CAAC,EAAE,qIAAqI;SAC7J,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACJ,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACjG,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC;AAAA,CAChD;AAED,2DAA2D;AAC3D,MAAM,UAAU,eAAe,CAAC,KAAa,EAAmC;IAC/E,OAAO,QAAQ,IAAI,KAAK,CAAC;AAAA,CACzB","sourcesContent":["/**\n * Renaming and removing a canvas extension.\n *\n * Design: `docs/canvas-extensions-design.md` §13.6. These exist because a canvas\n * has more identity than a file does, and it is spread across four places: the\n * directory name (which *is* the extension id, since discovery keys off\n * position), the `id` a canvas declares, its `displayName`, and whatever the\n * header comment tells the reader to type. Renaming by hand means getting all\n * four right, and getting the `id` wrong is not a typo — it drops the canvas the\n * person is looking at on the next reload, because the instance was opened\n * against a canvas that no longer exists.\n *\n * So the rewriting is deliberately narrow. It changes the three places the\n * scaffold puts the name and nothing else, then **reports every other line the\n * old name still appears on** rather than guessing at prose. A rename that\n * silently edited a description would be worse than one that admits what it left\n * behind.\n */\n\nimport { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CanvasSearchRoot, DiscoveredCanvasExtension } from \"./discovery.js\";\nimport { CANVAS_ENTRY_FILE } from \"./discovery.js\";\nimport { validateCanvasName } from \"./scaffold.js\";\n\n/** One line the rename changed. */\nexport interface CanvasRewrite {\n\t/** 1-based, so it can be read straight off an editor gutter. */\n\tline: number;\n\tbefore: string;\n\tafter: string;\n}\n\n/** What {@link renameCanvasExtension} did. */\nexport interface CanvasRenameResult {\n\tfrom: string;\n\tto: string;\n\t/** Workspace-relative where possible, for a message a person can act on. */\n\tdir: string;\n\trewrites: CanvasRewrite[];\n\t/**\n\t * Lines where the old name survives, because they are prose rather than\n\t * identity. Surfaced, never silently edited.\n\t */\n\tleftovers: number[];\n}\n\n/**\n * Why a canvas cannot be renamed or removed in place.\n *\n * `packaged` is the interesting one: a plugin's canvases are named by its\n * manifest rather than by where they sit, so moving the directory would either\n * do nothing or break the plugin. That is `/plugin`'s job, not this one's.\n */\nexport type CanvasLifecycleRefusal =\n\t| { reason: \"invalid-name\"; detail: string }\n\t| { reason: \"exists\"; detail: string }\n\t| { reason: \"packaged\"; detail: string }\n\t| { reason: \"unwritable\"; detail: string };\n\n/**\n * Whether this extension is one we may move or delete.\n *\n * The test is positional and deliberately so: an extension is ours to edit when\n * its directory sits *directly* inside one of the search roots, which is exactly\n * the layout `discoverCanvasExtensions` walks. Anything else arrived inside a\n * package — resolved through a manifest by `plugin-canvases.ts` — and its\n * location is that package's business.\n */\nexport function canvasHomeRoot(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasSearchRoot | undefined {\n\tconst parent = path.dirname(path.resolve(extension.dir));\n\treturn roots.find((root) => path.resolve(root.dir) === parent);\n}\n\n/**\n * Swap the old name for the new one where it is *identity* rather than prose.\n *\n * Two rules, and the first is the interesting one: **a string literal whose\n * entire content is the old name**. That covers `id`, `displayName`, `title` and\n * the scaffold's `const ID = \"…\"` without knowing any of their names, and it\n * does not touch a sentence that merely mentions the canvas — `\"the board is the\n * point of the board\"` is not the string `\"board\"`. A whole-word replacement\n * would have rewritten that sentence, which is worse than leaving it.\n *\n * The second rule keeps the instructions in a comment honest: `/canvas open\n * <old>` would otherwise tell the next reader to type a name that no longer\n * resolves.\n *\n * Anything else is left and reported. The scaffold names itself in exactly three\n * places, all covered here, so a canvas that has not been renamed by hand comes\n * through with nothing left over.\n */\nfunction rewriteEntrySource(source: string, from: string, to: string): { source: string; rewrites: CanvasRewrite[] } {\n\tconst name = escapeForRegExp(from);\n\tconst substitutions: [RegExp, string][] = [\n\t\t// A quoted string that is *only* the old name. Backticks included, but only\n\t\t// without interpolation — `${x}-${from}` is a computed value, not a literal.\n\t\t[new RegExp(`([\"'])${name}\\\\1`, \"g\"), `$1${to}$1`],\n\t\t[new RegExp(\"`\" + name + \"`\", \"g\"), `\\`${to}\\``],\n\t\t// Instructions in a comment: /canvas open <old>, /canvas reload <old>, …\n\t\t[new RegExp(`(/canvas\\\\s+(?:open|reload|rename|remove|close)\\\\s+)${name}\\\\b`, \"g\"), `$1${to}`],\n\t];\n\n\tconst before = source.split(\"\\n\");\n\tconst after = before.map((line) =>\n\t\tsubstitutions.reduce((text, [pattern, replacement]) => text.replace(pattern, replacement), line),\n\t);\n\n\tconst rewrites: CanvasRewrite[] = [];\n\tafter.forEach((line, index) => {\n\t\tif (line !== before[index]) rewrites.push({ line: index + 1, before: before[index] as string, after: line });\n\t});\n\treturn { source: after.join(\"\\n\"), rewrites };\n}\n\n/** Lines still mentioning `name` after the rewrite. */\nfunction remainingMentions(source: string, name: string): number[] {\n\tconst lines: number[] = [];\n\tsource.split(\"\\n\").forEach((line, index) => {\n\t\tif (line.includes(name)) lines.push(index + 1);\n\t});\n\treturn lines;\n}\n\nfunction escapeForRegExp(value: string): string {\n\treturn value.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\n/**\n * Rename a canvas extension: its directory, and the name inside its code.\n *\n * The caller is responsible for closing open instances first. This does not do\n * it, because closing is the *session's* concern and this module has no session —\n * but skipping it leaves an instance pointing at a directory that has moved.\n */\nexport function renameCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\tto: string,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRenameResult | CanvasLifecycleRefusal {\n\tconst invalid = validateCanvasName(to);\n\tif (invalid) return { reason: \"invalid-name\", detail: invalid };\n\tif (to === extension.id) return { reason: \"invalid-name\", detail: `it is already called \"${to}\"` };\n\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin, which names its canvases in its own manifest. Renaming it here would break the plugin; use /plugin instead.`,\n\t\t};\n\t}\n\n\tconst target = path.join(root.dir, to);\n\tif (existsSync(target)) return { reason: \"exists\", detail: `${target} already exists` };\n\n\tconst entry = path.join(extension.dir, CANVAS_ENTRY_FILE);\n\tlet source: string;\n\ttry {\n\t\tsource = readFileSync(entry, \"utf8\");\n\t} catch (cause) {\n\t\treturn {\n\t\t\treason: \"unwritable\",\n\t\t\tdetail: `could not read ${entry}: ${cause instanceof Error ? cause.message : String(cause)}`,\n\t\t};\n\t}\n\n\tconst rewritten = rewriteEntrySource(source, extension.id, to);\n\ttry {\n\t\t// Write before moving: if the write fails the directory has not moved and\n\t\t// nothing is half-renamed.\n\t\tif (rewritten.source !== source) writeFileSync(entry, rewritten.source, \"utf8\");\n\t\trenameSync(extension.dir, target);\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\n\treturn {\n\t\tfrom: extension.id,\n\t\tto,\n\t\tdir: target,\n\t\trewrites: rewritten.rewrites,\n\t\tleftovers: remainingMentions(rewritten.source, extension.id),\n\t};\n}\n\n/** What {@link removeCanvasExtension} deleted. */\nexport interface CanvasRemoveResult {\n\tid: string;\n\tdir: string;\n}\n\n/**\n * Delete a canvas extension's directory.\n *\n * As with rename, the caller closes open instances first — a deleted directory\n * whose child is still forked leaves a process serving code that no longer\n * exists on disk, which is the most confusing state of all.\n */\nexport function removeCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRemoveResult | CanvasLifecycleRefusal {\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin. Deleting its directory would leave the plugin broken rather than uninstalled; use /plugin instead.`,\n\t\t};\n\t}\n\ttry {\n\t\trmSync(extension.dir, { recursive: true, force: true });\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\treturn { id: extension.id, dir: extension.dir };\n}\n\n/** Narrow a lifecycle return value to its refusal case. */\nexport function isCanvasRefusal(value: object): value is CanvasLifecycleRefusal {\n\treturn \"reason\" in value;\n}\n"]}
+{"version":3,"file":"lifecycle.js","sourceRoot":"","sources":["../../../src/core/canvas/lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAqCnD;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAC7B,SAAoC,EACpC,KAAkC,EACH;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,CAAC;AAAA,CAC/D;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAS,kBAAkB,CAAC,MAAc,EAAE,IAAY,EAAE,EAAU,EAAiD;IACpH,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAuB;QACzC,4EAA4E;QAC5E,+EAA6E;QAC7E,CAAC,IAAI,MAAM,CAAC,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC;QAClD,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC;QAC7C,2EAAyE;QACzE,CAAC,IAAI,MAAM,CAAC,uDAAuD,IAAI,KAAK,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;KAC9F,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,CAChG,CAAC;IAEF,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC9B,IAAI,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAW,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAAA,CAC7G,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC;AAAA,CAC9C;AAED,uDAAuD;AACvD,SAAS,iBAAiB,CAAC,MAAc,EAAE,IAAY,EAAY;IAClE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAAA,CAC/C,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,eAAe,CAAC,KAAa,EAAU;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AAAA,CACpD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACpC,SAAoC,EACpC,EAAU,EACV,KAAkC,EACY;IAC9C,MAAM,OAAO,GAAG,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACvC,IAAI,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAChE,IAAI,EAAE,KAAK,SAAS,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,yBAAyB,EAAE,GAAG,EAAE,CAAC;IAEnG,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO;YACN,MAAM,EAAE,UAAU;YAClB,MAAM,EAAE,IAAI,SAAS,CAAC,EAAE,8IAA8I;SACtK,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvC,IAAI,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAC;IAExF,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IAC1D,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACJ,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO;YACN,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,kBAAkB,KAAK,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;SAC5F,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC;QACJ,0EAA0E;QAC1E,2BAA2B;QAC3B,IAAI,SAAS,CAAC,MAAM,KAAK,MAAM;YAAE,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChF,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACjG,CAAC;IAED,OAAO;QACN,IAAI,EAAE,SAAS,CAAC,EAAE;QAClB,EAAE;QACF,GAAG,EAAE,MAAM;QACX,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,SAAS,EAAE,iBAAiB,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;KAC5D,CAAC;AAAA,CACF;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACpC,SAAoC,EACpC,KAAkC,EACY;IAC9C,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,OAAO;YACN,MAAM,EAAE,UAAU;YAClB,MAAM,EAAE,IAAI,SAAS,CAAC,EAAE,qIAAqI;SAC7J,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACJ,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACjG,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC;AAAA,CAChD;AAED,2DAA2D;AAC3D,MAAM,UAAU,eAAe,CAAC,KAAa,EAAmC;IAC/E,OAAO,QAAQ,IAAI,KAAK,CAAC;AAAA,CACzB","sourcesContent":["/**\n * Renaming and removing a canvas extension.\n *\n * Design: `docs/canvas-extensions-design.md` §13.6. These exist because a canvas\n * has more identity than a file does, and it is spread across four places: the\n * directory name (which *is* the extension id, since discovery keys off\n * position), the `id` a canvas declares, its `displayName`, and whatever the\n * header comment tells the reader to type. Renaming by hand means getting all\n * four right, and getting the `id` wrong is not a typo — it drops the canvas the\n * person is looking at on the next reload, because the instance was opened\n * against a canvas that no longer exists.\n *\n * So the rewriting is deliberately narrow. It changes the three places the\n * scaffold puts the name and nothing else, then **reports every other line the\n * old name still appears on** rather than guessing at prose. A rename that\n * silently edited a description would be worse than one that admits what it left\n * behind.\n */\n\nimport { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from \"node:fs\";\nimport * as path from \"node:path\";\nimport type { CanvasSearchRoot, DiscoveredCanvasExtension } from \"./discovery.js\";\nimport { CANVAS_ENTRY_FILE } from \"./discovery.js\";\nimport { validateCanvasName } from \"./scaffold.js\";\n\n/** One line the rename changed. */\nexport interface CanvasRewrite {\n\t/** 1-based, so it can be read straight off an editor gutter. */\n\tline: number;\n\tbefore: string;\n\tafter: string;\n}\n\n/** What {@link renameCanvasExtension} did. */\nexport interface CanvasRenameResult {\n\tfrom: string;\n\tto: string;\n\t/** Workspace-relative where possible, for a message a person can act on. */\n\tdir: string;\n\trewrites: CanvasRewrite[];\n\t/**\n\t * Lines where the old name survives, because they are prose rather than\n\t * identity. Surfaced, never silently edited.\n\t */\n\tleftovers: number[];\n}\n\n/**\n * Why a canvas cannot be renamed or removed in place.\n *\n * `packaged` is the interesting one: a plugin's canvases are named by its\n * manifest rather than by where they sit, so moving the directory would either\n * do nothing or break the plugin. That is `/plugin`'s job, not this one's.\n */\nexport type CanvasLifecycleRefusal =\n\t| { reason: \"invalid-name\"; detail: string }\n\t| { reason: \"exists\"; detail: string }\n\t| { reason: \"packaged\"; detail: string }\n\t| { reason: \"unwritable\"; detail: string };\n\n/**\n * Whether this extension is one we may move or delete.\n *\n * The test is positional and deliberately so: an extension is ours to edit when\n * its directory sits *directly* inside one of the search roots, which is exactly\n * the layout `discoverCanvasExtensions` walks. Anything else arrived inside a\n * package — resolved through a manifest by `plugin-canvases.ts` — and its\n * location is that package's business.\n */\nexport function canvasHomeRoot(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasSearchRoot | undefined {\n\tconst parent = path.dirname(path.resolve(extension.dir));\n\treturn roots.find((root) => path.resolve(root.dir) === parent);\n}\n\n/**\n * Swap the old name for the new one where it is *identity* rather than prose.\n *\n * Two rules, and the first is the interesting one: **a string literal whose\n * entire content is the old name**. That covers `id`, `displayName`, `title` and\n * the scaffold's `const ID = \"…\"` without knowing any of their names, and it\n * does not touch a sentence that merely mentions the canvas — `\"the board is the\n * point of the board\"` is not the string `\"board\"`. A whole-word replacement\n * would have rewritten that sentence, which is worse than leaving it.\n *\n * The second rule keeps the instructions in a comment honest: `/canvas open\n * <old>` would otherwise tell the next reader to type a name that no longer\n * resolves.\n *\n * Anything else is left and reported. The scaffold names itself in exactly three\n * places, all covered here, so a canvas that has not been renamed by hand comes\n * through with nothing left over.\n */\nfunction rewriteEntrySource(source: string, from: string, to: string): { source: string; rewrites: CanvasRewrite[] } {\n\tconst name = escapeForRegExp(from);\n\tconst substitutions: [RegExp, string][] = [\n\t\t// A quoted string that is *only* the old name. Backticks included, but only\n\t\t// without interpolation — `${x}-${from}` is a computed value, not a literal.\n\t\t[new RegExp(`([\"'])${name}\\\\1`, \"g\"), `$1${to}$1`],\n\t\t[new RegExp(`\\`${name}\\``, \"g\"), `\\`${to}\\``],\n\t\t// Instructions in a comment: /canvas open <old>, /canvas reload <old>, …\n\t\t[new RegExp(`(/canvas\\\\s+(?:open|reload|rename|remove|close)\\\\s+)${name}\\\\b`, \"g\"), `$1${to}`],\n\t];\n\n\tconst before = source.split(\"\\n\");\n\tconst after = before.map((line) =>\n\t\tsubstitutions.reduce((text, [pattern, replacement]) => text.replace(pattern, replacement), line),\n\t);\n\n\tconst rewrites: CanvasRewrite[] = [];\n\tafter.forEach((line, index) => {\n\t\tif (line !== before[index]) rewrites.push({ line: index + 1, before: before[index] as string, after: line });\n\t});\n\treturn { source: after.join(\"\\n\"), rewrites };\n}\n\n/** Lines still mentioning `name` after the rewrite. */\nfunction remainingMentions(source: string, name: string): number[] {\n\tconst lines: number[] = [];\n\tsource.split(\"\\n\").forEach((line, index) => {\n\t\tif (line.includes(name)) lines.push(index + 1);\n\t});\n\treturn lines;\n}\n\nfunction escapeForRegExp(value: string): string {\n\treturn value.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\n/**\n * Rename a canvas extension: its directory, and the name inside its code.\n *\n * The caller is responsible for closing open instances first. This does not do\n * it, because closing is the *session's* concern and this module has no session —\n * but skipping it leaves an instance pointing at a directory that has moved.\n */\nexport function renameCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\tto: string,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRenameResult | CanvasLifecycleRefusal {\n\tconst invalid = validateCanvasName(to);\n\tif (invalid) return { reason: \"invalid-name\", detail: invalid };\n\tif (to === extension.id) return { reason: \"invalid-name\", detail: `it is already called \"${to}\"` };\n\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin, which names its canvases in its own manifest. Renaming it here would break the plugin; use /plugin instead.`,\n\t\t};\n\t}\n\n\tconst target = path.join(root.dir, to);\n\tif (existsSync(target)) return { reason: \"exists\", detail: `${target} already exists` };\n\n\tconst entry = path.join(extension.dir, CANVAS_ENTRY_FILE);\n\tlet source: string;\n\ttry {\n\t\tsource = readFileSync(entry, \"utf8\");\n\t} catch (cause) {\n\t\treturn {\n\t\t\treason: \"unwritable\",\n\t\t\tdetail: `could not read ${entry}: ${cause instanceof Error ? cause.message : String(cause)}`,\n\t\t};\n\t}\n\n\tconst rewritten = rewriteEntrySource(source, extension.id, to);\n\ttry {\n\t\t// Write before moving: if the write fails the directory has not moved and\n\t\t// nothing is half-renamed.\n\t\tif (rewritten.source !== source) writeFileSync(entry, rewritten.source, \"utf8\");\n\t\trenameSync(extension.dir, target);\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\n\treturn {\n\t\tfrom: extension.id,\n\t\tto,\n\t\tdir: target,\n\t\trewrites: rewritten.rewrites,\n\t\tleftovers: remainingMentions(rewritten.source, extension.id),\n\t};\n}\n\n/** What {@link removeCanvasExtension} deleted. */\nexport interface CanvasRemoveResult {\n\tid: string;\n\tdir: string;\n}\n\n/**\n * Delete a canvas extension's directory.\n *\n * As with rename, the caller closes open instances first — a deleted directory\n * whose child is still forked leaves a process serving code that no longer\n * exists on disk, which is the most confusing state of all.\n */\nexport function removeCanvasExtension(\n\textension: DiscoveredCanvasExtension,\n\troots: readonly CanvasSearchRoot[],\n): CanvasRemoveResult | CanvasLifecycleRefusal {\n\tconst root = canvasHomeRoot(extension, roots);\n\tif (!root) {\n\t\treturn {\n\t\t\treason: \"packaged\",\n\t\t\tdetail: `\"${extension.id}\" came from an installed plugin. Deleting its directory would leave the plugin broken rather than uninstalled; use /plugin instead.`,\n\t\t};\n\t}\n\ttry {\n\t\trmSync(extension.dir, { recursive: true, force: true });\n\t} catch (cause) {\n\t\treturn { reason: \"unwritable\", detail: cause instanceof Error ? cause.message : String(cause) };\n\t}\n\treturn { id: extension.id, dir: extension.dir };\n}\n\n/** Narrow a lifecycle return value to its refusal case. */\nexport function isCanvasRefusal(value: object): value is CanvasLifecycleRefusal {\n\treturn \"reason\" in value;\n}\n"]}

package/dist/core/embsearch/chunker.d.ts

@@ -11,6 +11,17 @@
  * sidecar triggers a clean rebuild of the store.
  */
 export declare const CHUNKER_VERSION = 2;
+/**
+ * Hard character cap per chunk.
+ *
+ * The "~256 tokens ≈ 1000 chars" this was set from does not hold for code:
+ * measured against the bundled tokenizer over this repo, 1000 chars is **313
+ * tokens** at the median, so MiniLM (256) truncates 85.8% of chunks and drops
+ * 20.3% of the corpus's tokens, while bge-small (512) drops 0.1%. Raising this
+ * is therefore not free in the way the old comment implied — it spends a
+ * budget that is already overdrawn on one model and nearly full on the other.
+ */
+export declare const CHUNK_MAX_CHARS = 1000;
 export interface Chunk {
     /** `relpath#index` — the id stored in the vector index. */
     id: string;
@@ -24,6 +35,11 @@ export interface Chunk {
 /**
  * Split `content` into chunks. `relPath` becomes the id prefix. Returns an
  * empty array for empty or binary-looking content.
+ *
+ * `maxChars` overrides {@link CHUNK_MAX_CHARS} for eval arms that sweep the
+ * window. Production never passes it; a caller that does is changing what the
+ * index contains and owes the store a distinct key, since nothing about a
+ * stored vector records the cap it was built under.
  */
-export declare function chunkFile(relPath: string, content: string): Chunk[];
+export declare function chunkFile(relPath: string, content: string, maxChars?: number): Chunk[];
 //# sourceMappingURL=chunker.d.ts.map

package/dist/core/embsearch/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,eAAe,IAAI,CAAC;AAmBjC,MAAM,WAAW,KAAK;IACrB,6DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;CAChB;AAQD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,KAAK,EAAE,CA8CnE","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 2;\n\n/**\n * Target lines per chunk.\n *\n * Halving this (30 lines / 500 chars) was measured and rejected. The theory\n * was dilution — a four-line answer sharing one vector with five neighbouring\n * methods — but sharper chunks cost reach: Recall@50 fell 79% -> 69% on\n * `auto +rr` and the boundary class went from 2 of 4 findable to 0 of 4.\n * A fixed top-k over smaller chunks retrieves less *content*: 50 chunks at\n * ~460 chars sees half the corpus that 50 at ~930 does. Recall@1 ticked up\n * (53% -> 55%), which is the sharpening, and it did not pay for the loss.\n */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/** Hard character cap per chunk (MiniLM truncates ~256 tokens ≈ 1000 chars). */\nconst CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nfunction looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n */\nexport function chunkFile(relPath: string, content: string): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > CHUNK_MAX_CHARS && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\t// Trim whole blank lines off both ends, and narrow the recorded range to\n\t\t// match. Trimming the joined string instead left `startLine`/`endLine`\n\t\t// claiming lines whose text was never embedded — 31% of chunks in this\n\t\t// repo, up to 3 lines each. Those lines were unfindable by the vector\n\t\t// leg while still being handed to the context assembler as if they were\n\t\t// part of the chunk.\n\t\tlet from = start;\n\t\tlet to = end; // exclusive\n\t\twhile (from < to && lines[from].trim() === \"\") from++;\n\t\twhile (to > from && lines[to - 1].trim() === \"\") to--;\n\t\tlet text = lines.slice(from, to).join(\"\\n\").trim();\n\t\tif (text.length > CHUNK_MAX_CHARS) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, CHUNK_MAX_CHARS);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: from + 1,\n\t\t\t\tendLine: to,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,eAAO,MAAM,eAAe,IAAI,CAAC;AAgBjC;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,WAAW,KAAK;IACrB,6DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAC;CAChB;AAQD;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAwB,GAAG,KAAK,EAAE,CA8CvG","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 2;\n\n/**\n * Target lines per chunk.\n *\n * Halving this (30 lines / 500 chars) was measured and rejected. The theory\n * was dilution — a four-line answer sharing one vector with five neighbouring\n * methods — but sharper chunks cost reach: Recall@50 fell 79% -> 69% on\n * `auto +rr` and the boundary class went from 2 of 4 findable to 0 of 4.\n * A fixed top-k over smaller chunks retrieves less *content*: 50 chunks at\n * ~460 chars sees half the corpus that 50 at ~930 does. Recall@1 ticked up\n * (53% -> 55%), which is the sharpening, and it did not pay for the loss.\n */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/**\n * Hard character cap per chunk.\n *\n * The \"~256 tokens ≈ 1000 chars\" this was set from does not hold for code:\n * measured against the bundled tokenizer over this repo, 1000 chars is **313\n * tokens** at the median, so MiniLM (256) truncates 85.8% of chunks and drops\n * 20.3% of the corpus's tokens, while bge-small (512) drops 0.1%. Raising this\n * is therefore not free in the way the old comment implied — it spends a\n * budget that is already overdrawn on one model and nearly full on the other.\n */\nexport const CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nfunction looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n *\n * `maxChars` overrides {@link CHUNK_MAX_CHARS} for eval arms that sweep the\n * window. Production never passes it; a caller that does is changing what the\n * index contains and owes the store a distinct key, since nothing about a\n * stored vector records the cap it was built under.\n */\nexport function chunkFile(relPath: string, content: string, maxChars: number = CHUNK_MAX_CHARS): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > maxChars && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\t// Trim whole blank lines off both ends, and narrow the recorded range to\n\t\t// match. Trimming the joined string instead left `startLine`/`endLine`\n\t\t// claiming lines whose text was never embedded — 31% of chunks in this\n\t\t// repo, up to 3 lines each. Those lines were unfindable by the vector\n\t\t// leg while still being handed to the context assembler as if they were\n\t\t// part of the chunk.\n\t\tlet from = start;\n\t\tlet to = end; // exclusive\n\t\twhile (from < to && lines[from].trim() === \"\") from++;\n\t\twhile (to > from && lines[to - 1].trim() === \"\") to--;\n\t\tlet text = lines.slice(from, to).join(\"\\n\").trim();\n\t\tif (text.length > maxChars) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, maxChars);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: from + 1,\n\t\t\t\tendLine: to,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}

package/dist/core/embsearch/chunker.js

@@ -25,8 +25,17 @@ export const CHUNKER_VERSION = 2;
 const CHUNK_LINES = 60;
 /** Overlapping lines between consecutive chunks, for context continuity. */
 const CHUNK_OVERLAP_LINES = 10;
-/** Hard character cap per chunk (MiniLM truncates ~256 tokens ≈ 1000 chars). */
-const CHUNK_MAX_CHARS = 1000;
+/**
+ * Hard character cap per chunk.
+ *
+ * The "~256 tokens ≈ 1000 chars" this was set from does not hold for code:
+ * measured against the bundled tokenizer over this repo, 1000 chars is **313
+ * tokens** at the median, so MiniLM (256) truncates 85.8% of chunks and drops
+ * 20.3% of the corpus's tokens, while bge-small (512) drops 0.1%. Raising this
+ * is therefore not free in the way the old comment implied — it spends a
+ * budget that is already overdrawn on one model and nearly full on the other.
+ */
+export const CHUNK_MAX_CHARS = 1000;
 /** Heuristic binary sniff: NUL byte in the first 8KB. */
 function looksBinary(content) {
     const probe = content.slice(0, 8192);
@@ -35,8 +44,13 @@ function looksBinary(content) {
 /**
  * Split `content` into chunks. `relPath` becomes the id prefix. Returns an
  * empty array for empty or binary-looking content.
+ *
+ * `maxChars` overrides {@link CHUNK_MAX_CHARS} for eval arms that sweep the
+ * window. Production never passes it; a caller that does is changing what the
+ * index contains and owes the store a distinct key, since nothing about a
+ * stored vector records the cap it was built under.
  */
-export function chunkFile(relPath, content) {
+export function chunkFile(relPath, content, maxChars = CHUNK_MAX_CHARS) {
     if (!content.trim() || looksBinary(content))
         return [];
     const lines = content.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n");
@@ -48,7 +62,7 @@ export function chunkFile(relPath, content) {
         let chars = 0;
         while (end < lines.length && end - start < CHUNK_LINES) {
             const lineLen = lines[end].length + 1;
-            if (chars + lineLen > CHUNK_MAX_CHARS && end > start)
+            if (chars + lineLen > maxChars && end > start)
                 break;
             chars += lineLen;
             end++;
@@ -66,10 +80,10 @@ export function chunkFile(relPath, content) {
         while (to > from && lines[to - 1].trim() === "")
             to--;
         let text = lines.slice(from, to).join("\n").trim();
-        if (text.length > CHUNK_MAX_CHARS) {
+        if (text.length > maxChars) {
             // Oversized chunk (e.g. long minified line): keep the prefix. The
             // underlying model would truncate anyway, so this stays bounded.
-            text = text.slice(0, CHUNK_MAX_CHARS);
+            text = text.slice(0, maxChars);
         }
         if (text) {
             chunks.push({

package/dist/core/embsearch/chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,4EAA4E;AAC5E,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,kFAAgF;AAChF,MAAM,eAAe,GAAG,IAAI,CAAC;AAa7B,yDAAyD;AACzD,SAAS,WAAW,CAAC,OAAe,EAAW;IAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAAA,CAChC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,OAAe,EAAW;IACpE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,WAAW,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU;IACzB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,YAAY;QAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,GAAG,KAAK,GAAG,WAAW,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACtC,IAAI,KAAK,GAAG,OAAO,GAAG,eAAe,IAAI,GAAG,GAAG,KAAK;gBAAE,MAAM;YAC5D,KAAK,IAAI,OAAO,CAAC;YACjB,GAAG,EAAE,CAAC;QACP,CAAC;QACD,yEAAyE;QACzE,uEAAuE;QACvE,yEAAuE;QACvE,sEAAsE;QACtE,wEAAwE;QACxE,qBAAqB;QACrB,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC,YAAY;QAC1B,OAAO,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,IAAI,EAAE,CAAC;QACtD,OAAO,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,EAAE,EAAE,CAAC;QACtD,IAAI,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YACnC,kEAAkE;YAClE,iEAAiE;YACjE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,OAAO,IAAI,KAAK,EAAE;gBACzB,IAAI;gBACJ,SAAS,EAAE,IAAI,GAAG,CAAC;gBACnB,OAAO,EAAE,EAAE;aACX,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACT,CAAC;QACD,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM;QAC/B,uDAAuD;QACvD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,mBAAmB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 2;\n\n/**\n * Target lines per chunk.\n *\n * Halving this (30 lines / 500 chars) was measured and rejected. The theory\n * was dilution — a four-line answer sharing one vector with five neighbouring\n * methods — but sharper chunks cost reach: Recall@50 fell 79% -> 69% on\n * `auto +rr` and the boundary class went from 2 of 4 findable to 0 of 4.\n * A fixed top-k over smaller chunks retrieves less *content*: 50 chunks at\n * ~460 chars sees half the corpus that 50 at ~930 does. Recall@1 ticked up\n * (53% -> 55%), which is the sharpening, and it did not pay for the loss.\n */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/** Hard character cap per chunk (MiniLM truncates ~256 tokens ≈ 1000 chars). */\nconst CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nfunction looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n */\nexport function chunkFile(relPath: string, content: string): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > CHUNK_MAX_CHARS && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\t// Trim whole blank lines off both ends, and narrow the recorded range to\n\t\t// match. Trimming the joined string instead left `startLine`/`endLine`\n\t\t// claiming lines whose text was never embedded — 31% of chunks in this\n\t\t// repo, up to 3 lines each. Those lines were unfindable by the vector\n\t\t// leg while still being handed to the context assembler as if they were\n\t\t// part of the chunk.\n\t\tlet from = start;\n\t\tlet to = end; // exclusive\n\t\twhile (from < to && lines[from].trim() === \"\") from++;\n\t\twhile (to > from && lines[to - 1].trim() === \"\") to--;\n\t\tlet text = lines.slice(from, to).join(\"\\n\").trim();\n\t\tif (text.length > CHUNK_MAX_CHARS) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, CHUNK_MAX_CHARS);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: from + 1,\n\t\t\t\tendLine: to,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/core/embsearch/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,4EAA4E;AAC5E,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAapC,yDAAyD;AACzD,SAAS,WAAW,CAAC,OAAe,EAAW;IAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAAA,CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,OAAe,EAAE,OAAe,EAAE,QAAQ,GAAW,eAAe,EAAW;IACxG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,WAAW,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU;IACzB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,YAAY;QAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,GAAG,KAAK,GAAG,WAAW,EAAE,CAAC;YACxD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YACtC,IAAI,KAAK,GAAG,OAAO,GAAG,QAAQ,IAAI,GAAG,GAAG,KAAK;gBAAE,MAAM;YACrD,KAAK,IAAI,OAAO,CAAC;YACjB,GAAG,EAAE,CAAC;QACP,CAAC;QACD,yEAAyE;QACzE,uEAAuE;QACvE,yEAAuE;QACvE,sEAAsE;QACtE,wEAAwE;QACxE,qBAAqB;QACrB,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC,YAAY;QAC1B,OAAO,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,IAAI,EAAE,CAAC;QACtD,OAAO,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,EAAE,EAAE,CAAC;QACtD,IAAI,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC5B,kEAAkE;YAClE,iEAAiE;YACjE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,OAAO,IAAI,KAAK,EAAE;gBACzB,IAAI;gBACJ,SAAS,EAAE,IAAI,GAAG,CAAC;gBACnB,OAAO,EAAE,EAAE;aACX,CAAC,CAAC;YACH,KAAK,EAAE,CAAC;QACT,CAAC;QACD,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM;QAC/B,uDAAuD;QACvD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,mBAAmB,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd","sourcesContent":["/**\n * File chunking for semantic indexing.\n *\n * Splits a file into overlapping line windows, each capped by characters so a\n * chunk stays within the embedding model's effective token window (MiniLM\n * truncates around 256 tokens ≈ 1000 chars). Chunk ids are `relpath#index`;\n * the id → line-range mapping is kept in the sidecar metadata (index-meta.ts)\n * so search hits can be rendered as `path:start-end`.\n *\n * Bump CHUNKER_VERSION when the strategy changes — a version mismatch in the\n * sidecar triggers a clean rebuild of the store.\n */\n\nexport const CHUNKER_VERSION = 2;\n\n/**\n * Target lines per chunk.\n *\n * Halving this (30 lines / 500 chars) was measured and rejected. The theory\n * was dilution — a four-line answer sharing one vector with five neighbouring\n * methods — but sharper chunks cost reach: Recall@50 fell 79% -> 69% on\n * `auto +rr` and the boundary class went from 2 of 4 findable to 0 of 4.\n * A fixed top-k over smaller chunks retrieves less *content*: 50 chunks at\n * ~460 chars sees half the corpus that 50 at ~930 does. Recall@1 ticked up\n * (53% -> 55%), which is the sharpening, and it did not pay for the loss.\n */\nconst CHUNK_LINES = 60;\n/** Overlapping lines between consecutive chunks, for context continuity. */\nconst CHUNK_OVERLAP_LINES = 10;\n/**\n * Hard character cap per chunk.\n *\n * The \"~256 tokens ≈ 1000 chars\" this was set from does not hold for code:\n * measured against the bundled tokenizer over this repo, 1000 chars is **313\n * tokens** at the median, so MiniLM (256) truncates 85.8% of chunks and drops\n * 20.3% of the corpus's tokens, while bge-small (512) drops 0.1%. Raising this\n * is therefore not free in the way the old comment implied — it spends a\n * budget that is already overdrawn on one model and nearly full on the other.\n */\nexport const CHUNK_MAX_CHARS = 1000;\n\nexport interface Chunk {\n\t/** `relpath#index` — the id stored in the vector index. */\n\tid: string;\n\t/** Text sent to the embedder. */\n\ttext: string;\n\t/** 1-based inclusive start line. */\n\tstartLine: number;\n\t/** 1-based inclusive end line. */\n\tendLine: number;\n}\n\n/** Heuristic binary sniff: NUL byte in the first 8KB. */\nfunction looksBinary(content: string): boolean {\n\tconst probe = content.slice(0, 8192);\n\treturn probe.includes(\"\\u0000\");\n}\n\n/**\n * Split `content` into chunks. `relPath` becomes the id prefix. Returns an\n * empty array for empty or binary-looking content.\n *\n * `maxChars` overrides {@link CHUNK_MAX_CHARS} for eval arms that sweep the\n * window. Production never passes it; a caller that does is changing what the\n * index contains and owes the store a distinct key, since nothing about a\n * stored vector records the cap it was built under.\n */\nexport function chunkFile(relPath: string, content: string, maxChars: number = CHUNK_MAX_CHARS): Chunk[] {\n\tif (!content.trim() || looksBinary(content)) return [];\n\tconst lines = content.replace(/\\r\\n/g, \"\\n\").replace(/\\r/g, \"\\n\").split(\"\\n\");\n\tconst chunks: Chunk[] = [];\n\tlet start = 0; // 0-based\n\tlet index = 0;\n\n\twhile (start < lines.length) {\n\t\tlet end = start; // exclusive\n\t\tlet chars = 0;\n\t\twhile (end < lines.length && end - start < CHUNK_LINES) {\n\t\t\tconst lineLen = lines[end].length + 1;\n\t\t\tif (chars + lineLen > maxChars && end > start) break;\n\t\t\tchars += lineLen;\n\t\t\tend++;\n\t\t}\n\t\t// Trim whole blank lines off both ends, and narrow the recorded range to\n\t\t// match. Trimming the joined string instead left `startLine`/`endLine`\n\t\t// claiming lines whose text was never embedded — 31% of chunks in this\n\t\t// repo, up to 3 lines each. Those lines were unfindable by the vector\n\t\t// leg while still being handed to the context assembler as if they were\n\t\t// part of the chunk.\n\t\tlet from = start;\n\t\tlet to = end; // exclusive\n\t\twhile (from < to && lines[from].trim() === \"\") from++;\n\t\twhile (to > from && lines[to - 1].trim() === \"\") to--;\n\t\tlet text = lines.slice(from, to).join(\"\\n\").trim();\n\t\tif (text.length > maxChars) {\n\t\t\t// Oversized chunk (e.g. long minified line): keep the prefix. The\n\t\t\t// underlying model would truncate anyway, so this stays bounded.\n\t\t\ttext = text.slice(0, maxChars);\n\t\t}\n\t\tif (text) {\n\t\t\tchunks.push({\n\t\t\t\tid: `${relPath}#${index}`,\n\t\t\t\ttext,\n\t\t\t\tstartLine: from + 1,\n\t\t\t\tendLine: to,\n\t\t\t});\n\t\t\tindex++;\n\t\t}\n\t\tif (end >= lines.length) break;\n\t\t// Step forward with overlap, but always make progress.\n\t\tstart = Math.max(end - CHUNK_OVERLAP_LINES, start + 1);\n\t}\n\treturn chunks;\n}\n"]}

package/dist/core/embsearch/client.d.ts

@@ -57,6 +57,15 @@ export interface EmbSearchClientOptions {
     binaryPath: string;
     /** Store directory passed as `--path`. */
     storePath: string;
+    /**
+     * Model directory passed as `--model` (onnx builds only): a dir holding
+     * `model.onnx`, `tokenizer.json` and `model.json`.
+     *
+     * Omitted, the daemon uses the model bundled into the binary. Set, the
+     * binary no longer determines which model produced a vector — which is why
+     * the eval harness records `info.model_id` rather than the binary version.
+     */
+    modelDir?: string;
     /** Metric for a freshly created store. Default: "cosine". */
     metric?: "cosine" | "dot" | "euclidean";
 }

package/dist/core/embsearch/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,oDAAoD;AACpD,MAAM,WAAW,sBAAsB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACb;AAED;wDACwD;AACxD,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,wEAAwE;AACxE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7D,MAAM,WAAW,sBAAsB;IACtC,6EAA6E;IAC7E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,CAAC;CACxC;AA4BD,qBAAa,eAAe;IAC3B,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAgB;IAEpC,YAAY,IAAI,EAAE,sBAAsB,EA+BvC;IAED,6DAA6D;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,OAAO,CAAC,QAAQ;IAqBhB,OAAO,CAAC,IAAI;IAQZ;;;;;;;;;;;;OAYG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAK,EAAE,SAAS,GAAE,eAAyB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAKlG;IAED;;;;;;;;OAQG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAG3G;IAED;;;;OAIG;IACG,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAGnF;IAED,qEAAqE;IAC/D,IAAI,IAAI,OAAO,CAAC,mBAAmB,CAAC,CASzC;IAED,yDAAyD;IACnD,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGzC;IAED,uDAAuD;IACjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;IAED,gDAAgD;IAC1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAED,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAI3B;CACD","sourcesContent":["/**\n * Client for the `embsearch` stdio daemon (vendored from\n * github.com/kolisachint/embeddingsearchtools ts/client.ts).\n *\n * Spawns `embsearch serve` once and talks newline-delimited JSON over its\n * stdin/stdout. The process stays alive so the model and index load a single\n * time; every query after startup is hot.\n */\n\nimport { type ChildProcessWithoutNullStreams, spawn } from \"child_process\";\n\nexport interface EmbSearchResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchDaemonInfo {\n\tmodelId: string;\n\tdim: number;\n\tcount: number;\n\t/**\n\t * Whether the open store carries a BM25 index alongside its vectors.\n\t *\n\t * Fixed when the store is created — passing `--hybrid` at a non-hybrid\n\t * store only warns — so this is the only reliable way to find out, and the\n\t * signal that an existing store has to be rebuilt rather than reused.\n\t */\n\thybrid?: boolean;\n\t/**\n\t * Whether this daemon can actually serve `rerank`, or `undefined` from a\n\t * daemon too old to say.\n\t *\n\t * Reported separately from the version because the two came apart:\n\t * released binaries carry the `rerank` op but no longer bundle the ~23 MB\n\t * cross-encoder weights, so a version check alone would advertise a\n\t * reranker that fails on first call.\n\t */\n\trerank?: boolean;\n}\n\n/** One candidate sent for cross-encoder scoring. */\nexport interface EmbSearchRerankPassage {\n\tid: string;\n\ttext: string;\n}\n\n/** A cross-encoder relevance logit. Higher is more relevant, but the scale is\n *  unnormalized and comparable only within one call. */\nexport interface EmbSearchRerankResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchBulkResult {\n\tinserted: number;\n\tupdated: number;\n}\n\n/** Which daemon-side retriever answers a query (embsearch >= 0.2.0). */\nexport type DaemonRetriever = \"dense\" | \"lexical\" | \"hybrid\";\n\nexport interface EmbSearchClientOptions {\n\t/** Open/create the store with a BM25 lexical index alongside the vectors. */\n\thybrid?: boolean;\n\t/** Path to the `embsearch` binary. */\n\tbinaryPath: string;\n\t/** Store directory passed as `--path`. */\n\tstorePath: string;\n\t/** Metric for a freshly created store. Default: \"cosine\". */\n\tmetric?: \"cosine\" | \"dot\" | \"euclidean\";\n}\n\ninterface Pending {\n\tresolve: (value: EmbSearchRawResponse) => void;\n\treject: (err: Error) => void;\n}\n\ninterface EmbSearchRawResponse {\n\tok: boolean;\n\terror?: string;\n\tresults?: EmbSearchResult[];\n\tinserted?: boolean;\n\tremoved?: boolean;\n\tcount?: number;\n\tinserted_count?: number;\n\tupdated_count?: number;\n\tmodel_id?: string;\n\tdim?: number;\n\t/** Cross-encoder scores. Separate from `results` because the daemon's\n\t *  logits are not comparable to retrieval scores. */\n\treranked?: EmbSearchRerankResult[];\n\t/** `info` only: whether the open store has a BM25 index. */\n\thybrid?: boolean;\n\t/** `info` only: whether `rerank` will work. Absent on daemons too old to\n\t *  report it. */\n\trerank?: boolean;\n}\n\nexport class EmbSearchClient {\n\tprivate proc: ChildProcessWithoutNullStreams;\n\tprivate queue: Pending[] = [];\n\tprivate buffer = \"\";\n\tprivate closed = false;\n\tprivate readyPromise: Promise<void>;\n\n\tconstructor(opts: EmbSearchClientOptions) {\n\t\tconst args = [\"serve\", \"--path\", opts.storePath];\n\t\tif (opts.metric) args.push(\"--metric\", opts.metric);\n\t\t// Hybrid-ness is fixed when a store is created: passing --hybrid against\n\t\t// an existing non-hybrid store warns and is ignored daemon-side, so a\n\t\t// hybrid store needs its own directory.\n\t\tif (opts.hybrid) args.push(\"--hybrid\");\n\n\t\tthis.proc = spawn(opts.binaryPath, args, { stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\n\t\tthis.proc.stdout.setEncoding(\"utf8\");\n\t\tthis.proc.stdout.on(\"data\", (chunk: string) => this.onStdout(chunk));\n\t\t// Swallow the informational stderr banner; errors surface via responses/exit.\n\t\tthis.proc.stderr.resume();\n\n\t\t// Readiness = the daemon answering a ping (probes the actual request loop).\n\t\tthis.readyPromise = this.send({ op: \"ping\" }).then(() => undefined);\n\t\t// Spawn failures reject the pending ping via the exit handler; mark the\n\t\t// promise handled so a failed spawn doesn't raise an unhandled rejection\n\t\t// before ready() is awaited.\n\t\tthis.readyPromise.catch(() => {});\n\n\t\tthis.proc.on(\"error\", (err) => {\n\t\t\tthis.closed = true;\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t\tthis.proc.on(\"exit\", (code) => {\n\t\t\tthis.closed = true;\n\t\t\tconst err = new Error(`embsearch daemon exited (code ${code})`);\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t}\n\n\t/** Resolves once the daemon has loaded the model + index. */\n\tready(): Promise<void> {\n\t\treturn this.readyPromise;\n\t}\n\n\tget isClosed(): boolean {\n\t\treturn this.closed;\n\t}\n\n\tprivate onStdout(chunk: string): void {\n\t\tthis.buffer += chunk;\n\t\t// Each response is one line; dispatch FIFO against the pending queue.\n\t\tfor (let nl = this.buffer.indexOf(\"\\n\"); nl !== -1; nl = this.buffer.indexOf(\"\\n\")) {\n\t\t\tconst line = this.buffer.slice(0, nl).trim();\n\t\t\tthis.buffer = this.buffer.slice(nl + 1);\n\t\t\tif (!line) continue;\n\t\t\tconst pending = this.queue.shift();\n\t\t\tif (!pending) continue;\n\t\t\tlet msg: EmbSearchRawResponse;\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line) as EmbSearchRawResponse;\n\t\t\t} catch {\n\t\t\t\tpending.reject(new Error(`bad response: ${line}`));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (msg.ok) pending.resolve(msg);\n\t\t\telse pending.reject(new Error(msg.error ?? \"unknown error\"));\n\t\t}\n\t}\n\n\tprivate send(req: Record<string, unknown>): Promise<EmbSearchRawResponse> {\n\t\tif (this.closed) return Promise.reject(new Error(\"embsearch client is closed\"));\n\t\treturn new Promise<EmbSearchRawResponse>((resolve, reject) => {\n\t\t\tthis.queue.push({ resolve, reject });\n\t\t\tthis.proc.stdin.write(`${JSON.stringify(req)}\\n`);\n\t\t});\n\t}\n\n\t/**\n\t * Search for the top-`k` matches for `text`.\n\t *\n\t * `retriever` selects which leg answers:\n\t *  - `dense` (default) — vector search, the historical behaviour;\n\t *  - `lexical` — BM25 only, raw scores, no embedding computed;\n\t *  - `hybrid` — both, pre-fused by the daemon's own RRF constant.\n\t *\n\t * `lexical` and `hybrid` need a store created with `--hybrid`. Prefer\n\t * `lexical` over `hybrid` when fusing here: `hybrid` collapses both legs\n\t * into one RRF score, discarding the per-retriever ranks the trace records\n\t * and preventing n-way fusion with the grep leg.\n\t */\n\tasync query(text: string, k = 10, retriever: DaemonRetriever = \"dense\"): Promise<EmbSearchResult[]> {\n\t\tconst res = await this.send(\n\t\t\tretriever === \"dense\" ? { op: \"query\", text, k } : { op: \"query\", text, k, retriever },\n\t\t);\n\t\treturn res.results ?? [];\n\t}\n\n\t/**\n\t * Score `passages` against `query` with the daemon's cross-encoder and\n\t * return the best `k`, best first.\n\t *\n\t * Passages are sent inline rather than referenced by id: the caller has the\n\t * exact spans it intends to show the model, and a cross-encoder scores the\n\t * text it is given, so sending anything else would score the wrong thing.\n\t * Requires an onnx build with reranker weights (embsearch >= 0.3.0).\n\t */\n\tasync rerank(query: string, passages: EmbSearchRerankPassage[], k: number): Promise<EmbSearchRerankResult[]> {\n\t\tconst res = await this.send({ op: \"rerank\", query, passages, k });\n\t\treturn res.reranked ?? [];\n\t}\n\n\t/**\n\t * Batched insert-or-replace. One embedding inference for the whole batch —\n\t * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a\n\t * concurrent query is not stuck behind a huge inference.\n\t */\n\tasync bulk(items: Array<{ id: string; text: string }>): Promise<EmbSearchBulkResult> {\n\t\tconst res = await this.send({ op: \"bulk\", items });\n\t\treturn { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };\n\t}\n\n\t/** Model id, dimensionality, and live vector count of the daemon. */\n\tasync info(): Promise<EmbSearchDaemonInfo> {\n\t\tconst res = await this.send({ op: \"info\" });\n\t\treturn {\n\t\t\tmodelId: res.model_id ?? \"\",\n\t\t\tdim: res.dim ?? 0,\n\t\t\tcount: res.count ?? 0,\n\t\t\thybrid: res.hybrid,\n\t\t\trerank: res.rerank,\n\t\t};\n\t}\n\n\t/** Remove a record. Resolves to `true` if it existed. */\n\tasync remove(id: string): Promise<boolean> {\n\t\tconst res = await this.send({ op: \"remove\", id });\n\t\treturn res.removed === true;\n\t}\n\n\t/** Reclaim tombstoned rows left behind by `remove`. */\n\tasync compact(): Promise<void> {\n\t\tawait this.send({ op: \"compact\" });\n\t}\n\n\t/** Persist the index to the store directory. */\n\tasync save(): Promise<void> {\n\t\tawait this.send({ op: \"save\" });\n\t}\n\n\t/** Shut the daemon down, closing stdin so it exits cleanly. */\n\tasync close(): Promise<void> {\n\t\tif (this.closed) return;\n\t\tthis.proc.stdin.end();\n\t\tawait new Promise<void>((resolve) => this.proc.on(\"exit\", () => resolve()));\n\t}\n}\n"]}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../src/core/embsearch/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;OAQG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,oDAAoD;AACpD,MAAM,WAAW,sBAAsB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACb;AAED;wDACwD;AACxD,MAAM,WAAW,qBAAqB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,wEAAwE;AACxE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7D,MAAM,WAAW,sBAAsB;IACtC,6EAA6E;IAC7E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,CAAC;CACxC;AA4BD,qBAAa,eAAe;IAC3B,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAgB;IAEpC,YAAY,IAAI,EAAE,sBAAsB,EAgCvC;IAED,6DAA6D;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,OAAO,CAAC,QAAQ;IAqBhB,OAAO,CAAC,IAAI;IAQZ;;;;;;;;;;;;OAYG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,SAAK,EAAE,SAAS,GAAE,eAAyB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAKlG;IAED;;;;;;;;OAQG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,sBAAsB,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAG3G;IAED;;;;OAIG;IACG,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAGnF;IAED,qEAAqE;IAC/D,IAAI,IAAI,OAAO,CAAC,mBAAmB,CAAC,CASzC;IAED,yDAAyD;IACnD,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAGzC;IAED,uDAAuD;IACjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;IAED,gDAAgD;IAC1C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAED,+DAA+D;IACzD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAI3B;CACD","sourcesContent":["/**\n * Client for the `embsearch` stdio daemon (vendored from\n * github.com/kolisachint/embeddingsearchtools ts/client.ts).\n *\n * Spawns `embsearch serve` once and talks newline-delimited JSON over its\n * stdin/stdout. The process stays alive so the model and index load a single\n * time; every query after startup is hot.\n */\n\nimport { type ChildProcessWithoutNullStreams, spawn } from \"child_process\";\n\nexport interface EmbSearchResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchDaemonInfo {\n\tmodelId: string;\n\tdim: number;\n\tcount: number;\n\t/**\n\t * Whether the open store carries a BM25 index alongside its vectors.\n\t *\n\t * Fixed when the store is created — passing `--hybrid` at a non-hybrid\n\t * store only warns — so this is the only reliable way to find out, and the\n\t * signal that an existing store has to be rebuilt rather than reused.\n\t */\n\thybrid?: boolean;\n\t/**\n\t * Whether this daemon can actually serve `rerank`, or `undefined` from a\n\t * daemon too old to say.\n\t *\n\t * Reported separately from the version because the two came apart:\n\t * released binaries carry the `rerank` op but no longer bundle the ~23 MB\n\t * cross-encoder weights, so a version check alone would advertise a\n\t * reranker that fails on first call.\n\t */\n\trerank?: boolean;\n}\n\n/** One candidate sent for cross-encoder scoring. */\nexport interface EmbSearchRerankPassage {\n\tid: string;\n\ttext: string;\n}\n\n/** A cross-encoder relevance logit. Higher is more relevant, but the scale is\n *  unnormalized and comparable only within one call. */\nexport interface EmbSearchRerankResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchBulkResult {\n\tinserted: number;\n\tupdated: number;\n}\n\n/** Which daemon-side retriever answers a query (embsearch >= 0.2.0). */\nexport type DaemonRetriever = \"dense\" | \"lexical\" | \"hybrid\";\n\nexport interface EmbSearchClientOptions {\n\t/** Open/create the store with a BM25 lexical index alongside the vectors. */\n\thybrid?: boolean;\n\t/** Path to the `embsearch` binary. */\n\tbinaryPath: string;\n\t/** Store directory passed as `--path`. */\n\tstorePath: string;\n\t/**\n\t * Model directory passed as `--model` (onnx builds only): a dir holding\n\t * `model.onnx`, `tokenizer.json` and `model.json`.\n\t *\n\t * Omitted, the daemon uses the model bundled into the binary. Set, the\n\t * binary no longer determines which model produced a vector — which is why\n\t * the eval harness records `info.model_id` rather than the binary version.\n\t */\n\tmodelDir?: string;\n\t/** Metric for a freshly created store. Default: \"cosine\". */\n\tmetric?: \"cosine\" | \"dot\" | \"euclidean\";\n}\n\ninterface Pending {\n\tresolve: (value: EmbSearchRawResponse) => void;\n\treject: (err: Error) => void;\n}\n\ninterface EmbSearchRawResponse {\n\tok: boolean;\n\terror?: string;\n\tresults?: EmbSearchResult[];\n\tinserted?: boolean;\n\tremoved?: boolean;\n\tcount?: number;\n\tinserted_count?: number;\n\tupdated_count?: number;\n\tmodel_id?: string;\n\tdim?: number;\n\t/** Cross-encoder scores. Separate from `results` because the daemon's\n\t *  logits are not comparable to retrieval scores. */\n\treranked?: EmbSearchRerankResult[];\n\t/** `info` only: whether the open store has a BM25 index. */\n\thybrid?: boolean;\n\t/** `info` only: whether `rerank` will work. Absent on daemons too old to\n\t *  report it. */\n\trerank?: boolean;\n}\n\nexport class EmbSearchClient {\n\tprivate proc: ChildProcessWithoutNullStreams;\n\tprivate queue: Pending[] = [];\n\tprivate buffer = \"\";\n\tprivate closed = false;\n\tprivate readyPromise: Promise<void>;\n\n\tconstructor(opts: EmbSearchClientOptions) {\n\t\tconst args = [\"serve\", \"--path\", opts.storePath];\n\t\tif (opts.metric) args.push(\"--metric\", opts.metric);\n\t\tif (opts.modelDir) args.push(\"--model\", opts.modelDir);\n\t\t// Hybrid-ness is fixed when a store is created: passing --hybrid against\n\t\t// an existing non-hybrid store warns and is ignored daemon-side, so a\n\t\t// hybrid store needs its own directory.\n\t\tif (opts.hybrid) args.push(\"--hybrid\");\n\n\t\tthis.proc = spawn(opts.binaryPath, args, { stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\n\t\tthis.proc.stdout.setEncoding(\"utf8\");\n\t\tthis.proc.stdout.on(\"data\", (chunk: string) => this.onStdout(chunk));\n\t\t// Swallow the informational stderr banner; errors surface via responses/exit.\n\t\tthis.proc.stderr.resume();\n\n\t\t// Readiness = the daemon answering a ping (probes the actual request loop).\n\t\tthis.readyPromise = this.send({ op: \"ping\" }).then(() => undefined);\n\t\t// Spawn failures reject the pending ping via the exit handler; mark the\n\t\t// promise handled so a failed spawn doesn't raise an unhandled rejection\n\t\t// before ready() is awaited.\n\t\tthis.readyPromise.catch(() => {});\n\n\t\tthis.proc.on(\"error\", (err) => {\n\t\t\tthis.closed = true;\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t\tthis.proc.on(\"exit\", (code) => {\n\t\t\tthis.closed = true;\n\t\t\tconst err = new Error(`embsearch daemon exited (code ${code})`);\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t}\n\n\t/** Resolves once the daemon has loaded the model + index. */\n\tready(): Promise<void> {\n\t\treturn this.readyPromise;\n\t}\n\n\tget isClosed(): boolean {\n\t\treturn this.closed;\n\t}\n\n\tprivate onStdout(chunk: string): void {\n\t\tthis.buffer += chunk;\n\t\t// Each response is one line; dispatch FIFO against the pending queue.\n\t\tfor (let nl = this.buffer.indexOf(\"\\n\"); nl !== -1; nl = this.buffer.indexOf(\"\\n\")) {\n\t\t\tconst line = this.buffer.slice(0, nl).trim();\n\t\t\tthis.buffer = this.buffer.slice(nl + 1);\n\t\t\tif (!line) continue;\n\t\t\tconst pending = this.queue.shift();\n\t\t\tif (!pending) continue;\n\t\t\tlet msg: EmbSearchRawResponse;\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line) as EmbSearchRawResponse;\n\t\t\t} catch {\n\t\t\t\tpending.reject(new Error(`bad response: ${line}`));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (msg.ok) pending.resolve(msg);\n\t\t\telse pending.reject(new Error(msg.error ?? \"unknown error\"));\n\t\t}\n\t}\n\n\tprivate send(req: Record<string, unknown>): Promise<EmbSearchRawResponse> {\n\t\tif (this.closed) return Promise.reject(new Error(\"embsearch client is closed\"));\n\t\treturn new Promise<EmbSearchRawResponse>((resolve, reject) => {\n\t\t\tthis.queue.push({ resolve, reject });\n\t\t\tthis.proc.stdin.write(`${JSON.stringify(req)}\\n`);\n\t\t});\n\t}\n\n\t/**\n\t * Search for the top-`k` matches for `text`.\n\t *\n\t * `retriever` selects which leg answers:\n\t *  - `dense` (default) — vector search, the historical behaviour;\n\t *  - `lexical` — BM25 only, raw scores, no embedding computed;\n\t *  - `hybrid` — both, pre-fused by the daemon's own RRF constant.\n\t *\n\t * `lexical` and `hybrid` need a store created with `--hybrid`. Prefer\n\t * `lexical` over `hybrid` when fusing here: `hybrid` collapses both legs\n\t * into one RRF score, discarding the per-retriever ranks the trace records\n\t * and preventing n-way fusion with the grep leg.\n\t */\n\tasync query(text: string, k = 10, retriever: DaemonRetriever = \"dense\"): Promise<EmbSearchResult[]> {\n\t\tconst res = await this.send(\n\t\t\tretriever === \"dense\" ? { op: \"query\", text, k } : { op: \"query\", text, k, retriever },\n\t\t);\n\t\treturn res.results ?? [];\n\t}\n\n\t/**\n\t * Score `passages` against `query` with the daemon's cross-encoder and\n\t * return the best `k`, best first.\n\t *\n\t * Passages are sent inline rather than referenced by id: the caller has the\n\t * exact spans it intends to show the model, and a cross-encoder scores the\n\t * text it is given, so sending anything else would score the wrong thing.\n\t * Requires an onnx build with reranker weights (embsearch >= 0.3.0).\n\t */\n\tasync rerank(query: string, passages: EmbSearchRerankPassage[], k: number): Promise<EmbSearchRerankResult[]> {\n\t\tconst res = await this.send({ op: \"rerank\", query, passages, k });\n\t\treturn res.reranked ?? [];\n\t}\n\n\t/**\n\t * Batched insert-or-replace. One embedding inference for the whole batch —\n\t * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a\n\t * concurrent query is not stuck behind a huge inference.\n\t */\n\tasync bulk(items: Array<{ id: string; text: string }>): Promise<EmbSearchBulkResult> {\n\t\tconst res = await this.send({ op: \"bulk\", items });\n\t\treturn { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };\n\t}\n\n\t/** Model id, dimensionality, and live vector count of the daemon. */\n\tasync info(): Promise<EmbSearchDaemonInfo> {\n\t\tconst res = await this.send({ op: \"info\" });\n\t\treturn {\n\t\t\tmodelId: res.model_id ?? \"\",\n\t\t\tdim: res.dim ?? 0,\n\t\t\tcount: res.count ?? 0,\n\t\t\thybrid: res.hybrid,\n\t\t\trerank: res.rerank,\n\t\t};\n\t}\n\n\t/** Remove a record. Resolves to `true` if it existed. */\n\tasync remove(id: string): Promise<boolean> {\n\t\tconst res = await this.send({ op: \"remove\", id });\n\t\treturn res.removed === true;\n\t}\n\n\t/** Reclaim tombstoned rows left behind by `remove`. */\n\tasync compact(): Promise<void> {\n\t\tawait this.send({ op: \"compact\" });\n\t}\n\n\t/** Persist the index to the store directory. */\n\tasync save(): Promise<void> {\n\t\tawait this.send({ op: \"save\" });\n\t}\n\n\t/** Shut the daemon down, closing stdin so it exits cleanly. */\n\tasync close(): Promise<void> {\n\t\tif (this.closed) return;\n\t\tthis.proc.stdin.end();\n\t\tawait new Promise<void>((resolve) => this.proc.on(\"exit\", () => resolve()));\n\t}\n}\n"]}

package/dist/core/embsearch/client.js

@@ -17,6 +17,8 @@ export class EmbSearchClient {
         const args = ["serve", "--path", opts.storePath];
         if (opts.metric)
             args.push("--metric", opts.metric);
+        if (opts.modelDir)
+            args.push("--model", opts.modelDir);
         // Hybrid-ness is fixed when a store is created: passing --hybrid against
         // an existing non-hybrid store warns and is ignored daemon-side, so a
         // hybrid store needs its own directory.

package/dist/core/embsearch/client.js.map

@@ -1 +1 @@
-{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../src/core/embsearch/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAuC,KAAK,EAAE,MAAM,eAAe,CAAC;AAyF3E,MAAM,OAAO,eAAe;IACnB,IAAI,CAAiC;IACrC,KAAK,GAAc,EAAE,CAAC;IACtB,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,GAAG,KAAK,CAAC;IACf,YAAY,CAAgB;IAEpC,YAAY,IAA4B,EAAE;QACzC,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACpD,yEAAyE;QACzE,sEAAsE;QACtE,wCAAwC;QACxC,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,8EAA8E;QAC9E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAE1B,4EAA4E;QAC5E,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpE,wEAAwE;QACxE,yEAAyE;QACzE,6BAA6B;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAA,CACpD,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,iCAAiC,IAAI,GAAG,CAAC,CAAC;YAChE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAA,CACpD,CAAC,CAAC;IAAA,CACH;IAED,6DAA6D;IAC7D,KAAK,GAAkB;QACtB,OAAO,IAAI,CAAC,YAAY,CAAC;IAAA,CACzB;IAED,IAAI,QAAQ,GAAY;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC;IAAA,CACnB;IAEO,QAAQ,CAAC,KAAa,EAAQ;QACrC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACrB,sEAAsE;QACtE,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,IAAI,GAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;gBACnD,SAAS;YACV,CAAC;YACD,IAAI,GAAG,CAAC,EAAE;gBAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;;gBAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC,CAAC;QAC9D,CAAC;IAAA,CACD;IAEO,IAAI,CAAC,GAA4B,EAAiC;QACzE,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,OAAO,CAAuB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;YAC7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAAA,CAClD,CAAC,CAAC;IAAA,CACH;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,GAAoB,OAAO,EAA8B;QACnG,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CACtF,CAAC;QACF,OAAO,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAAA,CACzB;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAkC,EAAE,CAAS,EAAoC;QAC5G,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;IAAA,CAC1B;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CAAC,KAA0C,EAAgC;QACpF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,cAAc,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;IAAA,CAC9E;IAED,qEAAqE;IACrE,KAAK,CAAC,IAAI,GAAiC;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5C,OAAO;YACN,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC3B,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC;YACjB,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;YACrB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;SAClB,CAAC;IAAA,CACF;IAED,yDAAyD;IACzD,KAAK,CAAC,MAAM,CAAC,EAAU,EAAoB;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC;IAAA,CAC5B;IAED,uDAAuD;IACvD,KAAK,CAAC,OAAO,GAAkB;QAC9B,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAAA,CACnC;IAED,gDAAgD;IAChD,KAAK,CAAC,IAAI,GAAkB;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAAA,CAChC;IAED,+DAA+D;IAC/D,KAAK,CAAC,KAAK,GAAkB;QAC5B,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAAA,CAC5E;CACD","sourcesContent":["/**\n * Client for the `embsearch` stdio daemon (vendored from\n * github.com/kolisachint/embeddingsearchtools ts/client.ts).\n *\n * Spawns `embsearch serve` once and talks newline-delimited JSON over its\n * stdin/stdout. The process stays alive so the model and index load a single\n * time; every query after startup is hot.\n */\n\nimport { type ChildProcessWithoutNullStreams, spawn } from \"child_process\";\n\nexport interface EmbSearchResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchDaemonInfo {\n\tmodelId: string;\n\tdim: number;\n\tcount: number;\n\t/**\n\t * Whether the open store carries a BM25 index alongside its vectors.\n\t *\n\t * Fixed when the store is created — passing `--hybrid` at a non-hybrid\n\t * store only warns — so this is the only reliable way to find out, and the\n\t * signal that an existing store has to be rebuilt rather than reused.\n\t */\n\thybrid?: boolean;\n\t/**\n\t * Whether this daemon can actually serve `rerank`, or `undefined` from a\n\t * daemon too old to say.\n\t *\n\t * Reported separately from the version because the two came apart:\n\t * released binaries carry the `rerank` op but no longer bundle the ~23 MB\n\t * cross-encoder weights, so a version check alone would advertise a\n\t * reranker that fails on first call.\n\t */\n\trerank?: boolean;\n}\n\n/** One candidate sent for cross-encoder scoring. */\nexport interface EmbSearchRerankPassage {\n\tid: string;\n\ttext: string;\n}\n\n/** A cross-encoder relevance logit. Higher is more relevant, but the scale is\n *  unnormalized and comparable only within one call. */\nexport interface EmbSearchRerankResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchBulkResult {\n\tinserted: number;\n\tupdated: number;\n}\n\n/** Which daemon-side retriever answers a query (embsearch >= 0.2.0). */\nexport type DaemonRetriever = \"dense\" | \"lexical\" | \"hybrid\";\n\nexport interface EmbSearchClientOptions {\n\t/** Open/create the store with a BM25 lexical index alongside the vectors. */\n\thybrid?: boolean;\n\t/** Path to the `embsearch` binary. */\n\tbinaryPath: string;\n\t/** Store directory passed as `--path`. */\n\tstorePath: string;\n\t/** Metric for a freshly created store. Default: \"cosine\". */\n\tmetric?: \"cosine\" | \"dot\" | \"euclidean\";\n}\n\ninterface Pending {\n\tresolve: (value: EmbSearchRawResponse) => void;\n\treject: (err: Error) => void;\n}\n\ninterface EmbSearchRawResponse {\n\tok: boolean;\n\terror?: string;\n\tresults?: EmbSearchResult[];\n\tinserted?: boolean;\n\tremoved?: boolean;\n\tcount?: number;\n\tinserted_count?: number;\n\tupdated_count?: number;\n\tmodel_id?: string;\n\tdim?: number;\n\t/** Cross-encoder scores. Separate from `results` because the daemon's\n\t *  logits are not comparable to retrieval scores. */\n\treranked?: EmbSearchRerankResult[];\n\t/** `info` only: whether the open store has a BM25 index. */\n\thybrid?: boolean;\n\t/** `info` only: whether `rerank` will work. Absent on daemons too old to\n\t *  report it. */\n\trerank?: boolean;\n}\n\nexport class EmbSearchClient {\n\tprivate proc: ChildProcessWithoutNullStreams;\n\tprivate queue: Pending[] = [];\n\tprivate buffer = \"\";\n\tprivate closed = false;\n\tprivate readyPromise: Promise<void>;\n\n\tconstructor(opts: EmbSearchClientOptions) {\n\t\tconst args = [\"serve\", \"--path\", opts.storePath];\n\t\tif (opts.metric) args.push(\"--metric\", opts.metric);\n\t\t// Hybrid-ness is fixed when a store is created: passing --hybrid against\n\t\t// an existing non-hybrid store warns and is ignored daemon-side, so a\n\t\t// hybrid store needs its own directory.\n\t\tif (opts.hybrid) args.push(\"--hybrid\");\n\n\t\tthis.proc = spawn(opts.binaryPath, args, { stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\n\t\tthis.proc.stdout.setEncoding(\"utf8\");\n\t\tthis.proc.stdout.on(\"data\", (chunk: string) => this.onStdout(chunk));\n\t\t// Swallow the informational stderr banner; errors surface via responses/exit.\n\t\tthis.proc.stderr.resume();\n\n\t\t// Readiness = the daemon answering a ping (probes the actual request loop).\n\t\tthis.readyPromise = this.send({ op: \"ping\" }).then(() => undefined);\n\t\t// Spawn failures reject the pending ping via the exit handler; mark the\n\t\t// promise handled so a failed spawn doesn't raise an unhandled rejection\n\t\t// before ready() is awaited.\n\t\tthis.readyPromise.catch(() => {});\n\n\t\tthis.proc.on(\"error\", (err) => {\n\t\t\tthis.closed = true;\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t\tthis.proc.on(\"exit\", (code) => {\n\t\t\tthis.closed = true;\n\t\t\tconst err = new Error(`embsearch daemon exited (code ${code})`);\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t}\n\n\t/** Resolves once the daemon has loaded the model + index. */\n\tready(): Promise<void> {\n\t\treturn this.readyPromise;\n\t}\n\n\tget isClosed(): boolean {\n\t\treturn this.closed;\n\t}\n\n\tprivate onStdout(chunk: string): void {\n\t\tthis.buffer += chunk;\n\t\t// Each response is one line; dispatch FIFO against the pending queue.\n\t\tfor (let nl = this.buffer.indexOf(\"\\n\"); nl !== -1; nl = this.buffer.indexOf(\"\\n\")) {\n\t\t\tconst line = this.buffer.slice(0, nl).trim();\n\t\t\tthis.buffer = this.buffer.slice(nl + 1);\n\t\t\tif (!line) continue;\n\t\t\tconst pending = this.queue.shift();\n\t\t\tif (!pending) continue;\n\t\t\tlet msg: EmbSearchRawResponse;\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line) as EmbSearchRawResponse;\n\t\t\t} catch {\n\t\t\t\tpending.reject(new Error(`bad response: ${line}`));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (msg.ok) pending.resolve(msg);\n\t\t\telse pending.reject(new Error(msg.error ?? \"unknown error\"));\n\t\t}\n\t}\n\n\tprivate send(req: Record<string, unknown>): Promise<EmbSearchRawResponse> {\n\t\tif (this.closed) return Promise.reject(new Error(\"embsearch client is closed\"));\n\t\treturn new Promise<EmbSearchRawResponse>((resolve, reject) => {\n\t\t\tthis.queue.push({ resolve, reject });\n\t\t\tthis.proc.stdin.write(`${JSON.stringify(req)}\\n`);\n\t\t});\n\t}\n\n\t/**\n\t * Search for the top-`k` matches for `text`.\n\t *\n\t * `retriever` selects which leg answers:\n\t *  - `dense` (default) — vector search, the historical behaviour;\n\t *  - `lexical` — BM25 only, raw scores, no embedding computed;\n\t *  - `hybrid` — both, pre-fused by the daemon's own RRF constant.\n\t *\n\t * `lexical` and `hybrid` need a store created with `--hybrid`. Prefer\n\t * `lexical` over `hybrid` when fusing here: `hybrid` collapses both legs\n\t * into one RRF score, discarding the per-retriever ranks the trace records\n\t * and preventing n-way fusion with the grep leg.\n\t */\n\tasync query(text: string, k = 10, retriever: DaemonRetriever = \"dense\"): Promise<EmbSearchResult[]> {\n\t\tconst res = await this.send(\n\t\t\tretriever === \"dense\" ? { op: \"query\", text, k } : { op: \"query\", text, k, retriever },\n\t\t);\n\t\treturn res.results ?? [];\n\t}\n\n\t/**\n\t * Score `passages` against `query` with the daemon's cross-encoder and\n\t * return the best `k`, best first.\n\t *\n\t * Passages are sent inline rather than referenced by id: the caller has the\n\t * exact spans it intends to show the model, and a cross-encoder scores the\n\t * text it is given, so sending anything else would score the wrong thing.\n\t * Requires an onnx build with reranker weights (embsearch >= 0.3.0).\n\t */\n\tasync rerank(query: string, passages: EmbSearchRerankPassage[], k: number): Promise<EmbSearchRerankResult[]> {\n\t\tconst res = await this.send({ op: \"rerank\", query, passages, k });\n\t\treturn res.reranked ?? [];\n\t}\n\n\t/**\n\t * Batched insert-or-replace. One embedding inference for the whole batch —\n\t * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a\n\t * concurrent query is not stuck behind a huge inference.\n\t */\n\tasync bulk(items: Array<{ id: string; text: string }>): Promise<EmbSearchBulkResult> {\n\t\tconst res = await this.send({ op: \"bulk\", items });\n\t\treturn { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };\n\t}\n\n\t/** Model id, dimensionality, and live vector count of the daemon. */\n\tasync info(): Promise<EmbSearchDaemonInfo> {\n\t\tconst res = await this.send({ op: \"info\" });\n\t\treturn {\n\t\t\tmodelId: res.model_id ?? \"\",\n\t\t\tdim: res.dim ?? 0,\n\t\t\tcount: res.count ?? 0,\n\t\t\thybrid: res.hybrid,\n\t\t\trerank: res.rerank,\n\t\t};\n\t}\n\n\t/** Remove a record. Resolves to `true` if it existed. */\n\tasync remove(id: string): Promise<boolean> {\n\t\tconst res = await this.send({ op: \"remove\", id });\n\t\treturn res.removed === true;\n\t}\n\n\t/** Reclaim tombstoned rows left behind by `remove`. */\n\tasync compact(): Promise<void> {\n\t\tawait this.send({ op: \"compact\" });\n\t}\n\n\t/** Persist the index to the store directory. */\n\tasync save(): Promise<void> {\n\t\tawait this.send({ op: \"save\" });\n\t}\n\n\t/** Shut the daemon down, closing stdin so it exits cleanly. */\n\tasync close(): Promise<void> {\n\t\tif (this.closed) return;\n\t\tthis.proc.stdin.end();\n\t\tawait new Promise<void>((resolve) => this.proc.on(\"exit\", () => resolve()));\n\t}\n}\n"]}
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../src/core/embsearch/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAuC,KAAK,EAAE,MAAM,eAAe,CAAC;AAkG3E,MAAM,OAAO,eAAe;IACnB,IAAI,CAAiC;IACrC,KAAK,GAAc,EAAE,CAAC;IACtB,MAAM,GAAG,EAAE,CAAC;IACZ,MAAM,GAAG,KAAK,CAAC;IACf,YAAY,CAAgB;IAEpC,YAAY,IAA4B,EAAE;QACzC,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvD,yEAAyE;QACzE,sEAAsE;QACtE,wCAAwC;QACxC,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,8EAA8E;QAC9E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAE1B,4EAA4E;QAC5E,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpE,wEAAwE;QACxE,yEAAyE;QACzE,6BAA6B;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAA,CACpD,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,iCAAiC,IAAI,GAAG,CAAC,CAAC;YAChE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;gBAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAA,CACpD,CAAC,CAAC;IAAA,CACH;IAED,6DAA6D;IAC7D,KAAK,GAAkB;QACtB,OAAO,IAAI,CAAC,YAAY,CAAC;IAAA,CACzB;IAED,IAAI,QAAQ,GAAY;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC;IAAA,CACnB;IAEO,QAAQ,CAAC,KAAa,EAAQ;QACrC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QACrB,sEAAsE;QACtE,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,IAAI,GAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;gBACnD,SAAS;YACV,CAAC;YACD,IAAI,GAAG,CAAC,EAAE;gBAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;;gBAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC,CAAC;QAC9D,CAAC;IAAA,CACD;IAEO,IAAI,CAAC,GAA4B,EAAiC;QACzE,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,OAAO,CAAuB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;YAC7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAAA,CAClD,CAAC,CAAC;IAAA,CACH;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,GAAoB,OAAO,EAA8B;QACnG,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CACtF,CAAC;QACF,OAAO,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAAA,CACzB;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAkC,EAAE,CAAS,EAAoC;QAC5G,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC;IAAA,CAC1B;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CAAC,KAA0C,EAAgC;QACpF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,cAAc,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;IAAA,CAC9E;IAED,qEAAqE;IACrE,KAAK,CAAC,IAAI,GAAiC;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5C,OAAO;YACN,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC3B,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC;YACjB,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;YACrB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;SAClB,CAAC;IAAA,CACF;IAED,yDAAyD;IACzD,KAAK,CAAC,MAAM,CAAC,EAAU,EAAoB;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAClD,OAAO,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC;IAAA,CAC5B;IAED,uDAAuD;IACvD,KAAK,CAAC,OAAO,GAAkB;QAC9B,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAAA,CACnC;IAED,gDAAgD;IAChD,KAAK,CAAC,IAAI,GAAkB;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAAA,CAChC;IAED,+DAA+D;IAC/D,KAAK,CAAC,KAAK,GAAkB;QAC5B,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAAA,CAC5E;CACD","sourcesContent":["/**\n * Client for the `embsearch` stdio daemon (vendored from\n * github.com/kolisachint/embeddingsearchtools ts/client.ts).\n *\n * Spawns `embsearch serve` once and talks newline-delimited JSON over its\n * stdin/stdout. The process stays alive so the model and index load a single\n * time; every query after startup is hot.\n */\n\nimport { type ChildProcessWithoutNullStreams, spawn } from \"child_process\";\n\nexport interface EmbSearchResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchDaemonInfo {\n\tmodelId: string;\n\tdim: number;\n\tcount: number;\n\t/**\n\t * Whether the open store carries a BM25 index alongside its vectors.\n\t *\n\t * Fixed when the store is created — passing `--hybrid` at a non-hybrid\n\t * store only warns — so this is the only reliable way to find out, and the\n\t * signal that an existing store has to be rebuilt rather than reused.\n\t */\n\thybrid?: boolean;\n\t/**\n\t * Whether this daemon can actually serve `rerank`, or `undefined` from a\n\t * daemon too old to say.\n\t *\n\t * Reported separately from the version because the two came apart:\n\t * released binaries carry the `rerank` op but no longer bundle the ~23 MB\n\t * cross-encoder weights, so a version check alone would advertise a\n\t * reranker that fails on first call.\n\t */\n\trerank?: boolean;\n}\n\n/** One candidate sent for cross-encoder scoring. */\nexport interface EmbSearchRerankPassage {\n\tid: string;\n\ttext: string;\n}\n\n/** A cross-encoder relevance logit. Higher is more relevant, but the scale is\n *  unnormalized and comparable only within one call. */\nexport interface EmbSearchRerankResult {\n\tid: string;\n\tscore: number;\n}\n\nexport interface EmbSearchBulkResult {\n\tinserted: number;\n\tupdated: number;\n}\n\n/** Which daemon-side retriever answers a query (embsearch >= 0.2.0). */\nexport type DaemonRetriever = \"dense\" | \"lexical\" | \"hybrid\";\n\nexport interface EmbSearchClientOptions {\n\t/** Open/create the store with a BM25 lexical index alongside the vectors. */\n\thybrid?: boolean;\n\t/** Path to the `embsearch` binary. */\n\tbinaryPath: string;\n\t/** Store directory passed as `--path`. */\n\tstorePath: string;\n\t/**\n\t * Model directory passed as `--model` (onnx builds only): a dir holding\n\t * `model.onnx`, `tokenizer.json` and `model.json`.\n\t *\n\t * Omitted, the daemon uses the model bundled into the binary. Set, the\n\t * binary no longer determines which model produced a vector — which is why\n\t * the eval harness records `info.model_id` rather than the binary version.\n\t */\n\tmodelDir?: string;\n\t/** Metric for a freshly created store. Default: \"cosine\". */\n\tmetric?: \"cosine\" | \"dot\" | \"euclidean\";\n}\n\ninterface Pending {\n\tresolve: (value: EmbSearchRawResponse) => void;\n\treject: (err: Error) => void;\n}\n\ninterface EmbSearchRawResponse {\n\tok: boolean;\n\terror?: string;\n\tresults?: EmbSearchResult[];\n\tinserted?: boolean;\n\tremoved?: boolean;\n\tcount?: number;\n\tinserted_count?: number;\n\tupdated_count?: number;\n\tmodel_id?: string;\n\tdim?: number;\n\t/** Cross-encoder scores. Separate from `results` because the daemon's\n\t *  logits are not comparable to retrieval scores. */\n\treranked?: EmbSearchRerankResult[];\n\t/** `info` only: whether the open store has a BM25 index. */\n\thybrid?: boolean;\n\t/** `info` only: whether `rerank` will work. Absent on daemons too old to\n\t *  report it. */\n\trerank?: boolean;\n}\n\nexport class EmbSearchClient {\n\tprivate proc: ChildProcessWithoutNullStreams;\n\tprivate queue: Pending[] = [];\n\tprivate buffer = \"\";\n\tprivate closed = false;\n\tprivate readyPromise: Promise<void>;\n\n\tconstructor(opts: EmbSearchClientOptions) {\n\t\tconst args = [\"serve\", \"--path\", opts.storePath];\n\t\tif (opts.metric) args.push(\"--metric\", opts.metric);\n\t\tif (opts.modelDir) args.push(\"--model\", opts.modelDir);\n\t\t// Hybrid-ness is fixed when a store is created: passing --hybrid against\n\t\t// an existing non-hybrid store warns and is ignored daemon-side, so a\n\t\t// hybrid store needs its own directory.\n\t\tif (opts.hybrid) args.push(\"--hybrid\");\n\n\t\tthis.proc = spawn(opts.binaryPath, args, { stdio: [\"pipe\", \"pipe\", \"pipe\"] });\n\n\t\tthis.proc.stdout.setEncoding(\"utf8\");\n\t\tthis.proc.stdout.on(\"data\", (chunk: string) => this.onStdout(chunk));\n\t\t// Swallow the informational stderr banner; errors surface via responses/exit.\n\t\tthis.proc.stderr.resume();\n\n\t\t// Readiness = the daemon answering a ping (probes the actual request loop).\n\t\tthis.readyPromise = this.send({ op: \"ping\" }).then(() => undefined);\n\t\t// Spawn failures reject the pending ping via the exit handler; mark the\n\t\t// promise handled so a failed spawn doesn't raise an unhandled rejection\n\t\t// before ready() is awaited.\n\t\tthis.readyPromise.catch(() => {});\n\n\t\tthis.proc.on(\"error\", (err) => {\n\t\t\tthis.closed = true;\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t\tthis.proc.on(\"exit\", (code) => {\n\t\t\tthis.closed = true;\n\t\t\tconst err = new Error(`embsearch daemon exited (code ${code})`);\n\t\t\tfor (const p of this.queue.splice(0)) p.reject(err);\n\t\t});\n\t}\n\n\t/** Resolves once the daemon has loaded the model + index. */\n\tready(): Promise<void> {\n\t\treturn this.readyPromise;\n\t}\n\n\tget isClosed(): boolean {\n\t\treturn this.closed;\n\t}\n\n\tprivate onStdout(chunk: string): void {\n\t\tthis.buffer += chunk;\n\t\t// Each response is one line; dispatch FIFO against the pending queue.\n\t\tfor (let nl = this.buffer.indexOf(\"\\n\"); nl !== -1; nl = this.buffer.indexOf(\"\\n\")) {\n\t\t\tconst line = this.buffer.slice(0, nl).trim();\n\t\t\tthis.buffer = this.buffer.slice(nl + 1);\n\t\t\tif (!line) continue;\n\t\t\tconst pending = this.queue.shift();\n\t\t\tif (!pending) continue;\n\t\t\tlet msg: EmbSearchRawResponse;\n\t\t\ttry {\n\t\t\t\tmsg = JSON.parse(line) as EmbSearchRawResponse;\n\t\t\t} catch {\n\t\t\t\tpending.reject(new Error(`bad response: ${line}`));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (msg.ok) pending.resolve(msg);\n\t\t\telse pending.reject(new Error(msg.error ?? \"unknown error\"));\n\t\t}\n\t}\n\n\tprivate send(req: Record<string, unknown>): Promise<EmbSearchRawResponse> {\n\t\tif (this.closed) return Promise.reject(new Error(\"embsearch client is closed\"));\n\t\treturn new Promise<EmbSearchRawResponse>((resolve, reject) => {\n\t\t\tthis.queue.push({ resolve, reject });\n\t\t\tthis.proc.stdin.write(`${JSON.stringify(req)}\\n`);\n\t\t});\n\t}\n\n\t/**\n\t * Search for the top-`k` matches for `text`.\n\t *\n\t * `retriever` selects which leg answers:\n\t *  - `dense` (default) — vector search, the historical behaviour;\n\t *  - `lexical` — BM25 only, raw scores, no embedding computed;\n\t *  - `hybrid` — both, pre-fused by the daemon's own RRF constant.\n\t *\n\t * `lexical` and `hybrid` need a store created with `--hybrid`. Prefer\n\t * `lexical` over `hybrid` when fusing here: `hybrid` collapses both legs\n\t * into one RRF score, discarding the per-retriever ranks the trace records\n\t * and preventing n-way fusion with the grep leg.\n\t */\n\tasync query(text: string, k = 10, retriever: DaemonRetriever = \"dense\"): Promise<EmbSearchResult[]> {\n\t\tconst res = await this.send(\n\t\t\tretriever === \"dense\" ? { op: \"query\", text, k } : { op: \"query\", text, k, retriever },\n\t\t);\n\t\treturn res.results ?? [];\n\t}\n\n\t/**\n\t * Score `passages` against `query` with the daemon's cross-encoder and\n\t * return the best `k`, best first.\n\t *\n\t * Passages are sent inline rather than referenced by id: the caller has the\n\t * exact spans it intends to show the model, and a cross-encoder scores the\n\t * text it is given, so sending anything else would score the wrong thing.\n\t * Requires an onnx build with reranker weights (embsearch >= 0.3.0).\n\t */\n\tasync rerank(query: string, passages: EmbSearchRerankPassage[], k: number): Promise<EmbSearchRerankResult[]> {\n\t\tconst res = await this.send({ op: \"rerank\", query, passages, k });\n\t\treturn res.reranked ?? [];\n\t}\n\n\t/**\n\t * Batched insert-or-replace. One embedding inference for the whole batch —\n\t * the fast path for bulk indexing. Keep batches modest (e.g. 32–64) so a\n\t * concurrent query is not stuck behind a huge inference.\n\t */\n\tasync bulk(items: Array<{ id: string; text: string }>): Promise<EmbSearchBulkResult> {\n\t\tconst res = await this.send({ op: \"bulk\", items });\n\t\treturn { inserted: res.inserted_count ?? 0, updated: res.updated_count ?? 0 };\n\t}\n\n\t/** Model id, dimensionality, and live vector count of the daemon. */\n\tasync info(): Promise<EmbSearchDaemonInfo> {\n\t\tconst res = await this.send({ op: \"info\" });\n\t\treturn {\n\t\t\tmodelId: res.model_id ?? \"\",\n\t\t\tdim: res.dim ?? 0,\n\t\t\tcount: res.count ?? 0,\n\t\t\thybrid: res.hybrid,\n\t\t\trerank: res.rerank,\n\t\t};\n\t}\n\n\t/** Remove a record. Resolves to `true` if it existed. */\n\tasync remove(id: string): Promise<boolean> {\n\t\tconst res = await this.send({ op: \"remove\", id });\n\t\treturn res.removed === true;\n\t}\n\n\t/** Reclaim tombstoned rows left behind by `remove`. */\n\tasync compact(): Promise<void> {\n\t\tawait this.send({ op: \"compact\" });\n\t}\n\n\t/** Persist the index to the store directory. */\n\tasync save(): Promise<void> {\n\t\tawait this.send({ op: \"save\" });\n\t}\n\n\t/** Shut the daemon down, closing stdin so it exits cleanly. */\n\tasync close(): Promise<void> {\n\t\tif (this.closed) return;\n\t\tthis.proc.stdin.end();\n\t\tawait new Promise<void>((resolve) => this.proc.on(\"exit\", () => resolve()));\n\t}\n}\n"]}

package/dist/core/embsearch/embsearch-service.d.ts

@@ -38,6 +38,25 @@ export interface EmbsearchServiceOptions {
     cwd: string;
     /** Explicit binary path (settings override). Default: "embsearch" from PATH. */
     binaryPath?: string;
+    /**
+     * Model directory handed to the daemon as `--model`, overriding the model
+     * bundled in the binary.
+     *
+     * Only the eval harness sets this, to score two embedding models from one
+     * binary. Pair it with a distinct `storeDir`: vectors from different models
+     * are incompatible, and the daemon refuses to open a store built by another
+     * model rather than mixing them.
+     */
+    modelDir?: string;
+    /**
+     * Override the chunker's character cap.
+     *
+     * Only the eval harness sets this, to sweep the chunk window. It changes
+     * what every vector in the store *is*, and nothing stored records it, so it
+     * must be paired with a distinct `storeDir` exactly as `modelDir` is —
+     * otherwise a run silently scores an index built at another cap.
+     */
+    chunkMaxChars?: number;
     /** Minimum indexable bytes before indexing kicks in. */
     thresholdBytes: number;
     /**
@@ -83,6 +102,13 @@ export declare class EmbsearchService {
     constructor(options: EmbsearchServiceOptions);
     getState(): EmbsearchState;
     /** Semantic search is usable (index ready, or still building with partial data). */
+    /**
+     * Model id reported by the running daemon, once it is up.
+     *
+     * This — not the binary's version — identifies which model produced the
+     * vectors in the store, because `--model` decouples the two.
+     */
+    modelId(): string | undefined;
     isAvailable(): boolean;
     private setState;
     /**
@@ -122,6 +148,17 @@ export declare class EmbsearchService {
      */
     rerank(query: string, passages: EmbSearchRerankPassage[], k: number): Promise<EmbSearchRerankResult[]>;
     private probeBinaryVersion;
+    /**
+     * What the store on disk says about itself, read straight from its manifest.
+     *
+     * `store-info` exists precisely for the case where the daemon will not open
+     * the store: a `serve` pairs a store with an embedder and refuses the pair
+     * when their models disagree, so at that moment nothing else can tell us
+     * what built it. Returns undefined when there is no readable store — which
+     * includes a binary too old to have the subcommand, and so degrades to the
+     * previous behaviour rather than guessing.
+     */
+    private probeStore;
     private resolveBinary;
     /**
      * Scan, threshold-check, and (when needed) index in the background.