@maxgfr/codeindex 2.9.0 → 2.10.0

package/README.md

@@ -89,6 +89,31 @@ default; disable with `--no-fuzzy` (CLI) or `fuzzy: false` (library/MCP
 `SearchOptions.fuzzy`); results carry an additive `fuzzyTerms` field when the
 fallback contributed.
 
+### Semantic search (deterministic static-embedding tier)
+
+`codeindex search "<query>" --repo . --semantic` RRF-fuses lexical BM25 with a
+**keyless, byte-deterministic** embedding tier. It uses a *static* embedding
+model (a `token → vector` lookup table, no neural forward pass, no wasm): the
+pure-JS encoder tokenizes → mean-pools → L2-normalizes → int8-quantizes
+(round-half-to-even), and ranking is a **pure integer dot product** — so encode
+and the `embeddings.bin` artifact are byte-identical across builds and platforms.
+
+It is **opt-in by asset**: with no model on disk the engine silently stays
+lexical, and `--semantic` without a model returns lexical results on **exit 0**
+(a stderr note only). Models are **never** shipped in the package; a model is
+resolved from `CODEINDEX_EMBED_DIR` or `<repo>/.codeindex/models/`.
+
+```sh
+codeindex embed status --repo .              # is a model active? (JSON)
+codeindex embed build  --repo . --out .codeindex   # write embeddings.bin
+codeindex search "http client retry" --repo . --semantic
+```
+
+`codeindex index` also writes `embeddings.bin` next to `graph.json` when a model
+is present. Fusion reuses the engine's `rrf` helper (k=60); `SCHEMA_VERSION` is
+untouched (a dedicated `EMBED_VERSION` keys the sidecar). Full details:
+[docs/SEMANTIC.md](./docs/SEMANTIC.md).
+
 ## Use as an MCP server
 
 `codeindex mcp` (or `node scripts/cli.mjs mcp`) serves the engine over stdio —

package/docs/MIGRATION.md

@@ -64,6 +64,35 @@ frequency in the corpus, so any query where every term already matched keeps
 producing byte-identical output — **no re-pin required**, no action needed
 from existing consumers. Pass `fuzzy: false` (or CLI `--no-fuzzy`) to opt out.
 
+## v2.10.0 — deterministic static-embedding tier (opt-in)
+
+New, purely additive, and **opt-in by asset presence** — nothing changes for a
+consumer that does not place a model. `SCHEMA_VERSION` is **untouched**;
+embeddings live in a separate `embeddings.bin` sidecar keyed by a dedicated
+`EMBED_VERSION`, so `graph.json` / `symbols.json` consumers are unaffected and
+**no re-pin is required**.
+
+- Activation mirrors the grammar tier: a model asset resolved via
+  `CODEINDEX_EMBED_DIR` or `<repo>/.codeindex/models/` (`resolveEmbedModelDir`).
+  No model → the engine stays lexical, silently. Models are **never** shipped in
+  the npm tarball (`files` unchanged; pack-smoke asserts no model asset).
+- `codeindex index` writes `embeddings.bin` next to `graph.json` **only** when a
+  model is present. `codeindex embed {status,build,pull}` and
+  `codeindex search --semantic` are the CLI surface; MCP gains `embed_status`
+  and a `semantic` property on `search`.
+- Determinism is the point: the pure-JS encoder (fold+lowercase → wordpiece →
+  mean-pool → L2-norm → int8 round-half-to-even at a fixed 1/127 scale) and the
+  **integer** dot-product ranking make encode and `embeddings.bin` byte-identical
+  across builds and platforms — goldens are possible. Fusion with lexical uses
+  the existing `rrf` helper (k=60), never a linear score blend.
+- New exports: `EMBED_VERSION`, `resolveEmbedModelDir`, `hasEmbedModel`,
+  `loadEmbedModel`, `encode`, `buildEmbeddingIndex`, `serializeEmbeddings`,
+  `deserializeEmbeddings`, `searchSemantic`, plus the v2.11-preview
+  `embedViaEndpoint`. See `docs/SEMANTIC.md`.
+
+`--semantic` without a model degrades to lexical results on **exit 0** (a stderr
+note only) — so wiring it on is safe before an asset exists.
+
 ## Per-skill mapping (what to replace with what)
 
 | Skill | Replace | With (engine export) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@maxgfr/codeindex",
-  "version": "2.9.0",
+  "version": "2.10.0",
   "description": "Self-contained, deterministic repo-indexing engine: walk + language detection + symbol/import extraction (tree-sitter AST with regex fallback) + import resolution + typed cross-file link-graph + analytics. Ships as a single zero-dependency engine.mjs that consumer tools vendor.",
   "type": "module",
   "packageManager": "pnpm@10.33.0",

package/scripts/engine.d.mts

@@ -1,4 +1,4 @@
-declare const ENGINE_VERSION = "2.9.0";
+declare const ENGINE_VERSION = "2.10.0";
 declare const SCHEMA_VERSION = 4;
 declare const EXTRACTOR_VERSION = 6;
 type FileKind = "code" | "doc" | "config" | "asset" | "other";
@@ -499,6 +499,61 @@ interface SearchResult {
 declare function subtokens(raw: string): string[];
 declare function searchIndex(scan: RepoScan, query: string, opts?: SearchOptions): SearchResult[];
 
+declare const EMBED_VERSION = 1;
+interface StaticEmbedModel {
+    modelId: string;
+    dim: number;
+    unk: string;
+    unkId: number;
+    vocabSize: number;
+    vocab: Map<string, number>;
+    weights: Float64Array;
+}
+declare function resolveEmbedModelDir(repo?: string): string | undefined;
+declare function hasEmbedModel(repo?: string): boolean;
+declare function loadEmbedModel(dir?: string): StaticEmbedModel | undefined;
+declare function resolveEmbedPullUrl(): string | undefined;
+
+declare function basicTokenize(text: string): string[];
+declare function wordpiece(word: string, model: StaticEmbedModel): number[];
+declare function tokenize(text: string, model: StaticEmbedModel): number[];
+declare function roundHalfToEven(x: number): number;
+declare function encode(model: StaticEmbedModel, text: string): Int8Array;
+declare function intDot(a: Int8Array, b: Int8Array): number;
+
+interface EmbeddingRecord {
+    file: string;
+    symbol?: string;
+    line?: number;
+    vec: Int8Array;
+}
+interface EmbeddingIndex {
+    embedVersion: number;
+    modelId: string;
+    dim: number;
+    records: EmbeddingRecord[];
+}
+declare function buildEmbeddingIndex(scan: RepoScan, model: StaticEmbedModel): EmbeddingIndex;
+declare function serializeEmbeddings(index: EmbeddingIndex): Uint8Array;
+declare function deserializeEmbeddings(bytes: Uint8Array): EmbeddingIndex;
+
+interface SemanticSearchOptions extends SearchOptions {
+    model?: StaticEmbedModel;
+    rrfK?: number;
+}
+interface SemanticSearchResult extends SearchResult {
+    semanticSymbol?: string;
+}
+declare function searchSemantic(scan: RepoScan, query: string, index: EmbeddingIndex | undefined, opts?: SemanticSearchOptions): SemanticSearchResult[];
+
+interface EmbedEndpointOptions {
+    url?: string;
+    timeoutMs?: number;
+    headers?: Record<string, string>;
+}
+declare function resolveEmbedEndpoint(opts?: EmbedEndpointOptions): string | undefined;
+declare function embedViaEndpoint(texts: string[], opts?: EmbedEndpointOptions): Promise<number[][]>;
+
 type RuleSeverity = "error" | "warn";
 interface ForbiddenEdgeRule {
     name: string;
@@ -623,4 +678,4 @@ declare function rrf<T>(lists: T[][], keyOf: (item: T) => string, k?: number): M
 
 declare function runCli(argv: string[]): Promise<void>;
 
-export { type ArchRule, type BuildIndexOptions, type BuiltinRule, type CallerEntry, type CallerIndex, type CallerIndexOptions, type CallerSite, type ChangeCoupling, type CodeInfo, type CodeSymbol, type CouplingOptions, DEFAULT_MAX_FILES, type DeadSymbol, type DiffFile, type DiffSpec, ENGINE_VERSION, EXTRACTOR_VERSION, type Edge, type EdgeKind, type EditResult, type FileCategory, type FileKind, type FileNode, type FileRecord, type FindSymbolOptions, type ForbiddenEdgeRule, type Graph, type GrepOptions, type Hotspot, type Hunk, type IgnoreRule, type IndexArtifacts, MARKDOWN_EXT, type MarkdownInfo, type MermaidOptions, type ModuleInfo, type ModuleNode, type RawRef, type RenderScipOptions, type RepoMapOptions, type RepoScan, type Resolution, type ResolveContext, type RiskHotspot, type RuleSeverity, type RuleViolation, SCHEMA_VERSION, type ScanOptions, type SearchHit, type SearchOptions, type SearchResult, type ShResult, type SurpriseEdge, type SymbolComplexity, type SymbolIndex, type SymbolMatch, type SymbolReferences, type TestMap, type Tier, type WalkOptions, type WalkResult, type WalkedFile, type WorkspaceInfo, type WorkspaceKind, type WorkspacePackage, allGrammarKeys, applyCentrality, betweennessOf, buildCallerIndex, buildGraph, buildIndexArtifacts, buildModules, buildResolveContext, buildSymbolIndex, byKey, byStr, categorize, changeCoupling, changedSince, checkRules, classify, clip, clipInline, communityOf, compileGlobs, complexityOfSource, computeImportPairs, computeSurprises, computeSymbolRefs, computeTestMap, deleteMemory, detectCommunities, detectWorkspaces, diffFiles, diffHunks, enclosingSymbol, ensureGrammars, escapeRegExp, extToLang, extractAst, extractCode, extractMarkdown, extractSymbols, findDeadCode, findReferences, findSymbol, foldText, gitChurn, grammarKeyForExt, grammarReady, grepRepo, have, headCommit, insertAfterSymbol, insertBeforeSymbol, isCode, isDoc, isGitWorktree, isIgnored, isSurprising, isTestFile, isTestPath, keywords, languageOf, listMemories, pagerankOf, parseGitignore, parseRules, rankHotspots, rankedKeywords, readMemory, readText, renderGraphJson, renderMermaid, renderRepoMap, renderScip, renderSymbolsJson, replaceSymbolBody, resolveBaseRef, resolveCallEdges, resolveDocLink, resolveImport, resolveUniqueSymbol, riskHotspots, rrf, runCli, runMcpServer, scanRepo, searchIndex, sh, sha1, shortHash, slugify, subtokens, symbolComplexity, symbolsOverview, testsForModule, tierForPath, uniqueSymbolDefs, untestedModules, untrackedFiles, walk, writeMemory };
+export { type ArchRule, type BuildIndexOptions, type BuiltinRule, type CallerEntry, type CallerIndex, type CallerIndexOptions, type CallerSite, type ChangeCoupling, type CodeInfo, type CodeSymbol, type CouplingOptions, DEFAULT_MAX_FILES, type DeadSymbol, type DiffFile, type DiffSpec, EMBED_VERSION, ENGINE_VERSION, EXTRACTOR_VERSION, type Edge, type EdgeKind, type EditResult, type EmbedEndpointOptions, type EmbeddingIndex, type EmbeddingRecord, type FileCategory, type FileKind, type FileNode, type FileRecord, type FindSymbolOptions, type ForbiddenEdgeRule, type Graph, type GrepOptions, type Hotspot, type Hunk, type IgnoreRule, type IndexArtifacts, MARKDOWN_EXT, type MarkdownInfo, type MermaidOptions, type ModuleInfo, type ModuleNode, type RawRef, type RenderScipOptions, type RepoMapOptions, type RepoScan, type Resolution, type ResolveContext, type RiskHotspot, type RuleSeverity, type RuleViolation, SCHEMA_VERSION, type ScanOptions, type SearchHit, type SearchOptions, type SearchResult, type SemanticSearchOptions, type SemanticSearchResult, type ShResult, type StaticEmbedModel, type SurpriseEdge, type SymbolComplexity, type SymbolIndex, type SymbolMatch, type SymbolReferences, type TestMap, type Tier, type WalkOptions, type WalkResult, type WalkedFile, type WorkspaceInfo, type WorkspaceKind, type WorkspacePackage, allGrammarKeys, applyCentrality, basicTokenize, betweennessOf, buildCallerIndex, buildEmbeddingIndex, buildGraph, buildIndexArtifacts, buildModules, buildResolveContext, buildSymbolIndex, byKey, byStr, categorize, changeCoupling, changedSince, checkRules, classify, clip, clipInline, communityOf, compileGlobs, complexityOfSource, computeImportPairs, computeSurprises, computeSymbolRefs, computeTestMap, deleteMemory, deserializeEmbeddings, detectCommunities, detectWorkspaces, diffFiles, diffHunks, embedViaEndpoint, enclosingSymbol, encode, ensureGrammars, escapeRegExp, extToLang, extractAst, extractCode, extractMarkdown, extractSymbols, findDeadCode, findReferences, findSymbol, foldText, gitChurn, grammarKeyForExt, grammarReady, grepRepo, hasEmbedModel, have, headCommit, insertAfterSymbol, insertBeforeSymbol, intDot, isCode, isDoc, isGitWorktree, isIgnored, isSurprising, isTestFile, isTestPath, keywords, languageOf, listMemories, loadEmbedModel, pagerankOf, parseGitignore, parseRules, rankHotspots, rankedKeywords, readMemory, readText, renderGraphJson, renderMermaid, renderRepoMap, renderScip, renderSymbolsJson, replaceSymbolBody, resolveBaseRef, resolveCallEdges, resolveDocLink, resolveEmbedEndpoint, resolveEmbedModelDir, resolveEmbedPullUrl, resolveImport, resolveUniqueSymbol, riskHotspots, roundHalfToEven, rrf, runCli, runMcpServer, scanRepo, searchIndex, searchSemantic, serializeEmbeddings, sh, sha1, shortHash, slugify, subtokens, symbolComplexity, symbolsOverview, testsForModule, tierForPath, tokenize, uniqueSymbolDefs, untestedModules, untrackedFiles, walk, wordpiece, writeMemory };