codesift-mcp 0.1.0 → 0.2.1

package/dist/retrieval/semantic-handlers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-handlers.js","sourceRoot":"","sources":["../../src/retrieval/semantic-handlers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,OAAO,EACL,cAAc,EACd,sBAAsB,EACtB,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,oBAAoB,EACpB,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AAExG,SAAS,qBAAqB,CAAC,OAAiI;IAC9J,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC;IAChD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACtG,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACtD,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAqBD,KAAK,UAAU,mBAAmB,CAChC,IAAY,EACZ,KAAsF;IAEtF,MAAM,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAC5F,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC/D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC3D,MAAM,EAAE,UAAU,EAAE,mBAAmB,EAAE,YAAY,EAAE,qBAAqB,EAAE,GAC5E,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IAE5C,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,iHAAiH,CAClH,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,IAAI,aAAa,CAAC;IAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC;IACrC,MAAM,YAAY,GAAG,KAAK,CAAC,aAAa,IAAI,IAAI,CAAC;IAEjD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;IAC3E,MAAM,aAAa,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,gBAAgB,EAAE,eAAe,CAAC,CAAC;IAEjG,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC1D,IAAI,MAAM,GAAkC,IAAI,CAAC;IACjD,IAAI,eAAe,GAAqC,IAAI,CAAC;IAE7D,IAAI,QAAQ,EAAE,CAAC;QACb,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC5C,UAAU,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC7C,mBAAmB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SAChE,CAAC,CAAC;IACL,CAAC;IAED,MAAM,eAAe,GAAG,MAAM;QAC5B,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,IAAI,GAAG,EAA8B,CAAC;IAE1C,MAAM,gBAAgB,GAAG,eAAe,IAAI,IAAI,GAAG,EAAwB,CAAC;IAC5E,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,gBAAgB,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;IAE/G,OAAO;QACL,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ;QAChC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,kBAAkB;QAC5D,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,gBAAgB;KACjD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,KAA8C;IAE9C,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAEnD,+CAA+C;IAC/C,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,eAAe,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,kBAAkB,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;aACpC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC;aAClB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QAErB,IAAI,QAAQ,GAAG,MAAM,CAAC;QACtB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YACjE,QAAQ,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,IAAI,GAAG,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;IACxE,CAAC;IAED,4CAA4C;IAC5C,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;IACpF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAEjE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,aAAa,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,4DAA4D,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,IAAI,oBAAoB,CAAC;IAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,UAAU,EAAE,aAAa,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;IAE/G,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAE1E,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,YAAY,CAAC,UAAU,CAAC,EAAE,kBAAkB,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACtG,wEAAwE;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IACtG,MAAM,IAAI,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC;IAC9C,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,8EAA8E;AAC9E,qDAAqD;AACrD,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,KAA4C;IAE5C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAC;IAEhE,iFAAiF;IACjF,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC3C,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC;QAChC,UAAU,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;KACnF,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,aAAa,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,eAAe;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,GAAG,CAAC,CAAC;IAGzF,gCAAgC;IAChC,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,kBAAkB,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAE3F,qEAAqE;IACrE,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqE,CAAC;IAClG,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACtE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,IAAI,GAAG,CAAC,YAAY,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QACzD,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAChD,iEAAiE;QACjE,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAE,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC/B,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;iBAAM,IAAI,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;gBACxC,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;gBACjF,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;SAClC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC;SAClB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAErB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;QACjE,MAAM,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;IAC5E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;AAClF,CAAC"}

package/dist/search/bm25.d.ts

@@ -1,5 +1,5 @@
 import type { CodeSymbol, SearchResult } from "../types.js";
-type FieldName = "name" | "signature" | "docstring" | "body";
+type FieldName = "name" | "signature" | "docstring" | "body" | "comments";
 export interface BM25Index {
     /** Per-field inverted index: token -> Map<symbolId, termFrequency> */
     fields: Record<FieldName, Map<string, Map<string, number>>>;
@@ -9,6 +9,10 @@ export interface BM25Index {
     docCount: number;
     /** Symbol lookup by ID */
     symbols: Map<string, CodeSymbol>;
+    /** Import centrality: file -> log-scaled importer count (for search ranking bonus) */
+    centrality: Map<string, number>;
+    /** Pre-computed per-document field lengths (avoids O(n*m) recomputation per search) */
+    fieldLengths: Map<string, Record<FieldName, number>>;
 }
 /**
  * General-purpose tokenizer for signature, docstring, and body text.
@@ -18,5 +22,6 @@ export interface BM25Index {
 export declare function tokenizeText(text: string): string[];
 export declare function buildBM25Index(symbols: CodeSymbol[]): BM25Index;
 export declare function searchBM25(index: BM25Index, query: string, topK: number, fieldWeights: Record<FieldName, number>): SearchResult[];
+export declare function applyCutoff(results: SearchResult[]): SearchResult[];
 export {};
 //# sourceMappingURL=bm25.d.ts.map

package/dist/search/bm25.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAe5D,KAAK,SAAS,GAAG,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,MAAM,CAAC;AAE7D,MAAM,WAAW,SAAS;IACxB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5D,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3C,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAClC;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBnD;AAqBD,wBAAgB,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,SAAS,CAgD/D;AAED,wBAAgB,UAAU,CACxB,KAAK,EAAE,SAAS,EAChB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,GACtC,YAAY,EAAE,CAiGhB"}
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAe5D,KAAK,SAAS,GAAG,MAAM,GAAG,WAAW,GAAG,WAAW,GAAG,MAAM,GAAG,UAAU,CAAC;AAE1E,MAAM,WAAW,SAAS;IACxB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5D,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3C,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACjC,sFAAsF;IACtF,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,uFAAuF;IACvF,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAqBnD;AA8CD,wBAAgB,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,SAAS,CAoF/D;AAED,wBAAgB,UAAU,CACxB,KAAK,EAAE,SAAS,EAChB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,GACtC,YAAY,EAAE,CAwFhB;AAKD,wBAAgB,WAAW,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CAWnE"}

package/dist/search/bm25.js

@@ -35,15 +35,37 @@ export function tokenizeText(text) {
     return tokens;
 }
 function getFieldTokens(symbol) {
+    const source = symbol.source?.slice(0, BODY_CHAR_LIMIT) ?? "";
+    const { code, comments } = splitCodeAndComments(source);
     return {
         name: tokenizeIdentifier(symbol.name),
         signature: symbol.signature ? tokenizeText(symbol.signature) : [],
         docstring: symbol.docstring ? tokenizeText(symbol.docstring) : [],
-        body: symbol.source
-            ? tokenizeText(symbol.source.slice(0, BODY_CHAR_LIMIT))
-            : [],
+        body: source ? tokenizeText(code) : [],
+        comments: comments ? tokenizeText(comments) : [],
     };
 }
+/**
+ * Split source into code (logic) vs inline comments.
+ * Strips single-line (//) and multi-line comments from code,
+ * collects them into a separate string.
+ *
+ * Limitation: regex-based, so `//` inside string literals (e.g. URLs)
+ * may be misclassified as comments. Acceptable for BM25 scoring where
+ * a few misclassified tokens have negligible impact on ranking.
+ */
+function splitCodeAndComments(source) {
+    const commentParts = [];
+    // Match // comments and /* ... */ blocks
+    const stripped = source.replace(/\/\/[^\n]*/g, (m) => {
+        commentParts.push(m);
+        return "";
+    }).replace(/\/\*[\s\S]*?\*\//g, (m) => {
+        commentParts.push(m);
+        return "";
+    });
+    return { code: stripped, comments: commentParts.join(" ") };
+}
 function countTermFrequencies(tokens) {
     const tf = new Map();
     for (const token of tokens) {
@@ -52,26 +74,31 @@ function countTermFrequencies(tokens) {
     return tf;
 }
 export function buildBM25Index(symbols) {
-    const fieldNames = ["name", "signature", "docstring", "body"];
+    const fieldNames = ["name", "signature", "docstring", "body", "comments"];
     const fields = {
         name: new Map(),
         signature: new Map(),
         docstring: new Map(),
         body: new Map(),
+        comments: new Map(),
     };
     const totalFieldLengths = {
         name: 0,
         signature: 0,
         docstring: 0,
         body: 0,
+        comments: 0,
     };
     const symbolMap = new Map();
+    const fieldLengths = new Map();
     for (const symbol of symbols) {
         symbolMap.set(symbol.id, symbol);
         const fieldTokens = getFieldTokens(symbol);
+        const lengths = { name: 0, signature: 0, docstring: 0, body: 0, comments: 0 };
         for (const field of fieldNames) {
             const tokens = fieldTokens[field];
             totalFieldLengths[field] += tokens.length;
+            lengths[field] = tokens.length;
             const tf = countTermFrequencies(tokens);
             for (const [token, freq] of tf) {
                 let postings = fields[field].get(token);
@@ -82,6 +109,7 @@ export function buildBM25Index(symbols) {
                 postings.set(symbol.id, freq);
             }
         }
+        fieldLengths.set(symbol.id, lengths);
     }
     const docCount = symbols.length;
     const avgFieldLengths = {
@@ -89,8 +117,37 @@ export function buildBM25Index(symbols) {
         signature: docCount > 0 ? totalFieldLengths.signature / docCount : 0,
         docstring: docCount > 0 ? totalFieldLengths.docstring / docCount : 0,
         body: docCount > 0 ? totalFieldLengths.body / docCount : 0,
+        comments: docCount > 0 ? totalFieldLengths.comments / docCount : 0,
     };
-    return { fields, avgFieldLengths, docCount, symbols: symbolMap };
+    // Compute import centrality: count how many files import each file
+    // Heuristic: scan symbol source for import/require patterns pointing to files in the index
+    const importCount = new Map();
+    const allFiles = new Set();
+    for (const sym of symbols)
+        allFiles.add(sym.file);
+    for (const sym of symbols) {
+        if (!sym.source)
+            continue;
+        // Quick regex for import paths (captures relative paths)
+        const importRe = /from\s+['"]\.?\.\/([\w/.-]+)['"]/g;
+        let match;
+        while ((match = importRe.exec(sym.source)) !== null) {
+            const imported = match[1];
+            // Try to match against known files
+            for (const file of allFiles) {
+                if (file.includes(imported)) {
+                    importCount.set(file, (importCount.get(file) ?? 0) + 1);
+                    break;
+                }
+            }
+        }
+    }
+    // Log-scale centrality: avoids a single highly-imported utility from dominating
+    const centrality = new Map();
+    for (const [file, count] of importCount) {
+        centrality.set(file, Math.log2(1 + count));
+    }
+    return { fields, avgFieldLengths, docCount, symbols: symbolMap, centrality, fieldLengths };
 }
 export function searchBM25(index, query, topK, fieldWeights) {
     if (index.docCount === 0 || !query.trim()) {
@@ -100,34 +157,13 @@ export function searchBM25(index, query, topK, fieldWeights) {
     if (queryTokens.length === 0) {
         return [];
     }
-    const fieldNames = ["name", "signature", "docstring", "body"];
+    const fieldNames = ["name", "signature", "docstring", "body", "comments"];
     // Accumulate scores per document
     const scores = new Map();
     // Track which query tokens matched per document
     const matchedTokens = new Map();
-    // Pre-compute field lengths per document per field
-    // We derive field length from the sum of term frequencies in each field's postings
-    const fieldLengths = new Map();
-    for (const [symbolId] of index.symbols) {
-        const lengths = {
-            name: 0,
-            signature: 0,
-            docstring: 0,
-            body: 0,
-        };
-        fieldLengths.set(symbolId, lengths);
-    }
-    // Compute field lengths by summing all term frequencies per doc per field
-    for (const field of fieldNames) {
-        for (const [, postings] of index.fields[field]) {
-            for (const [symbolId, freq] of postings) {
-                const lengths = fieldLengths.get(symbolId);
-                if (lengths) {
-                    lengths[field] += freq;
-                }
-            }
-        }
-    }
+    // Use precomputed field lengths from index (built once at index time)
+    const { fieldLengths } = index;
     for (const qToken of queryTokens) {
         for (const field of fieldNames) {
             const postings = index.fields[field].get(qToken);
@@ -152,12 +188,23 @@ export function searchBM25(index, query, topK, fieldWeights) {
             }
         }
     }
-    // Demote test file symbols so production code ranks above test helpers
+    // Centrality bonus: symbols in frequently-imported files get a tiebreaker
+    const maxCentrality = Math.max(1, ...index.centrality.values());
     for (const [symbolId, score] of scores) {
         const symbol = index.symbols.get(symbolId);
-        if (symbol && isTestFile(symbol.file)) {
-            scores.set(symbolId, score * TEST_FILE_SCORE_MULTIPLIER);
+        if (!symbol)
+            continue;
+        let adjusted = score;
+        // Centrality: 0-10% bonus scaled by file import popularity
+        const fileCentrality = index.centrality.get(symbol.file) ?? 0;
+        if (fileCentrality > 0) {
+            adjusted += score * 0.1 * (fileCentrality / maxCentrality);
         }
+        // Demote test file symbols so production code ranks above test helpers
+        if (isTestFile(symbol.file)) {
+            adjusted *= TEST_FILE_SCORE_MULTIPLIER;
+        }
+        scores.set(symbolId, adjusted);
     }
     // Sort by score descending, take top-K
     const sorted = [...scores.entries()]
@@ -176,4 +223,20 @@ export function searchBM25(index, query, topK, fieldWeights) {
     }
     return results;
 }
+const CUTOFF_THRESHOLD = 0.15;
+const CUTOFF_MIN_RESULTS = 3;
+export function applyCutoff(results) {
+    if (results.length <= CUTOFF_MIN_RESULTS)
+        return results;
+    const topScore = results[0]?.score ?? 0;
+    if (topScore <= 0)
+        return results;
+    const threshold = topScore * CUTOFF_THRESHOLD;
+    for (let i = CUTOFF_MIN_RESULTS; i < results.length; i++) {
+        if ((results[i]?.score ?? 0) < threshold) {
+            return results.slice(0, i);
+        }
+    }
+    return results;
+}
 //# sourceMappingURL=bm25.js.map

package/dist/search/bm25.js.map

@@ -1 +1 @@
-{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,kBAAkB;AAClB,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAEf,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAevC;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,kEAAkE;QAClE,MAAM,QAAQ,GAAG,IAAI;aAClB,OAAO,CAAC,oBAAoB,EAAE,QAAQ,CAAC;aACvC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC;aAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;QAEf,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB;IACxC,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,IAAI,EAAE,MAAM,CAAC,MAAM;YACjB,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;YACvD,CAAC,CAAC,EAAE;KACP,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAgB;IAC5C,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE3E,MAAM,MAAM,GAAwD;QAClE,IAAI,EAAE,IAAI,GAAG,EAAE;QACf,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,IAAI,EAAE,IAAI,GAAG,EAAE;KAChB,CAAC;IAEF,MAAM,iBAAiB,GAA8B;QACnD,IAAI,EAAE,CAAC;QACP,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,IAAI,EAAE,CAAC;KACR,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAE3C,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,iBAAiB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;YAE1C,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;YACxC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACrC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,MAAM,eAAe,GAA8B;QACjD,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;KAC3D,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAgB,EAChB,KAAa,EACb,IAAY,EACZ,YAAuC;IAEvC,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAE3E,iCAAiC;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,gDAAgD;IAChD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAErD,mDAAmD;IACnD,mFAAmF;IACnF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqC,CAAC;IAElE,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,OAAO,GAA8B;YACzC,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,IAAI,EAAE,CAAC;SACR,CAAC;QACF,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,0EAA0E;IAC1E,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACxC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAEnC,KAAK,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACtC,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACjE,MAAM,UAAU,GAAG,GAAG,GAAG,OAAO,GAAG,MAAM,CAAC;gBAE1C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;gBAE/D,IAAI,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACxC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,GAAG,0BAA0B,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM;YACN,KAAK;YACL,OAAO,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../src/search/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,kBAAkB;AAClB,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAEf,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAmBvC;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,uCAAuC;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,kEAAkE;QAClE,MAAM,QAAQ,GAAG,IAAI;aAClB,OAAO,CAAC,oBAAoB,EAAE,QAAQ,CAAC;aACvC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC;aAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;QAEf,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YAChC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC;IAC9D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAExD,OAAO;QACL,IAAI,EAAE,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;QACjE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;QACtC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;KACjD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAAC,MAAc;IAC1C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,yCAAyC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE;QACnD,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,EAAE;QACpC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAgB;IAC5C,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAwD;QAClE,IAAI,EAAE,IAAI,GAAG,EAAE;QACf,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,IAAI,EAAE,IAAI,GAAG,EAAE;QACf,QAAQ,EAAE,IAAI,GAAG,EAAE;KACpB,CAAC;IAEF,MAAM,iBAAiB,GAA8B;QACnD,IAAI,EAAE,CAAC;QACP,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,IAAI,EAAE,CAAC;QACP,QAAQ,EAAE,CAAC;KACZ,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAsB,CAAC;IAChD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAqC,CAAC;IAElE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,OAAO,GAA8B,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAEzG,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,iBAAiB,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAE/B,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;YACxC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACrC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QACD,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,MAAM,eAAe,GAA8B;QACjD,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1D,QAAQ,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;KACnE,CAAC;IAEF,mEAAmE;IACnE,2FAA2F;IAC3F,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,OAAO;QAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,MAAM;YAAE,SAAS;QAC1B,yDAAyD;QACzD,MAAM,QAAQ,GAAG,mCAAmC,CAAC;QACrD,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YAC3B,mCAAmC;YACnC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC5B,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxD,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QACxC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAgB,EAChB,KAAa,EACb,IAAY,EACZ,YAAuC;IAEvC,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACxC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEvF,iCAAiC;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,gDAAgD;IAChD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAErD,sEAAsE;IACtE,MAAM,EAAE,YAAY,EAAE,GAAG,KAAK,CAAC;IAE/B,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAEnC,KAAK,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACtC,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBACjE,MAAM,UAAU,GAAG,GAAG,GAAG,OAAO,GAAG,MAAM,CAAC;gBAE1C,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;gBAE/D,IAAI,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;oBACrB,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBACxC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,2DAA2D;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,IAAI,KAAK,GAAG,GAAG,GAAG,CAAC,cAAc,GAAG,aAAa,CAAC,CAAC;QAC7D,CAAC;QAED,uEAAuE;QACvE,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,IAAI,0BAA0B,CAAC;QACzC,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,OAAO,CAAC,IAAI,CAAC;YACX,MAAM;YACN,KAAK;YACL,OAAO,EAAE,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,MAAM,UAAU,WAAW,CAAC,OAAuB;IACjD,IAAI,OAAO,CAAC,MAAM,IAAI,kBAAkB;QAAE,OAAO,OAAO,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;IACxC,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IAClC,MAAM,SAAS,GAAG,QAAQ,GAAG,gBAAgB,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,kBAAkB,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,SAAS,EAAE,CAAC;YACzC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/search/chunker.d.ts

@@ -6,4 +6,14 @@ import type { CodeChunk } from "../types.js";
  * non-code extension).
  */
 export declare function chunkFile(file: string, content: string, repo: string): CodeChunk[];
+/**
+ * Chunk a file at symbol boundaries instead of fixed character count.
+ * Each symbol = one chunk. Preamble (imports) = separate chunk.
+ * Falls back to chunkFile when no symbols provided.
+ */
+export declare function chunkBySymbols(file: string, content: string, repo: string, symbols: Array<{
+    name: string;
+    start_line: number;
+    end_line: number;
+}>): CodeChunk[];
 //# sourceMappingURL=chunker.d.ts.map

package/dist/search/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/search/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAyB7C;;;;;GAKG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,GACX,SAAS,EAAE,CAoDb"}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/search/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAoC7C;;;;;GAKG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,GACX,SAAS,EAAE,CA2Cb;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,GACrE,SAAS,EAAE,CAiDb"}

package/dist/search/chunker.js

@@ -3,7 +3,7 @@
 // ---------------------------------------------------------------------------
 const CHUNK_TOKENS = 400; // target tokens per chunk
 const OVERLAP_TOKENS = 80; // overlap between consecutive chunks
-const CHARS_PER_TOKEN = 4; // rough approximation
+const CHARS_PER_TOKEN = 4; // intentionally high for chunk sizing (smaller chunks = safe)
 const CHUNK_CHARS = CHUNK_TOKENS * CHARS_PER_TOKEN; // 1600
 const OVERLAP_CHARS = OVERLAP_TOKENS * CHARS_PER_TOKEN; // 320
 const MAX_FILE_BYTES = 50_000; // skip files > 50KB
@@ -12,6 +12,17 @@ const SKIP_EXTENSIONS = new Set([
     ".json", ".lock", ".md", ".yaml", ".yml",
     ".env", ".txt", ".svg", ".png", ".wasm",
 ]);
+function shouldSkipChunking(file, content) {
+    const dotIdx = file.lastIndexOf(".");
+    const ext = dotIdx !== -1 ? file.slice(dotIdx) : "";
+    if (SKIP_EXTENSIONS.has(ext))
+        return true;
+    if (content.length > MAX_FILE_BYTES)
+        return true;
+    if (content.includes("\0"))
+        return true;
+    return false;
+}
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -22,16 +33,7 @@ const SKIP_EXTENSIONS = new Set([
  * non-code extension).
  */
 export function chunkFile(file, content, repo) {
-    // Skip non-code file types
-    const dotIdx = file.lastIndexOf(".");
-    const ext = dotIdx !== -1 ? file.slice(dotIdx) : "";
-    if (SKIP_EXTENSIONS.has(ext))
-        return [];
-    // Skip files that are too large
-    if (content.length > MAX_FILE_BYTES)
-        return [];
-    // Skip binary files (presence of null bytes is a reliable signal)
-    if (content.includes("\0"))
+    if (shouldSkipChunking(file, content))
         return [];
     const lines = content.split("\n");
     const totalLines = lines.length;
@@ -66,6 +68,56 @@ export function chunkFile(file, content, repo) {
     }
     return chunks;
 }
+/**
+ * Chunk a file at symbol boundaries instead of fixed character count.
+ * Each symbol = one chunk. Preamble (imports) = separate chunk.
+ * Falls back to chunkFile when no symbols provided.
+ */
+export function chunkBySymbols(file, content, repo, symbols) {
+    if (shouldSkipChunking(file, content))
+        return [];
+    if (symbols.length === 0)
+        return chunkFile(file, content, repo);
+    const lines = content.split("\n");
+    const chunks = [];
+    // Sort symbols by start_line
+    const sorted = [...symbols].sort((a, b) => a.start_line - b.start_line);
+    // Preamble: lines before first symbol (imports, comments)
+    const firstStart = sorted[0]?.start_line ?? 1;
+    if (firstStart > 1) {
+        const text = lines.slice(0, firstStart - 1).join("\n");
+        if (text.trim().length > 0) {
+            chunks.push({
+                id: `${repo}:${file}:1`,
+                file,
+                startLine: 1,
+                endLine: firstStart - 1,
+                text,
+                tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+            });
+        }
+    }
+    // One chunk per symbol
+    for (const sym of sorted) {
+        const start = sym.start_line - 1; // 0-based
+        const end = Math.min(sym.end_line, lines.length); // 1-based inclusive
+        const symLines = lines.slice(start, end);
+        const text = symLines.join("\n");
+        if (text.trim().length === 0)
+            continue;
+        // Cap very large symbols
+        const cappedText = text.length > MAX_FILE_BYTES ? text.slice(0, MAX_FILE_BYTES) : text;
+        chunks.push({
+            id: `${repo}:${file}:${sym.start_line}`,
+            file,
+            startLine: sym.start_line,
+            endLine: end,
+            text: cappedText,
+            tokenCount: Math.ceil(cappedText.length / CHARS_PER_TOKEN),
+        });
+    }
+    return chunks;
+}
 // ---------------------------------------------------------------------------
 // Internal helpers
 // ---------------------------------------------------------------------------

package/dist/search/chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/search/chunker.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,GAAG,CAAC,CAAW,0BAA0B;AAC9D,MAAM,cAAc,GAAG,EAAE,CAAC,CAAU,qCAAqC;AACzE,MAAM,eAAe,GAAG,CAAC,CAAC,CAAU,sBAAsB;AAE1D,MAAM,WAAW,GAAG,YAAY,GAAG,eAAe,CAAC,CAAG,OAAO;AAC7D,MAAM,aAAa,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,MAAM;AAE9D,MAAM,cAAc,GAAG,MAAM,CAAC,CAAM,oBAAoB;AAExD,6DAA6D;AAC7D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;IACxC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;CACxC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,OAAe,EACf,IAAY;IAEZ,2BAA2B;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,gCAAgC;IAChC,IAAI,OAAO,CAAC,MAAM,GAAG,cAAc;QAAE,OAAO,EAAE,CAAC;IAE/C,kEAAkE;IAClE,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAEhC,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,2EAA2E;IAC3E,wEAAwE;IACxE,MAAM,eAAe,GAAa,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACxD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,eAAe,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAElC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,sCAAsC;IAE1D,OAAO,UAAU,GAAG,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,WAAW,EAAE,UAAU,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAEjD,2CAA2C;QAC3C,MAAM,SAAS,GAAG,gBAAgB,CAAC,UAAU,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,GAAG,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC;QAEpE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;QAE5D,MAAM,EAAE,GAAG,GAAG,IAAI,IAAI,IAAI,IAAI,SAAS,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAEhE,oEAAoE;QACpE,MAAM,OAAO,GAAG,WAAW,GAAG,aAAa,CAAC;QAC5C,UAAU,IAAI,OAAO,CAAC;QAEtB,uEAAuE;QACvE,IAAI,UAAU,IAAI,UAAU;YAAE,MAAM;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,gBAAgB,CAAC,UAAkB,EAAE,eAAyB;IACrE,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;IAEpC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;YACvD,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC"}
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/search/chunker.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,GAAG,CAAC,CAAW,0BAA0B;AAC9D,MAAM,cAAc,GAAG,EAAE,CAAC,CAAU,qCAAqC;AACzE,MAAM,eAAe,GAAG,CAAC,CAAC,CAAU,8DAA8D;AAElG,MAAM,WAAW,GAAG,YAAY,GAAG,eAAe,CAAC,CAAG,OAAO;AAC7D,MAAM,aAAa,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,MAAM;AAE9D,MAAM,cAAc,GAAG,MAAM,CAAC,CAAM,oBAAoB;AAExD,6DAA6D;AAC7D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;IACxC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;CACxC,CAAC,CAAC;AAEH,SAAS,kBAAkB,CAAC,IAAY,EAAE,OAAe;IACvD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpD,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,cAAc;QAAE,OAAO,IAAI,CAAC;IACjD,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAExC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,OAAe,EACf,IAAY;IAEZ,IAAI,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAEhC,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,2EAA2E;IAC3E,wEAAwE;IACxE,MAAM,eAAe,GAAa,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACxD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,eAAe,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAElC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,sCAAsC;IAE1D,OAAO,UAAU,GAAG,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,WAAW,EAAE,UAAU,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAEjD,2CAA2C;QAC3C,MAAM,SAAS,GAAG,gBAAgB,CAAC,UAAU,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,GAAG,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC;QAEpE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;QAE5D,MAAM,EAAE,GAAG,GAAG,IAAI,IAAI,IAAI,IAAI,SAAS,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAEhE,oEAAoE;QACpE,MAAM,OAAO,GAAG,WAAW,GAAG,aAAa,CAAC;QAC5C,UAAU,IAAI,OAAO,CAAC;QAEtB,uEAAuE;QACvE,IAAI,UAAU,IAAI,UAAU;YAAE,MAAM;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,OAAe,EACf,IAAY,EACZ,OAAsE;IAEtE,IAAI,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAEhE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,6BAA6B;IAC7B,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAExE,0DAA0D;IAC1D,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC;IAC9C,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,GAAG,IAAI,IAAI,IAAI,IAAI;gBACvB,IAAI;gBACJ,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,UAAU,GAAG,CAAC;gBACvB,IAAI;gBACJ,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC;aACrD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,UAAU;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,oBAAoB;QACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEvC,yBAAyB;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEvF,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE;YACvC,IAAI;YACJ,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,OAAO,EAAE,GAAG;YACZ,IAAI,EAAE,UAAU;YAChB,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,eAAe,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,gBAAgB,CAAC,UAAkB,EAAE,eAAyB;IACrE,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;IAEpC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,IAAI,UAAU,EAAE,CAAC;YACvD,EAAE,GAAG,GAAG,CAAC;QACX,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/search/reranker.d.ts

@@ -0,0 +1,15 @@
+import type { SearchResult, CodeChunk } from "../types.js";
+/**
+ * Rerank SearchResult[] using a cross-encoder model.
+ * Returns results reordered by cross-encoder score.
+ * Falls back to original order if the model is unavailable.
+ */
+export declare function rerankResults(query: string, results: SearchResult[], topN?: number, model?: string): Promise<SearchResult[]>;
+/**
+ * Rerank chunk IDs using a cross-encoder model.
+ * Returns reordered chunk IDs.
+ */
+export declare function rerankChunkIds(query: string, chunkIds: string[], chunks: Map<string, CodeChunk>, topN?: number, model?: string): Promise<string[]>;
+/** Reset caches for testing. */
+export declare function _resetReranker(): void;
+//# sourceMappingURL=reranker.d.ts.map

package/dist/search/reranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../../src/search/reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAmD3D;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,YAAY,EAAE,EACvB,IAAI,CAAC,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,EAAE,CAAC,CAiCzB;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAAE,EAClB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,EAC9B,IAAI,CAAC,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,EAAE,CAAC,CAkCnB;AAYD,gCAAgC;AAChC,wBAAgB,cAAc,IAAI,IAAI,CAIrC"}

package/dist/search/reranker.js

@@ -0,0 +1,126 @@
+const DEFAULT_RERANK_TOP_N = 50;
+const DEFAULT_MODEL = "Xenova/ms-marco-MiniLM-L-6-v2";
+const pipelineCache = new Map();
+const failedModels = new Set();
+let loadWarned = false;
+async function loadPipeline(model) {
+    const modelName = model ?? DEFAULT_MODEL;
+    const cached = pipelineCache.get(modelName);
+    if (cached)
+        return cached;
+    if (failedModels.has(modelName))
+        return null;
+    try {
+        // @ts-expect-error — optional dependency, may not be installed
+        const transformers = await import("@huggingface/transformers");
+        const pipelineFn = transformers.pipeline ?? transformers.default?.pipeline;
+        if (!pipelineFn) {
+            failedModels.add(modelName);
+            return null;
+        }
+        const classifier = await pipelineFn("text-classification", modelName, {
+            quantized: true,
+        });
+        const rerankerFn = async (pairs) => {
+            // Batch: send all inputs at once for better throughput
+            const inputs = pairs.map(([q, t]) => `${q} [SEP] ${t}`);
+            const outputs = await classifier(inputs, { topk: 1 });
+            // Normalize: pipeline returns single object for 1 input, array for N
+            const results = [];
+            for (let i = 0; i < pairs.length; i++) {
+                const out = Array.isArray(outputs[i]) ? outputs[i][0] : outputs[i] ?? outputs;
+                const score = out?.score ?? 0;
+                results.push({ score: typeof score === "number" ? score : 0 });
+            }
+            return results;
+        };
+        pipelineCache.set(modelName, rerankerFn);
+        return rerankerFn;
+    }
+    catch {
+        failedModels.add(modelName);
+        return null;
+    }
+}
+/**
+ * Rerank SearchResult[] using a cross-encoder model.
+ * Returns results reordered by cross-encoder score.
+ * Falls back to original order if the model is unavailable.
+ */
+export async function rerankResults(query, results, topN, model) {
+    if (results.length <= 1)
+        return results;
+    const limit = Math.min(topN ?? DEFAULT_RERANK_TOP_N, results.length);
+    const candidates = results.slice(0, limit);
+    const remainder = results.slice(limit);
+    const reranker = await loadPipeline(model);
+    if (!reranker) {
+        if (!loadWarned) {
+            loadWarned = true;
+            console.error("[codesift] Cross-encoder reranking unavailable. Install @huggingface/transformers for improved search quality.");
+        }
+        return results;
+    }
+    const pairs = candidates.map((r) => {
+        const text = buildCandidateText(r);
+        return [query, text];
+    });
+    const scores = await reranker(pairs);
+    const scored = candidates.map((r, i) => ({
+        result: r,
+        ceScore: scores[i]?.score ?? 0,
+    }));
+    scored.sort((a, b) => b.ceScore - a.ceScore);
+    return [...scored.map((s) => s.result), ...remainder];
+}
+/**
+ * Rerank chunk IDs using a cross-encoder model.
+ * Returns reordered chunk IDs.
+ */
+export async function rerankChunkIds(query, chunkIds, chunks, topN, model) {
+    if (chunkIds.length <= 1)
+        return chunkIds;
+    const limit = Math.min(topN ?? DEFAULT_RERANK_TOP_N, chunkIds.length);
+    const candidates = chunkIds.slice(0, limit);
+    const remainder = chunkIds.slice(limit);
+    const reranker = await loadPipeline(model);
+    if (!reranker) {
+        if (!loadWarned) {
+            loadWarned = true;
+            console.error("[codesift] Cross-encoder reranking unavailable. Install @huggingface/transformers for improved search quality.");
+        }
+        return chunkIds;
+    }
+    const pairs = candidates.map((id) => {
+        const chunk = chunks.get(id);
+        const text = chunk?.text ?? id;
+        return [query, text];
+    });
+    const scores = await reranker(pairs);
+    const scored = candidates.map((id, i) => ({
+        id,
+        ceScore: scores[i]?.score ?? 0,
+    }));
+    scored.sort((a, b) => b.ceScore - a.ceScore);
+    return [...scored.map((s) => s.id), ...remainder];
+}
+function buildCandidateText(r) {
+    const parts = [];
+    if (r.symbol.kind)
+        parts.push(r.symbol.kind);
+    parts.push(r.symbol.name);
+    if (r.symbol.signature)
+        parts.push(r.symbol.signature);
+    if (r.symbol.source)
+        parts.push(r.symbol.source.slice(0, 500));
+    else if (r.symbol.docstring)
+        parts.push(r.symbol.docstring.slice(0, 200));
+    return parts.join(" ");
+}
+/** Reset caches for testing. */
+export function _resetReranker() {
+    pipelineCache.clear();
+    failedModels.clear();
+    loadWarned = false;
+}
+//# sourceMappingURL=reranker.js.map

package/dist/search/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../src/search/reranker.ts"],"names":[],"mappings":"AAEA,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,aAAa,GAAG,+BAA+B,CAAC;AAItD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAsB,CAAC;AACpD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;AACvC,IAAI,UAAU,GAAG,KAAK,CAAC;AAEvB,KAAK,UAAU,YAAY,CAAC,KAAc;IACxC,MAAM,SAAS,GAAG,KAAK,IAAI,aAAa,CAAC;IAEzC,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7C,IAAI,CAAC;QACH,+DAA+D;QAC/D,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,IAAI,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC;QAC3E,IAAI,CAAC,UAAU,EAAE,CAAC;YAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAAC,OAAO,IAAI,CAAC;QAAC,CAAC;QAE9D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,qBAAqB,EAAE,SAAS,EAAE;YACpE,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAe,KAAK,EAAE,KAAiB,EAAE,EAAE;YACzD,uDAAuD;YACvD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YACxD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YAEtD,qEAAqE;YACrE,MAAM,OAAO,GAA6B,EAAE,CAAC;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC;gBAC9E,MAAM,KAAK,GAAG,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC,CAAC;QAEF,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,OAAuB,EACvB,IAAa,EACb,KAAc;IAEd,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IAExC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,IAAI,CAAC;YAClB,OAAO,CAAC,KAAK,CACX,gHAAgH,CACjH,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;QACnC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACvC,MAAM,EAAE,CAAC;QACT,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;KAC/B,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,QAAkB,EAClB,MAA8B,EAC9B,IAAa,EACb,KAAc;IAEd,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtE,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAExC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,UAAU,GAAG,IAAI,CAAC;YAClB,OAAO,CAAC,KAAK,CACX,gHAAgH,CACjH,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAClC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,EAAE;QACF,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;KAC/B,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAE7C,OAAO,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,kBAAkB,CAAC,CAAe;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACvD,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;SAC1D,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1E,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,gCAAgC;AAChC,MAAM,UAAU,cAAc;IAC5B,aAAa,CAAC,KAAK,EAAE,CAAC;IACtB,YAAY,CAAC,KAAK,EAAE,CAAC;IACrB,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC"}

package/dist/search/semantic.d.ts

@@ -6,7 +6,7 @@ export interface EmbeddingProvider {
 }
 /**
  * Build a searchable text string from a symbol for embedding.
- * Format: "{kind} {name}\n{signature}\n{docstring first line}\n{body first 200 chars}"
+ * Format: "{kind} {name}\n{signature}\n{docstring first line}\n{body first N chars}"
  */
 export declare function buildSymbolText(symbol: CodeSymbol): string;
 export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;

package/dist/search/semantic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../src/search/semantic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM5D,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAiB1D;AAMD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAezE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,cAAc,EAAE,YAAY,EAC5B,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACrC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAChC,IAAI,EAAE,MAAM,GACX,YAAY,EAAE,CAoBhB;AAoBD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,mBAAmB;IACjC,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,MAAM;IAIpB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAyBlD;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,4BAA4B;IAC1C,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,MAAM;IAIpB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAwBlD;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,sBAAsB;IACpC,QAAQ,CAAC,UAAU,OAAO;IAC1B,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM;IAIrB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA4BlD;AAMD,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,EACxC,MAAM,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAChG,iBAAiB,CAenB"}
+{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../src/search/semantic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAU5D,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAiB1D;AAMD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAiBzE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,cAAc,EAAE,YAAY,EAC5B,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACrC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAChC,IAAI,EAAE,MAAM,GACX,YAAY,EAAE,CAoBhB;AAsDD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,mBAAmB;IACjC,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,MAAM;IAIpB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAQlD;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,4BAA4B;IAC1C,QAAQ,CAAC,UAAU,QAAQ;IAC3B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,MAAM;IAIpB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAQlD;AAMD,qBAAa,cAAe,YAAW,iBAAiB;IACtD,QAAQ,CAAC,KAAK,sBAAsB;IACpC,QAAQ,CAAC,UAAU,OAAO;IAC1B,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM;IAIrB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA+BlD;AAMD,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,EACxC,MAAM,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAChG,iBAAiB,CAenB"}