@mnemonik/shared 6.48.0 → 6.51.0

package/dist/codeScanner.d.ts

@@ -36,6 +36,7 @@ export interface CodeChunk {
         size: number;
         signature?: string;
         symbolName?: string;
+        symbolKind?: string;
     };
 }
 export interface ScanOptions {
@@ -43,6 +44,13 @@ export interface ScanOptions {
     minChunkSize?: number;
     ignorePatterns?: string[];
     includeExtensions?: string[];
+    /**
+     * Byte ceiling above which a file is chunked heuristically instead of parsed.
+     * Defaults to `MAX_AST_PARSE_BYTES`; exposed so a caller (and the routing
+     * test) can drive the degradation path through production code rather than a
+     * mock.
+     */
+    maxAstParseBytes?: number;
 }
 /**
  * Directory basenames the scanner never indexes, across every ecosystem —
@@ -62,6 +70,39 @@ export declare const BUILT_IN_IGNORE_DIRS: readonly string[];
  * for older or misbehaving daemons). Accepts OS-native or POSIX separators.
  */
 export declare function isSecretFile(relPath: string): boolean;
+/**
+ * Every file type the scanner will chunk, by extension. Matched
+ * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
+ * also cover `.R`, `.SQL`, `.PS1` and the uppercase `.C`/`.H` of older trees.
+ *
+ * Exported because this list IS the product's language coverage: a project
+ * written in something absent from it indexes zero code, and `code_search`
+ * reports that as "no indexed matches" rather than "I do not read this
+ * language" — invisible to the developer and to us. Coverage is a separate
+ * dial from parse quality: a language with no structured extractor falls
+ * through to `chunkRaw`, which is what Go, Java and C already get in
+ * production, and crude chunks beat no chunks by an enormous margin.
+ *
+ * Data formats (`.json`, `.yaml`, `.toml`, `.lock`) are deliberately absent:
+ * the ones that carry meaning are already collected verbatim by
+ * `AUTHORITY_FILE_MATCHERS`, and blanket-indexing the extension would pull in
+ * lockfiles and generated output. `.tfvars` is absent for a different reason —
+ * `terraform.tfvars` is a conventional home for provider credentials and
+ * `isSecretFile` now excludes it outright.
+ */
+export declare const DEFAULT_INCLUDE_EXTENSIONS: readonly string[];
+/**
+ * Language string for a file path or a bare extension, `'unknown'` when the
+ * extension is unmapped. A free function rather than a method because callers
+ * that never scan anything (AST grammar resolution, server-side symbol
+ * preference) need the same answer without constructing a scanner.
+ */
+export declare function languageForExtension(filePathOrExt: string): string;
+/**
+ * True when `relPath` is collected verbatim as authority content and must not
+ * additionally be chunked. Accepts OS-native or POSIX separators.
+ */
+export declare function isAuthorityOnlyPath(relPath: string): boolean;
 /**
  * Matchers for authority manifest / config / CI files whose verbatim content
  * is collected by `collectAuthorityFiles` and pushed to the server for
@@ -101,9 +142,46 @@ export declare function makeIgnoreMatcher(extraPatterns?: readonly string[]): (r
  * (callers only test child directories).
  */
 export declare function isGitBoundary(dirPath: string): Promise<boolean>;
+/**
+ * Log which grammars this install ships — ONCE per process, at daemon start,
+ * never per file.
+ *
+ * A missing grammar silently degrades every file of that language to the
+ * heuristic chunker. Per-file warnings would say so 30,000 times and drown the
+ * log; saying nothing is how a half-broken install looks healthy. One startup
+ * line naming the vendored grammars, and a WARNING when an artifact is missing,
+ * is the whole contract.
+ *
+ * Deliberately a `statSync` of the artifacts (`astArtifactReport`) and not a
+ * load of them (`astCapabilityReport`). Loading all 18 to print this line costs
+ * ~690 ms and ~75 MB of RSS that is never returned — web-tree-sitter exposes no
+ * `Language.delete` — which is a permanent tax on a daemon watching a pure
+ * TypeScript repo, paid to pre-answer a question about seventeen languages it
+ * will never see. Grammars load lazily instead, on the first file of a language,
+ * and the two failure modes only a load can detect (`wasm_load_failed`,
+ * `query_compile_failed`) are warned there, once per language, by the
+ * `grammar_unavailable` branch in `chunkFile`.
+ *
+ * Cheap to call repeatedly: this latches, and the report instantiates nothing.
+ */
+export declare function logAstCapabilityOnce(): Promise<void>;
 export declare class CodeScanner {
     private options;
+    /**
+     * `includeExtensions` folded to lowercase for matching. The gate used to
+     * compare `extname()` verbatim while `detectLanguage` lowercased, so a
+     * project spelling its files the canonical way — `.R` for R, `.SQL`/`.PS1`
+     * on Windows, `.C`/`.H` in older C trees — indexed zero of them and nothing
+     * said why.
+     */
+    private readonly includeExtensionSet;
     constructor(options?: ScanOptions);
+    /**
+     * The chunkable-file gate, shared by every walker and by the explicit
+     * file-list path so all three agree on what exists. `relPath` is the path
+     * relative to the scan root (OS-native separators accepted).
+     */
+    private isChunkable;
     /**
      * Read `.gitignore` + `.mnemonikignore` in `absDir` and compile them into one
      * ignore layer (the two files are unioned — `.mnemonikignore` is just an
@@ -231,6 +309,34 @@ export declare class CodeScanner {
      */
     private static readonly MAX_FILE_SIZE;
     private parseFile;
+    /**
+     * Hold every AST chunk at or under `maxChunkSize`, or answer `null` when this
+     * file cannot be held there on line boundaries.
+     *
+     * The AST chunker bounds only the spans it invents (`MAX_RAW_SPAN_CHARS`); a
+     * chunk that came from a definition is exactly as big as the definition, and a
+     * span that is one enormous line is left whole on purpose. Both used to reach
+     * the wire verbatim, where `scanChunkSchema` caps content at 500,000 chars and
+     * rejects the WHOLE push if any chunk is over — so one generated file stopped
+     * all scanning for that push — and where anything past 8191 tokens is only
+     * partially embedded.
+     *
+     * The invariants, all of which the split preserves:
+     * - content stays verbatim contiguous source: `lines[startLine-1..endLine-1]`
+     *   joined by '\n', never a join of disjoint regions and never an elision;
+     * - total coverage is unchanged: the pieces tile the original range, in order,
+     *   with no gap and no overlap;
+     * - the definition's identity rides on the piece that carries its signature —
+     *   the first one — and the continuations are plain `raw` spans, because a
+     *   continuation is not the definition and must not claim its name.
+     *
+     * `null` (the caller degrades the whole file to the heuristic chunker) is
+     * reserved for the one shape a line-boundary split cannot fix: a single line
+     * longer than the cap, as every generated `*_pb2.py` has. Cutting mid-line
+     * would give two chunks the same `filePath:startLine-endLine` staleness key,
+     * which is the collision the chunker refuses everywhere else.
+     */
+    private boundAstChunks;
     /**
      * Detect language from file extension
      */

package/dist/codeScanner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"codeScanner.d.ts","sourceRoot":"","sources":["../src/codeScanner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAeH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,sBAAsB,QAAmB,CAAC;AAEvD,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAcD;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAqDjD,CAAC;AAmEF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAQrD;AA4CD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAiBvE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,QAA4B,CAAC;AACzD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAID;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,aAAa,GAAE,SAAS,MAAM,EAAO,GACpC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAmB9B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAWrE;AAcD,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAwB;IAEvC,YAAY,OAAO,GAAE,WAAgB,EAEpC;IAED;;;;;;;OAOG;YACW,eAAe;IA4B7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAqBtB;;;;OAIG;YACW,uBAAuB;IAwBrC;;;;;OAKG;YACW,uBAAuB;IAerC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEvC;;;;;;OAMG;YACW,cAAc;IAM5B;;;OAGG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG1D;IAED;;;;;;;OAOG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAKrD;IAED;;;;;;;;;;;;;;;OAeG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnD;IAED;;;;;;OAMG;IACG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAQ3F;YAEa,aAAa;IA+D3B;;;OAGG;IACG,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAuC3E;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW;IAYnB;;;OAGG;YACW,iBAAiB;IAiF/B;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IA6BpB;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAA0B;YAEjD,SAAS;IAoGvB;;OAEG;IACH,OAAO,CAAC,cAAc;IA4BtB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoFrB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0GzB;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAoE/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqD3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,QAAQ;IA8FhB;;OAEG;IACH,OAAO,CAAC,IAAI;IAIZ;;;;OAIG;IACG,qBAAqB,CACzB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAGjE;IAED;;;;;;OAMG;IACG,+BAA+B,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC,CAmFD;CACF"}
+{"version":3,"file":"codeScanner.d.ts","sourceRoot":"","sources":["../src/codeScanner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiBH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,sBAAsB,QAAmB,CAAC;AAEvD,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QAKpB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAcD;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAqDjD,CAAC;AA4EF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAQrD;AAUD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,0BAA0B,EAAE,SAAS,MAAM,EAqFvD,CAAC;AAsFF;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,CAalE;AAwCD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAI5D;AAUD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAmBvE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,QAA4B,CAAC;AACzD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAID;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,aAAa,GAAE,SAAS,MAAM,EAAO,GACpC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAmB9B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAWrE;AAgCD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC,CAiBpD;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAsB;IAE1D,YAAY,OAAO,GAAE,WAAgB,EAKpC;IAED;;;;OAIG;IACH,OAAO,CAAC,WAAW;IASnB;;;;;;;OAOG;YACW,eAAe;IA4B7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAqBtB;;;;OAIG;YACW,uBAAuB;IAwBrC;;;;;OAKG;YACW,uBAAuB;IAerC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEvC;;;;;;OAMG;YACW,cAAc;IAM5B;;;OAGG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG1D;IAED;;;;;;;OAOG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAKrD;IAED;;;;;;;;;;;;;;;OAeG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnD;IAED;;;;;;OAMG;IACG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAQ3F;YAEa,aAAa;IA+D3B;;;OAGG;IACG,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAsC3E;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW;IAYnB;;;OAGG;YACW,iBAAiB;IAgF/B;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IA6BpB;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAA0B;YAEjD,SAAS;IAyNvB;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,OAAO,CAAC,cAAc;IAoDtB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoFrB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0GzB;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAoE/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqD3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,QAAQ;IA8FhB;;OAEG;IACH,OAAO,CAAC,IAAI;IAIZ;;;;OAIG;IACG,qBAAqB,CACzB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAGjE;IAED;;;;;;OAMG;IACG,+BAA+B,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC,CAmFD;CACF"}