@mnemonik/shared 6.47.0 → 6.50.0

package/dist/codeScanner.d.ts

@@ -1,6 +1,27 @@
 /**
  * Code Scanner - Parse and chunk source files for embedding
  */
+/**
+ * The one file-size ceiling in Mnemonik. A file at or below this is scanned,
+ * chunked, AND pushed verbatim; a file above it is uniformly outside the
+ * system — not scanned, not chunked, not citable. "Scanned ⟺ verifiable".
+ *
+ * This is deliberately a single exported constant rather than a number
+ * repeated per layer. Three ceilings previously existed and had drifted:
+ * the scan ceiling here (10 MB), a 5 MB whole-file push cap in the scanner
+ * daemon, and a 5 MB `content` cap in the server's scanPushSchema. Files
+ * between 5 and 10 MB were therefore indexed as chunks whose source content
+ * the server never received — chunks that could be retrieved but never
+ * verified against, a silent fidelity split. Every consumer now imports this:
+ *
+ * - `CodeScanner.MAX_FILE_SIZE` / `collectAuthorityFilesWithStatus` (this file)
+ * - `MAX_PUSH_CONTENT_BYTES` (packages/scanner/src/daemon.ts)
+ * - `scanFileSchema.content` (src/server/routes/schemas.ts)
+ *
+ * Raising it means widening the stored-content pipe end to end; do not raise
+ * one site alone.
+ */
+export declare const MAX_SCANNED_FILE_BYTES: number;
 export interface CodeChunk {
     content: string;
     filePath: string;
@@ -34,6 +55,46 @@ export interface ScanOptions {
  * project-specific cases (`vendor/`, custom dirs) hierarchically.
  */
 export declare const BUILT_IN_IGNORE_DIRS: readonly string[];
+/**
+ * True when `relPath` names a file whose contents are credentials. Applied
+ * by `shouldIgnore` (every walker: directory scan, explicit file lists, and
+ * the authority-file walk) and by `makeIgnoreMatcher` (server-side backstop
+ * for older or misbehaving daemons). Accepts OS-native or POSIX separators.
+ */
+export declare function isSecretFile(relPath: string): boolean;
+/**
+ * Every file type the scanner will chunk, by extension. Matched
+ * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
+ * also cover `.R`, `.SQL`, `.PS1` and the uppercase `.C`/`.H` of older trees.
+ *
+ * Exported because this list IS the product's language coverage: a project
+ * written in something absent from it indexes zero code, and `code_search`
+ * reports that as "no indexed matches" rather than "I do not read this
+ * language" — invisible to the developer and to us. Coverage is a separate
+ * dial from parse quality: a language with no structured extractor falls
+ * through to `chunkRaw`, which is what Go, Java and C already get in
+ * production, and crude chunks beat no chunks by an enormous margin.
+ *
+ * Data formats (`.json`, `.yaml`, `.toml`, `.lock`) are deliberately absent:
+ * the ones that carry meaning are already collected verbatim by
+ * `AUTHORITY_FILE_MATCHERS`, and blanket-indexing the extension would pull in
+ * lockfiles and generated output. `.tfvars` is absent for a different reason —
+ * `terraform.tfvars` is a conventional home for provider credentials and
+ * `isSecretFile` now excludes it outright.
+ */
+export declare const DEFAULT_INCLUDE_EXTENSIONS: readonly string[];
+/**
+ * Language string for a file path or a bare extension, `'unknown'` when the
+ * extension is unmapped. A free function rather than a method because callers
+ * that never scan anything (AST grammar resolution, server-side symbol
+ * preference) need the same answer without constructing a scanner.
+ */
+export declare function languageForExtension(filePathOrExt: string): string;
+/**
+ * True when `relPath` is collected verbatim as authority content and must not
+ * additionally be chunked. Accepts OS-native or POSIX separators.
+ */
+export declare function isAuthorityOnlyPath(relPath: string): boolean;
 /**
  * Matchers for authority manifest / config / CI files whose verbatim content
  * is collected by `collectAuthorityFiles` and pushed to the server for
@@ -75,7 +136,21 @@ export declare function makeIgnoreMatcher(extraPatterns?: readonly string[]): (r
 export declare function isGitBoundary(dirPath: string): Promise<boolean>;
 export declare class CodeScanner {
     private options;
+    /**
+     * `includeExtensions` folded to lowercase for matching. The gate used to
+     * compare `extname()` verbatim while `detectLanguage` lowercased, so a
+     * project spelling its files the canonical way — `.R` for R, `.SQL`/`.PS1`
+     * on Windows, `.C`/`.H` in older C trees — indexed zero of them and nothing
+     * said why.
+     */
+    private readonly includeExtensionSet;
     constructor(options?: ScanOptions);
+    /**
+     * The chunkable-file gate, shared by every walker and by the explicit
+     * file-list path so all three agree on what exists. `relPath` is the path
+     * relative to the scan root (OS-native separators accepted).
+     */
+    private isChunkable;
     /**
      * Read `.gitignore` + `.mnemonikignore` in `absDir` and compile them into one
      * ignore layer (the two files are unioned — `.mnemonikignore` is just an
@@ -196,8 +271,10 @@ export declare class CodeScanner {
      */
     private shouldIgnore;
     /**
-     * Parse a file and extract code chunks
-     * Added 10MB file size limit
+     * Parse a file and extract code chunks.
+     * Size limit is the shared `MAX_SCANNED_FILE_BYTES` ceiling — the same
+     * number the daemon and the server's scanPushSchema enforce, so anything
+     * chunked here can always be shipped and stored verbatim.
      */
     private static readonly MAX_FILE_SIZE;
     private parseFile;

package/dist/codeScanner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"codeScanner.d.ts","sourceRoot":"","sources":["../src/codeScanner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAeH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAcD;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAqDjD,CAAC;AAuEF;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAiBvE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,QAA4B,CAAC;AACzD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAID;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,aAAa,GAAE,SAAS,MAAM,EAAO,GACpC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAkB9B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAWrE;AAcD,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAwB;IAEvC,YAAY,OAAO,GAAE,WAAgB,EAEpC;IAED;;;;;;;OAOG;YACW,eAAe;IA4B7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAqBtB;;;;OAIG;YACW,uBAAuB;IAwBrC;;;;;OAKG;YACW,uBAAuB;IAerC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEvC;;;;;;OAMG;YACW,cAAc;IAM5B;;;OAGG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG1D;IAED;;;;;;;OAOG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAKrD;IAED;;;;;;;;;;;;;;;OAeG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnD;IAED;;;;;;OAMG;IACG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAQ3F;YAEa,aAAa;IA+D3B;;;OAGG;IACG,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAuC3E;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW;IAYnB;;;OAGG;YACW,iBAAiB;IAiF/B;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAyBpB;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAoB;YAE3C,SAAS;IAoGvB;;OAEG;IACH,OAAO,CAAC,cAAc;IA4BtB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoFrB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0GzB;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAoE/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqD3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,QAAQ;IA8FhB;;OAEG;IACH,OAAO,CAAC,IAAI;IAIZ;;;;OAIG;IACG,qBAAqB,CACzB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAGjE;IAED;;;;;;OAMG;IACG,+BAA+B,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC,CAyED;CACF"}
+{"version":3,"file":"codeScanner.d.ts","sourceRoot":"","sources":["../src/codeScanner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAeH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,sBAAsB,QAAmB,CAAC;AAEvD,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAcD;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,EAAE,SAAS,MAAM,EAqDjD,CAAC;AA4EF;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAQrD;AAUD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,0BAA0B,EAAE,SAAS,MAAM,EAqFvD,CAAC;AAsFF;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,CAalE;AAwCD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAI5D;AASD;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAmBvE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,QAA4B,CAAC;AACzD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAID;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,aAAa,GAAE,SAAS,MAAM,EAAO,GACpC,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAmB9B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAWrE;AAcD,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAsB;IAE1D,YAAY,OAAO,GAAE,WAAgB,EAKpC;IAED;;;;OAIG;IACH,OAAO,CAAC,WAAW;IASnB;;;;;;;OAOG;YACW,eAAe;IA4B7B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAqBtB;;;;OAIG;YACW,uBAAuB;IAwBrC;;;;;OAKG;YACW,uBAAuB;IAerC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEvC;;;;;;OAMG;YACW,cAAc;IAM5B;;;OAGG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG1D;IAED;;;;;;;OAOG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAKrD;IAED;;;;;;;;;;;;;;;OAeG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnD;IAED;;;;;;OAMG;IACG,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAQ3F;YAEa,aAAa;IA+D3B;;;OAGG;IACG,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAsC3E;IAED;;;;;;;;;OASG;IACH,OAAO,CAAC,WAAW;IAYnB;;;OAGG;YACW,iBAAiB;IAgF/B;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IA6BpB;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAA0B;YAEjD,SAAS;IAoGvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoFrB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0GzB;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAoE/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqD3B;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,QAAQ;IA8FhB;;OAEG;IACH,OAAO,CAAC,IAAI;IAIZ;;;;OAIG;IACG,qBAAqB,CACzB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAGjE;IAED;;;;;;OAMG;IACG,+BAA+B,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC,CAmFD;CACF"}