@mnemonik/shared 6.47.0 → 6.50.0

package/src/codeScanner.ts

@@ -6,7 +6,7 @@ import { readdir, readFile, stat, lstat, realpath } from 'fs/promises';
 import { join, relative, extname, sep } from 'path';
 import { createHash } from 'crypto';
 import ignore, { type Ignore } from 'ignore';
-import { debug as logDebug } from './logger.js';
+import { debug as logDebug, warn as logWarn } from './logger.js';
 import { withTimeout } from './asyncUtils.js';
 import { scrubSecrets } from './secretPatterns.js';
 
@@ -15,6 +15,28 @@ import { scrubSecrets } from './secretPatterns.js';
  */
 const FILE_OP_TIMEOUT_MS = 5000;
 
+/**
+ * The one file-size ceiling in Mnemonik. A file at or below this is scanned,
+ * chunked, AND pushed verbatim; a file above it is uniformly outside the
+ * system — not scanned, not chunked, not citable. "Scanned ⟺ verifiable".
+ *
+ * This is deliberately a single exported constant rather than a number
+ * repeated per layer. Three ceilings previously existed and had drifted:
+ * the scan ceiling here (10 MB), a 5 MB whole-file push cap in the scanner
+ * daemon, and a 5 MB `content` cap in the server's scanPushSchema. Files
+ * between 5 and 10 MB were therefore indexed as chunks whose source content
+ * the server never received — chunks that could be retrieved but never
+ * verified against, a silent fidelity split. Every consumer now imports this:
+ *
+ * - `CodeScanner.MAX_FILE_SIZE` / `collectAuthorityFilesWithStatus` (this file)
+ * - `MAX_PUSH_CONTENT_BYTES` (packages/scanner/src/daemon.ts)
+ * - `scanFileSchema.content` (src/server/routes/schemas.ts)
+ *
+ * Raising it means widening the stored-content pipe end to end; do not raise
+ * one site alone.
+ */
+export const MAX_SCANNED_FILE_BYTES = 10 * 1024 * 1024; // 10MB
+
 export interface CodeChunk {
   content: string;
   filePath: string;
@@ -129,7 +151,10 @@ export const BUILT_IN_IGNORE_DIRS: readonly string[] = [
  * generically by the isGitBoundary rule below, not a path convention.)
  */
 const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
-  '.env',
+  // NOTE: the `.env` family and every other secret-bearing FILE KIND lives in
+  // `isSecretFile` below, not here. This glob engine cannot express negation
+  // (`.env.example` must stay collectable) and its unanchored substring
+  // matching would make `.env.*` also swallow `.environment.ts`.
   '.DS_Store',
   '*.log',
   '*.lock',
@@ -143,6 +168,66 @@ const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
   '/tests/fixtures/*',
 ];
 
+/**
+ * Files whose CONTENT is a credential rather than code. Today these are
+ * excluded only INCIDENTALLY — `.pem`, `.key` and friends carry no
+ * allowlisted extension, so nothing chunks them. That protection evaporates
+ * the moment the extension allowlist widens (as it did for `.sh`/`.lua`), so
+ * the exclusion is made explicit here and becomes a defended invariant:
+ * unlike `ignorePatterns`, this predicate is NOT caller-overridable.
+ *
+ * Expressed as a predicate rather than glob strings because `shouldIgnore`'s
+ * engine has no negation and matches globs as unanchored substrings — neither
+ * property can state "every `.env.*` EXCEPT `.env.example`".
+ *
+ * Precision notes (over-exclusion is a fidelity failure in its own right):
+ * - `.env` family: `/^\.env(\..+)?$/` matches `.env`, `.env.local`,
+ *   `.env.production` — and deliberately NOT `.envrc` or `.environment.ts`.
+ * - `.env.example` is carved out: it is authority-collected on purpose
+ *   (AUTHORITY_FILE_MATCHERS) and holds placeholder values, not secrets.
+ *   It is the doc-truth env_vars authority; excluding it would blind that
+ *   extractor exactly as the old `process.env` over-scrub once did.
+ * - `credentials` is matched ONLY directly inside a dot-directory
+ *   (`.aws/credentials`, `.docker/credentials`). A bare `credentials/` is a
+ *   real, common SOURCE directory name (gRPC, auth libraries) and
+ *   `shouldIgnore` cannot tell a file from a directory at its call sites —
+ *   an unqualified rule would silently delete that subtree from the index.
+ * - `.tfvars` is excluded whole rather than per-basename: the credential file
+ *   is conventionally `terraform.tfvars` but the name is free-form, and the
+ *   variables that are NOT secret are visible in `.tf` anyway.
+ */
+const SECRET_FILE_PLACEHOLDER_BASENAMES: ReadonlySet<string> = new Set(['.env.example']);
+
+const SECRET_FILE_BASENAME_PATTERNS: readonly RegExp[] = [
+  /^\.env(\..+)?$/i, // .env, .env.local, .env.production.local
+  /^id_(?:rsa|dsa|ecdsa|ed25519)/i, // ssh private keys (and .pub siblings)
+  /^\.(?:npmrc|netrc|pgpass)$/i, // registry / ftp / postgres password files
+  /^kubeconfig$/i,
+  /\.(?:pem|key|p12|pfx|keystore|jks|kubeconfig)$/i,
+  // Terraform variable files: `terraform.tfvars` is the conventional home for
+  // provider credentials, and unlike the entries above nothing else was
+  // keeping it out — it is excluded here rather than merely left off
+  // `DEFAULT_INCLUDE_EXTENSIONS`, so a future widening of the allowlist cannot
+  // quietly opt a project into embedding its own cloud keys.
+  /\.tfvars(\.json)?$/i,
+];
+
+/**
+ * True when `relPath` names a file whose contents are credentials. Applied
+ * by `shouldIgnore` (every walker: directory scan, explicit file lists, and
+ * the authority-file walk) and by `makeIgnoreMatcher` (server-side backstop
+ * for older or misbehaving daemons). Accepts OS-native or POSIX separators.
+ */
+export function isSecretFile(relPath: string): boolean {
+  if (!relPath) return false;
+  const segments = relPath.split(sep).join('/').split('/');
+  const base = segments[segments.length - 1] ?? '';
+  if (!base || SECRET_FILE_PLACEHOLDER_BASENAMES.has(base)) return false;
+  if (SECRET_FILE_BASENAME_PATTERNS.some((re) => re.test(base))) return true;
+  const parent = segments[segments.length - 2];
+  return base === 'credentials' && parent !== undefined && parent.startsWith('.');
+}
+
 /**
  * Per-directory ignore files the walkers honor, unioned. `.gitignore` is the
  * project's own declaration of generated/vendored paths; `.mnemonikignore`
@@ -151,38 +236,271 @@ const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
  */
 const IGNORE_FILE_NAMES = ['.gitignore', '.mnemonikignore'] as const;
 
+/**
+ * Every file type the scanner will chunk, by extension. Matched
+ * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
+ * also cover `.R`, `.SQL`, `.PS1` and the uppercase `.C`/`.H` of older trees.
+ *
+ * Exported because this list IS the product's language coverage: a project
+ * written in something absent from it indexes zero code, and `code_search`
+ * reports that as "no indexed matches" rather than "I do not read this
+ * language" — invisible to the developer and to us. Coverage is a separate
+ * dial from parse quality: a language with no structured extractor falls
+ * through to `chunkRaw`, which is what Go, Java and C already get in
+ * production, and crude chunks beat no chunks by an enormous margin.
+ *
+ * Data formats (`.json`, `.yaml`, `.toml`, `.lock`) are deliberately absent:
+ * the ones that carry meaning are already collected verbatim by
+ * `AUTHORITY_FILE_MATCHERS`, and blanket-indexing the extension would pull in
+ * lockfiles and generated output. `.tfvars` is absent for a different reason —
+ * `terraform.tfvars` is a conventional home for provider credentials and
+ * `isSecretFile` now excludes it outright.
+ */
+export const DEFAULT_INCLUDE_EXTENSIONS: readonly string[] = [
+  '.ts',
+  '.tsx',
+  '.js',
+  '.jsx',
+  '.py',
+  '.rs',
+  '.go',
+  '.java',
+  '.c',
+  '.cpp',
+  '.h',
+  '.cs',
+  '.rb',
+  '.php',
+  '.swift',
+  '.kt',
+  // Shell and Lua are the working languages of whole real projects —
+  // deployment tooling, container entrypoints, imapfilter/nginx/redis
+  // configuration. Omitting them meant such a project indexed ZERO code and
+  // code_search could not answer anything about it, while reporting that as
+  // "no indexed matches" rather than as missing coverage.
+  '.sh',
+  '.bash',
+  '.zsh',
+  '.lua',
+  '.md',
+
+  // ── Variants of languages already on the list. `.mjs`/`.cjs`/`.mts`/`.cts`/
+  // `.pyi` inherit their language string from a listed extension, so their
+  // absence was oversight rather than policy. The C++ spellings are a choice,
+  // not an inheritance: `.h` stays 'c' (C headers dominate, and reading a C++
+  // header as C is the safer default), while the unambiguously-C++ spellings
+  // resolve to 'cpp'.
+  '.mjs',
+  '.cjs',
+  '.mts',
+  '.cts',
+  '.pyi',
+  '.hpp',
+  '.hh',
+  '.hxx',
+  '.cc',
+  '.cxx',
+  '.kts',
+
+  // ── Languages the scanner could not read at all.
+  '.dart',
+  '.m',
+  '.mm',
+  '.scala',
+  '.sc',
+  '.ex',
+  '.exs',
+  '.erl',
+  '.hrl',
+  '.hs',
+  '.jl',
+  '.ml',
+  '.mli',
+  '.clj',
+  '.cljs',
+  '.cljc',
+  '.groovy',
+  '.gradle',
+  '.ps1',
+  '.psm1',
+  '.pl',
+  '.pm',
+  '.r',
+  '.sol',
+  '.zig',
+  '.vue',
+  '.svelte',
+
+  // ── Infrastructure and schema DSLs, where real logic lives and where "how
+  // is this deployed?" and "what does this table hold?" go unanswered today.
+  '.tf',
+  '.sql',
+  '.proto',
+  '.graphql',
+  '.gql',
+  '.cmake',
+  '.nix',
+  '.bzl',
+];
+
+/**
+ * Extension → language string, the value carried on every chunk and on the
+ * wire (`/scan/push` accepts any non-empty string up to 50 chars).
+ *
+ * A string no grammar claims is correct and expected: the AST layer resolves
+ * such a language to `null` and the heuristic chunker takes over. Keep this in
+ * sync with `DEFAULT_INCLUDE_EXTENSIONS` — an allowlisted extension that lands
+ * on 'unknown' still gets chunked, but nothing downstream can reason about it.
+ *
+ * Two extensions are genuinely ambiguous and are resolved rather than fudged:
+ * `.m` is Objective-C here, not MATLAB, because a repo carrying `.m` alongside
+ * `.h`/`.mm` is overwhelmingly an Apple-platform project; `.pl` is Perl, not
+ * Prolog, on the same frequency argument.
+ */
+const EXTENSION_LANGUAGES: Readonly<Record<string, string>> = {
+  '.ts': 'typescript',
+  '.tsx': 'typescript',
+  '.mts': 'typescript',
+  '.cts': 'typescript',
+  '.js': 'javascript',
+  '.jsx': 'javascript',
+  '.mjs': 'javascript',
+  '.cjs': 'javascript',
+  '.py': 'python',
+  '.pyi': 'python',
+  '.rs': 'rust',
+  '.go': 'go',
+  '.java': 'java',
+  '.c': 'c',
+  '.h': 'c',
+  '.cpp': 'cpp',
+  '.cc': 'cpp',
+  '.cxx': 'cpp',
+  '.hpp': 'cpp',
+  '.hh': 'cpp',
+  '.hxx': 'cpp',
+  '.cs': 'csharp',
+  '.rb': 'ruby',
+  '.php': 'php',
+  '.swift': 'swift',
+  '.kt': 'kotlin',
+  '.kts': 'kotlin',
+  '.sh': 'shell',
+  '.bash': 'shell',
+  '.zsh': 'shell',
+  '.lua': 'lua',
+  '.md': 'markdown',
+  '.dart': 'dart',
+  '.m': 'objc',
+  '.mm': 'objc',
+  '.scala': 'scala',
+  '.sc': 'scala',
+  '.ex': 'elixir',
+  '.exs': 'elixir',
+  '.erl': 'erlang',
+  '.hrl': 'erlang',
+  '.hs': 'haskell',
+  '.jl': 'julia',
+  '.ml': 'ocaml',
+  '.mli': 'ocaml',
+  '.clj': 'clojure',
+  '.cljs': 'clojure',
+  '.cljc': 'clojure',
+  '.groovy': 'groovy',
+  '.gradle': 'groovy',
+  '.ps1': 'powershell',
+  '.psm1': 'powershell',
+  '.pl': 'perl',
+  '.pm': 'perl',
+  '.r': 'r',
+  '.sol': 'solidity',
+  '.zig': 'zig',
+  '.vue': 'vue',
+  '.svelte': 'svelte',
+  '.tf': 'terraform',
+  '.sql': 'sql',
+  '.proto': 'protobuf',
+  '.graphql': 'graphql',
+  '.gql': 'graphql',
+  '.cmake': 'cmake',
+  '.nix': 'nix',
+  '.bzl': 'starlark',
+};
+
+/**
+ * Language string for a file path or a bare extension, `'unknown'` when the
+ * extension is unmapped. A free function rather than a method because callers
+ * that never scan anything (AST grammar resolution, server-side symbol
+ * preference) need the same answer without constructing a scanner.
+ */
+export function languageForExtension(filePathOrExt: string): string {
+  // `extname` FIRST. A dotfile that carries a real extension ('.eslintrc.js',
+  // '.mocharc.cjs', '.prettierrc.ts') starts with '.' and contains no '/', so
+  // a bare-extension-first reading swallowed the whole name and answered
+  // 'unknown' — while the same file spelled 'src/.eslintrc.js' answered
+  // 'javascript'. Those extensions are allowlisted, so the files are indexed
+  // and reach the server with exactly the root-relative spelling that failed.
+  const fromPath = EXTENSION_LANGUAGES[extname(filePathOrExt).toLowerCase()];
+  if (fromPath !== undefined) return fromPath;
+  // Fallback: the argument IS the extension ('.dart'), for which `extname`
+  // returns ''. Every key contains a leading dot and no separator, so a real
+  // path can never collide here.
+  return EXTENSION_LANGUAGES[filePathOrExt.toLowerCase()] ?? 'unknown';
+}
+
+/**
+ * THE definition of "this path is a SQL migration the schema_columns authority
+ * collects verbatim". One predicate, referenced by both halves of the deal —
+ * `AUTHORITY_FILE_MATCHERS` (collect it) and `isAuthorityOnlyPath` (therefore
+ * do not chunk it) — because two hand-written regexes drifted once already and
+ * the failure is silent in both directions.
+ *
+ * Root-anchored and CASE-SENSITIVE on purpose: it mirrors the server-side
+ * extractor, which does `listFiles('migrations/')` (LIKE 'migrations/%') then
+ * `endsWith('.sql')`, both case-sensitive. `Migrations/001.sql` (the .NET/EF
+ * Core spelling) and `migrations/002.SQL` are NOT collected, so they must not
+ * be suppressed from chunking either — that would index them nowhere. Same
+ * reason a nested `packages/x/migrations/y.sql` is left alone.
+ */
+const isMigrationSqlAuthorityPath = (posixRelPath: string): boolean =>
+  /^migrations\/.*\.sql$/.test(posixRelPath);
+
+/**
+ * Predicates for paths whose verbatim content is ALREADY shipped by
+ * `collectAuthorityFiles` and that carry no additional value as embedded code
+ * chunks. Checked at the extension gate rather than in `shouldIgnore`, because
+ * `shouldIgnore` also guards the authority walk and must keep letting these
+ * through.
+ *
+ * `migrations/**.sql` is the live case: adding `.sql` to the allowlist without
+ * this exclusion would dual-collect every migration — once verbatim, once
+ * chunked and embedded. On this repo alone that is 208 files of append-only
+ * DDL (140 forward, the rest rollback/manual), embedded to answer questions
+ * the authority path already answers exactly.
+ *
+ * INVARIANT: every predicate here must also appear in
+ * `AUTHORITY_FILE_MATCHERS`, so no path can be excluded from chunking unless
+ * the authority path definitely collects it.
+ */
+const AUTHORITY_ONLY_PATH_PREDICATES: readonly ((posixRelPath: string) => boolean)[] = [
+  isMigrationSqlAuthorityPath,
+];
+
+/**
+ * True when `relPath` is collected verbatim as authority content and must not
+ * additionally be chunked. Accepts OS-native or POSIX separators.
+ */
+export function isAuthorityOnlyPath(relPath: string): boolean {
+  if (!relPath) return false;
+  const posix = relPath.split(sep).join('/');
+  return AUTHORITY_ONLY_PATH_PREDICATES.some((matches) => matches(posix));
+}
+
 const DEFAULT_OPTIONS: Required<ScanOptions> = {
   maxChunkSize: 8000, // ~2000 tokens
   minChunkSize: 100,
   ignorePatterns: [...BUILT_IN_IGNORE_DIRS, ...BUILT_IN_IGNORE_FILE_PATTERNS],
-  includeExtensions: [
-    '.ts',
-    '.tsx',
-    '.js',
-    '.jsx',
-    '.py',
-    '.rs',
-    '.go',
-    '.java',
-    '.c',
-    '.cpp',
-    '.h',
-    '.cs',
-    '.rb',
-    '.php',
-    '.swift',
-    '.kt',
-    // Shell and Lua are the working languages of whole real projects —
-    // deployment tooling, container entrypoints, imapfilter/nginx/redis
-    // configuration. Omitting them meant such a project indexed ZERO code and
-    // code_search could not answer anything about it, while reporting that as
-    // "no indexed matches" rather than as missing coverage.
-    '.sh',
-    '.bash',
-    '.zsh',
-    '.lua',
-    '.md',
-  ],
+  includeExtensions: [...DEFAULT_INCLUDE_EXTENSIONS],
 };
 
 /**
@@ -205,8 +523,10 @@ export const AUTHORITY_FILE_MATCHERS: Array<(relPath: string) => boolean> = [
   // SQL migrations: the schema_columns authority extractor reads every .sql
   // under `migrations/` (listFiles('migrations/') -> LIKE 'migrations/%' then
   // .endsWith('.sql')). Without collecting these, that authority is empty and
-  // every schema_table_enumeration claim falls to unverifiable.
-  (p) => /^migrations\/.*\.sql$/.test(p),
+  // every schema_table_enumeration claim falls to unverifiable. Shared with
+  // `AUTHORITY_ONLY_PATH_PREDICATES` by reference, not by a copied regex, so
+  // collection and the chunking exclusion cannot disagree.
+  isMigrationSqlAuthorityPath,
 ];
 
 /**
@@ -243,6 +563,7 @@ export function makeIgnoreMatcher(
     if (!relPath) return false;
     const norm = relPath.split(sep).join('/');
     if (norm.split('/').some((s) => BUILT_IN_IGNORE_DIR_SET.has(s))) return true;
+    if (isSecretFile(norm)) return true;
     if (ig) {
       try {
         return ig.ignores(norm);
@@ -295,8 +616,34 @@ type IgnoreStack = IgnoreLayer[];
 export class CodeScanner {
   private options: Required<ScanOptions>;
 
+  /**
+   * `includeExtensions` folded to lowercase for matching. The gate used to
+   * compare `extname()` verbatim while `detectLanguage` lowercased, so a
+   * project spelling its files the canonical way — `.R` for R, `.SQL`/`.PS1`
+   * on Windows, `.C`/`.H` in older C trees — indexed zero of them and nothing
+   * said why.
+   */
+  private readonly includeExtensionSet: ReadonlySet<string>;
+
   constructor(options: ScanOptions = {}) {
     this.options = { ...DEFAULT_OPTIONS, ...options };
+    this.includeExtensionSet = new Set(
+      this.options.includeExtensions.map((ext) => ext.toLowerCase())
+    );
+  }
+
+  /**
+   * The chunkable-file gate, shared by every walker and by the explicit
+   * file-list path so all three agree on what exists. `relPath` is the path
+   * relative to the scan root (OS-native separators accepted).
+   */
+  private isChunkable(absOrRelPath: string, relPath: string): boolean {
+    if (!this.includeExtensionSet.has(extname(absOrRelPath).toLowerCase())) return false;
+    if (isAuthorityOnlyPath(relPath)) {
+      logDebug('Skipping chunking for authority-collected path', { relPath });
+      return false;
+    }
+    return true;
   }
 
   /**
@@ -548,7 +895,7 @@ export class CodeScanner {
           await this.traversePaths(fullPath, rootPath, out, depth + 1, walk, localStack);
         } else if (stats.isFile()) {
           if (this.ignoredByStack(relativePath, false, localStack)) continue;
-          if (this.options.includeExtensions.includes(extname(fullPath))) {
+          if (this.isChunkable(fullPath, relativePath)) {
             out.push(relativePath);
           }
         }
@@ -591,8 +938,7 @@ export class CodeScanner {
           continue;
         }
 
-        const ext = extname(filePath);
-        if (this.options.includeExtensions.includes(ext)) {
+        if (this.isChunkable(filePath, fileRel)) {
           const fileChunks = await this.parseFile(filePath, rootPath || filePath);
           chunks.push(...fileChunks);
         }
@@ -698,8 +1044,7 @@ export class CodeScanner {
           await this.traverseDirectory(fullPath, rootPath, chunks, depth + 1, walk, localStack);
         } else if (stats.isFile()) {
           if (this.ignoredByStack(relativePath, false, localStack)) continue;
-          const ext = extname(fullPath);
-          if (this.options.includeExtensions.includes(ext)) {
+          if (this.isChunkable(fullPath, relativePath)) {
             const fileChunks = await this.parseFile(fullPath, rootPath);
             chunks.push(...fileChunks);
           }
@@ -720,6 +1065,10 @@ export class CodeScanner {
    *   (e.g., '.env' should not match '.environment.ts')
    */
   private shouldIgnore(path: string): boolean {
+    // Credential-bearing file kinds first, and outside the overridable
+    // `ignorePatterns` list — a caller-supplied pattern set must not be able
+    // to opt a project back into indexing its own private keys.
+    if (isSecretFile(path)) return true;
     const segments = path.split('/');
     return this.options.ignorePatterns.some((pattern) => {
       if (pattern.includes('*')) {
@@ -745,10 +1094,12 @@ export class CodeScanner {
   }
 
   /**
-   * Parse a file and extract code chunks
-   * Added 10MB file size limit
+   * Parse a file and extract code chunks.
+   * Size limit is the shared `MAX_SCANNED_FILE_BYTES` ceiling — the same
+   * number the daemon and the server's scanPushSchema enforce, so anything
+   * chunked here can always be shipped and stored verbatim.
    */
-  private static readonly MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB
+  private static readonly MAX_FILE_SIZE = MAX_SCANNED_FILE_BYTES;
 
   private async parseFile(filePath: string, rootPath: string): Promise<CodeChunk[]> {
     try {
@@ -854,31 +1205,7 @@ export class CodeScanner {
    * Detect language from file extension
    */
   private detectLanguage(filePath: string): string {
-    const ext = extname(filePath).toLowerCase();
-    const langMap: Record<string, string> = {
-      '.ts': 'typescript',
-      '.tsx': 'typescript',
-      '.js': 'javascript',
-      '.jsx': 'javascript',
-      '.py': 'python',
-      '.rs': 'rust',
-      '.go': 'go',
-      '.java': 'java',
-      '.c': 'c',
-      '.cpp': 'cpp',
-      '.h': 'c',
-      '.cs': 'csharp',
-      '.rb': 'ruby',
-      '.php': 'php',
-      '.swift': 'swift',
-      '.kt': 'kotlin',
-      '.sh': 'shell',
-      '.bash': 'shell',
-      '.zsh': 'shell',
-      '.lua': 'lua',
-      '.md': 'markdown',
-    };
-    return langMap[ext] || 'unknown';
+    return languageForExtension(filePath);
   }
 
   /**
@@ -1396,9 +1723,19 @@ export class CodeScanner {
         } else if (isFile && AUTHORITY_FILE_MATCHERS.some((m) => m(rel))) {
           try {
             let content = await readFile(full, 'utf-8');
-            // C2: skip files whose content exceeds the server's 5MB cap —
-            // manifests are tiny; an oversized one is anomalous.
-            if (content.length > 5_000_000) continue;
+            // Authority files honor the same single ceiling as chunked files
+            // (MAX_SCANNED_FILE_BYTES). Manifests are tiny; one this large is
+            // anomalous, so say so rather than dropping it silently — a
+            // missing authority file otherwise looks identical to a project
+            // that simply has no manifest.
+            if (content.length > MAX_SCANNED_FILE_BYTES) {
+              logWarn('Authority file exceeds the scan ceiling; excluded from push', {
+                path: rel,
+                size: content.length,
+                limit: MAX_SCANNED_FILE_BYTES,
+              });
+              continue;
+            }
             // NUL sanitation (ingestion boundary, daemon side): Postgres
             // `text` columns reject the literal NUL byte (U+0000). Authority
             // files are shipped verbatim (no chunking), so strip here before

package/src/index.ts

@@ -9,16 +9,26 @@ export { MCP_INSTRUCTIONS, MCP_INSTRUCTIONS_RAW, getMcpInstructions } from './in
 export { USAGE_GUIDE } from './usageGuide.js';
 export {
   CodeScanner,
+  MAX_SCANNED_FILE_BYTES,
   AUTHORITY_FILE_MATCHERS,
   BUILT_IN_IGNORE_DIRS,
+  DEFAULT_INCLUDE_EXTENSIONS,
+  languageForExtension,
   makeIgnoreMatcher,
   isGitBoundary,
   FIXTURE_PATH_RE,
   isFixturePath,
+  isSecretFile,
+  isAuthorityOnlyPath,
   type CodeChunk,
   type ScanOptions,
 } from './codeScanner.js';
-export { SECRET_PATTERNS, SECRET_REDACTION_PLACEHOLDER, scrubSecrets } from './secretPatterns.js';
+export {
+  SECRET_PATTERNS,
+  SECRET_REDACTION_PLACEHOLDER,
+  scrubSecrets,
+  redactHighEntropyTokens,
+} from './secretPatterns.js';
 export {
   FETCH_TIMEOUT_MS,
   TELEMETRY_TIMEOUT_MS,