codedeep-mcp 0.1.0

package/dist/indexer/pipeline.js

@@ -0,0 +1,342 @@
+import { createHash } from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import { isAbsolute, join, relative } from 'node:path';
+import { errMsg, log } from '../logger.js';
+import { LANGUAGE_UNKNOWN } from '../types.js';
+import { extractSymbols } from './extractor.js';
+import { initParser, parseFile } from './parser.js';
+import { compileExcludeMatcher, detectLanguage, isBinaryByContent, isBinaryByExtension, refineHeaderLanguage, scanProject, toPosix, } from './scanner.js';
+const BATCH_SIZE = 50;
+// The ONE no-change policy, applied by both indexChanged's scan diff and
+// indexFile's single-file path. mtime alone misses content swaps under
+// coarse-resolution filesystems or `cp -p` / archive extraction that
+// preserves timestamps; comparing size catches the common case cheaply.
+// indexFile additionally hash-verifies (see indexFileInner) because it
+// runs in response to an explicit fs event.
+// `language` is the freshly DETECTED language for the path: an upgrade
+// that teaches the scanner a new extension (e.g. `.java`) reclassifies
+// files a cached index recorded as 'unknown' — those must re-index even
+// though their bytes never changed, or the new language stays inert on
+// every warm cache.
+function isUnchanged(prev, mtimeMs, size, language) {
+    return (prev !== undefined &&
+        prev.lastModified === mtimeMs &&
+        prev.size === size &&
+        prev.language === language);
+}
+function hashContent(content) {
+    return createHash('sha1').update(content).digest('hex').slice(0, 16);
+}
+export class Indexer {
+    config;
+    index;
+    cachePath;
+    matchExclude;
+    indexing = false;
+    done = 0;
+    total = 0;
+    ready = false;
+    // Whether the most recent indexAll/indexChanged saw a COMPLETE scan.
+    // A partial scan (transient readdir failure) resolves successfully but
+    // preserves unseen cached entries — the watcher must know the rescan
+    // it requested may not have covered everything.
+    lastScanCompleteFlag = true;
+    get lastScanComplete() {
+        return this.lastScanCompleteFlag;
+    }
+    constructor(config, index) {
+        this.config = config;
+        this.index = index;
+        this.cachePath = join(config.cacheDir, 'index.json');
+        this.matchExclude = compileExcludeMatcher(config.exclude);
+    }
+    get isIndexing() {
+        return this.indexing;
+    }
+    get progress() {
+        return { done: this.done, total: this.total };
+    }
+    // indexAll/indexChanged resolve `true` when the work ran and `false`
+    // when the concurrency guard dropped the request; indexFile returns the
+    // richer IndexFileResult so the watcher can tell mutation from idle
+    // work and retry what deserves retrying.
+    async indexAll() {
+        return this.runGuarded(async () => {
+            await initParser();
+            const { files: current, complete } = await scanProject(this.config);
+            this.lastScanCompleteFlag = complete;
+            this.total = current.length;
+            await this.processBatched(current);
+            // Prune cached entries no longer in the scan, but only when the scan
+            // was complete. A partial scan (transient readdir/stat failure) would
+            // otherwise drop valid symbols until the next clean run.
+            if (complete) {
+                const currentPaths = new Set(current.map((f) => f.path));
+                for (const existing of this.index.getAllFiles()) {
+                    if (!currentPaths.has(existing.path)) {
+                        this.index.removeFile(existing.path);
+                    }
+                }
+            }
+            else {
+                log.warn('Indexer.indexAll: scan incomplete; preserving cached entries not seen in this scan');
+            }
+            await this.persist();
+            this.ready = true;
+        });
+    }
+    async indexChanged() {
+        return this.runGuarded(async () => {
+            await initParser();
+            const { files: current, complete } = await scanProject(this.config);
+            this.lastScanCompleteFlag = complete;
+            const previous = new Map(this.index.getAllFiles().map((f) => [f.path, f]));
+            const toIndex = [];
+            for (const f of current) {
+                const prev = previous.get(f.path);
+                if (!isUnchanged(prev, f.lastModified, f.size, f.language)) {
+                    toIndex.push(f);
+                }
+                previous.delete(f.path);
+            }
+            let deletedCount = 0;
+            if (complete) {
+                deletedCount = previous.size;
+                for (const stalePath of previous.keys()) {
+                    this.index.removeFile(stalePath);
+                }
+            }
+            else if (previous.size > 0) {
+                log.warn(`Indexer.indexChanged: scan incomplete; preserving ${previous.size} cached entries not seen`);
+            }
+            if (toIndex.length === 0 && deletedCount === 0) {
+                log.debug('Indexer: indexChanged found no changes');
+                this.ready = true;
+                return;
+            }
+            this.total = toIndex.length;
+            await this.processBatched(toIndex);
+            await this.persist();
+            this.ready = true;
+        });
+    }
+    // Does NOT call save() — callers debounce events and batch persistence themselves.
+    async indexFile(rawPath) {
+        let outcome = 'noop';
+        const ran = await this.runGuarded(async () => {
+            outcome = await this.indexFileInner(rawPath);
+        });
+        return ran ? outcome : 'dropped';
+    }
+    async indexFileInner(rawPath) {
+        await initParser();
+        // Canonicalize to a project-relative POSIX path so the cache key
+        // aligns with the scanner's `src/a.ts` form regardless of whether
+        // the watcher emits an absolute path, a `./`-prefix, or Windows
+        // backslashes. Mismatched keys would orphan stale symbols and
+        // create duplicate entries on update.
+        const projectRoot = this.config.projectRoot;
+        const absInput = isAbsolute(rawPath) ? rawPath : join(projectRoot, rawPath);
+        const relPath = toPosix(relative(projectRoot, absInput));
+        if (relPath === '' || relPath === '..' || relPath.startsWith('../')) {
+            log.debug(`Indexer.indexFile: skip ${rawPath} (outside project root)`);
+            return 'noop';
+        }
+        const removed = () => this.index.removeFile(relPath) ? 'removed' : 'noop';
+        if (this.matchExclude(relPath) || isBinaryByExtension(relPath)) {
+            return removed();
+        }
+        // Stat before language detection so deletions and size-cap rejections
+        // remove cached entries even for unknown-language files.
+        const absPath = join(this.config.projectRoot, relPath);
+        let stats;
+        try {
+            stats = await fs.lstat(absPath);
+        }
+        catch (err) {
+            log.debug(`Indexer.indexFile: stat failed for ${relPath} (${errMsg(err)}); treated as deletion`);
+            return removed();
+        }
+        if (stats.isSymbolicLink()) {
+            log.debug(`Indexer.indexFile: skip ${relPath} (symlink)`);
+            return removed();
+        }
+        // FIFOs/sockets/devices must not reach isBinaryByContent — opening
+        // a writer-less named pipe blocks forever and would wedge the
+        // watcher's flush chain. (Directories land here too when called
+        // directly; the watcher routes those to a rescan first.)
+        if (!stats.isFile()) {
+            log.debug(`Indexer.indexFile: skip ${relPath} (not a regular file)`);
+            return removed();
+        }
+        // Mirror the scanner's maxFiles cap for files not already indexed —
+        // without this, watcher events could grow the index unboundedly
+        // past the configured bound until the next full scan prunes it.
+        if (this.config.maxFiles > 0 &&
+            !this.index.hasFile(relPath) &&
+            this.index.fileCount >= this.config.maxFiles) {
+            log.debug(`Indexer.indexFile: skip ${relPath} (index at maxFiles=${this.config.maxFiles})`);
+            return 'cap-skipped';
+        }
+        // Metadata-only events (and the trailing event of an atomic-save
+        // pair) would otherwise pay a full parse + index update + save. But
+        // mtime+size alone cannot distinguish an atomic-save echo from a
+        // REAL second same-size edit landing in the same coarse-mtime tick
+        // (HFS+/FAT/NFS report whole seconds) — an explicit fs event fired,
+        // so verify by content hash (read without parse) before skipping.
+        let language = detectLanguage(relPath) ?? LANGUAGE_UNKNOWN;
+        // A `.h` mapped to 'cpp' may be an Objective-C header — content-sniff it (no-op
+        // unless `.h` AND objc configured). MUST run before isUnchanged (which compares
+        // the stored language), so a `.h` re-classified by a heuristic tweak re-indexes.
+        if (language === 'cpp') {
+            language = await refineHeaderLanguage(absPath, language, new Set(this.config.languages));
+        }
+        const existing = this.index.getFile(relPath);
+        if (isUnchanged(existing, stats.mtimeMs, stats.size, language)) {
+            if (existing?.contentHash !== undefined) {
+                try {
+                    const content = await fs.readFile(absPath, 'utf8');
+                    if (hashContent(content) === existing.contentHash) {
+                        log.debug(`Indexer.indexFile: ${relPath} unchanged; skipping`);
+                        return 'noop';
+                    }
+                    // Same stat fingerprint, different bytes — fall through and
+                    // re-index for real.
+                }
+                catch (err) {
+                    log.debug(`Indexer.indexFile: hash check read failed for ${relPath} (${errMsg(err)}); treated as deletion`);
+                    return removed();
+                }
+            }
+            else {
+                // No stored hash (unknown-language entry) — stat match suffices;
+                // these files carry no symbols to go stale.
+                log.debug(`Indexer.indexFile: ${relPath} unchanged; skipping`);
+                return 'noop';
+            }
+        }
+        if (stats.size > this.config.maxFileSize) {
+            log.debug(`Indexer.indexFile: skip ${relPath} (size ${stats.size} > maxFileSize ${this.config.maxFileSize})`);
+            return removed();
+        }
+        if (language !== LANGUAGE_UNKNOWN &&
+            !this.config.languages.includes(language)) {
+            return removed();
+        }
+        if (language === LANGUAGE_UNKNOWN) {
+            try {
+                if (await isBinaryByContent(absPath)) {
+                    return removed();
+                }
+            }
+            catch (err) {
+                log.warn(`Indexer.indexFile: byte check failed for ${relPath}: ${errMsg(err)}`);
+                return removed();
+            }
+        }
+        const file = {
+            path: relPath,
+            language,
+            size: stats.size,
+            lastModified: stats.mtimeMs,
+            lastIndexed: 0,
+            symbolCount: 0,
+        };
+        this.total = 1;
+        const result = await this.processFile(file);
+        this.done = 1;
+        return result;
+    }
+    // Resolves `false` when a run is already in flight (the request is
+    // dropped, not queued); `true` when the work ran to completion.
+    async runGuarded(work) {
+        if (this.indexing) {
+            log.warn('Indexer: indexing already in progress; refusing concurrent run');
+            return false;
+        }
+        this.indexing = true;
+        this.done = 0;
+        this.total = 0;
+        try {
+            await work();
+        }
+        finally {
+            this.indexing = false;
+        }
+        return true;
+    }
+    async processBatched(files) {
+        for (let i = 0; i < files.length; i += BATCH_SIZE) {
+            const end = Math.min(i + BATCH_SIZE, files.length);
+            for (let j = i; j < end; j++) {
+                await this.processFile(files[j]);
+                this.done++;
+            }
+            log.debug(`Indexed ${this.done}/${this.total} files`);
+            // Yield to the event loop so concurrent MCP requests can be served.
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    }
+    async processFile(file) {
+        // Recorded for audit but never parsed — keeps overview's "Other files"
+        // count accurate without invoking the parser on unsupported grammars.
+        if (file.language === LANGUAGE_UNKNOWN) {
+            this.index.updateFile({ ...file, lastIndexed: Date.now() }, [], [], []);
+            return 'indexed';
+        }
+        const absPath = join(this.config.projectRoot, file.path);
+        const removed = () => this.index.removeFile(file.path) ? 'removed' : 'noop';
+        let content;
+        try {
+            content = await fs.readFile(absPath, 'utf8');
+        }
+        catch (err) {
+            log.warn(`Indexer: failed to read ${file.path}: ${errMsg(err)}`);
+            return removed();
+        }
+        let tree;
+        try {
+            tree = parseFile(content, file.language);
+        }
+        catch (err) {
+            log.warn(`Indexer: parseFile threw for ${file.path}: ${errMsg(err)}`);
+            return removed();
+        }
+        if (!tree) {
+            log.warn(`Indexer: parser returned null for ${file.path} (language=${file.language})`);
+            return removed();
+        }
+        try {
+            let result;
+            try {
+                result = extractSymbols(tree, content, file);
+            }
+            catch (err) {
+                log.warn(`Indexer: extractSymbols threw for ${file.path}: ${errMsg(err)}`);
+                return removed();
+            }
+            const annotated = {
+                ...file,
+                lastIndexed: Date.now(),
+                symbolCount: result.symbols.length,
+                contentHash: hashContent(content),
+            };
+            this.index.updateFile(annotated, result.symbols, result.references, result.imports);
+            return 'indexed';
+        }
+        finally {
+            // tree-sitter trees hold WASM memory that JS GC won't reclaim.
+            tree.delete();
+        }
+    }
+    async persist() {
+        try {
+            await this.index.save(this.cachePath);
+            log.debug(`Indexer: saved cache to ${this.cachePath}`);
+        }
+        catch (err) {
+            log.error(`Indexer: failed to save cache: ${errMsg(err)}`);
+            // Do not rethrow — the in-memory index remains usable.
+        }
+    }
+}

package/dist/indexer/scanner.js

@@ -0,0 +1,279 @@
+import { open, readdir, stat } from 'node:fs/promises';
+import { join, relative, sep, posix } from 'node:path';
+import picomatch from 'picomatch';
+import { LANGUAGE_UNKNOWN } from '../types.js';
+import { log } from '../logger.js';
+const BYTE_CHECK_BUF_SIZE = 8192;
+const LANGUAGE_BY_EXT = {
+    '.ts': 'typescript',
+    '.tsx': 'tsx',
+    '.js': 'javascript',
+    '.jsx': 'javascript',
+    '.mjs': 'javascript',
+    '.cjs': 'javascript',
+    '.py': 'python',
+    '.java': 'java',
+    '.go': 'go',
+    '.rs': 'rust',
+    '.swift': 'swift',
+    '.kt': 'kotlin',
+    '.kts': 'kotlin',
+    '.dart': 'dart',
+    '.cs': 'csharp',
+    '.php': 'php',
+    '.rb': 'ruby',
+    '.rake': 'ruby',
+    '.gemspec': 'ruby',
+    // C: `.c` source files use the dedicated tree-sitter-c grammar — NOT
+    // tree-sitter-cpp, which errors on K&R old-style functions and mis-parses C
+    // code that uses C++ keywords as identifiers (`int new;`, `int class;`).
+    // `'c'` then dispatches to the C++ extractor (tree-sitter-c and -cpp produce
+    // byte-identical ASTs for the C subset — see extractor.ts).
+    '.c': 'c',
+    // C++: the C++-specific source/header extensions plus `.h`. `.h` is
+    // ambiguous (C or C++) but C++ is the dominant case for this tool's
+    // audience and tree-sitter-cpp parses C headers fine as a superset — so a C
+    // header maps to 'cpp', not 'c' (only `.c` is C-specific).
+    '.cpp': 'cpp',
+    '.cc': 'cpp',
+    '.cxx': 'cpp',
+    '.hpp': 'cpp',
+    '.hh': 'cpp',
+    '.hxx': 'cpp',
+    '.ipp': 'cpp',
+    '.tpp': 'cpp',
+    '.h': 'cpp',
+    // Objective-C: `.m` is unambiguous ObjC. `.h` stays mapped to 'cpp' above and is
+    // content-sniffed (refineHeaderLanguage) — ObjC headers are also `.h` and hold the
+    // bulk of the API (@protocol + @property are header-exclusive). `.mm` (ObjC++) is
+    // intentionally UNMAPPED (it needs a separate grammar; tree-sitter-objc errors on
+    // the C++ parts) — it falls through to LANGUAGE_UNKNOWN.
+    '.m': 'objc',
+};
+// Line-anchored Objective-C markers used to refine an ambiguous `.h` from cpp→objc.
+// Anchoring kills the false positives a substring match would hit: `//@interface`
+// comments, a `"#import"` string literal, and C++23's bare `import std;` (which has
+// neither the leading `#` of `#import` nor the `@` of `@import`). `#import` is
+// ObjC-exclusive and present in essentially every ObjC header.
+const OBJC_HEADER_MARKERS = [
+    // `#import` is the dominant ObjC header signal (also a niche MSVC C++ directive, so
+    // not strictly ObjC-exclusive — but a C++ `.h` using MSVC `#import` is rare and the
+    // mis-route is recall-only). All markers are LINE-ANCHORED so a `//@interface`
+    // comment or a `"#import"` string literal never matches.
+    /^\s*#\s*import\b/m,
+    /^\s*@(?:interface|protocol|implementation|class|import)\b/m,
+    /^\s*NS_ASSUME_NONNULL_(?:BEGIN|END)\b/m,
+    // `typedef NS_ENUM(NSInteger, Foo)` is the real shape, so allow a leading
+    // `typedef` (still line-anchored — a `// NS_ENUM(...)` comment stays excluded).
+    /^\s*(?:typedef\s+)?NS_(?:ENUM|OPTIONS)\s*\(/m,
+];
+const BINARY_EXT = new Set([
+    '.png', '.jpg', '.jpeg', '.gif', '.ico',
+    '.woff', '.woff2', '.ttf', '.eot',
+    '.mp3', '.mp4',
+    '.zip', '.tar', '.gz',
+    '.wasm', '.pdf',
+    '.exe', '.dll', '.so', '.dylib', '.o', '.a',
+    '.class', '.pyc', '.pyo', '.jar', '.war',
+]);
+const GLOB_CHARS = /[*?[\]{}!]/;
+export function toPosix(p) {
+    return sep === '/' ? p : p.split(sep).join('/');
+}
+export function detectLanguage(filename) {
+    const ext = posix.extname(toPosix(filename)).toLowerCase();
+    return LANGUAGE_BY_EXT[ext] ?? null;
+}
+// A `.h` file is ambiguous: a C/C++ header maps to 'cpp' (above), but an Objective-C
+// header is also `.h` and holds the public API (@protocol + @property are
+// header-exclusive). tree-sitter-cpp errors on every ObjC header, and tree-sitter-objc
+// wrecks C++ headers, so a blanket route is wrong either way. Refine the detected
+// language by content: ONLY a `.h`-that-resolved-to-'cpp', and ONLY when 'objc' is a
+// configured language (so disabling objc keeps `.h`→cpp), read the first 8KB and route
+// to 'objc' iff a line-anchored ObjC marker matches. Self-heals: `isUnchanged` compares
+// the stored language, so a later heuristic tweak re-indexes a re-classified `.h` with
+// no schema bump. Everything else returns `language` unchanged (one I/O-free fast path).
+export async function refineHeaderLanguage(absPath, language, langSet) {
+    // Only the ambiguous `.h` extension is sniffed. `.cpp`/`.hpp`/`.cc`/… also map to
+    // 'cpp' but are unambiguously C++ — they must NOT pay the read or risk a misroute.
+    if (language !== 'cpp' ||
+        !langSet.has('objc') ||
+        posix.extname(toPosix(absPath)).toLowerCase() !== '.h') {
+        return language;
+    }
+    let head;
+    try {
+        const fh = await open(absPath, 'r');
+        try {
+            const buf = Buffer.alloc(BYTE_CHECK_BUF_SIZE);
+            const { bytesRead } = await fh.read(buf, 0, BYTE_CHECK_BUF_SIZE, 0);
+            head = buf.toString('utf8', 0, bytesRead);
+        }
+        finally {
+            await fh.close();
+        }
+    }
+    catch {
+        // A read failure here is non-fatal for routing — keep the path-based 'cpp'.
+        return language;
+    }
+    return OBJC_HEADER_MARKERS.some((re) => re.test(head)) ? 'objc' : language;
+}
+export function isBinaryByExtension(filename) {
+    return BINARY_EXT.has(posix.extname(toPosix(filename)).toLowerCase());
+}
+// Reads up to 8KB from absPath and returns true on the first null byte.
+// Mirrors git's "any NUL in the prefix means binary" heuristic. Used only
+// for unknown-extension files; trusted source extensions skip this I/O.
+export async function isBinaryByContent(absPath) {
+    const fh = await open(absPath, 'r');
+    try {
+        const buf = Buffer.alloc(BYTE_CHECK_BUF_SIZE);
+        const { bytesRead } = await fh.read(buf, 0, BYTE_CHECK_BUF_SIZE, 0);
+        for (let i = 0; i < bytesRead; i++) {
+            if (buf[i] === 0)
+                return true;
+        }
+        return false;
+    }
+    finally {
+        await fh.close();
+    }
+}
+export function compileExcludeMatcher(patterns) {
+    const expanded = [];
+    for (const p of patterns) {
+        const hasSlash = p.includes('/');
+        const hasGlob = GLOB_CHARS.test(p);
+        if (!hasSlash && !hasGlob) {
+            expanded.push(p, `${p}/**`, `**/${p}`, `**/${p}/**`);
+        }
+        else if (!hasSlash && hasGlob) {
+            expanded.push(p, `**/${p}`);
+        }
+        else {
+            expanded.push(p);
+        }
+    }
+    const isMatch = picomatch(expanded, { dot: true });
+    return (relPath) => isMatch(relPath);
+}
+export function depthOf(relPath) {
+    let n = 0;
+    for (let i = 0; i < relPath.length; i++) {
+        if (relPath.charCodeAt(i) === 47 /* '/' */)
+            n++;
+    }
+    return n;
+}
+export function compareShallowFirst(a, b) {
+    const da = depthOf(a.path);
+    const db = depthOf(b.path);
+    if (da !== db)
+        return da - db;
+    return a.path < b.path ? -1 : a.path > b.path ? 1 : 0;
+}
+async function* walk(root, dir, matchExclude, state) {
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch (err) {
+        if (dir === root)
+            throw err;
+        // A transient failure here (EACCES/EMFILE/network FS) hides files
+        // from this scan. Mark the scan incomplete so callers don't
+        // mistake the omission for a deletion.
+        state.complete = false;
+        log.warn(`scanner: readdir failed for ${dir}: ${err.message}`);
+        return;
+    }
+    for (const entry of entries) {
+        if (entry.isSymbolicLink())
+            continue;
+        const absPath = join(dir, entry.name);
+        const relPath = toPosix(relative(root, absPath));
+        if (entry.isDirectory()) {
+            if (matchExclude(relPath))
+                continue;
+            yield* walk(root, absPath, matchExclude, state);
+        }
+        else if (entry.isFile()) {
+            yield { absPath, relPath };
+        }
+    }
+}
+export async function scanProject(config) {
+    const matchExclude = compileExcludeMatcher(config.exclude);
+    const langSet = new Set(config.languages);
+    const root = config.projectRoot;
+    const cap = config.maxFiles > 0 ? config.maxFiles : Number.MAX_SAFE_INTEGER;
+    // parseable claims every cap slot first; unknowns fill residual budget.
+    // Without this split, readdir order alone could let overview-only files
+    // exhaust a tight maxFiles before src/ is even walked.
+    const parseable = [];
+    const unknown = [];
+    const state = { complete: true };
+    for await (const { absPath, relPath } of walk(root, root, matchExclude, state)) {
+        if (parseable.length >= cap) {
+            log.warn(`scanner: reached maxFiles=${cap}; remaining files skipped`);
+            break;
+        }
+        if (matchExclude(relPath))
+            continue;
+        if (isBinaryByExtension(relPath))
+            continue;
+        let language = detectLanguage(relPath) ?? LANGUAGE_UNKNOWN;
+        // A `.h` mapped to 'cpp' may be an Objective-C header — content-sniff it (no-op
+        // unless the ext is `.h` AND objc is configured). Done before the langSet gate so
+        // a refined 'objc' is kept iff objc is enabled (refineHeaderLanguage self-gates).
+        if (language === 'cpp')
+            language = await refineHeaderLanguage(absPath, language, langSet);
+        // Recognized-but-unconfigured languages are dropped; unknown files are
+        // kept (subject to residual budget) so overview can surface them.
+        if (language !== LANGUAGE_UNKNOWN && !langSet.has(language))
+            continue;
+        if (language === LANGUAGE_UNKNOWN) {
+            if (unknown.length >= cap - parseable.length)
+                continue;
+            try {
+                if (await isBinaryByContent(absPath))
+                    continue;
+            }
+            catch (err) {
+                state.complete = false;
+                log.warn(`scanner: byte check failed for ${relPath}: ${err.message}`);
+                continue;
+            }
+        }
+        let stats;
+        try {
+            stats = await stat(absPath);
+        }
+        catch (err) {
+            state.complete = false;
+            log.warn(`scanner: stat failed for ${relPath}: ${err.message}`);
+            continue;
+        }
+        if (stats.size > config.maxFileSize) {
+            log.debug(`scanner: skip ${relPath} (size ${stats.size} > maxFileSize ${config.maxFileSize})`);
+            continue;
+        }
+        const fileInfo = {
+            path: relPath,
+            language,
+            size: stats.size,
+            lastModified: stats.mtimeMs,
+            lastIndexed: 0,
+            symbolCount: 0,
+        };
+        if (language === LANGUAGE_UNKNOWN)
+            unknown.push(fileInfo);
+        else
+            parseable.push(fileInfo);
+    }
+    const remaining = Math.max(0, cap - parseable.length);
+    const results = parseable.concat(unknown.slice(0, remaining));
+    results.sort(compareShallowFirst);
+    return { files: results, complete: state.complete };
+}