@aiwg/cli 2026.10.3 → 2026.10.5

package/dist/src/artifacts/index-files.js

@@ -10,6 +10,7 @@ import fs from 'fs';
 import path from 'path';
 import { DEFAULT_INDEX_EXTENSIONS, GRAPH_CONFIGS, resolveGraphScanDir, } from './types.js';
 import { DEFAULT_PROJECT_AIWG_DIR, resolveProjectAiwgDir, } from '../config/project-artifacts.js';
+import { dropGitignored } from '../config/gitignore.js';
 import { workspaceLinkedFiles } from '../smiths/context-pipeline/workspace-context.js';
 import { discoverProjectLocalBundles } from '../extensions/project-local-discovery.js';
 function pathContains(parent, child) {
@@ -32,16 +33,18 @@ export function indexPathFor(cwd, fullPath, graph) {
         return toPosixPath(relative);
     return fullPath;
 }
-export function findArtifactFiles(dir, extensions = DEFAULT_INDEX_EXTENSIONS) {
-    return walkArtifactFiles(dir, extensions, new Set());
+export function findArtifactFiles(dir, extensions = DEFAULT_INDEX_EXTENSIONS, acceptPath = () => true) {
+    return walkArtifactFiles(dir, extensions, new Set(), acceptPath);
 }
 /** Recursively find indexable files, excluding hidden directories such as .index. */
-function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
+function walkArtifactFiles(dir, extensions, seenRealDirs, acceptPath, knownStat) {
     const results = [];
-    if (!fs.existsSync(dir))
+    if (!fs.existsSync(dir) || !acceptPath(dir, knownStat))
         return results;
     let realDir;
     try {
+        if ((knownStat ?? fs.lstatSync(dir)).isSymbolicLink())
+            return results;
         realDir = fs.realpathSync(dir);
     }
     catch {
@@ -55,8 +58,8 @@ function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
         const fullPath = path.join(dir, entry.name);
         let stat;
         try {
-            stat = fs.statSync(fullPath);
-            if (boundary && !pathContains(boundary, fs.realpathSync(fullPath)))
+            stat = fs.lstatSync(fullPath);
+            if (stat.isSymbolicLink() || !acceptPath(fullPath, stat, dir))
                 continue;
         }
         catch {
@@ -65,7 +68,7 @@ function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
         if (stat.isDirectory()) {
             if (entry.name.startsWith('.'))
                 continue;
-            results.push(...walkArtifactFiles(fullPath, extensions, seenRealDirs, boundary));
+            results.push(...walkArtifactFiles(fullPath, extensions, seenRealDirs, acceptPath, stat));
         }
         else if (stat.isFile() && extensions.some(extension => entry.name.endsWith(extension))) {
             results.push(fullPath);
@@ -73,22 +76,96 @@ function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
     }
     return results;
 }
+/** Runtime and acquisition scratch beneath either the local or relocated artifact root. */
+export const INDEX_SCRATCH_DIRS = [
+    'working', 'sessions', 'scratch', 'tmp', 'temp', 'cache', '.index', '.cache', '.storage-cache',
+];
 /** Return the exact current source-file set used by a standard graph build. */
-export async function collectGraphIndexFiles(cwd, graph) {
+export async function collectGraphIndexFiles(cwd, graph, options = {}) {
     const config = graph ? GRAPH_CONFIGS[graph] : undefined;
     if (graph && !config) {
         throw new Error(`Unknown graph: ${graph}`);
     }
-    const scanDirs = config
-        ? config.scanDirs.map(directory => resolveGraphScanDir(cwd, directory))
-        : [resolveProjectAiwgDir(cwd)];
-    const extensions = config?.extensions ?? [...DEFAULT_INDEX_EXTENSIONS];
+    const artifactRoot = resolveProjectAiwgDir(cwd);
+    const excludedDirs = [path.join(cwd, DEFAULT_PROJECT_AIWG_DIR), artifactRoot]
+        .flatMap(root => INDEX_SCRATCH_DIRS.map(name => path.resolve(root, name)));
+    const excludedPaths = new Set(excludedDirs);
+    // A directory's ancestors are shared by thousands of sibling artifacts.
+    // Cache only within this scan so the next build still notices new symlinks.
+    const safePaths = new Map();
+    const roots = [artifactRoot, cwd].map(root => path.resolve(root));
+    const artifactPrefix = roots[0].endsWith(path.sep) ? roots[0] : roots[0] + path.sep;
+    const acceptPath = (file, knownStat, parent) => {
+        // The walker already checked this parent and rejected symlink entries.
+        // A regular file cannot introduce an excluded ancestor; only an exact
+        // excluded path remains to check. Avoid normalizing and memoizing every
+        // leaf merely to repeat its parent's proof thousands of times.
+        if (parent && knownStat?.isFile() && safePaths.get(parent) === true) {
+            if (process.platform === 'win32'
+                ? excludedDirs.some(dir => pathContains(dir, file)) : excludedPaths.has(file))
+                return false;
+            options.fileStats?.set(file, knownStat);
+            return true;
+        }
+        const inputPath = file;
+        file = path.resolve(file);
+        if (excludedDirs.some(dir => process.platform === 'win32'
+            ? pathContains(dir, file) : file === dir || file.startsWith(dir + path.sep)))
+            return false;
+        if (knownStat && !knownStat.isSymbolicLink() && safePaths.get(path.dirname(file)) === true) {
+            if (knownStat.isFile())
+                options.fileStats?.set(inputPath, knownStat);
+            safePaths.set(file, true);
+            return true;
+        }
+        // Explicit scopes and workspace links can start below a symlink. Check
+        // their ancestors too, stopping at the authorized root (which may be external).
+        const inArtifactRoot = process.platform === 'win32'
+            ? pathContains(roots[0], file) : file === roots[0] || file.startsWith(artifactPrefix);
+        const root = inArtifactRoot ? roots[0] : roots[1];
+        let current = file;
+        const checked = [];
+        let safe = true;
+        while (pathContains(root, current)) {
+            if (safePaths.has(current)) {
+                safe = safePaths.get(current);
+                break;
+            }
+            checked.push(current);
+            try {
+                const stat = current === file && knownStat ? knownStat : fs.lstatSync(current);
+                if (stat.isSymbolicLink()) {
+                    safe = false;
+                    break;
+                }
+                if (current === file && stat.isFile())
+                    options.fileStats?.set(inputPath, stat);
+            }
+            catch {
+                safe = false;
+                break;
+            }
+            if (current === root)
+                break;
+            current = path.dirname(current);
+        }
+        for (const candidate of checked)
+            safePaths.set(candidate, safe);
+        return safe;
+    };
+    const scanDirs = options.scope
+        ? [path.resolve(cwd, options.scope)]
+        : config
+            ? config.scanDirs.map(directory => resolveGraphScanDir(cwd, directory))
+            : [artifactRoot];
+    const extensions = options.scope ? DEFAULT_INDEX_EXTENSIONS : config?.extensions ?? DEFAULT_INDEX_EXTENSIONS;
     const files = new Set();
+    const ignoreRoots = new Set([cwd, ...scanDirs.filter(dir => !pathContains(cwd, dir))]);
     for (const scanDir of scanDirs) {
-        for (const file of findArtifactFiles(scanDir, extensions))
+        for (const file of findArtifactFiles(scanDir, extensions, acceptPath))
             files.add(file);
     }
-    if (!graph || graph === 'project') {
+    if (!options.scope && (!graph || graph === 'project')) {
         // Additional search roots authorize bundle payloads, not the surrounding
         // external corpus. Reuse deployment's validated source resolution, including
         // plugin payloads, and prevent payload symlinks from escaping that boundary.
@@ -96,8 +173,9 @@ export async function collectGraphIndexFiles(cwd, graph) {
         for (const bundle of bundles) {
             if (scanDirs.some(scanDir => pathContains(scanDir, bundle.artifactPath)))
                 continue;
-            const boundary = fs.realpathSync(bundle.artifactPath);
-            for (const file of walkArtifactFiles(bundle.artifactPath, extensions, new Set(), boundary)) {
+            if (!pathContains(cwd, bundle.artifactPath))
+                ignoreRoots.add(bundle.artifactPath);
+            for (const file of findArtifactFiles(bundle.artifactPath, extensions, acceptPath)) {
                 files.add(file);
             }
         }
@@ -107,10 +185,46 @@ export async function collectGraphIndexFiles(cwd, graph) {
             ...await workspaceLinkedFiles(cwd),
         ];
         for (const contextFile of contextFiles) {
-            if (extensions.some(extension => contextFile.endsWith(extension)))
+            if (acceptPath(contextFile) && extensions.some(extension => contextFile.endsWith(extension)))
                 files.add(contextFile);
         }
     }
-    return [...files];
+    let sources = [...files];
+    if (!options.includeIgnored) {
+        // Many projects ignore the whole artifact root (`.aiwg/`). Honoring that
+        // would empty the index, so ignore rules apply only beneath a scan root
+        // that is not itself ignored; runtime directories are excluded regardless.
+        const exemptRoots = [];
+        if (!options.scope) {
+            // Sibling roots use the same Git context. Probe them together without
+            // merging external roots or changing which ignored roots are exempt.
+            const byParent = new Map();
+            for (const dir of scanDirs.filter(dir => fs.existsSync(dir))) {
+                const parent = path.dirname(dir);
+                const siblings = byParent.get(parent) ?? [];
+                siblings.push(dir);
+                byParent.set(parent, siblings);
+            }
+            for (const [parent, siblings] of byParent) {
+                const kept = new Set(dropGitignored(parent, siblings));
+                exemptRoots.push(...siblings.filter(dir => !kept.has(dir)));
+            }
+        }
+        const isExempt = (file) => exemptRoots.some(dir => pathContains(dir, file));
+        // Git rejects paths outside its worktree. Filter authorized external roots
+        // separately so they cannot disable ignore filtering for local sources.
+        for (const root of ignoreRoots) {
+            const candidates = sources.filter(file => pathContains(root, file) && !isExempt(file));
+            const kept = new Set(dropGitignored(root, candidates));
+            sources = sources.filter(file => !pathContains(root, file) || isExempt(file) || kept.has(file));
+        }
+    }
+    if (options.fileStats) {
+        const selected = new Set(sources);
+        for (const file of options.fileStats.keys())
+            if (!selected.has(file))
+                options.fileStats.delete(file);
+    }
+    return sources;
 }
 //# sourceMappingURL=index-files.js.map

package/dist/src/artifacts/index-preparation.js

@@ -0,0 +1,130 @@
+/** Bounded artifact-local preparation; ordered graph reduction stays in index-builder. */
+import os from 'node:os';
+import { fork } from 'node:child_process';
+import { signalChild } from '../process/signal-child.mjs';
+const MIB = 1024 ** 2;
+const WORKER_HEAP_MB = 256;
+// Leave room for V8's young generation, executable code, IPC and native allocations.
+const WORKER_BUDGET_BYTES = 512 * MIB;
+// Parsing and metadata extraction amplify input size; do large files only after the pool exits.
+const MAX_WORKER_ARTIFACT_BYTES = MIB;
+export function resolveIndexWorkers(requested, env = process.env) {
+    const value = requested ?? env.AIWG_INDEX_WORKERS;
+    if (requested === undefined && (value === undefined || value === ''))
+        return undefined;
+    if ((typeof value === 'string' && !/^\d+$/.test(value))
+        || !Number.isSafeInteger(Number(value)) || Number(value) < 0) {
+        throw new Error(`${requested === undefined ? 'AIWG_INDEX_WORKERS' : '--index-workers'} must be a nonnegative safe integer (0 = serial)`);
+    }
+    return Number(value);
+}
+export function indexPreparationWorkers(jobs, requested, cores = os.availableParallelism(), memory = Math.min(os.freemem(), process.availableMemory?.() ?? Infinity), totalBytes = 0) {
+    requested = resolveIndexWorkers(requested);
+    if (jobs < 64 || cores < 3 || requested === 0 || requested === 1)
+        return 0;
+    // Process startup costs dominate lightweight scans even at thousands of files.
+    // Explicit requests bypass only this conservative automatic workload gate.
+    if (requested === undefined && (jobs < 1000 || !Number.isFinite(totalBytes) || totalBytes < 16 * MIB))
+        return 0;
+    const byMemory = Math.floor((memory - 512 * MIB) / WORKER_BUDGET_BYTES);
+    const bounded = Math.max(0, Math.min(requested ?? 4, 4, cores - 1, byMemory, jobs));
+    return bounded >= 2 ? bounded : 0;
+}
+export async function runIndexPreparation(jobs, serial, moduleUrl, requested, onDiagnostic) {
+    const diagnostics = [];
+    const diagnose = (message) => { diagnostics.push(message); onDiagnostic?.(message); };
+    const eligible = jobs.map((job, index) => ({ job, index })).filter(({ job }) => Number.isFinite(job.sizeBytes) && job.sizeBytes >= 0 && job.sizeBytes <= MAX_WORKER_ARTIFACT_BYTES);
+    const workerCount = indexPreparationWorkers(eligible.length, requested, undefined, undefined, eligible.reduce((total, { job }) => total + job.sizeBytes, 0));
+    if (!workerCount)
+        return { results: jobs.map(serial), workerCount: 0, diagnostics };
+    const results = new Array(jobs.length), workers = [];
+    const closed = [];
+    let next = 0, completed = 0, stopped = false;
+    try {
+        await new Promise((resolve, reject) => {
+            // Stop dispatch on a process failure. Reap the entire pool before retrying
+            // unfinished work in the main isolate, avoiding overlapping memory peaks.
+            const recover = (message) => {
+                if (stopped)
+                    return;
+                stopped = true;
+                diagnose(`${message}; retrying unfinished preparation serially`);
+                resolve();
+            };
+            const dispatch = (worker) => {
+                if (stopped || next >= eligible.length)
+                    return;
+                const { index, job } = eligible[next++];
+                try {
+                    worker.send({ index, job }, error => { if (error)
+                        recover(`Index preparation IPC failed for ${job.relativePath}: ${error.message}`); });
+                }
+                catch (error) {
+                    recover(`Index preparation IPC failed for ${job.relativePath}: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            };
+            for (let index = 0; index < workerCount && !stopped; index++) {
+                // A worker_threads resourceLimit is not a process boundary: V8 can abort
+                // Node during allocation before its isolate-limit error is delivered.
+                // A child heap fatal/SIGABRT is instead an ordinary, recoverable exit.
+                const worker = fork(new URL('./index-worker.mjs', import.meta.url), [moduleUrl], {
+                    execArgv: [`--max-old-space-size=${WORKER_HEAP_MB}`, '--max-semi-space-size=16'],
+                    stdio: ['ignore', 'ignore', 'pipe', 'ipc'], serialization: 'advanced',
+                });
+                workers.push(worker);
+                closed.push(new Promise(done => worker.once('close', () => done())));
+                let stderr = '';
+                let activePath = '<startup>';
+                worker.stderr?.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-8192); });
+                worker.on('error', error => recover(`Index preparation worker failed for ${activePath}: ${error.message}`));
+                worker.on('exit', (code, signal) => {
+                    const oom = /heap out of memory|ERR_WORKER_OUT_OF_MEMORY/i.test(stderr) ? ' (heap out of memory)' : '';
+                    recover(`Index preparation worker exited early for ${activePath} (${signal ?? code})${oom}`);
+                });
+                worker.on('message', (message) => {
+                    if (stopped)
+                        return;
+                    if (message.error) {
+                        stopped = true;
+                        reject(new Error(message.error));
+                        return;
+                    }
+                    results[message.index] = message.result;
+                    completed++;
+                    if (completed === eligible.length) {
+                        stopped = true;
+                        resolve();
+                    }
+                    else {
+                        activePath = eligible[next]?.job.relativePath ?? '<idle>';
+                        dispatch(worker);
+                    }
+                });
+                activePath = eligible[next]?.job.relativePath ?? '<idle>';
+                dispatch(worker);
+            }
+        });
+    }
+    finally {
+        stopped = true;
+        for (const worker of workers)
+            if (worker.exitCode === null && worker.signalCode === null)
+                signalChild(worker, 'SIGKILL');
+        await Promise.all(closed);
+    }
+    const largeCount = jobs.length - eligible.length;
+    if (largeCount)
+        diagnose(`${largeCount} artifact(s) above the worker size threshold prepared serially`);
+    for (let index = 0; index < jobs.length; index++) {
+        if (!results[index]) {
+            try {
+                results[index] = serial(jobs[index]);
+            }
+            catch (error) {
+                throw new Error(`${jobs[index].relativePath}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+    }
+    return { results, workerCount, diagnostics };
+}
+//# sourceMappingURL=index-preparation.js.map