@aiwg/cli 2026.10.1 → 2026.10.4

package/dist/src/artifacts/index-files.js

@@ -10,6 +10,7 @@ import fs from 'fs';
 import path from 'path';
 import { DEFAULT_INDEX_EXTENSIONS, GRAPH_CONFIGS, resolveGraphScanDir, } from './types.js';
 import { DEFAULT_PROJECT_AIWG_DIR, resolveProjectAiwgDir, } from '../config/project-artifacts.js';
+import { dropGitignored } from '../config/gitignore.js';
 import { workspaceLinkedFiles } from '../smiths/context-pipeline/workspace-context.js';
 import { discoverProjectLocalBundles } from '../extensions/project-local-discovery.js';
 function pathContains(parent, child) {
@@ -32,16 +33,18 @@ export function indexPathFor(cwd, fullPath, graph) {
         return toPosixPath(relative);
     return fullPath;
 }
-export function findArtifactFiles(dir, extensions = DEFAULT_INDEX_EXTENSIONS) {
-    return walkArtifactFiles(dir, extensions, new Set());
+export function findArtifactFiles(dir, extensions = DEFAULT_INDEX_EXTENSIONS, acceptPath = () => true) {
+    return walkArtifactFiles(dir, extensions, new Set(), acceptPath);
 }
 /** Recursively find indexable files, excluding hidden directories such as .index. */
-function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
+function walkArtifactFiles(dir, extensions, seenRealDirs, acceptPath, knownStat) {
     const results = [];
-    if (!fs.existsSync(dir))
+    if (!fs.existsSync(dir) || !acceptPath(dir, knownStat))
         return results;
     let realDir;
     try {
+        if ((knownStat ?? fs.lstatSync(dir)).isSymbolicLink())
+            return results;
         realDir = fs.realpathSync(dir);
     }
     catch {
@@ -55,8 +58,8 @@ function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
         const fullPath = path.join(dir, entry.name);
         let stat;
         try {
-            stat = fs.statSync(fullPath);
-            if (boundary && !pathContains(boundary, fs.realpathSync(fullPath)))
+            stat = fs.lstatSync(fullPath);
+            if (stat.isSymbolicLink() || !acceptPath(fullPath, stat, dir))
                 continue;
         }
         catch {
@@ -65,7 +68,7 @@ function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
         if (stat.isDirectory()) {
             if (entry.name.startsWith('.'))
                 continue;
-            results.push(...walkArtifactFiles(fullPath, extensions, seenRealDirs, boundary));
+            results.push(...walkArtifactFiles(fullPath, extensions, seenRealDirs, acceptPath, stat));
         }
         else if (stat.isFile() && extensions.some(extension => entry.name.endsWith(extension))) {
             results.push(fullPath);
@@ -73,22 +76,96 @@ function walkArtifactFiles(dir, extensions, seenRealDirs, boundary) {
     }
     return results;
 }
+/** Runtime and acquisition scratch beneath either the local or relocated artifact root. */
+export const INDEX_SCRATCH_DIRS = [
+    'working', 'sessions', 'scratch', 'tmp', 'temp', 'cache', '.index', '.cache', '.storage-cache',
+];
 /** Return the exact current source-file set used by a standard graph build. */
-export async function collectGraphIndexFiles(cwd, graph) {
+export async function collectGraphIndexFiles(cwd, graph, options = {}) {
     const config = graph ? GRAPH_CONFIGS[graph] : undefined;
     if (graph && !config) {
         throw new Error(`Unknown graph: ${graph}`);
     }
-    const scanDirs = config
-        ? config.scanDirs.map(directory => resolveGraphScanDir(cwd, directory))
-        : [resolveProjectAiwgDir(cwd)];
-    const extensions = config?.extensions ?? [...DEFAULT_INDEX_EXTENSIONS];
+    const artifactRoot = resolveProjectAiwgDir(cwd);
+    const excludedDirs = [path.join(cwd, DEFAULT_PROJECT_AIWG_DIR), artifactRoot]
+        .flatMap(root => INDEX_SCRATCH_DIRS.map(name => path.resolve(root, name)));
+    const excludedPaths = new Set(excludedDirs);
+    // A directory's ancestors are shared by thousands of sibling artifacts.
+    // Cache only within this scan so the next build still notices new symlinks.
+    const safePaths = new Map();
+    const roots = [artifactRoot, cwd].map(root => path.resolve(root));
+    const artifactPrefix = roots[0].endsWith(path.sep) ? roots[0] : roots[0] + path.sep;
+    const acceptPath = (file, knownStat, parent) => {
+        // The walker already checked this parent and rejected symlink entries.
+        // A regular file cannot introduce an excluded ancestor; only an exact
+        // excluded path remains to check. Avoid normalizing and memoizing every
+        // leaf merely to repeat its parent's proof thousands of times.
+        if (parent && knownStat?.isFile() && safePaths.get(parent) === true) {
+            if (process.platform === 'win32'
+                ? excludedDirs.some(dir => pathContains(dir, file)) : excludedPaths.has(file))
+                return false;
+            options.fileStats?.set(file, knownStat);
+            return true;
+        }
+        const inputPath = file;
+        file = path.resolve(file);
+        if (excludedDirs.some(dir => process.platform === 'win32'
+            ? pathContains(dir, file) : file === dir || file.startsWith(dir + path.sep)))
+            return false;
+        if (knownStat && !knownStat.isSymbolicLink() && safePaths.get(path.dirname(file)) === true) {
+            if (knownStat.isFile())
+                options.fileStats?.set(inputPath, knownStat);
+            safePaths.set(file, true);
+            return true;
+        }
+        // Explicit scopes and workspace links can start below a symlink. Check
+        // their ancestors too, stopping at the authorized root (which may be external).
+        const inArtifactRoot = process.platform === 'win32'
+            ? pathContains(roots[0], file) : file === roots[0] || file.startsWith(artifactPrefix);
+        const root = inArtifactRoot ? roots[0] : roots[1];
+        let current = file;
+        const checked = [];
+        let safe = true;
+        while (pathContains(root, current)) {
+            if (safePaths.has(current)) {
+                safe = safePaths.get(current);
+                break;
+            }
+            checked.push(current);
+            try {
+                const stat = current === file && knownStat ? knownStat : fs.lstatSync(current);
+                if (stat.isSymbolicLink()) {
+                    safe = false;
+                    break;
+                }
+                if (current === file && stat.isFile())
+                    options.fileStats?.set(inputPath, stat);
+            }
+            catch {
+                safe = false;
+                break;
+            }
+            if (current === root)
+                break;
+            current = path.dirname(current);
+        }
+        for (const candidate of checked)
+            safePaths.set(candidate, safe);
+        return safe;
+    };
+    const scanDirs = options.scope
+        ? [path.resolve(cwd, options.scope)]
+        : config
+            ? config.scanDirs.map(directory => resolveGraphScanDir(cwd, directory))
+            : [artifactRoot];
+    const extensions = options.scope ? DEFAULT_INDEX_EXTENSIONS : config?.extensions ?? DEFAULT_INDEX_EXTENSIONS;
     const files = new Set();
+    const ignoreRoots = new Set([cwd, ...scanDirs.filter(dir => !pathContains(cwd, dir))]);
     for (const scanDir of scanDirs) {
-        for (const file of findArtifactFiles(scanDir, extensions))
+        for (const file of findArtifactFiles(scanDir, extensions, acceptPath))
             files.add(file);
     }
-    if (!graph || graph === 'project') {
+    if (!options.scope && (!graph || graph === 'project')) {
         // Additional search roots authorize bundle payloads, not the surrounding
         // external corpus. Reuse deployment's validated source resolution, including
         // plugin payloads, and prevent payload symlinks from escaping that boundary.
@@ -96,8 +173,9 @@ export async function collectGraphIndexFiles(cwd, graph) {
         for (const bundle of bundles) {
             if (scanDirs.some(scanDir => pathContains(scanDir, bundle.artifactPath)))
                 continue;
-            const boundary = fs.realpathSync(bundle.artifactPath);
-            for (const file of walkArtifactFiles(bundle.artifactPath, extensions, new Set(), boundary)) {
+            if (!pathContains(cwd, bundle.artifactPath))
+                ignoreRoots.add(bundle.artifactPath);
+            for (const file of findArtifactFiles(bundle.artifactPath, extensions, acceptPath)) {
                 files.add(file);
             }
         }
@@ -107,10 +185,46 @@ export async function collectGraphIndexFiles(cwd, graph) {
             ...await workspaceLinkedFiles(cwd),
         ];
         for (const contextFile of contextFiles) {
-            if (extensions.some(extension => contextFile.endsWith(extension)))
+            if (acceptPath(contextFile) && extensions.some(extension => contextFile.endsWith(extension)))
                 files.add(contextFile);
         }
     }
-    return [...files];
+    let sources = [...files];
+    if (!options.includeIgnored) {
+        // Many projects ignore the whole artifact root (`.aiwg/`). Honoring that
+        // would empty the index, so ignore rules apply only beneath a scan root
+        // that is not itself ignored; runtime directories are excluded regardless.
+        const exemptRoots = [];
+        if (!options.scope) {
+            // Sibling roots use the same Git context. Probe them together without
+            // merging external roots or changing which ignored roots are exempt.
+            const byParent = new Map();
+            for (const dir of scanDirs.filter(dir => fs.existsSync(dir))) {
+                const parent = path.dirname(dir);
+                const siblings = byParent.get(parent) ?? [];
+                siblings.push(dir);
+                byParent.set(parent, siblings);
+            }
+            for (const [parent, siblings] of byParent) {
+                const kept = new Set(dropGitignored(parent, siblings));
+                exemptRoots.push(...siblings.filter(dir => !kept.has(dir)));
+            }
+        }
+        const isExempt = (file) => exemptRoots.some(dir => pathContains(dir, file));
+        // Git rejects paths outside its worktree. Filter authorized external roots
+        // separately so they cannot disable ignore filtering for local sources.
+        for (const root of ignoreRoots) {
+            const candidates = sources.filter(file => pathContains(root, file) && !isExempt(file));
+            const kept = new Set(dropGitignored(root, candidates));
+            sources = sources.filter(file => !pathContains(root, file) || isExempt(file) || kept.has(file));
+        }
+    }
+    if (options.fileStats) {
+        const selected = new Set(sources);
+        for (const file of options.fileStats.keys())
+            if (!selected.has(file))
+                options.fileStats.delete(file);
+    }
+    return sources;
 }
 //# sourceMappingURL=index-files.js.map

package/dist/src/artifacts/index-preparation.js

@@ -0,0 +1,65 @@
+/** Bounded artifact-local preparation; ordered graph reduction stays in index-builder. */
+import os from 'node:os';
+import { Worker } from 'node:worker_threads';
+export function indexPreparationWorkers(jobs, requested, cores = os.availableParallelism(), memory = Math.min(os.freemem(), process.availableMemory?.() ?? Infinity), totalBytes = 0) {
+    if (requested !== undefined && (!Number.isInteger(requested) || requested < 0))
+        throw new Error('Index workers must be a nonnegative integer');
+    if (jobs < 64 || cores < 3 || requested === 0 || requested === 1)
+        return 0;
+    // Thread startup costs dominate lightweight scans even at thousands of files.
+    // Explicit thread requests bypass only this conservative automatic workload gate.
+    if (requested === undefined && (jobs < 1000 || !Number.isFinite(totalBytes) || totalBytes < 16 * 1024 ** 2))
+        return 0;
+    const byMemory = Math.floor((memory - 512 * 1024 ** 2) / (256 * 1024 ** 2));
+    const bounded = Math.max(0, Math.min(requested ?? 4, 4, cores - 1, byMemory, jobs));
+    return bounded >= 2 ? bounded : 0;
+}
+export async function runIndexPreparation(jobs, serial, moduleUrl, requested) {
+    const workerCount = indexPreparationWorkers(jobs.length, requested, undefined, undefined, jobs.reduce((total, job) => total + job.sizeBytes, 0));
+    if (!workerCount)
+        return { results: jobs.map(serial), workerCount: 0 };
+    const results = new Array(jobs.length), workers = [];
+    let next = 0, completed = 0, stopped = false;
+    try {
+        await new Promise((resolve, reject) => {
+            const fail = (error) => { stopped = true; reject(error); };
+            const dispatch = (worker) => {
+                if (!stopped && next < jobs.length) {
+                    const index = next++;
+                    worker.postMessage({ index, job: jobs[index] });
+                }
+            };
+            for (let index = 0; index < workerCount; index++) {
+                const worker = new Worker(new URL('./index-worker.mjs', import.meta.url), {
+                    workerData: { moduleUrl }, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 256 },
+                });
+                workers.push(worker);
+                worker.on('error', fail);
+                worker.on('exit', code => { if (!stopped && completed < jobs.length)
+                    fail(new Error(`Index preparation worker exited early (${code})`)); });
+                worker.on('message', message => {
+                    if (stopped)
+                        return;
+                    if (message.error) {
+                        fail(new Error(message.error));
+                        return;
+                    }
+                    results[message.index] = message.result;
+                    completed++;
+                    if (completed === jobs.length) {
+                        stopped = true;
+                        resolve();
+                    }
+                    else
+                        dispatch(worker);
+                });
+                dispatch(worker);
+            }
+        });
+        return { results, workerCount };
+    }
+    finally {
+        await Promise.all(workers.map(worker => worker.terminate()));
+    }
+}
+//# sourceMappingURL=index-preparation.js.map