gitnexus 1.6.11-rc.2 → 1.6.11-rc.21

package/dist/storage/file-hash.d.ts

@@ -27,6 +27,11 @@ export declare const computeFileHash: (absPath: string) => Promise<string | null
  * fail to read are omitted from the result map.
  */
 export declare const computeFileHashes: (repoPath: string, relPaths: readonly string[]) => Promise<Map<string, string>>;
+/** Like {@link computeFileHashes}, but keeps paths whose content could not be read. */
+export declare const computeFileHashesDetailed: (repoPath: string, relPaths: readonly string[]) => Promise<{
+    hashes: Map<string, string>;
+    unreadable: string[];
+}>;
 /** Result of comparing the current on-disk hashes against stored ones. */
 export interface FileHashDiff {
     /** Files whose content hash differs from stored. */

package/dist/storage/file-hash.js

@@ -39,18 +39,28 @@ export const computeFileHash = async (absPath) => {
  * fail to read are omitted from the result map.
  */
 export const computeFileHashes = async (repoPath, relPaths) => {
-    const out = new Map();
+    const { hashes } = await computeFileHashesDetailed(repoPath, relPaths);
+    return hashes;
+};
+/** Like {@link computeFileHashes}, but keeps paths whose content could not be read. */
+export const computeFileHashesDetailed = async (repoPath, relPaths) => {
+    const hashes = new Map();
+    const unreadable = [];
     const BATCH = 100;
     for (const batch of chunk(relPaths, BATCH)) {
         const results = await Promise.all(batch.map(async (rel) => {
             const h = await computeFileHash(path.join(repoPath, rel));
-            return h ? [rel, h] : null;
+            return { rel, h };
         }));
-        for (const r of results)
-            if (r)
-                out.set(r[0], r[1]);
+        for (const { rel, h } of results) {
+            if (h)
+                hashes.set(rel, h);
+            else
+                unreadable.push(rel);
+        }
     }
-    return out;
+    unreadable.sort();
+    return { hashes, unreadable };
 };
 /**
  * Diff a current hash map against a previously stored one.

package/dist/storage/fs-atomic.d.ts

@@ -28,3 +28,27 @@ export declare function retryRename(src: string, dst: string, attempts?: number)
  * retry backoff (and, under a lock, make everyone else wait for it).
  */
 export declare function writeFileAtomic(targetPath: string, data: string, attempts?: number): Promise<void>;
+/**
+ * Publish `src` at `dst` without ever writing through the inode currently
+ * named by `dst` (#3090). Fast path is `fs.link`. Any failure (EEXIST, EXDEV,
+ * EPERM, unsupported hardlinks) copies into a unique sibling and
+ * {@link retryRename}s over `dst`. POSIX rename of a file over an existing
+ * file unlinks the old name; it does not open or truncate that inode.
+ *
+ * Invariant: may replace the destination directory entry; must never modify
+ * the inode that entry currently references. There is no dest unlink and no
+ * in-place `copyFile(src, dst)`.
+ */
+export declare function linkOrCopyFile(src: string, dst: string): Promise<void>;
+/**
+ * Binary sibling of {@link writeFileAtomic}. Same random tmp + `'wx'` + `0o600`
+ * + rename contract; used for optional V8 cache sidecars that cannot go through
+ * a UTF-8 `writeFile`.
+ */
+export declare function writeFileAtomicBytes(targetPath: string, data: Uint8Array, attempts?: number): Promise<void>;
+/**
+ * Sync binary publish for parse workers. Same exclusive-tmp + mode contract as
+ * {@link writeFileAtomicBytes}; rename is not retried (the worker is not the
+ * Windows multi-reader case `retryRename` exists for).
+ */
+export declare function writeFileAtomicBytesSync(targetPath: string, data: Uint8Array): void;

package/dist/storage/fs-atomic.js

@@ -6,7 +6,7 @@
  * core/group/ import (the established direction is core/group/ -> storage/,
  * e.g. core/group/service.ts already imports loadMeta from here).
  */
-import fsp from 'fs/promises';
+import { closeSync, openSync, promises as fsp, renameSync, unlinkSync, writeSync } from 'node:fs';
 import { randomBytes } from 'crypto';
 const RETRY_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
 /**
@@ -52,7 +52,7 @@ export async function retryRename(src, dst, attempts = 3) {
  * retry backoff (and, under a lock, make everyone else wait for it).
  */
 export async function writeFileAtomic(targetPath, data, attempts) {
-    const tmpPath = `${targetPath}.tmp.${randomBytes(8).toString('hex')}`;
+    const tmpPath = tmpBeside(targetPath);
     const handle = await fsp.open(tmpPath, 'wx', 0o600);
     try {
         try {
@@ -68,3 +68,87 @@ export async function writeFileAtomic(targetPath, data, attempts) {
         throw err;
     }
 }
+function tmpBeside(targetPath) {
+    return `${targetPath}.tmp.${randomBytes(8).toString('hex')}`;
+}
+/**
+ * Publish `src` at `dst` without ever writing through the inode currently
+ * named by `dst` (#3090). Fast path is `fs.link`. Any failure (EEXIST, EXDEV,
+ * EPERM, unsupported hardlinks) copies into a unique sibling and
+ * {@link retryRename}s over `dst`. POSIX rename of a file over an existing
+ * file unlinks the old name; it does not open or truncate that inode.
+ *
+ * Invariant: may replace the destination directory entry; must never modify
+ * the inode that entry currently references. There is no dest unlink and no
+ * in-place `copyFile(src, dst)`.
+ */
+export async function linkOrCopyFile(src, dst) {
+    try {
+        await fsp.link(src, dst);
+        return;
+    }
+    catch {
+        /* any link failure → publish a new inode via tmp + rename */
+    }
+    const tmp = tmpBeside(dst);
+    try {
+        await fsp.copyFile(src, tmp);
+        await retryRename(tmp, dst);
+    }
+    catch (err) {
+        await fsp.unlink(tmp).catch(() => { });
+        throw err;
+    }
+}
+/**
+ * Binary sibling of {@link writeFileAtomic}. Same random tmp + `'wx'` + `0o600`
+ * + rename contract; used for optional V8 cache sidecars that cannot go through
+ * a UTF-8 `writeFile`.
+ */
+export async function writeFileAtomicBytes(targetPath, data, attempts) {
+    const tmpPath = tmpBeside(targetPath);
+    const handle = await fsp.open(tmpPath, 'wx', 0o600);
+    try {
+        try {
+            await handle.writeFile(data);
+        }
+        finally {
+            await handle.close();
+        }
+        await retryRename(tmpPath, targetPath, attempts);
+    }
+    catch (err) {
+        await fsp.unlink(tmpPath).catch(() => { });
+        throw err;
+    }
+}
+/**
+ * Sync binary publish for parse workers. Same exclusive-tmp + mode contract as
+ * {@link writeFileAtomicBytes}; rename is not retried (the worker is not the
+ * Windows multi-reader case `retryRename` exists for).
+ */
+export function writeFileAtomicBytesSync(targetPath, data) {
+    const tmpPath = tmpBeside(targetPath);
+    const fd = openSync(tmpPath, 'wx', 0o600);
+    try {
+        try {
+            let offset = 0;
+            while (offset < data.byteLength) {
+                offset += writeSync(fd, data, offset);
+            }
+        }
+        finally {
+            closeSync(fd);
+        }
+        renameSync(tmpPath, targetPath);
+    }
+    catch (err) {
+        try {
+            unlinkSync(tmpPath);
+        }
+        catch {
+            /* leftover tmp is unlinked on the next exclusive create */
+        }
+        throw err;
+    }
+}

package/dist/storage/git.d.ts

@@ -1,15 +1,26 @@
 /**
  * True when the working tree has uncommitted changes that analyze would
- * re-index, even at a matching HEAD. Excludes the paths GitNexus writes during
- * analyze (.gitnexus/, .claude/, .cursor/, AGENTS.md, CLAUDE.md, and the
- * repo-local .agents/ mirror) so its own output never counts as dirty
- * (regression vs PR #1233 behavior). The entire .agents/ tree is excluded,
- * matching the .claude/ treatment, because the skill mirror writes across
- * .agents/skills/ and deeper paths. Conservative on any git failure. Shared
- * so `analyze`'s fast-path gate and `status`'s freshness report agree on what
- * "dirty" means.
+ * re-index, even at a matching HEAD. Excludes GITNEXUS_MANAGED_PATHS so
+ * GitNexus's own analyze output never counts as dirty (regression vs PR #1233
+ * behavior); whole directory trees are excluded, not just their root entries,
+ * because the skill mirror writes across .agents/skills/ and deeper paths.
+ * Conservative on any git failure.
+ *
+ * This drives `analyze`'s up-to-date fast path. It is deliberately coarse:
+ * a false "dirty" here costs only a hash diff that finds nothing. `status`
+ * reaches for the per-file comparison in core/index-content-drift.ts instead,
+ * because there the same false positive is a verdict the user cannot clear
+ * (#3077), and falls back to this only when that comparison cannot run.
  */
 export declare const isWorkingTreeDirty: (repoPath: string) => boolean;
+/**
+ * Repo-relative paths `git status` reports as dirty or untracked, using the
+ * same managed-path excludes as {@link isWorkingTreeDirty}, plus tracked paths
+ * whose assume-unchanged or skip-worktree bits can hide content changes from
+ * porcelain. `null` means either query failed — callers must not treat that as
+ * a clean tree.
+ */
+export declare const listWorkingTreeDirtyPaths: (repoPath: string) => string[] | null;
 /**
  * Snapshot, per candidate file, whether it is safe for `selfCommitContextFiles`
  * to auto-commit — call this BEFORE `analyze` writes AGENTS.md/CLAUDE.md.

package/dist/storage/git.js

@@ -4,37 +4,27 @@ import path from 'path';
 import os from 'os';
 import { logger } from '../core/logger.js';
 import { toZeroBasedLine } from '../core/ingestion/utils/line-base.js';
+import { GITNEXUS_MANAGED_PATH_EXCLUDES, isGitNexusManagedPath } from './gitnexus-managed-paths.js';
 // Git utilities for repository detection, commit tracking, and diff analysis
 const chompGitOutput = (value) => value.toString().replace(/\r?\n$/, '');
+const GIT_PATH_LIST_MAX_BUFFER = 64 * 1024 * 1024;
 /**
  * True when the working tree has uncommitted changes that analyze would
- * re-index, even at a matching HEAD. Excludes the paths GitNexus writes during
- * analyze (.gitnexus/, .claude/, .cursor/, AGENTS.md, CLAUDE.md, and the
- * repo-local .agents/ mirror) so its own output never counts as dirty
- * (regression vs PR #1233 behavior). The entire .agents/ tree is excluded,
- * matching the .claude/ treatment, because the skill mirror writes across
- * .agents/skills/ and deeper paths. Conservative on any git failure. Shared
- * so `analyze`'s fast-path gate and `status`'s freshness report agree on what
- * "dirty" means.
+ * re-index, even at a matching HEAD. Excludes GITNEXUS_MANAGED_PATHS so
+ * GitNexus's own analyze output never counts as dirty (regression vs PR #1233
+ * behavior); whole directory trees are excluded, not just their root entries,
+ * because the skill mirror writes across .agents/skills/ and deeper paths.
+ * Conservative on any git failure.
+ *
+ * This drives `analyze`'s up-to-date fast path. It is deliberately coarse:
+ * a false "dirty" here costs only a hash diff that finds nothing. `status`
+ * reaches for the per-file comparison in core/index-content-drift.ts instead,
+ * because there the same false positive is a verdict the user cannot clear
+ * (#3077), and falls back to this only when that comparison cannot run.
  */
 export const isWorkingTreeDirty = (repoPath) => {
     try {
-        const out = execFileSync('git', [
-            'status',
-            '--porcelain',
-            '--',
-            '.',
-            ':(exclude).gitnexus',
-            ':(exclude).gitnexus/**',
-            ':(exclude).claude',
-            ':(exclude).claude/**',
-            ':(exclude).cursor',
-            ':(exclude).cursor/**',
-            ':(exclude)AGENTS.md',
-            ':(exclude)CLAUDE.md',
-            ':(exclude).agents',
-            ':(exclude).agents/**',
-        ], {
+        const out = execFileSync('git', ['status', '--porcelain', '--', '.', ...GITNEXUS_MANAGED_PATH_EXCLUDES], {
             cwd: repoPath,
             stdio: ['ignore', 'pipe', 'ignore'],
             windowsHide: true,
@@ -46,6 +36,75 @@ export const isWorkingTreeDirty = (repoPath) => {
         return true; // conservative on git failure
     }
 };
+const parsePorcelainPaths = (porcelain) => {
+    const paths = new Set();
+    const records = porcelain.split('\0');
+    for (let i = 0; i < records.length; i++) {
+        const record = records[i];
+        if (record.length < 4)
+            continue;
+        const status = record.slice(0, 2);
+        paths.add(record.slice(3));
+        // In porcelain v1 `-z` mode, rename/copy source and destination paths are
+        // separate NUL records (with no human-facing ` -> ` delimiter). Keep both:
+        // either side may be present in the previous coverage set.
+        if (status.includes('R') || status.includes('C')) {
+            const pairedPath = records[++i];
+            if (pairedPath)
+                paths.add(pairedPath);
+        }
+    }
+    return [...paths];
+};
+const gitPathListExec = {
+    stdio: ['ignore', 'pipe', 'ignore'],
+    encoding: 'utf8',
+    maxBuffer: GIT_PATH_LIST_MAX_BUFFER,
+};
+const listHiddenIndexPaths = (repoPath) => {
+    const out = execFileSync('git', ['ls-files', '-v', '-z', '--'], {
+        cwd: repoPath,
+        windowsHide: true,
+        ...gitPathListExec,
+    });
+    const paths = [];
+    for (const record of out.split('\0')) {
+        if (record.length < 3 || record[1] !== ' ')
+            continue;
+        const tag = record[0];
+        // `S` marks skip-worktree. With `-v`, an assume-unchanged entry's
+        // ordinary tag is lower-cased (`H` -> `h`, `S` -> `s`, etc.).
+        if (tag === 'S' || (tag >= 'a' && tag <= 'z'))
+            paths.push(record.slice(2));
+    }
+    return paths;
+};
+/**
+ * Repo-relative paths `git status` reports as dirty or untracked, using the
+ * same managed-path excludes as {@link isWorkingTreeDirty}, plus tracked paths
+ * whose assume-unchanged or skip-worktree bits can hide content changes from
+ * porcelain. `null` means either query failed — callers must not treat that as
+ * a clean tree.
+ */
+export const listWorkingTreeDirtyPaths = (repoPath) => {
+    try {
+        const out = execFileSync('git', [
+            'status',
+            '--porcelain=v1',
+            '-z',
+            '--untracked-files=all',
+            '--',
+            '.',
+            ...GITNEXUS_MANAGED_PATH_EXCLUDES,
+        ], { cwd: repoPath, windowsHide: true, ...gitPathListExec });
+        return [
+            ...new Set([...parsePorcelainPaths(out), ...listHiddenIndexPaths(repoPath)].filter((rel) => !isGitNexusManagedPath(rel))),
+        ];
+    }
+    catch {
+        return null;
+    }
+};
 /**
  * Snapshot, per candidate file, whether it is safe for `selfCommitContextFiles`
  * to auto-commit — call this BEFORE `analyze` writes AGENTS.md/CLAUDE.md.

package/dist/storage/gitnexus-managed-paths.d.ts

@@ -0,0 +1,36 @@
+/**
+ * The paths GitNexus itself writes during `analyze`.
+ *
+ * `analyze` rewrites the stats blocks in AGENTS.md/CLAUDE.md and refreshes the
+ * agent skill mirrors as its final step — after it has recorded the per-file
+ * hashes for the run. Counting its own output as a repository change makes
+ * every completed run look immediately out of date, which is the regression
+ * PR #1233 introduced and #1233's fix excluded these paths to prevent.
+ *
+ * Two freshness checks depend on this list agreeing: `isWorkingTreeDirty`
+ * (analyze's up-to-date fast-path gate) and the per-file comparison behind
+ * `status`. They used to hold separate copies of it, so a path added to one
+ * silently became a permanent "stale" verdict in the other. One list, imported
+ * by both.
+ */
+/**
+ * Repository-root-relative. A directory entry covers everything beneath it;
+ * a file entry matches only itself. Prefix collisions are NOT matches —
+ * `.agentsrc` is an ordinary file, not part of the `.agents` tree.
+ */
+export declare const GITNEXUS_MANAGED_PATHS: readonly [".gitnexus", ".claude", ".cursor", ".agents", "AGENTS.md", "CLAUDE.md"];
+/**
+ * Git pathspecs excluding {@link GITNEXUS_MANAGED_PATHS} from a `git status`
+ * run rooted at the repository. Patterns include `./` so they match only at
+ * the repo root: a slash-free `:(exclude)AGENTS.md` would also drop
+ * `docs/AGENTS.md`, which {@link isGitNexusManagedPath} does not treat as
+ * managed. Both forms are emitted per entry: the root path itself, and `/**`
+ * for directory contents.
+ */
+export declare const GITNEXUS_MANAGED_PATH_EXCLUDES: readonly string[];
+/**
+ * True when a repository-relative path is GitNexus's own output. Mirrors the
+ * pathspec semantics above: root-relative, whole path segments only, so
+ * neither `.agentsrc` nor a nested `subdir/.agents/` is treated as managed.
+ */
+export declare const isGitNexusManagedPath: (relPath: string) => boolean;

package/dist/storage/gitnexus-managed-paths.js

@@ -0,0 +1,46 @@
+/**
+ * The paths GitNexus itself writes during `analyze`.
+ *
+ * `analyze` rewrites the stats blocks in AGENTS.md/CLAUDE.md and refreshes the
+ * agent skill mirrors as its final step — after it has recorded the per-file
+ * hashes for the run. Counting its own output as a repository change makes
+ * every completed run look immediately out of date, which is the regression
+ * PR #1233 introduced and #1233's fix excluded these paths to prevent.
+ *
+ * Two freshness checks depend on this list agreeing: `isWorkingTreeDirty`
+ * (analyze's up-to-date fast-path gate) and the per-file comparison behind
+ * `status`. They used to hold separate copies of it, so a path added to one
+ * silently became a permanent "stale" verdict in the other. One list, imported
+ * by both.
+ */
+/**
+ * Repository-root-relative. A directory entry covers everything beneath it;
+ * a file entry matches only itself. Prefix collisions are NOT matches —
+ * `.agentsrc` is an ordinary file, not part of the `.agents` tree.
+ */
+export const GITNEXUS_MANAGED_PATHS = [
+    '.gitnexus',
+    '.claude',
+    '.cursor',
+    '.agents',
+    'AGENTS.md',
+    'CLAUDE.md',
+];
+/**
+ * Git pathspecs excluding {@link GITNEXUS_MANAGED_PATHS} from a `git status`
+ * run rooted at the repository. Patterns include `./` so they match only at
+ * the repo root: a slash-free `:(exclude)AGENTS.md` would also drop
+ * `docs/AGENTS.md`, which {@link isGitNexusManagedPath} does not treat as
+ * managed. Both forms are emitted per entry: the root path itself, and `/**`
+ * for directory contents.
+ */
+export const GITNEXUS_MANAGED_PATH_EXCLUDES = GITNEXUS_MANAGED_PATHS.flatMap((managed) => [`:(exclude,glob)./${managed}`, `:(exclude,glob)./${managed}/**`]);
+/**
+ * True when a repository-relative path is GitNexus's own output. Mirrors the
+ * pathspec semantics above: root-relative, whole path segments only, so
+ * neither `.agentsrc` nor a nested `subdir/.agents/` is treated as managed.
+ */
+export const isGitNexusManagedPath = (relPath) => {
+    const normalized = relPath.replace(/\\/g, '/');
+    return GITNEXUS_MANAGED_PATHS.some((managed) => normalized === managed || normalized.startsWith(`${managed}/`));
+};

package/dist/storage/parse-cache.d.ts

@@ -6,10 +6,12 @@
  * does is skip the tree-sitter worker dispatch when a chunk's contents
  * haven't changed since the last run.
  *
- * Granularity: chunk-level. The parse phase chunks files into ~20MB byte
- * budgets. The cache key is `sha256(joined(filePath:contentHash for each
- * file in the chunk, sorted))`. A change to a single file invalidates only
- * that file's chunk — typically 1 of ~50 chunks on a 1000-file repo.
+ * Granularity: chunk-level. Files are assigned to a stable
+ * `(language, hash(path) mod 128)` bucket, then packed to a 2 MiB (or
+ * operator) byte budget *inside* that bucket. Membership does not depend
+ * on worker count. The cache key is `sha256(joined(filePath:contentHash
+ * for each file in the chunk, sorted))`. A content edit invalidates only
+ * that file's pack; add/delete/rename only the affected bucket.
  *
  * Why not per-file:
  * - Workers process sub-batches and emit aggregated `ParseWorkerResult`s.
@@ -23,6 +25,22 @@
  */
 import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.js';
 export declare const PARSE_CACHE_VERSION: string;
+/** Stable parse-cache bucket count (#3088). Changing this requires SCHEMA_BUMP. */
+export declare const PARSE_CACHE_BUCKET_COUNT = 128;
+/** Bucket id for cache membership: `sha256(path) mod N` without IEEE-754 truncation. */
+export declare const parseCacheBucketId: (filePath: string) => number;
+export type ParseCachePackFile = {
+    path: string;
+    size: number;
+    language: string;
+};
+/**
+ * Pack files into parse-cache chunks: group by (language, bucket id), sort
+ * paths inside the group, then cut at `byteBudget`. Bucket visit order is
+ * the lexicographic order of `${language}\\0${bucketId}` keys (deterministic,
+ * independent of scan order and worker count).
+ */
+export declare const packParseCacheChunks: (files: readonly ParseCachePackFile[], byteBudget: number) => string[][];
 /** Runtime view: keyed Map for fast lookup; mutated in place during a run. */
 export interface ParseCache {
     version: string;