@ankurhalder/codebase-snapshot 4.0.6 → 4.1.0

package/dist/{types-x7aogSer.d.ts → types-BMP676ER.d.ts}

@@ -1,5 +1,21 @@
 type Mode = "generate" | "validate" | "report";
 type OutputFormat = "text" | "json";
+/**
+ * How files are discovered.
+ *
+ * `auto` lists files through git when git can do it and walks the tree otherwise; `git` and `walk`
+ * force one path. It is part of the manifest's recorded settings because it decides which files a
+ * snapshot contains — the same repository scanned both ways is two different snapshots.
+ */
+type DiscoveryMode = "auto" | "git" | "walk";
+/**
+ * How a file's bytes became text.
+ *
+ * Absent from a `FileEntry` means plain UTF-8, which is the overwhelmingly common case. The other
+ * values are recorded because they change what the artifact actually contains: `utf-16le`/`utf-16be`
+ * files were transcoded, and `utf-8-lossy` files carry U+FFFD where their undecodable bytes were.
+ */
+type TextEncoding = "utf-8" | "utf-8-lossy" | "utf-16le" | "utf-16be";
 /**
  * How much of a file made it into the snapshot.
  * `full`    – the complete file body was emitted.
@@ -64,7 +80,15 @@ type SkipReason = "binary" | "read-error" | "not-file"
 /** Excluded by a built-in rule: the ignore list, a preset, or an output dir. */
  | "default-ignore"
 /** Excluded by the user's own `excludePatterns` or `include` allow-list. */
- | "excluded";
+ | "excluded"
+/**
+ * A directory that belongs to another repository: a nested checkout or a submodule.
+ *
+ * A submodule is a single gitlink entry, so `stat` reports a directory where a file was
+ * expected — which used to be recorded as the generic `not-file`, implying the path was a
+ * socket or a FIFO rather than a repository whose contents are simply somebody else's.
+ */
+ | "nested-repository";
 interface SkipRecord {
     /** Path as it may be shown in an artifact: secret spans are masked. */
     path: string;
@@ -99,6 +123,11 @@ interface FileEntry {
     summaryQuality?: SummaryQuality;
     /** True when the summary alone was too large for the part budget and was dropped. */
     summaryOmitted?: boolean;
+    /**
+     * How the file's bytes were decoded. Absent means plain UTF-8. Present when the file was
+     * transcoded from UTF-16, or when it was not valid UTF-8 and lost bytes to U+FFFD.
+     */
+    encoding?: TextEncoding;
     /** Emitted body after redaction/truncation. Never serialized into the manifest. */
     content?: string;
 }
@@ -215,6 +244,8 @@ interface ManifestConfigSnapshot {
     previewTailLines: number;
     includeMonolith: boolean;
     includeSensitiveFiles: boolean;
+    /** How files were discovered; `undefined` on manifests written before this was recorded. */
+    discovery?: DiscoveryMode;
     artifacts: ResolvedArtifacts;
     include: string[];
     preset: string;
@@ -368,6 +399,12 @@ interface ResolvedSnapshotConfig {
     includeMonolith: boolean;
     maxMonolithBytes: number | null;
     includeSensitiveFiles: boolean;
+    /**
+     * How files are discovered; `auto` picks git when it can list files and walks the tree otherwise.
+     * Optional because a resolved config can arrive from an older caller that never named it — every
+     * reader treats `undefined` as `auto`, so the effective mode is never guessed twice.
+     */
+    discovery?: DiscoveryMode;
     /** Empty means "no allow-list"; every discovered file is a candidate. */
     include: string[];
     /** The preset that was resolved (detected or forced). */
@@ -400,6 +437,8 @@ interface CliOptions {
 interface ValidationDrift {
     missingInSnapshot: string[];
     missingOnDisk: string[];
+    /** Files that exist but could not be read, so their content could not be verified. */
+    unreadable: string[];
     hashMismatches: Array<{
         path: string;
         diskHash: string;
@@ -424,4 +463,4 @@ interface ValidationResult {
     warnings: string[];
 }
 
-export type { ArtifactSelection as A, ByteSize as B, CliOptions as C, FileEntry as F, HooksConfig as H, InclusionMode as I, JsonSummary as J, KindRule as K, LayerRule as L, MarkdownSummary as M, OutputFormat as O, PartMetadata as P, ResolvedSnapshotConfig as R, SourceSummary as S, ValidationResult as V, SnapshotConfig as a, SecretRule as b, SecretFinding as c, Manifest as d, SkipRecord as e, BudgetConfig as f, ManifestArtifact as g, ManifestConfigSnapshot as h, ManifestIndexes as i, ManifestTotals as j, Mode as k, PartFileEntry as l, ResolvedArtifacts as m, ResolvedBudgets as n, SecretRuleInput as o, SkipReason as p, SummaryQuality as q, ValidationDrift as r };
+export type { ArtifactSelection as A, ByteSize as B, CliOptions as C, DiscoveryMode as D, FileEntry as F, HooksConfig as H, InclusionMode as I, JsonSummary as J, KindRule as K, LayerRule as L, MarkdownSummary as M, OutputFormat as O, PartMetadata as P, ResolvedSnapshotConfig as R, SecretRule as S, TextEncoding as T, ValidationResult as V, SkipRecord as a, SourceSummary as b, SnapshotConfig as c, SecretFinding as d, Manifest as e, BudgetConfig as f, ManifestArtifact as g, ManifestConfigSnapshot as h, ManifestIndexes as i, ManifestTotals as j, Mode as k, PartFileEntry as l, ResolvedArtifacts as m, ResolvedBudgets as n, SecretRuleInput as o, SkipReason as p, SummaryQuality as q, ValidationDrift as r };

package/docs/agent.md

@@ -117,4 +117,6 @@ When you parse the manifest, remember what is conditional: `manifest.files[]` ca
 `relativePath`, `layer`, `kind`, `language`, `part`, `inclusionMode` and `hashSha256` for
 every included file, but `summary` and `summaryQuality` only for a language the tool can
 analyse. A `.gitignore` or plain-text entry has neither key, and a file whose summary was too
-large for its part carries `summaryOmitted: true` instead of a `summary`.
+large for its part carries `summaryOmitted: true` instead of a `summary`. `encoding` appears only
+when a file was not plain UTF-8: `utf-16le`/`utf-16be` were transcoded, and `utf-8-lossy` means
+the body you are reading contains `U+FFFD` where undecodable bytes were.

package/docs/cli.md

@@ -175,6 +175,7 @@ With `--format json` the same answer is one object on stdout, with only the keys
 | `layer`, `kind`, `language` | The classification, present only when `included` is `true`. |
 | `layerMatch`, `kindMatch` | Which rule chose the layer or kind. A glob is printed as written (`src/api/**`); a `RegExp` as its literal form (`/\.tsx$/i`); a function matcher, which is what the presets use, as `function (custom predicate)`. Absent when no rule matched and the `other` fallback was used. |
 | `bytes`, `lines` | Size of the file on disk. |
+| `encoding` | Present only when the file is not plain UTF-8 — `utf-16le`, `utf-16be` or `utf-8-lossy`. |
 | `secretCount`, `secrets` | How many secret rules fire, and the first 20 as `{ rule, line }` — never the matched text. |
 | `part` | The part filename from the last written manifest, or `null` when it is not in that snapshot. |
 | `inclusionMode` | `full` or `preview`, when a part entry exists. |
@@ -193,6 +194,20 @@ Compares two snapshots and lists added, removed and changed files. Either argume
 snapshot directory or a `manifest.json`. Useful in review: "what changed in the context we
 hand to the model?"
 
+Comparison is per file (`relativePath` + `sha256`), so on its own it cannot see a run that
+repacked the *same* sources differently. `diff` therefore also compares the two manifests'
+`generator.version`, `config.digest`, content fingerprint, part count and layer totals, and
+prints a `payload/configuration differs:` section when the file set is identical but the
+emitted payload — or the configuration that produced it — is not. That covers `--large-file-bytes`
+(a preview becomes a full body), the preview head/tail counts, redaction rules, artifact
+selection and per-layer budgets, none of which move a source hash. The per-file lists and the
+JSON fields `added` / `removed` / `changed` / `unchanged` are unchanged; the extra data is
+under `payload` in `--format json`.
+
+Manifests written before 4.0.5 have no `config.digest`. In that case `diff` compares the
+content fingerprint and the part count only, and prints `configuration not compared: a
+manifest has no config digest (written before 4.0.5).` rather than implying the settings match.
+
 ### `describe`
 
 ```bash
@@ -242,13 +257,14 @@ narrowest cases of the same rule.
 | `--format <text\|json>` | enum | `text` | every command | `json` writes a pure JSON payload to stdout. |
 | `--dry-run` | flag | off | generate, init | Compute everything, print it, write nothing. |
 | `--watch` | flag | off | generate | Regenerate on every change until interrupted. |
-| `--no-config` | flag | off | generate, validate, report, explain | Ignore every `snapshot.config.*` file and run on the built-in defaults. |
+| `--no-config` | flag | off | generate, validate, report, explain, diff | Ignore every `snapshot.config.*` file and run on the built-in defaults. `diff` never loads a config anyway and accepts the flag for consistency. |
+| `--discovery <mode>` | enum | `auto` | generate, validate, explain | How files are found: `auto` lists through git when git works and refuses when a `.git` directory cannot be listed; `git` requires git; `walk` scans the tree and reads `.gitignore` files directly. The escape hatch for a broken `.git`. |
 | `--preset <name>` | name | detected | *init only* | Force a preset instead of detecting one. |
 | `--include <globs>` | list | unset | *init only* | Allow-list recorded in the generated config. |
 | `--exclude <globs>` | list | unset | *init only* | Exclude globs recorded in the generated config. |
 | `--yes` | flag | off | *init only* | Never prompt; accept every default. |
 | `--minimal` | flag | off | *init only* | Write only settings that differ from the defaults. |
-| `--force` | flag | off | *init only* | Overwrite an existing config. |
+| `--force` | flag | off | *init only* | Replace an existing `.ts`/`.js`/`.mjs` config in place. A JSON-family config is refused, naming the file to remove. |
 | `--install` | flag | off | *skill only* | Write the skill into the repository instead of printing it. |
 | `--target <name>` | enum | `all` | *skill only* | `claude`, `agents`, `cursor` or `all`. |
 | `-h`, `--help` | flag | — | every command | Print usage. |
@@ -263,6 +279,21 @@ config mismatch.
 Every option also accepts the `--option=value` form. That form is required for negative
 numbers, which otherwise look like flags.
 
+### `--mode`
+
+`--mode <command>` names the command instead of passing it positionally, which is handy when
+a wrapper builds the argument list. `--mode validate` and `codebase-snapshot validate` are the
+same run. It accepts the nine command names, and it is deliberately strict: asking for two
+different commands is an error rather than a silent winner.
+
+```
+❌ --mode validate contradicts the command "generate". Pass one or the other.
+❌ Unknown --mode "bogus". Expected one of: generate, validate, check, report, init, explain, diff, describe, skill.
+```
+
+`--mode` is not listed in `describe`'s option table or `--help`, on purpose: an agent has no
+reason to use it, and the positional form is the documented one.
+
 ## Exit codes
 
 | Code | Meaning |
@@ -297,7 +328,7 @@ npx @ankurhalder/codebase-snapshot generate --out-dir .context --max-part-bytes
 npx @ankurhalder/codebase-snapshot generate --only-artifacts parts,manifest
 
 # One file per part, with everything else left alone
-npx @ankurhalder/codebase-snapshot validate
+npx @ankurhalder/codebase-snapshot generate --only-artifacts parts
 
 # Why is this file missing?
 npx @ankurhalder/codebase-snapshot explain src/generated/schema.ts

package/docs/configuration.md

@@ -64,12 +64,26 @@ see [the trust boundary](../SECURITY.md#trust-boundary).
 export default {
   include: [],                    // allow-list; empty means "everything"
   excludePatterns: [],            // extra excludes, applied on top of the preset's
+  discovery: "auto",              // "auto" | "git" | "walk" — how files are found
 };
 ```
 
 - `include` is an allow-list of globs. When non-empty, only matching files are considered.
+- `exclude` is accepted as an alias of `excludePatterns` and means the same thing; the resolved
+  config always records it under `excludePatterns`.
 - `excludePatterns` removes files everywhere. Presets contribute their own ecosystem ignores
   on top; those are not removable, only addable to.
+- `discovery` chooses how the file list is produced, and is recorded in `manifest.config`, so
+  changing it shows up as config drift:
+  - `"auto"` (default) — list through `git ls-files` when git can do it; if a `.git` directory
+    exists but git cannot list the files, refuse rather than silently switch methods.
+  - `"git"` — require git; a directory without a usable repository is an error.
+  - `"walk"` — scan the tree and read `.gitignore` files directly (nested ones included), still
+    skipping credential files. Use it when a `.git` directory cannot be listed. It cannot
+    reproduce git's own ignore sources (`.git/info/exclude`, your global `core.excludesFile`),
+    git's inclusion of files that are tracked although a `.gitignore` now matches them, or its
+    treatment of a submodule as a single entry. The same setting is available per run as
+    [`--discovery`](cli.md#options).
 
 Built-in ignores always apply: `.git`, `node_modules`, `.next`, `dist`, `build`, lockfiles,
 minified assets and binaries. Sensitive files are handled separately (see
@@ -270,14 +284,17 @@ snapshot the marker describes.
 export default {
   hooks: {
     afterGenerate: (manifest) => {
-      // runs after a successful generate (and after a dry run)
+      // runs after a real generate; a dry run does not call it
     },
   },
 };
 ```
 
 A throwing hook is downgraded to a warning on the manifest — it never loses a snapshot that
-was already written.
+was already written. A hook may return a promise: it is not awaited (this API is synchronous),
+and a rejection is recorded as a warning on the re-shipped manifest and marker. Anything the
+hook writes to stdout is redirected to stderr for the duration of the call, so `--format json`
+stays parseable.
 
 A hook may write its own files into the output directory; that is safe across reruns because
 ownership is established by the marker rather than by the filenames present.
@@ -311,6 +328,7 @@ export default {
   // ── Scope ────────────────────────────────────────────────
   include: [],
   excludePatterns: [],
+  discovery: "auto",                 // "auto" | "git" | "walk"
 
   // ── Artifacts ────────────────────────────────────────────
   artifacts: {
@@ -364,5 +382,12 @@ npx codebase-snapshot init --preset go --max-part-bytes 200k \
 ```
 
 `--minimal` writes only what differs from the defaults. `--dry-run` prints the config and
-writes nothing. `--force` overwrites an existing config; without it, an existing config is
-left alone and the command exits `1`.
+writes nothing. `--force` replaces an existing ES-module config (`.ts`, `.js`, `.mjs`) in
+place; without it, an existing config is left alone and the command exits `1`.
+
+It will not replace a JSON-family config — `snapshot.config.json`, `.snapshotrc`,
+`.snapshotrc.json`, or the `"snapshot"` key in `package.json` — because this wizard writes an
+ES-module file and a new file beside the old one would silently shadow it. Those cases exit `1`
+with a message naming the file to delete or rename. Note that this means `--force` writing
+`snapshot.config.ts` does shadow a `"snapshot"` key in `package.json`, since a dedicated config
+file always wins.

package/docs/install.md

@@ -53,13 +53,15 @@ most reproducible option:
 ```json
 {
   "scripts": {
-    "snapshot": "codebase-snapshot generate",
-    "snapshot:validate": "codebase-snapshot validate"
+    "snapshot": "npx @ankurhalder/codebase-snapshot generate",
+    "snapshot:validate": "npx @ankurhalder/codebase-snapshot validate"
   }
 }
 ```
 
-`codebase-snapshot init` writes these entries for you.
+`codebase-snapshot init` writes these entries for you, in exactly this `npx` form: the bare
+`codebase-snapshot` command only exists on `PATH` inside a client runner or a script that has
+already resolved the package, and `init` does not assume the install has happened yet.
 
 ## One-off runners for other clients
 

package/docs/output.md

@@ -129,9 +129,18 @@ values — the hash is the whole 64-character digest, never elided.
 - Only `isPreviewTruncated` tells you content is missing; `isLarge` alone does not.
 - `hashSha256` is the hash of the source file on disk — `validate` recomputes it to detect
   drift. It is the full 64-character digest.
+- `encoding` is present **only** when the file is not plain UTF-8; its absence means clean UTF-8.
+  `utf-16le` and `utf-16be` mean the file was transcoded to UTF-8 before it was emitted, so a
+  UTF-16 source file is readable in the parts instead of vanishing as "binary". `utf-8-lossy`
+  means the file is not valid UTF-8 at all: its undecodable byte spans became `U+FFFD`, the count
+  is reported in `warnings`, and the file is kept rather than dropped. The hash still covers the
+  raw bytes, so a lossy file validates as current — `encoding` is how you learn that the body you
+  are reading is not byte-exact.
 - `summary` and `summaryQuality` are **conditional**. Both keys are present for a language
   with an analyser and both are absent for a shapeless one — a `.gitignore`, a `.txt`. Treat a
-  missing `summary` as "no structural information", not as an empty one.
+  missing `summary` as "no structural information", not as an empty one. A file larger than
+  2 MB gets no summary at all: parsing it costs more memory than the file itself, and its body is
+  a head/tail preview at that size anyway. `manifest.warnings` names those files.
 - `summary` has one of three shapes. Code files get the thirteen keys above (`directives`,
   `imports`, `exports`, `reExports`, `functions`, `classes`, `interfaces`, `types`, `enums`,
   `variables`, `reactComponents`, `hooks`, `routeHandlers`), plus `parseErrors` when the parser
@@ -160,12 +169,13 @@ Reasons, and what each one means:
 | `default-ignore` | A built-in rule caught it, the preset's ecosystem ignores did (`node_modules`, `__pycache__`, `target`, …), or it sat inside an output directory. |
 | `sensitive-file` | Its name is credential-bearing (`.env`, `*.pem`, `.npmrc`, …). |
 | `sensitive-path` | The path itself matched a secret rule. The file is skipped and the path is masked in every artifact. |
-| `binary` | It is not text. |
+| `binary` | It is not text: it holds a NUL byte outside a UTF-16 stream, or more than 1% control characters (a 4 KB blob of `0x07` is valid UTF-8 but is not text). `detail` carries the size and, when there is one to give, the reason: the control-byte count (`Size: 13 B; 5 control byte(s)`), or the offset of a NUL found past the 8 MB head scan (`Size: 9000010 B; NUL byte at offset 9000000`). A NUL inside the head scan is caught by the cheap extension-and-prefix check, so its `detail` is the size alone. |
 | `symlink` | It is a link, and links are not followed. A directory link names its target in `detail`. |
 | `outside-root` | It is a link whose target resolves outside the scanned root. |
 | `gitignore` | A `.gitignore` excluded it, in a scan that is not reading git's index. |
 | `read-error` | It could not be read. |
-| `not-file` | It is a directory, socket or FIFO. |
+| `not-file` | It is a directory, socket or FIFO. A directory that belongs to another repository is reported as `nested-repository` instead. |
+| `nested-repository` | It is a nested checkout or a submodule: a directory whose contents are tracked by a different repository, so they are not part of this codebase. |
 
 Every path the scanner discovers is recorded — nothing is dropped silently, in either discovery
 mode. Inside a git repository the candidate list comes from git, which applies `.gitignore`
@@ -191,14 +201,17 @@ example). See [Troubleshooting](troubleshooting.md#a-file-is-missing-from-the-sn
   replacement is complete.
 - **The tool refuses to replace a directory it did not write.** Pointing `--out-dir` at
   `src/` is refused with exit `1` and the directory is left untouched — a mistyped output
-  path cannot delete source code. Empty directories, directories containing only
-  `.gitkeep`/`.gitignore`/`.DS_Store`, and directories holding a complete snapshot this build
-  wrote are accepted.
+  path cannot delete source code. Empty directories, directories that hold nothing but
+  `.DS_Store`, and directories holding a complete snapshot this build wrote are accepted.
+  A `.gitkeep` or `.gitignore` inside the directory is not accepted, because the generator never
+  writes either into an output directory: a directory that holds one is somebody's data, so the
+  run refuses instead of deleting it.
 - **Ownership is the marker, not a filename.** `.codebase-snapshot` is a JSON record carrying
   this generator's name and `version`, the marker's `schemaVersion`, a `createdAt` timestamp,
-  a per-snapshot `id`, the run `state` (`staging` while it is being written, `complete` once it
-  is whole), the `manifestSha256` digest of the manifest it shipped, and `entries` — the
-  inventory of everything it wrote. A zero-byte marker, a marker that is not JSON, a marker
+  a `pid`, a per-snapshot `id`, the run `state` (`staging` while it is being written, `complete`
+  once it is whole), the `manifestSha256` digest of the manifest it shipped, and `entries` — the
+  inventory of everything it wrote. `pid` is what lets a later run tell a crashed run's staging
+  directory and lock from a live one. A zero-byte marker, a marker that is not JSON, a marker
   written by 3.0.0 (no `id`), a bare `manifest.json`, and a directory with files you added
   after the run are all refused.
   See [Troubleshooting](troubleshooting.md#refusing-to-replace-outdir-).

package/docs/troubleshooting.md

@@ -83,8 +83,13 @@ and the digest of the manifest it shipped. A file with the right name is not eno
 | carries our marker but you added files | `it carries this generator's ownership marker, but these entries are not part of the snapshot it recorded: custom.txt. Remove them yourself, or point outDir at a new or empty directory.` |
 
 **Fix** — point `--out-dir` somewhere new, or delete the directory yourself. Do not delete
-your source. Accepted without complaint: a missing directory, an empty directory, one
-holding only `.gitkeep`/`.gitignore`/`.DS_Store`, and a complete snapshot this build wrote.
+your source. Accepted without complaint: a missing directory, an empty directory, one holding
+nothing but `.DS_Store`, and a complete snapshot this build wrote.
+
+`.gitkeep` and `.gitignore` are **not** accepted, even in a directory that holds nothing else.
+Both are files you create — the generator never writes either into an output directory — so a
+directory holding one is somebody's data and is refused rather than replaced. Move the file out,
+or point `--out-dir` somewhere new.
 
 A snapshot written by 3.0.0 or earlier is **not** upgraded in place: its marker has three keys
 and no `id`, so the new build cannot prove it wrote the directory and refuses. Delete the old
@@ -113,6 +118,44 @@ settings and still report success.
 loadable on your Node version: a `.ts` config needs Node ≥ 22.6 **and** an ES-module project.
 `codebase-snapshot init` picks the right filename for the project automatically.
 
+### `This directory is a git repository, but git could not list its files.`
+
+```
+❌ Failed to generate snapshot: This directory is a git repository, but git could not list its files. A plain walk honours .gitignore files, but git also applies .git/info/exclude and your global excludes file, includes files that are tracked although a .gitignore now matches them, and treats a submodule as a single entry. Fix the repository (git status should work), run outside git, or pass --discovery walk to accept those differences.
+```
+
+`git` is present and a `.git` directory exists, but it cannot list the files — an empty or
+half-initialised `.git`, a permissions problem, or a damaged index. The tool refuses rather
+than silently choosing between two discovery methods: which files are in the snapshot would
+otherwise depend on whether git happened to work.
+
+**Fix** — make `git status` work in that directory (repair or re-clone the repository), generate
+from a copy that has no `.git` directory, or pass `--discovery walk` to scan the tree anyway.
+The walk reads every `.gitignore` it meets, nested ones included, and still skips credential
+files; what it cannot reproduce is git's own ignore sources (`.git/info/exclude` and your global
+`core.excludesFile`), git's inclusion of files that are tracked although a `.gitignore` now
+matches them, and its treatment of a submodule as one entry instead of a tree.
+
+`--discovery git` says the opposite — require git — so it refuses in a directory that has no
+usable repository:
+
+```
+❌ Failed to generate snapshot: Discovery mode is `git`, but git could not list this directory's files. Run inside a working repository, or pass --discovery walk (or auto) to scan the tree directly.
+```
+
+### `Another generate run is already using the staging directory …`
+
+```
+❌ Failed to generate snapshot: Another generate run is already using the staging directory /repo/snapshot.tmp. Wait for it to finish, or delete /repo/codebase-snapshot-1a2b3c4d5e6f7890.lock if no other run is active.
+```
+
+A second `generate` against the same repository started while the first still held the staging
+directory — two terminals, or a CI job and a developer at once. The lock is advisory and on
+disk, so a run that was killed cannot release it.
+
+**Fix** — wait for the other run, then retry. If no other run is active, the named lock file is
+stale: delete it, and the next run proceeds.
+
 ### `Refusing to generate: …`
 
 ```
@@ -195,7 +238,7 @@ npx codebase-snapshot explain src/generated/schema.ts
 | :--- | :--- | :--- |
 | `sensitive-file` | `.env`, `*.pem`, `.npmrc`, `.ssh/`, `.tfstate`, … | `--include-sensitive-files` if you really want it. |
 | `sensitive-path` | The path itself matched a secret rule, so the file is skipped and its path is masked in every artifact. | Rename the file; a path that looks like a key is treated as one. |
-| `binary` | Not text. | Nothing to do. |
+| `binary` | Not text: it contains a NUL byte outside a UTF-16 stream, or is more than 1% control characters. The `detail` names the file size and the reason — including the offset of the first NUL when that byte sits past the 8 MB head scan. | Nothing to do. A UTF-16 file is *not* binary — it is transcoded and reported with `encoding`, not skipped. |
 | `default-ignore` | A built-in rule, the preset's ecosystem ignores (`node_modules`, `__pycache__`, `target`, `vendor`, …), or the output directory. | A preset's ignores are not configurable; a custom `excludePatterns` can only remove more. |
 | `excluded` | Your own `excludePatterns`, or a path outside a non-empty `include` allow-list. | Remove the pattern, or widen `include`. |
 | `symlink` | Links are not followed (a directory link names its target in `detail`). | Snapshot the target. |
@@ -203,6 +246,7 @@ npx codebase-snapshot explain src/generated/schema.ts
 | `gitignore` | A `.gitignore` excluded it, in a scan that is not reading git's index. | Remove the entry, or add the path to `include`. |
 | `read-error` | Unreadable file. | Check permissions. |
 | `not-file` | A directory, socket, FIFO. | Expected. |
+| `nested-repository` | A nested checkout or submodule — a directory tracked by a different repository. | Expected; snapshot that repository separately if you need its code. |
 
 Inside a git repository the candidate list comes from git, which applies `.gitignore` before
 the scanner runs; those ignored paths are listed separately and recorded too, so a `.gitignore`d
@@ -238,6 +282,19 @@ under a `- Warnings: N` line in the summary, and none of them stops a run:
 - `every artifact is disabled: only the .codebase-snapshot ownership marker will be written.`
 - A `hooks.afterGenerate` error is downgraded to a warning: the snapshot it was called for is
   already written and is not lost.
+- `N file(s) exceed 2 MB and were not parsed for a structural summary: …` — the named files are
+  emitted as head/tail previews without an `imports`/`exports` summary. Parsing them costs several
+  times their own size in memory, which is how a single 50 MB source file used to reach 3.3 GB of
+  resident memory. Nothing to fix; raise the part budget if the preview is too small.
+- `N file(s) are UTF-16LE and were transcoded to UTF-8: …` — those files were UTF-16 and are
+  now ordinary text in the snapshot. They are listed in `manifest.files[]` with
+  `encoding: "utf-16le"`, or `"utf-16be"`. Nothing to fix; the warning exists because the bytes
+  in the artifact are not the bytes on disk.
+- `N file(s) are not valid UTF-8; M undecodable byte(s) became U+FFFD: …` — the named files are
+  not UTF-8 at all (Latin-1 is the usual cause). They are kept, with `encoding: "utf-8-lossy"`,
+  but the undecodable spans are substituted, so **do not hand the emitted body to a model as if
+  it were the source**. Convert the file to UTF-8 (`iconv -f WINDOWS-1252 -t UTF-8 file > file.tmp`)
+  and regenerate. The snapshot still validates as current, because the hash covers the raw bytes.
 
 `validate` produces its own warnings, printed with a `⚠ ` prefix and never written to the
 manifest, because they describe the state of the repository rather than the snapshot:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ankurhalder/codebase-snapshot",
-  "version": "4.0.6",
+  "version": "4.1.0",
   "description": "Production-grade LLM codebase snapshot generator, auditor, and AST partitioner",
   "keywords": [
     "codebase",