codebase-snapshot 0.9.9 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +873 -0
- package/README.md +3 -3
- package/dist/bin.js +6343 -3897
- package/dist/{chunk-KUVFTJHK.js → chunk-DUZK5VL2.js} +3036 -1049
- package/dist/{chunk-HQBEQ67K.js → chunk-J4W74N4Y.js} +496 -114
- package/dist/cli.cjs +6282 -3859
- package/dist/cli.d.cts +282 -3
- package/dist/cli.d.ts +282 -3
- package/dist/cli.js +198 -87
- package/dist/generate-worker-D_mUjgXw.d.cts +73 -0
- package/dist/generate-worker-D_mUjgXw.d.ts +73 -0
- package/dist/index.cjs +2981 -1057
- package/dist/index.d.cts +337 -25
- package/dist/index.d.ts +337 -25
- package/dist/index.js +1 -3
- package/dist/mcp.cjs +6426 -5065
- package/dist/mcp.d.cts +136 -68
- package/dist/mcp.d.ts +136 -68
- package/dist/mcp.js +2 -4
- package/dist/{types-Ca4PmkiQ.d.cts → types-DIX6zWSt.d.cts} +130 -2
- package/dist/{types-Ca4PmkiQ.d.ts → types-DIX6zWSt.d.ts} +130 -2
- package/docs/agent.md +1 -1
- package/docs/cli.md +26 -6
- package/docs/configuration.md +48 -9
- package/docs/how-to-feed-a-repo-to-claude.md +61 -0
- package/docs/mcp.md +36 -10
- package/docs/output.md +49 -4
- package/docs/stale-context-is-your-bug.md +13 -0
- package/docs/vs-repomix.md +1 -1
- package/package.json +5 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
|
-
import { S as SecretRule, D as DiscoveryMode, a as SkipRecord, J as JsonSummary, M as MarkdownSummary, b as SourceSummary, R as ResolvedSnapshotConfig, c as SnapshotConfig, B as ByteSize, C as CliOptions,
|
|
2
|
-
export { f as BudgetConfig, H as HooksConfig, I as InclusionMode, g as ManifestArtifact, h as ManifestConfigSnapshot, i as ManifestIndexes, j as ManifestTotals, k as Mode, O as OutputFormat, l as PartFileEntry, m as ResolvedArtifacts, n as ResolvedBudgets, o as SecretRuleInput, p as SkipReason, q as SummaryQuality, r as ValidationDrift } from './types-
|
|
1
|
+
import { S as SecretRule, D as DiscoveryMode, a as SkipRecord, J as JsonSummary, M as MarkdownSummary, b as SourceSummary, K as KindRule, L as LayerRule, R as ResolvedSnapshotConfig, c as SnapshotConfig, B as ByteSize, C as CliOptions, P as PartMetadata, F as FileEntry, T as TextEncoding, d as SecretFinding, V as ValidationResult, e as Manifest, A as ArtifactSelection } from './types-DIX6zWSt.js';
|
|
2
|
+
export { f as BudgetConfig, H as HooksConfig, I as InclusionMode, g as ManifestArtifact, h as ManifestConfigSnapshot, i as ManifestIndexes, j as ManifestTotals, k as Mode, O as OutputFormat, l as PartFileEntry, m as ResolvedArtifacts, n as ResolvedBudgets, o as SecretRuleInput, p as SkipReason, q as SummaryQuality, r as ValidationDrift } from './types-DIX6zWSt.js';
|
|
3
|
+
export { w as workerEntry } from './generate-worker-D_mUjgXw.js';
|
|
4
|
+
|
|
5
|
+
interface GitInfo {
|
|
6
|
+
branch: string;
|
|
7
|
+
commit: string;
|
|
8
|
+
isDirty: boolean;
|
|
9
|
+
}
|
|
3
10
|
|
|
4
11
|
interface CollectOptions {
|
|
5
12
|
/**
|
|
@@ -39,7 +46,23 @@ interface CollectResult {
|
|
|
39
46
|
files: string[];
|
|
40
47
|
skipped: SkipRecord[];
|
|
41
48
|
}
|
|
49
|
+
declare function getGitInfo(cwd: string, fallbackBranch?: string): GitInfo;
|
|
42
50
|
declare function isSensitivePath(relativePath: string): boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Reports a secret-shaped value in a *path*.
|
|
53
|
+
*
|
|
54
|
+
* File names are emitted verbatim into `project-tree.md`, `navigation.md`,
|
|
55
|
+
* `manifest.files[].relativePath`, the `## Skipped` table of the audit and
|
|
56
|
+
* `skipped[].path`, so a token in a name leaks exactly like a token in a body —
|
|
57
|
+
* and content redaction never sees it.
|
|
58
|
+
*
|
|
59
|
+
* `extraRules` are the caller's custom patterns. A `valueGroup` rule is masked
|
|
60
|
+
* whole here: a path has no `key: value` context to anchor on.
|
|
61
|
+
*/
|
|
62
|
+
declare function findSecretInPath(relativePath: string, extraRules?: readonly SecretRule[]): {
|
|
63
|
+
rules: string[];
|
|
64
|
+
masked: string;
|
|
65
|
+
} | null;
|
|
43
66
|
/**
|
|
44
67
|
* Translates a glob (`**`, `*`, `?`, `{a,b}`, `[abc]`) into a RegExp. A pattern
|
|
45
68
|
* without a leading `/` matches at any depth, mirroring `.gitignore` semantics.
|
|
@@ -47,6 +70,12 @@ declare function isSensitivePath(relativePath: string): boolean;
|
|
|
47
70
|
* Brace groups and character classes are supported because they are what people
|
|
48
71
|
* actually write: `*.{ts,tsx}` previously matched nothing at all, since `{` and
|
|
49
72
|
* `}` were escaped as literals.
|
|
73
|
+
*
|
|
74
|
+
* Compiled once per distinct pattern. `matchesExclude` calls this for every
|
|
75
|
+
* (file, include), (file, exclude), (file, layer rule) and (file, kind rule)
|
|
76
|
+
* pair, so a 20 000-file repository with 55 patterns compiled over a million
|
|
77
|
+
* identical RegExp objects per run. The result is a pure function of the string,
|
|
78
|
+
* so the cache cannot change an answer.
|
|
50
79
|
*/
|
|
51
80
|
declare function globToRegExp(glob: string): RegExp;
|
|
52
81
|
/**
|
|
@@ -67,9 +96,19 @@ declare function matchesExclude(relativePath: string, pattern: string): boolean;
|
|
|
67
96
|
declare function collectFiles(cwd: string, options?: CollectOptions): CollectResult;
|
|
68
97
|
|
|
69
98
|
/**
|
|
70
|
-
* Strips `//` and block comments the way JSONC does, without touching
|
|
71
|
-
* inside a string literal.
|
|
72
|
-
*
|
|
99
|
+
* Strips `//` and block comments the way JSONC does, plus trailing commas, without touching
|
|
100
|
+
* anything inside a string literal.
|
|
101
|
+
*
|
|
102
|
+
* `JSON.parse` alone made every `.jsonc` file — and every `tsconfig.json` an editor had
|
|
103
|
+
* reformatted — report `invalid-json`, losing its key shape entirely. Two things this file's
|
|
104
|
+
* matcher did that `src/config.ts`'s did not, and both are JSONC, not extensions of it:
|
|
105
|
+
*
|
|
106
|
+
* - **Trailing commas.** `{ "a": 1, }` is valid JSONC and invalid JSON.
|
|
107
|
+
* - **`\r`-only line endings.** A `//` comment ended only at `\n`, so on a CR-delimited file it
|
|
108
|
+
* swallowed everything to the end of the document.
|
|
109
|
+
*
|
|
110
|
+
* A block comment keeps the line breaks it spans. Dropping them renumbered every line after it,
|
|
111
|
+
* and the only thing a reader has to place a summary key is the line number.
|
|
73
112
|
*/
|
|
74
113
|
declare function stripJsonComments(content: string): string;
|
|
75
114
|
declare function summarizeJson(content: string): JsonSummary;
|
|
@@ -138,16 +177,29 @@ interface ConfigLoadResult {
|
|
|
138
177
|
/** True when this Node can `import()` a `.ts` file natively (type stripping). */
|
|
139
178
|
declare function supportsTypeScriptImports(): boolean;
|
|
140
179
|
/**
|
|
141
|
-
*
|
|
180
|
+
* Every key a configuration file may carry.
|
|
142
181
|
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
182
|
+
* This is `SnapshotConfig` plus the spellings that are read through casts: `discovery`
|
|
183
|
+
* (validated separately), `exclude` (the flag spelling of `excludePatterns`) and
|
|
184
|
+
* `secretRules` (the library spelling of a rule list, accepted by `resolveConfig` and
|
|
185
|
+
* therefore needed here too — omitting it made a config file that used it fail with
|
|
186
|
+
* "unknown setting(s): secretRules" while the same key worked when passed in code).
|
|
146
187
|
*/
|
|
188
|
+
declare const CONFIG_KEYS: ReadonlySet<string>;
|
|
147
189
|
declare function loadConfigFileDetailed(cwd: string): Promise<ConfigLoadResult>;
|
|
148
190
|
declare function loadConfigFile(cwd: string): Promise<SnapshotConfig | null>;
|
|
149
191
|
/** Smallest usable part budget: a part header plus one section must fit. */
|
|
150
192
|
declare const MIN_MAX_PART_BYTES = 4096;
|
|
193
|
+
/**
|
|
194
|
+
* The same floor expressed in the unit `--max-part-tokens` is written in.
|
|
195
|
+
*
|
|
196
|
+
* `bytesForTokens` multiplies by four, so 1024 tokens and 4096 bytes are the same budget. A
|
|
197
|
+
* smaller token budget used to be raised to the floor without a word: `--max-part-tokens 200`
|
|
198
|
+
* was accepted, reported as 1024 tokens, and the setting the user wrote never applied. Rule 1
|
|
199
|
+
* says an impossible budget is an error, so this is the number the CLI and the config both
|
|
200
|
+
* refuse to go below.
|
|
201
|
+
*/
|
|
202
|
+
declare const MIN_MAX_PART_TOKENS: number;
|
|
151
203
|
/** Every artifact this package can write, in the order it is written. */
|
|
152
204
|
declare const ARTIFACT_NAMES: readonly ["readme", "manifest", "parts", "navigation", "projectTree", "audit", "monolith"];
|
|
153
205
|
/**
|
|
@@ -173,8 +225,27 @@ declare function resolveConfig(userConfig: SnapshotConfig | null, cliOptions?: P
|
|
|
173
225
|
* `"src/app/**"` now means "exactly `src/app` or below": the previous
|
|
174
226
|
* implementation stripped `**` **and** the trailing slash, so the pattern also
|
|
175
227
|
* matched `src/apple/` and `src/application/`.
|
|
228
|
+
*
|
|
229
|
+
* Everything else is answered by {@link matchesExclude}, which is the one implementation of
|
|
230
|
+
* "does this pattern select this path": glob metacharacters (including `{a,b}` and `[abc]`,
|
|
231
|
+
* which used to be tested for `*`/`?` only and then silently compared as literal text) go
|
|
232
|
+
* through the glob translator, and a plain pattern means a whole **path segment** sequence
|
|
233
|
+
* rather than a substring. `match: "test"` used to select `src/contest/a.ts`, and a layer rule
|
|
234
|
+
* that matched nothing classified every file under it as `other`.
|
|
176
235
|
*/
|
|
177
236
|
declare function matchRule(filePath: string, matcher: string | RegExp | ((filePath: string) => boolean)): boolean;
|
|
237
|
+
declare function classifyLayer(filePath: string, layers: LayerRule[]): string;
|
|
238
|
+
declare function classifyKind(filePath: string, kinds: KindRule[]): string;
|
|
239
|
+
/**
|
|
240
|
+
* Digest of every setting that changes what a snapshot contains.
|
|
241
|
+
*
|
|
242
|
+
* Masked config values cannot be compared directly: two different secrets both
|
|
243
|
+
* mask to the same placeholder, and `validate` would then report "current" for a
|
|
244
|
+
* snapshot generated with different rules. A digest of the raw effective settings
|
|
245
|
+
* answers "is this snapshot still configured the same way" without ever printing
|
|
246
|
+
* a value.
|
|
247
|
+
*/
|
|
248
|
+
declare function configDigest(config: ResolvedSnapshotConfig): string;
|
|
178
249
|
|
|
179
250
|
declare function defineConfig(config: SnapshotConfig): SnapshotConfig;
|
|
180
251
|
|
|
@@ -194,6 +265,14 @@ interface PartitionedPart {
|
|
|
194
265
|
* large file produced a part 13.5x over the configured budget.
|
|
195
266
|
*/
|
|
196
267
|
declare function partitionFiles(files: FileEntry[], maxPartBytes: number, contentFingerprint?: string): PartitionedPart[];
|
|
268
|
+
/**
|
|
269
|
+
* Hard-clips any part that is still above `budget`, re-hashing what it wrote.
|
|
270
|
+
*
|
|
271
|
+
* Exported so the clip path can be tested at all: the partitioner splits oversized sections
|
|
272
|
+
* rather than clipping them, so no ordinary fixture reaches this branch, and the stale token
|
|
273
|
+
* count it used to leave behind was therefore invisible to every end-to-end test.
|
|
274
|
+
*/
|
|
275
|
+
declare function enforceBudget(parts: PartitionedPart[], budget: number): PartitionedPart[];
|
|
197
276
|
/**
|
|
198
277
|
* Re-numbers a set of parts as one sequence and rewrites their headers.
|
|
199
278
|
*
|
|
@@ -202,6 +281,22 @@ declare function partitionFiles(files: FileEntry[], maxPartBytes: number, conten
|
|
|
202
281
|
* contain duplicate filenames and every `Previous:`/`Next:` link would lie.
|
|
203
282
|
*/
|
|
204
283
|
declare function renumberParts(parts: PartitionedPart[], contentFingerprint?: string): PartitionedPart[];
|
|
284
|
+
/**
|
|
285
|
+
* Paths reach the `## Files` list, the `### FILE:` heading and the navigation
|
|
286
|
+
* index, so a control character in a name is rendered as `\uXXXX` instead of
|
|
287
|
+
* being emitted raw: a bidi override visually reorders the line, a C1 CSI drives
|
|
288
|
+
* the terminal, and a newline would break the line-oriented layout. Escaping
|
|
289
|
+
* also keeps the `## Files` fence safe (see {@link fenceForText}). Paths are
|
|
290
|
+
* legal UTF-8, so this is reachable with an ordinary `git mv`.
|
|
291
|
+
*/
|
|
292
|
+
/**
|
|
293
|
+
* The form of a path that is written into a part.
|
|
294
|
+
*
|
|
295
|
+
* Exported because the reader has to compare against it: a section heading holds this
|
|
296
|
+
* form, not the manifest's `relativePath`, and the two differ for any path with a
|
|
297
|
+
* control character or one longer than {@link DISPLAY_PATH_MAX}.
|
|
298
|
+
*/
|
|
299
|
+
declare const displayPath: (value: string) => string;
|
|
205
300
|
/**
|
|
206
301
|
* Chooses a fence longer than any backtick run in `content`, so a file that
|
|
207
302
|
* contains a fenced block cannot break out of its own code block.
|
|
@@ -238,6 +333,10 @@ interface PreviewResult {
|
|
|
238
333
|
*
|
|
239
334
|
* Previously `index.ts` recorded 0 for an empty file while `preview.ts` recorded
|
|
240
335
|
* 1, and every newline-terminated file was over-counted by one.
|
|
336
|
+
*
|
|
337
|
+
* Counted by scanning rather than by `split`: this runs on every file body and on every part, and
|
|
338
|
+
* normalising first would allocate a second copy of a multi-megabyte string to answer a question
|
|
339
|
+
* about its terminator count.
|
|
241
340
|
*/
|
|
242
341
|
declare function countLines(content: string): number;
|
|
243
342
|
/**
|
|
@@ -379,18 +478,6 @@ declare function isBinaryFile(absolutePath: string): boolean;
|
|
|
379
478
|
|
|
380
479
|
declare function detectLanguage(filePath: string): string;
|
|
381
480
|
|
|
382
|
-
/**
|
|
383
|
-
* Redacts secrets from `content`.
|
|
384
|
-
*
|
|
385
|
-
* - Replaces only the matched span, so unrelated code sharing the line survives
|
|
386
|
-
* (the previous implementation discarded the whole line).
|
|
387
|
-
* - When a rule declares `valueGroup`, only that capture group is masked, so
|
|
388
|
-
* `dbPassword: "…"` keeps its key and stays parseable.
|
|
389
|
-
* - Masks PEM private-key blocks as a unit; per-line rules can only see the
|
|
390
|
-
* `BEGIN` header, which left every body line of a real key in the output.
|
|
391
|
-
* - Records `{ rule, line, path }` only. No excerpt of the source is retained,
|
|
392
|
-
* because that excerpt was being written verbatim into `reports/audit.md`.
|
|
393
|
-
*/
|
|
394
481
|
declare function scanAndRedactSecrets(content: string, rules: SecretRule[], filePath?: string): {
|
|
395
482
|
redacted: string;
|
|
396
483
|
findings: SecretFinding[];
|
|
@@ -422,7 +509,7 @@ declare function validateSnapshot(configInput: ResolvedSnapshotConfig | Snapshot
|
|
|
422
509
|
* `package.json` carries the same version; `npm run verify` compares the two.
|
|
423
510
|
*/
|
|
424
511
|
declare const GENERATOR_NAME = "codebase-snapshot";
|
|
425
|
-
declare const GENERATOR_VERSION = "0.9.
|
|
512
|
+
declare const GENERATOR_VERSION = "0.9.14";
|
|
426
513
|
|
|
427
514
|
/** The one endpoint this package will ever request. */
|
|
428
515
|
declare const PACKAGE_LATEST_URL = "https://registry.npmjs.org/codebase-snapshot/latest";
|
|
@@ -467,7 +554,13 @@ interface UpdateCheckOptions {
|
|
|
467
554
|
fetchLatest?: (url: string, timeoutMs: number) => Promise<string | null>;
|
|
468
555
|
enabled?: boolean;
|
|
469
556
|
}
|
|
470
|
-
/**
|
|
557
|
+
/**
|
|
558
|
+
* True when `candidate` is a version worth telling somebody about.
|
|
559
|
+
*
|
|
560
|
+
* A prerelease is never announced — being offered `1.0.0-rc.1` while running `0.9.12` is not an
|
|
561
|
+
* upgrade, and a check that says so is wrong every time it fires. A stable release *is* newer than
|
|
562
|
+
* the prerelease of itself, so somebody on `0.9.12-beta.1` is told about `0.9.12`.
|
|
563
|
+
*/
|
|
471
564
|
declare function isNewer(candidate: string, current: string): boolean;
|
|
472
565
|
/** The per-user cache directory, following each platform's convention. */
|
|
473
566
|
declare function cacheDirectory(env?: NodeJS.ProcessEnv, platform?: NodeJS.Platform): string;
|
|
@@ -480,7 +573,18 @@ declare function cacheFilePath(env?: NodeJS.ProcessEnv): string;
|
|
|
480
573
|
* spends a CI job's network and pollutes a machine-readable stream.
|
|
481
574
|
*/
|
|
482
575
|
declare function shouldCheck(argv: readonly string[], env: NodeJS.ProcessEnv): boolean;
|
|
483
|
-
/**
|
|
576
|
+
/**
|
|
577
|
+
* One GET with a hard deadline. Resolves null for every failure.
|
|
578
|
+
*
|
|
579
|
+
* The deadline is a **wall clock**, not an idle-socket timeout. `request.setTimeout` only fires
|
|
580
|
+
* when the socket goes quiet, so a slow or trickling proxy — one that sends a byte every second —
|
|
581
|
+
* kept the CLI process alive long past `REQUEST_TIMEOUT_MS`. The timer below is absolute: when it
|
|
582
|
+
* fires the request is destroyed and the answer is "could not determine", which is what the notice
|
|
583
|
+
* already treats as "say nothing".
|
|
584
|
+
*
|
|
585
|
+
* The scheme picks the transport so the deadline can be exercised against a local server; the
|
|
586
|
+
* default URL is https either way.
|
|
587
|
+
*/
|
|
484
588
|
declare function fetchLatestVersion(url?: string, timeoutMs?: number): Promise<string | null>;
|
|
485
589
|
/**
|
|
486
590
|
* Returns a notice when a newer version is published, or null.
|
|
@@ -651,10 +755,19 @@ interface InitOptions {
|
|
|
651
755
|
/** Force a preset instead of detecting one. */
|
|
652
756
|
preset?: string;
|
|
653
757
|
maxPartBytes?: ByteSize;
|
|
758
|
+
/** Ceiling in tokens; narrows `maxPartBytes` rather than replacing it. */
|
|
759
|
+
maxPartTokens?: number;
|
|
760
|
+
largeFileBytes?: ByteSize;
|
|
761
|
+
maxMonolithBytes?: ByteSize;
|
|
762
|
+
discovery?: DiscoveryMode;
|
|
763
|
+
skeleton?: boolean;
|
|
764
|
+
sarif?: boolean;
|
|
654
765
|
include?: string[];
|
|
655
766
|
exclude?: string[];
|
|
656
767
|
skipArtifacts?: string[];
|
|
657
768
|
onlyArtifacts?: string[];
|
|
769
|
+
/** Turn the monolith on without disabling anything else (`--include-monolith`). */
|
|
770
|
+
includeMonolith?: boolean;
|
|
658
771
|
includeSensitiveFiles?: boolean;
|
|
659
772
|
/** Accept every default without prompting. */
|
|
660
773
|
yes?: boolean;
|
|
@@ -698,6 +811,14 @@ interface InitResult {
|
|
|
698
811
|
dependency: InitDependency;
|
|
699
812
|
/** Human-readable notes about anything that was skipped or could not be done. */
|
|
700
813
|
notes: string[];
|
|
814
|
+
/**
|
|
815
|
+
* The output directory the written config names.
|
|
816
|
+
*
|
|
817
|
+
* Returned so the caller can bake it into the pre-commit hook: the hook is a shell script and
|
|
818
|
+
* cannot read the config, and hardcoding `snapshot` there made a gate that never ran for any
|
|
819
|
+
* project that chose a different directory.
|
|
820
|
+
*/
|
|
821
|
+
outDir: string;
|
|
701
822
|
}
|
|
702
823
|
/**
|
|
703
824
|
* Runs the interactive setup, or writes a config from flags.
|
|
@@ -791,7 +912,12 @@ interface WizardAnswers {
|
|
|
791
912
|
preset: string;
|
|
792
913
|
branch: string;
|
|
793
914
|
maxPartBytes: ByteSize;
|
|
915
|
+
maxPartTokens: number | null;
|
|
794
916
|
largeFileBytes: ByteSize;
|
|
917
|
+
maxMonolithBytes: ByteSize | null;
|
|
918
|
+
discovery: DiscoveryMode;
|
|
919
|
+
skeleton: boolean;
|
|
920
|
+
sarif: boolean;
|
|
795
921
|
include: string[];
|
|
796
922
|
excludePatterns: string[];
|
|
797
923
|
artifacts: Required<ArtifactSelection>;
|
|
@@ -853,6 +979,184 @@ declare function profileForPath(path: string): LanguageProfile | null;
|
|
|
853
979
|
*/
|
|
854
980
|
declare function summarizeWithProfile(path: string, content: string): HeuristicResult | null;
|
|
855
981
|
|
|
982
|
+
/**
|
|
983
|
+
* A deterministic token estimate for emitted text.
|
|
984
|
+
*
|
|
985
|
+
* Why a local estimate at all: the snapshot exists to be pasted into a model's context, and
|
|
986
|
+
* every budget in this tool is expressed in bytes. Bytes are the right unit for a filesystem
|
|
987
|
+
* and the wrong one for a context window — the arithmetic that converts them is in
|
|
988
|
+
* `docs/codebase-token-budget.md` and every reader has to repeat it by hand. A number in the
|
|
989
|
+
* manifest that says "this part is about 8,400 tokens" is the answer to the question an
|
|
990
|
+
* agent actually asks before reading it.
|
|
991
|
+
*
|
|
992
|
+
* Why not a real tokenizer: this package has exactly one runtime dependency (`typescript`)
|
|
993
|
+
* and that is a feature, not an oversight. A BPE vocabulary is a few megabytes of data that
|
|
994
|
+
* would have to ship, version and be kept current. This counts the structure that BPE
|
|
995
|
+
* tokenizers follow closely on source code:
|
|
996
|
+
*
|
|
997
|
+
* - a run of word characters costs `round(length / 6)`, minimum one, because common words in
|
|
998
|
+
* source (`export`, `const`, `return`, `value`) are a single token in every mainstream
|
|
999
|
+
* vocabulary and long identifiers are two or three;
|
|
1000
|
+
* - every other non-space character costs one, because punctuation is almost always its own
|
|
1001
|
+
* token (`=`, `(`, `,` and `)` are separate);
|
|
1002
|
+
* - a *space* costs nothing, because BPE merges the leading space into the next token — the
|
|
1003
|
+
* sequence `" const"` is one token, not two;
|
|
1004
|
+
* - a newline costs one, which is what an indented block boundary actually is.
|
|
1005
|
+
*
|
|
1006
|
+
* It is an estimate and it says so. The constants were set against `cl100k_base` on this
|
|
1007
|
+
* repository's own sources and land within about 10% either way; the method above is stated
|
|
1008
|
+
* so a reader can judge the number rather than being asked to trust it.
|
|
1009
|
+
*/
|
|
1010
|
+
/**
|
|
1011
|
+
* Estimates the token count of `text`.
|
|
1012
|
+
*
|
|
1013
|
+
* Deterministic: the same string always produces the same number, on every platform, with no
|
|
1014
|
+
* vocabulary to load and no network. That is what makes it safe to record in a manifest and
|
|
1015
|
+
* compare between runs.
|
|
1016
|
+
*/
|
|
1017
|
+
declare function estimateTokens(text: string): number;
|
|
1018
|
+
/**
|
|
1019
|
+
* Converts a token budget into the byte budget the partitioner understands.
|
|
1020
|
+
*
|
|
1021
|
+
* The partitioner is byte-bounded because bytes are what it measures and writes. A user who
|
|
1022
|
+
* thinks in tokens gets a byte ceiling that corresponds to their number, using the same
|
|
1023
|
+
* ratio the estimate is built on — and `maxPartBytes` still applies, so the smaller of the
|
|
1024
|
+
* two wins and the manifest records both.
|
|
1025
|
+
*/
|
|
1026
|
+
declare function bytesForTokens(tokens: number): number;
|
|
1027
|
+
/** Renders a token count for a header or a report: `8400` → `8,400`. */
|
|
1028
|
+
declare function formatTokens(count: number): string;
|
|
1029
|
+
|
|
1030
|
+
/** Marks a hook this tool wrote, so a re-run may replace it and a user can recognise it. */
|
|
1031
|
+
declare const HOOK_MARKER = "# codebase-snapshot: pre-commit snapshot check";
|
|
1032
|
+
interface HookInstallResult {
|
|
1033
|
+
/** Absolute path of the hook file, or `null` when there was nowhere to write it. */
|
|
1034
|
+
path: string | null;
|
|
1035
|
+
/** True when this call wrote (or rewrote) the hook. */
|
|
1036
|
+
written: boolean;
|
|
1037
|
+
/** Human-readable outcomes, always at least one. */
|
|
1038
|
+
notes: string[];
|
|
1039
|
+
}
|
|
1040
|
+
/**
|
|
1041
|
+
* Installs (or refreshes) the pre-commit hook.
|
|
1042
|
+
*
|
|
1043
|
+
* Idempotent: a hook carrying {@link HOOK_MARKER} is replaced with the current body, so an
|
|
1044
|
+
* upgrade of this package updates the check. Anyone else's hook is untouched.
|
|
1045
|
+
*/
|
|
1046
|
+
declare function installPreCommitHook(cwd: string, outDir?: string): HookInstallResult;
|
|
1047
|
+
|
|
1048
|
+
interface ChangedScopeRequest {
|
|
1049
|
+
/** A git ref to compare the working tree against (`main`, `HEAD~3`, a sha). */
|
|
1050
|
+
changedSince?: string;
|
|
1051
|
+
/** Only what is staged for the next commit. */
|
|
1052
|
+
staged?: boolean;
|
|
1053
|
+
}
|
|
1054
|
+
interface ChangedScope {
|
|
1055
|
+
/** Repository-relative paths, POSIX separators. */
|
|
1056
|
+
paths: ReadonlySet<string>;
|
|
1057
|
+
/** How the set was produced, for the manifest and the summary line. */
|
|
1058
|
+
detail: string;
|
|
1059
|
+
}
|
|
1060
|
+
/**
|
|
1061
|
+
* Resolves the request into a path set.
|
|
1062
|
+
*
|
|
1063
|
+
* `--changed-since <ref>` is the working tree against that ref — committed, staged and
|
|
1064
|
+
* unstaged changes all count, because all three are what a reviewer is looking at — plus
|
|
1065
|
+
* untracked files, which `git diff` never lists and which are exactly what a new feature
|
|
1066
|
+
* starts with. `--staged` is the index against `HEAD`. Both together are a union.
|
|
1067
|
+
*/
|
|
1068
|
+
declare function resolveChangedScope(cwd: string, request: ChangedScopeRequest): ChangedScope;
|
|
1069
|
+
|
|
1070
|
+
/** One-pass lookup tables for relative-import resolution.
|
|
1071
|
+
*
|
|
1072
|
+
* `resolveSpecifier` used to fall back to scanning every known file for every
|
|
1073
|
+
* unresolved import — 100k files × 100k imports — and built a RegExp from the
|
|
1074
|
+
* candidate path, so a directory named `[unmatched` aborted the run and `[id]`
|
|
1075
|
+
* matched unrelated files. Both are gone: every candidate is a map lookup, and
|
|
1076
|
+
* no path is ever used as a pattern.
|
|
1077
|
+
*
|
|
1078
|
+
* Exported so a consumer can answer "what imports this file" from the snapshot's own
|
|
1079
|
+
* index using the same rules the generator used. `manifest.indexes.imports.byFile`
|
|
1080
|
+
* records the *raw* specifiers a file imports (`"./user.js"`), and a second
|
|
1081
|
+
* implementation of the resolution would eventually disagree with this one — leaving
|
|
1082
|
+
* two accounts of one graph.
|
|
1083
|
+
*/
|
|
1084
|
+
interface ImportResolver {
|
|
1085
|
+
byPath: Map<string, string>;
|
|
1086
|
+
byPathFolded: Map<string, string>;
|
|
1087
|
+
byStem: Map<string, string>;
|
|
1088
|
+
byStemFolded: Map<string, string>;
|
|
1089
|
+
}
|
|
1090
|
+
/** Anything that knows its own repository-relative path; `FileEntry` is the common case. */
|
|
1091
|
+
interface ImportCandidate {
|
|
1092
|
+
relativePath: string;
|
|
1093
|
+
}
|
|
1094
|
+
declare function buildImportResolver(files: readonly ImportCandidate[]): ImportResolver;
|
|
1095
|
+
/** Best-effort resolution of a relative import to an included file. */
|
|
1096
|
+
declare function resolveSpecifier(fromFile: string, specifier: string, resolver: ImportResolver): string | null;
|
|
1097
|
+
/**
|
|
1098
|
+
* True when `imports` — the specifiers one file imports — reach anything in `changedPaths`.
|
|
1099
|
+
*
|
|
1100
|
+
* One hop, and only one hop. The test is against the **changed set**, never against a set that
|
|
1101
|
+
* is being extended while it is tested: doing that walked the graph transitively, so a changed
|
|
1102
|
+
* `a.ts` pulled in `b.ts` (which imports it) and then `c.ts` (which imports `b.ts`), and whether
|
|
1103
|
+
* the two-hop file was included depended on whether its name happened to sort after `b.ts`. The
|
|
1104
|
+
* same repository produced different snapshots depending on file names.
|
|
1105
|
+
*/
|
|
1106
|
+
declare function importsChangedFile(importerPath: string, imports: readonly string[], changedPaths: ReadonlySet<string>, resolver: ImportResolver): boolean;
|
|
1107
|
+
|
|
1108
|
+
/** Import specifiers from text already decoded, or `[]` when the language has no importer. */
|
|
1109
|
+
declare function importsOfText(relativePath: string, text: string): string[];
|
|
1110
|
+
/**
|
|
1111
|
+
* Import specifiers of one file on disk.
|
|
1112
|
+
*
|
|
1113
|
+
* A file this cannot read, a binary file, an oversized file and a file the parser rejects all
|
|
1114
|
+
* contribute `[]`, which is what the generator saw for them too — so the answer stays consistent
|
|
1115
|
+
* with the snapshot rather than becoming a third opinion about it.
|
|
1116
|
+
*/
|
|
1117
|
+
declare function importsOfFile(relativePath: string, absolutePath: string): string[];
|
|
1118
|
+
|
|
1119
|
+
interface SkeletonResult {
|
|
1120
|
+
text: string;
|
|
1121
|
+
/** How many bodies were replaced; zero means the file had none and was left alone. */
|
|
1122
|
+
collapsed: number;
|
|
1123
|
+
}
|
|
1124
|
+
/**
|
|
1125
|
+
* Replaces the body of every function-like node, innermost-first, by text span.
|
|
1126
|
+
*
|
|
1127
|
+
* Spans are collected and then applied from the end of the file backwards so that an earlier
|
|
1128
|
+
* replacement never invalidates a later span's offsets.
|
|
1129
|
+
*/
|
|
1130
|
+
/**
|
|
1131
|
+
* True when a filename is JavaScript-with-JSX, in any letter case.
|
|
1132
|
+
*
|
|
1133
|
+
* One function for one question, because two spellings of it is exactly how `.TSX` came to be
|
|
1134
|
+
* eligible for a skeleton and then parsed as plain TypeScript, where JSX is a syntax error.
|
|
1135
|
+
* Exported so the decision itself can be tested: collapsing a body is span-based and produces the
|
|
1136
|
+
* same text under either script kind for simple files, so an output comparison cannot prove which
|
|
1137
|
+
* parser ran.
|
|
1138
|
+
*/
|
|
1139
|
+
declare function isTsxFileName(fileName: string): boolean;
|
|
1140
|
+
declare function skeletonizeTypeScript(source: string, fileName?: string): SkeletonResult;
|
|
1141
|
+
/**
|
|
1142
|
+
* True for a path this module can compress.
|
|
1143
|
+
*
|
|
1144
|
+
* `.d.ts` and friends are excluded first: they are declarations, so there is nothing to collapse,
|
|
1145
|
+
* and parsing them was pure cost. Everything else the TypeScript parser can read is included —
|
|
1146
|
+
* JavaScript and JSX included, which is most of the code in most repositories.
|
|
1147
|
+
*/
|
|
1148
|
+
declare function isSkeletonizable(filePath: string): boolean;
|
|
1149
|
+
|
|
1150
|
+
/**
|
|
1151
|
+
* Renders the findings as a SARIF log.
|
|
1152
|
+
*
|
|
1153
|
+
* Every result carries the file and line, a `level` of `note` — a redaction is the tool doing
|
|
1154
|
+
* its job, not a build failure — and the rule that matched. `partialFingerprints` gives Code
|
|
1155
|
+
* Scanning a stable identity for an alert across commits, built from the path, the rules and
|
|
1156
|
+
* the line, so re-running `generate` does not close and reopen every alert.
|
|
1157
|
+
*/
|
|
1158
|
+
declare function generateSarifReport(manifest: Manifest, findings: readonly SecretFinding[], rules: readonly SecretRule[], toolVersion: string): string;
|
|
1159
|
+
|
|
856
1160
|
declare const SCHEMA_VERSION = "3.0.0";
|
|
857
1161
|
|
|
858
1162
|
interface GenerateOptions {
|
|
@@ -884,6 +1188,14 @@ declare function resolveSafeOutputDirs(config: {
|
|
|
884
1188
|
* directory per run rather than a single snapshot.
|
|
885
1189
|
*/
|
|
886
1190
|
declare function assertReplaceable(target: string, label: "outDir" | "tmpDir", secretRules?: readonly SecretRule[]): void;
|
|
1191
|
+
/**
|
|
1192
|
+
* Where the run lock for a staging root lives.
|
|
1193
|
+
*
|
|
1194
|
+
* Exported because the lock is the one piece of state that can outlive a crashed run and
|
|
1195
|
+
* wedge the next one, and the error the user sees names this path — a diagnostic that
|
|
1196
|
+
* has to be transcribed from a hash by hand is not a diagnostic.
|
|
1197
|
+
*/
|
|
1198
|
+
declare function lockPathFor(stagingRoot: string): string;
|
|
887
1199
|
/**
|
|
888
1200
|
* Generates the snapshot.
|
|
889
1201
|
*
|
|
@@ -895,4 +1207,4 @@ declare function generateSnapshot(configInput?: SnapshotConfig | ResolvedSnapsho
|
|
|
895
1207
|
/** Promise-returning wrapper for consumers that prefer an async call site. */
|
|
896
1208
|
declare function generateSnapshotAsync(configInput?: SnapshotConfig | ResolvedSnapshotConfig | null, options?: GenerateOptions): Promise<Manifest>;
|
|
897
1209
|
|
|
898
|
-
export { ARTIFACT_NAMES, ArtifactSelection, ByteSize, CHECK_INTERVAL_MS, CliOptions, DEFAULT_ANSWERS, DEFAULT_SECRET_PATTERNS, DiscoveryMode, type ExplainResult, FAILURE_INTERVAL_MS, FileEntry, type FirstRunFacts, type FirstRunPlan, type FirstRunRefusal, GENERATOR_NAME, GENERATOR_VERSION, type GenerateOptions, type HeuristicResult, type InitDependency, type InitOptions, type InitResult, type InstallKind, JsonSummary, KindRule, LANGUAGE_PROFILES, type LanguageProfile, LayerRule, MIN_MAX_PART_BYTES, Manifest, type ManifestDiff, MarkdownSummary, PACKAGE_LATEST_URL, PRESETS, PartMetadata, PromptBack, PromptCancelled, REQUEST_TIMEOUT_MS, ResolvedSnapshotConfig, SCHEMA_VERSION, SecretFinding, SecretRule, SkipRecord, SnapshotConfig, SourceSummary, TextEncoding, type UpdateCheckOptions, type UpdateNotice, ValidationResult, type WizardAnswers, anchorForPath, assertReplaceable, assertUsableConfigInput, cacheDirectory, cacheFilePath, checkForUpdate, collectFiles, compareManifests, countInvalidUtf8Bytes, countLines, decodeText, defineConfig, detectLanguage, detectPreset, displayWidth, explainPath, fenceForContent, fenceForText, fetchLatestVersion, firstRunMessage, formatUpdateNotice, generateProjectTree, generateSnapshot, generateSnapshotAsync, getPreset, globToRegExp, hasPriorSnapshot, installKind, isBinaryBuffer, isBinaryExtension, isBinaryFile, isEffectivelyEmpty, isInteractive, isNewer, isResolvedConfig, isSensitivePath, listPresetNames, loadConfigFile, loadConfigFileDetailed, loadManifest, markResolvedConfig, matchRule, matchesExclude, normalizeResolvedConfig, parseByteSize, partitionFiles, planFirstRun, profileForPath, renderConfig, renderDiffText, renderExplainText, renumberParts, resolveConfig, resolveSafeOutputDirs, runInit, scanAndRedactSecrets, shouldCheck, stripJsonComments, summarizeJson, summarizeMarkdown, summarizeTypeScriptSource, summarizeWithProfile, supportsTypeScriptImports, truncatePreview, truncateToBytes, updateCommand, validateSnapshot, wrapLines };
|
|
1210
|
+
export { ARTIFACT_NAMES, ArtifactSelection, ByteSize, CHECK_INTERVAL_MS, CONFIG_KEYS, type ChangedScope, type ChangedScopeRequest, CliOptions, DEFAULT_ANSWERS, DEFAULT_SECRET_PATTERNS, DiscoveryMode, type ExplainResult, FAILURE_INTERVAL_MS, FileEntry, type FirstRunFacts, type FirstRunPlan, type FirstRunRefusal, GENERATOR_NAME, GENERATOR_VERSION, type GenerateOptions, HOOK_MARKER, type HeuristicResult, type HookInstallResult, type ImportCandidate, type ImportResolver, type InitDependency, type InitOptions, type InitResult, type InstallKind, JsonSummary, KindRule, LANGUAGE_PROFILES, type LanguageProfile, LayerRule, MIN_MAX_PART_BYTES, MIN_MAX_PART_TOKENS, Manifest, type ManifestDiff, MarkdownSummary, PACKAGE_LATEST_URL, PRESETS, PartMetadata, PromptBack, PromptCancelled, REQUEST_TIMEOUT_MS, ResolvedSnapshotConfig, SCHEMA_VERSION, SecretFinding, SecretRule, SkipRecord, SnapshotConfig, SourceSummary, TextEncoding, type UpdateCheckOptions, type UpdateNotice, ValidationResult, type WizardAnswers, anchorForPath, assertReplaceable, assertUsableConfigInput, buildImportResolver, bytesForTokens, cacheDirectory, cacheFilePath, checkForUpdate, classifyKind, classifyLayer, collectFiles, compareManifests, configDigest, countInvalidUtf8Bytes, countLines, decodeText, defineConfig, detectLanguage, detectPreset, displayPath, displayWidth, enforceBudget, estimateTokens, explainPath, fenceForContent, fenceForText, fetchLatestVersion, findSecretInPath, firstRunMessage, formatTokens, formatUpdateNotice, generateProjectTree, generateSarifReport, generateSnapshot, generateSnapshotAsync, getGitInfo, getPreset, globToRegExp, hasPriorSnapshot, importsChangedFile, importsOfFile, importsOfText, installKind, installPreCommitHook, isBinaryBuffer, isBinaryExtension, isBinaryFile, isEffectivelyEmpty, isInteractive, isNewer, isResolvedConfig, isSensitivePath, isSkeletonizable, isTsxFileName, listPresetNames, loadConfigFile, loadConfigFileDetailed, loadManifest, lockPathFor, markResolvedConfig, matchRule, matchesExclude, normalizeResolvedConfig, parseByteSize, partitionFiles, planFirstRun, profileForPath, renderConfig, renderDiffText, renderExplainText, renumberParts, resolveChangedScope, resolveConfig, resolveSafeOutputDirs, resolveSpecifier, runInit, scanAndRedactSecrets, shouldCheck, skeletonizeTypeScript, stripJsonComments, summarizeJson, summarizeMarkdown, summarizeTypeScriptSource, summarizeWithProfile, supportsTypeScriptImports, truncatePreview, truncateToBytes, updateCommand, validateSnapshot, wrapLines };
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1 @@
|
|
|
1
|
-
export { ARTIFACT_NAMES, CHECK_INTERVAL_MS, DEFAULT_ANSWERS, DEFAULT_SECRET_PATTERNS, FAILURE_INTERVAL_MS, GENERATOR_NAME, GENERATOR_VERSION, LANGUAGE_PROFILES, MIN_MAX_PART_BYTES, PACKAGE_LATEST_URL, PRESETS, PromptBack, PromptCancelled, REQUEST_TIMEOUT_MS, SCHEMA_VERSION, anchorForPath, assertReplaceable, assertUsableConfigInput, cacheDirectory, cacheFilePath, checkForUpdate, collectFiles, compareManifests, countInvalidUtf8Bytes, countLines, decodeText, defineConfig, detectLanguage, detectPreset, displayWidth, explainPath, fenceForContent, fenceForText, fetchLatestVersion, firstRunMessage, formatUpdateNotice, generateProjectTree, generateSnapshot, generateSnapshotAsync, getPreset, globToRegExp, hasPriorSnapshot, installKind, isBinaryBuffer, isBinaryExtension, isBinaryFile, isEffectivelyEmpty, isInteractive, isNewer, isResolvedConfig, isSensitivePath, listPresetNames, loadConfigFile, loadConfigFileDetailed, loadManifest, markResolvedConfig, matchRule, matchesExclude, normalizeResolvedConfig, parseByteSize, partitionFiles, planFirstRun, profileForPath, renderConfig, renderDiffText, renderExplainText, renumberParts, resolveConfig, resolveSafeOutputDirs, runInit, scanAndRedactSecrets, shouldCheck, stripJsonComments, summarizeJson, summarizeMarkdown, summarizeTypeScriptSource, summarizeWithProfile, supportsTypeScriptImports, truncatePreview, truncateToBytes, updateCommand, validateSnapshot, wrapLines } from './chunk-
|
|
2
|
-
//# sourceMappingURL=index.js.map
|
|
3
|
-
//# sourceMappingURL=index.js.map
|
|
1
|
+
export { ARTIFACT_NAMES, CHECK_INTERVAL_MS, CONFIG_KEYS, DEFAULT_ANSWERS, DEFAULT_SECRET_PATTERNS, FAILURE_INTERVAL_MS, GENERATOR_NAME, GENERATOR_VERSION, HOOK_MARKER, LANGUAGE_PROFILES, MIN_MAX_PART_BYTES, MIN_MAX_PART_TOKENS, PACKAGE_LATEST_URL, PRESETS, PromptBack, PromptCancelled, REQUEST_TIMEOUT_MS, SCHEMA_VERSION, anchorForPath, assertReplaceable, assertUsableConfigInput, buildImportResolver, bytesForTokens, cacheDirectory, cacheFilePath, checkForUpdate, classifyKind, classifyLayer, collectFiles, compareManifests, configDigest, countInvalidUtf8Bytes, countLines, decodeText, defineConfig, detectLanguage, detectPreset, displayPath, displayWidth, enforceBudget, estimateTokens, explainPath, fenceForContent, fenceForText, fetchLatestVersion, findSecretInPath, firstRunMessage, formatTokens, formatUpdateNotice, generateProjectTree, generateSarifReport, generateSnapshot, generateSnapshotAsync, getGitInfo, getPreset, globToRegExp, hasPriorSnapshot, importsChangedFile, importsOfFile, importsOfText, installKind, installPreCommitHook, isBinaryBuffer, isBinaryExtension, isBinaryFile, isEffectivelyEmpty, isInteractive, isNewer, isResolvedConfig, isSensitivePath, isSkeletonizable, isTsxFileName, listPresetNames, loadConfigFile, loadConfigFileDetailed, loadManifest, lockPathFor, markResolvedConfig, matchRule, matchesExclude, normalizeResolvedConfig, parseByteSize, partitionFiles, planFirstRun, profileForPath, renderConfig, renderDiffText, renderExplainText, renumberParts, resolveChangedScope, resolveConfig, resolveSafeOutputDirs, resolveSpecifier, runInit, scanAndRedactSecrets, shouldCheck, skeletonizeTypeScript, stripJsonComments, summarizeJson, summarizeMarkdown, summarizeTypeScriptSource, summarizeWithProfile, supportsTypeScriptImports, truncatePreview, truncateToBytes, updateCommand, validateSnapshot, workerEntry, wrapLines } from './chunk-DUZK5VL2.js';
|