@vibe-agent-toolkit/utils 0.2.0-rc.2 → 0.2.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -15
- package/dist/compare-code-units.d.ts +28 -0
- package/dist/compare-code-units.d.ts.map +1 -0
- package/dist/compare-code-units.js +33 -0
- package/dist/compare-code-units.js.map +1 -0
- package/dist/crawl-timing.d.ts +657 -0
- package/dist/crawl-timing.d.ts.map +1 -0
- package/dist/crawl-timing.js +765 -0
- package/dist/crawl-timing.js.map +1 -0
- package/dist/crawl.d.ts +2 -4
- package/dist/crawl.d.ts.map +1 -1
- package/dist/crawl.js +2 -4
- package/dist/crawl.js.map +1 -1
- package/dist/file-crawler.d.ts +22 -0
- package/dist/file-crawler.d.ts.map +1 -1
- package/dist/file-crawler.js +31 -7
- package/dist/file-crawler.js.map +1 -1
- package/dist/fs-utils.d.ts +17 -0
- package/dist/fs-utils.d.ts.map +1 -1
- package/dist/fs-utils.js +69 -0
- package/dist/fs-utils.js.map +1 -1
- package/dist/fs.d.ts +3 -1
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +5 -1
- package/dist/fs.js.map +1 -1
- package/dist/git-run.d.ts +167 -0
- package/dist/git-run.d.ts.map +1 -0
- package/dist/git-run.js +154 -0
- package/dist/git-run.js.map +1 -0
- package/dist/git-snapshot.d.ts +181 -0
- package/dist/git-snapshot.d.ts.map +1 -0
- package/dist/git-snapshot.js +232 -0
- package/dist/git-snapshot.js.map +1 -0
- package/dist/git-tracker.d.ts +94 -1
- package/dist/git-tracker.d.ts.map +1 -1
- package/dist/git-tracker.js +135 -3
- package/dist/git-tracker.js.map +1 -1
- package/dist/git-utils.d.ts +38 -0
- package/dist/git-utils.d.ts.map +1 -1
- package/dist/git-utils.js +103 -80
- package/dist/git-utils.js.map +1 -1
- package/dist/git.d.ts +12 -4
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +12 -4
- package/dist/git.js.map +1 -1
- package/dist/gitignore-checker.d.ts.map +1 -1
- package/dist/gitignore-checker.js +6 -3
- package/dist/gitignore-checker.js.map +1 -1
- package/dist/index.d.ts +24 -18
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -30
- package/dist/index.js.map +1 -1
- package/dist/path-core.d.ts +1 -1
- package/dist/path-core.js +1 -1
- package/dist/project-utils.d.ts.map +1 -1
- package/dist/project-utils.js +6 -3
- package/dist/project-utils.js.map +1 -1
- package/dist/project.d.ts +7 -5
- package/dist/project.d.ts.map +1 -1
- package/dist/project.js +7 -5
- package/dist/project.js.map +1 -1
- package/dist/safe-exec.d.ts +38 -10
- package/dist/safe-exec.d.ts.map +1 -1
- package/dist/safe-exec.js +88 -19
- package/dist/safe-exec.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/spawn-claude.d.ts +28 -0
- package/dist/skill-test/spawn-claude.d.ts.map +1 -1
- package/dist/skill-test/spawn-claude.js +51 -2
- package/dist/skill-test/spawn-claude.js.map +1 -1
- package/dist/skill-test/transcript.d.ts +41 -2
- package/dist/skill-test/transcript.d.ts.map +1 -1
- package/dist/skill-test/transcript.js +47 -3
- package/dist/skill-test/transcript.js.map +1 -1
- package/dist/test-helpers.d.ts +185 -13
- package/dist/test-helpers.d.ts.map +1 -1
- package/dist/test-helpers.js +274 -27
- package/dist/test-helpers.js.map +1 -1
- package/dist/testing.d.ts +73 -3
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +93 -3
- package/dist/testing.js.map +1 -1
- package/dist/text-content.d.ts +197 -0
- package/dist/text-content.d.ts.map +1 -0
- package/dist/text-content.js +348 -0
- package/dist/text-content.js.map +1 -0
- package/dist/text-file.d.ts +45 -0
- package/dist/text-file.d.ts.map +1 -0
- package/dist/text-file.js +53 -0
- package/dist/text-file.js.map +1 -0
- package/dist/text.d.ts +16 -0
- package/dist/text.d.ts.map +1 -0
- package/dist/text.js +16 -0
- package/dist/text.js.map +1 -0
- package/dist/timing-dump.d.ts +136 -0
- package/dist/timing-dump.d.ts.map +1 -0
- package/dist/timing-dump.js +230 -0
- package/dist/timing-dump.js.map +1 -0
- package/eslint/README.md +105 -3
- package/eslint/index.cjs +50 -0
- package/eslint/index.d.cts +14 -4
- package/eslint/rules/no-bare-symlink-in-tests.cjs +200 -0
- package/eslint/rules/no-process-exit-in-phase.cjs +117 -0
- package/eslint/rules/no-raw-text-decode.cjs +296 -0
- package/eslint/rules/no-self-package-import.cjs +157 -0
- package/package.json +13 -8
- package/dist/link-auth/build-headers.d.ts +0 -34
- package/dist/link-auth/build-headers.d.ts.map +0 -1
- package/dist/link-auth/build-headers.js +0 -58
- package/dist/link-auth/build-headers.js.map +0 -1
- package/dist/link-auth/expand-macro.d.ts +0 -38
- package/dist/link-auth/expand-macro.d.ts.map +0 -1
- package/dist/link-auth/expand-macro.js +0 -133
- package/dist/link-auth/expand-macro.js.map +0 -1
- package/dist/link-auth/macros.yaml +0 -50
- package/dist/link-auth/resolve-token.d.ts +0 -83
- package/dist/link-auth/resolve-token.d.ts.map +0 -1
- package/dist/link-auth/resolve-token.js +0 -101
- package/dist/link-auth/resolve-token.js.map +0 -1
- package/dist/link-auth/resolve.d.ts +0 -102
- package/dist/link-auth/resolve.d.ts.map +0 -1
- package/dist/link-auth/resolve.js +0 -66
- package/dist/link-auth/resolve.js.map +0 -1
- package/dist/link-auth/rewrite.d.ts +0 -52
- package/dist/link-auth/rewrite.d.ts.map +0 -1
- package/dist/link-auth/rewrite.js +0 -102
- package/dist/link-auth/rewrite.js.map +0 -1
- package/dist/link-auth/select-provider.d.ts +0 -30
- package/dist/link-auth/select-provider.d.ts.map +0 -1
- package/dist/link-auth/select-provider.js +0 -55
- package/dist/link-auth/select-provider.js.map +0 -1
- package/dist/link-auth/template.d.ts +0 -40
- package/dist/link-auth/template.d.ts.map +0 -1
- package/dist/link-auth/template.js +0 -89
- package/dist/link-auth/template.js.map +0 -1
- package/dist/link-auth/transforms.d.ts +0 -46
- package/dist/link-auth/transforms.d.ts.map +0 -1
- package/dist/link-auth/transforms.js +0 -52
- package/dist/link-auth/transforms.js.map +0 -1
- package/dist/template-entry.d.ts +0 -10
- package/dist/template-entry.d.ts.map +0 -1
- package/dist/template-entry.js +0 -10
- package/dist/template-entry.js.map +0 -1
- package/dist/template.d.ts +0 -7
- package/dist/template.d.ts.map +0 -1
- package/dist/template.js +0 -18
- package/dist/template.js.map +0 -1
package/dist/testing.d.ts
CHANGED
|
@@ -1,8 +1,78 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Fixture primitives shared by suites in more than one package.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* a
|
|
4
|
+
* Deliberately framework-free: nothing here imports `vitest`, so this module
|
|
5
|
+
* stays a plain function library rather than something that registers hooks as a
|
|
6
|
+
* side effect of being imported. Each suite owns its own `beforeEach`/
|
|
7
|
+
* `afterEach` and calls these from inside them — two lines, which is below
|
|
8
|
+
* anything worth sharing, while the part that is genuinely identical (mkdtemp,
|
|
9
|
+
* write the literal, remove the tree) lives here once.
|
|
10
|
+
*
|
|
11
|
+
* It earned a home in `utils` the ordinary way: two packages needed it, not one
|
|
12
|
+
* package speculating that a second might.
|
|
13
|
+
*
|
|
14
|
+
* ⛔ **Do not import `vitest` here to share the hook wrapper itself.**
|
|
15
|
+
* `subpath-purity.test.ts` pins this entry at an EMPTY third-party set, and that
|
|
16
|
+
* pin is load-bearing: `./testing` is a published subpath, so a test framework
|
|
17
|
+
* reached from it becomes a runtime requirement for every adopter who imports
|
|
18
|
+
* it. {@link replantableCorpus} exists precisely so the per-test wrapper each
|
|
19
|
+
* suite still writes is three lines rather than twelve.
|
|
6
20
|
*/
|
|
7
21
|
export * from './test-helpers.js';
|
|
22
|
+
/** A planted fixture tree and the means to remove it. */
|
|
23
|
+
export interface TempCorpus {
|
|
24
|
+
/** Absolute path to the tree's root. */
|
|
25
|
+
root: string;
|
|
26
|
+
/** Remove the tree. Safe to call when it is already gone. */
|
|
27
|
+
cleanup: () => void;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Write a literal corpus into a fresh temp directory.
|
|
31
|
+
*
|
|
32
|
+
* 🪤 Fixture files must NOT be byte-identical to each other when the suite is
|
|
33
|
+
* about content-addressed behaviour: VAT's blobs are keyed on their bytes plus
|
|
34
|
+
* the parser kind, so two files with the same content collapse into ONE blob and
|
|
35
|
+
* assertions then describe whichever path sorted first. Give each fixture a
|
|
36
|
+
* distinguishing marker line.
|
|
37
|
+
*
|
|
38
|
+
* ⚠️ The root is minted per call, so a suite that plants per test must call this
|
|
39
|
+
* per test — a root captured once and reused across tests survives its own
|
|
40
|
+
* `cleanup`.
|
|
41
|
+
*
|
|
42
|
+
* @param prefix - `mkdtemp` prefix, so a leaked directory names its own suite
|
|
43
|
+
* @param corpus - Fixture name to file content; written verbatim as UTF-8.
|
|
44
|
+
* Names may include forward-slash subpaths only if their parents already exist
|
|
45
|
+
* @returns The tree's root and its teardown
|
|
46
|
+
*/
|
|
47
|
+
export declare function createTempCorpus(prefix: string, corpus: Readonly<Record<string, string>>): TempCorpus;
|
|
48
|
+
/** A corpus that can be planted and cleared repeatedly — one tree per test. */
|
|
49
|
+
export interface ReplantableCorpus {
|
|
50
|
+
/** Plant a fresh tree. Drive from the suite's `beforeEach`. */
|
|
51
|
+
plant: () => void;
|
|
52
|
+
/** Remove the current tree, if any. Drive from the suite's `afterEach`. */
|
|
53
|
+
clear: () => void;
|
|
54
|
+
/** The root minted by the most recent {@link plant}. */
|
|
55
|
+
root: () => string;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Hold a per-test corpus root, so a suite's hook wrapper is three lines.
|
|
59
|
+
*
|
|
60
|
+
* ⚠️ **The root comes back through a GETTER, never as a value.** A per-test
|
|
61
|
+
* fixture is reminted for every test, so a root captured at registration time is
|
|
62
|
+
* `undefined` in the first test and stale in every one after it. That mistake is
|
|
63
|
+
* the reason this holder exists rather than each suite keeping its own `let`.
|
|
64
|
+
*
|
|
65
|
+
* ⭐ Framework-free on purpose: it takes no hooks and registers none, so the
|
|
66
|
+
* suite still owns its own `beforeEach`/`afterEach` and this module keeps the
|
|
67
|
+
* empty third-party set its purity pin asserts. See the module docstring.
|
|
68
|
+
*
|
|
69
|
+
* 🪤 `root()` before the first `plant()` THROWS by name. The shape it replaced
|
|
70
|
+
* read an uninitialised `let` and raised `Cannot read properties of undefined`
|
|
71
|
+
* from inside the fixture, which names neither the suite nor the missing hook.
|
|
72
|
+
*
|
|
73
|
+
* @param prefix - `mkdtemp` prefix, so a leaked directory names its own suite
|
|
74
|
+
* @param corpus - Fixture name to file content, as {@link createTempCorpus} takes it
|
|
75
|
+
* @returns Plant/clear/root, to be driven from the caller's own hooks
|
|
76
|
+
*/
|
|
77
|
+
export declare function replantableCorpus(prefix: string, corpus: Readonly<Record<string, string>>): ReplantableCorpus;
|
|
8
78
|
//# sourceMappingURL=testing.d.ts.map
|
package/dist/testing.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAUH,cAAc,mBAAmB,CAAC;AAElC,yDAAyD;AACzD,MAAM,WAAW,UAAU;IACzB,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,6DAA6D;IAC7D,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACvC,UAAU,CAOZ;AAED,+EAA+E;AAC/E,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,wDAAwD;IACxD,IAAI,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GACvC,iBAAiB,CAyBnB"}
|
package/dist/testing.js
CHANGED
|
@@ -1,8 +1,98 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Fixture primitives shared by suites in more than one package.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* a
|
|
4
|
+
* Deliberately framework-free: nothing here imports `vitest`, so this module
|
|
5
|
+
* stays a plain function library rather than something that registers hooks as a
|
|
6
|
+
* side effect of being imported. Each suite owns its own `beforeEach`/
|
|
7
|
+
* `afterEach` and calls these from inside them — two lines, which is below
|
|
8
|
+
* anything worth sharing, while the part that is genuinely identical (mkdtemp,
|
|
9
|
+
* write the literal, remove the tree) lives here once.
|
|
10
|
+
*
|
|
11
|
+
* It earned a home in `utils` the ordinary way: two packages needed it, not one
|
|
12
|
+
* package speculating that a second might.
|
|
13
|
+
*
|
|
14
|
+
* ⛔ **Do not import `vitest` here to share the hook wrapper itself.**
|
|
15
|
+
* `subpath-purity.test.ts` pins this entry at an EMPTY third-party set, and that
|
|
16
|
+
* pin is load-bearing: `./testing` is a published subpath, so a test framework
|
|
17
|
+
* reached from it becomes a runtime requirement for every adopter who imports
|
|
18
|
+
* it. {@link replantableCorpus} exists precisely so the per-test wrapper each
|
|
19
|
+
* suite still writes is three lines rather than twelve.
|
|
6
20
|
*/
|
|
21
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
22
|
+
import { normalizedTmpdir, safePath } from './path-utils.js';
|
|
23
|
+
// The rest of the testing surface. ⚠️ This re-export is the ENTIRETY of what
|
|
24
|
+
// `./testing` used to be, and dropping it silently removed `detachGitEnv`,
|
|
25
|
+
// `setupSyncTempDirSuite` and every sibling from a published subpath. A module
|
|
26
|
+
// that is both a definition site and a barrel loses the barrel first.
|
|
7
27
|
export * from './test-helpers.js';
|
|
28
|
+
/**
|
|
29
|
+
* Write a literal corpus into a fresh temp directory.
|
|
30
|
+
*
|
|
31
|
+
* 🪤 Fixture files must NOT be byte-identical to each other when the suite is
|
|
32
|
+
* about content-addressed behaviour: VAT's blobs are keyed on their bytes plus
|
|
33
|
+
* the parser kind, so two files with the same content collapse into ONE blob and
|
|
34
|
+
* assertions then describe whichever path sorted first. Give each fixture a
|
|
35
|
+
* distinguishing marker line.
|
|
36
|
+
*
|
|
37
|
+
* ⚠️ The root is minted per call, so a suite that plants per test must call this
|
|
38
|
+
* per test — a root captured once and reused across tests survives its own
|
|
39
|
+
* `cleanup`.
|
|
40
|
+
*
|
|
41
|
+
* @param prefix - `mkdtemp` prefix, so a leaked directory names its own suite
|
|
42
|
+
* @param corpus - Fixture name to file content; written verbatim as UTF-8.
|
|
43
|
+
* Names may include forward-slash subpaths only if their parents already exist
|
|
44
|
+
* @returns The tree's root and its teardown
|
|
45
|
+
*/
|
|
46
|
+
export function createTempCorpus(prefix, corpus) {
|
|
47
|
+
const root = mkdtempSync(safePath.join(normalizedTmpdir(), prefix));
|
|
48
|
+
for (const [name, content] of Object.entries(corpus)) {
|
|
49
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- a fixture name from the caller's own literal corpus, under this call's fresh mkdtemp root
|
|
50
|
+
writeFileSync(safePath.join(root, name), content, 'utf8');
|
|
51
|
+
}
|
|
52
|
+
return { root, cleanup: () => rmSync(root, { recursive: true, force: true }) };
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Hold a per-test corpus root, so a suite's hook wrapper is three lines.
|
|
56
|
+
*
|
|
57
|
+
* ⚠️ **The root comes back through a GETTER, never as a value.** A per-test
|
|
58
|
+
* fixture is reminted for every test, so a root captured at registration time is
|
|
59
|
+
* `undefined` in the first test and stale in every one after it. That mistake is
|
|
60
|
+
* the reason this holder exists rather than each suite keeping its own `let`.
|
|
61
|
+
*
|
|
62
|
+
* ⭐ Framework-free on purpose: it takes no hooks and registers none, so the
|
|
63
|
+
* suite still owns its own `beforeEach`/`afterEach` and this module keeps the
|
|
64
|
+
* empty third-party set its purity pin asserts. See the module docstring.
|
|
65
|
+
*
|
|
66
|
+
* 🪤 `root()` before the first `plant()` THROWS by name. The shape it replaced
|
|
67
|
+
* read an uninitialised `let` and raised `Cannot read properties of undefined`
|
|
68
|
+
* from inside the fixture, which names neither the suite nor the missing hook.
|
|
69
|
+
*
|
|
70
|
+
* @param prefix - `mkdtemp` prefix, so a leaked directory names its own suite
|
|
71
|
+
* @param corpus - Fixture name to file content, as {@link createTempCorpus} takes it
|
|
72
|
+
* @returns Plant/clear/root, to be driven from the caller's own hooks
|
|
73
|
+
*/
|
|
74
|
+
export function replantableCorpus(prefix, corpus) {
|
|
75
|
+
let planted;
|
|
76
|
+
return {
|
|
77
|
+
plant: () => {
|
|
78
|
+
// 🪤 Removes any tree still standing FIRST. Nested `describe` blocks each
|
|
79
|
+
// get their own `beforeEach` and vitest runs outer-then-inner before a
|
|
80
|
+
// single `afterEach`, so a double plant is ordinary rather than exotic.
|
|
81
|
+
// Overwriting the handle without this would leak the first tree for the
|
|
82
|
+
// process's lifetime, silently and only on the suites that nest.
|
|
83
|
+
planted?.cleanup();
|
|
84
|
+
planted = createTempCorpus(prefix, corpus);
|
|
85
|
+
},
|
|
86
|
+
clear: () => {
|
|
87
|
+
planted?.cleanup();
|
|
88
|
+
planted = undefined;
|
|
89
|
+
},
|
|
90
|
+
root: () => {
|
|
91
|
+
if (planted === undefined) {
|
|
92
|
+
throw new Error(`replantableCorpus('${prefix}'): root() before plant() — the suite is missing its beforeEach`);
|
|
93
|
+
}
|
|
94
|
+
return planted.root;
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
8
98
|
//# sourceMappingURL=testing.js.map
|
package/dist/testing.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE7D,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,sEAAsE;AACtE,cAAc,mBAAmB,CAAC;AAUlC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAc,EACd,MAAwC;IAExC,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACpE,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACrD,gKAAgK;QAChK,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AAYD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAc,EACd,MAAwC;IAExC,IAAI,OAA+B,CAAC;IACpC,OAAO;QACL,KAAK,EAAE,GAAG,EAAE;YACV,0EAA0E;YAC1E,uEAAuE;YACvE,wEAAwE;YACxE,wEAAwE;YACxE,iEAAiE;YACjE,OAAO,EAAE,OAAO,EAAE,CAAC;YACnB,OAAO,GAAG,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC;QACD,KAAK,EAAE,GAAG,EAAE;YACV,OAAO,EAAE,OAAO,EAAE,CAAC;YACnB,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;QACD,IAAI,EAAE,GAAG,EAAE;YACT,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACb,sBAAsB,MAAM,iEAAiE,CAC9F,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* **The one way VAT turns file bytes into text.**
|
|
3
|
+
*
|
|
4
|
+
* `local/no-raw-text-decode` enforces that — but only where it is registered,
|
|
5
|
+
* which today is `packages/utils/src` and `packages/resources/src`. Those two
|
|
6
|
+
* own the seam and every corpus-document read; the rest of the repo is a
|
|
7
|
+
* migration ledger, not a covered claim, and `eslint.config.js` carries it in
|
|
8
|
+
* cost order. Do not read this docstring as "nothing else in the repo decodes".
|
|
9
|
+
*
|
|
10
|
+
* ## Where this lives, and why it is a `utils` primitive
|
|
11
|
+
*
|
|
12
|
+
* Bytes-to-text is a pure function of its argument. It knows nothing about
|
|
13
|
+
* content keys, parse caches or the projection — `readContentWithKey` in
|
|
14
|
+
* `@vibe-agent-toolkit/resources` *composes* this with a raw-bytes content key,
|
|
15
|
+
* and the key is the projection concept, not the decode.
|
|
16
|
+
*
|
|
17
|
+
* It has to live here for a harder reason than tidiness. `resources` depends on
|
|
18
|
+
* `utils` and `utils` must never depend on `resources`, so a seam in `resources`
|
|
19
|
+
* with a lint rule shipped from `utils` would flag `utils`' own reads
|
|
20
|
+
* (`gitignore-checker.ts` reads an adopter's `.gitignore`; `project-utils.ts`
|
|
21
|
+
* reads an adopter's `package.json`) while giving them no legal way to comply.
|
|
22
|
+
* The rule would then be widened with exemptions until it meant nothing. Placing
|
|
23
|
+
* the primitive at the bottom of the arrow is what makes the guardrail
|
|
24
|
+
* enforceable.
|
|
25
|
+
*
|
|
26
|
+
* `./text` is also deliberately **pure** — no `node:*` import at all — so bytes
|
|
27
|
+
* from a git blob, an HTTP body or a zip entry decode through the same function
|
|
28
|
+
* as bytes from disk. The file-reading half is in `./fs` (`readTextContent`),
|
|
29
|
+
* where everything that touches `node:fs` lives.
|
|
30
|
+
*
|
|
31
|
+
* ## The defect this exists for
|
|
32
|
+
*
|
|
33
|
+
* `readContentWithKey` (in `@vibe-agent-toolkit/resources`) used to call
|
|
34
|
+
* `bytes.toString('utf-8')` unconditionally. Measured end to end, on a real
|
|
35
|
+
* `working-tree-encoding=UTF-16` checkout (see `resources`'
|
|
36
|
+
* `test/system/git-hostile-config.system.test.ts`):
|
|
37
|
+
*
|
|
38
|
+
* ```text
|
|
39
|
+
* bytes on disk 40 B, BOM fe ff, UTF-16BE
|
|
40
|
+
* decoded "��# \0D\0o\0c\0…" NUL-interleaved mojibake
|
|
41
|
+
* looksBinary true — the decoded string carries NULs
|
|
42
|
+
* projection BLOB_NOT_TEXT, no blob row, 0 sections, 0 references
|
|
43
|
+
* the same doc 1 heading, 1 link, from its UTF-8 bytes
|
|
44
|
+
* ```
|
|
45
|
+
*
|
|
46
|
+
* So VAT could not read a UTF-16 document *at all*, and the reason it matters is
|
|
47
|
+
* not exotic: **PowerShell 5.1's `Out-File` and `>` write UTF-16LE by default**,
|
|
48
|
+
* so a Windows-authored document lands squarely in that hole.
|
|
49
|
+
*
|
|
50
|
+
* ## What is a fact here, and what is an assumption
|
|
51
|
+
*
|
|
52
|
+
* The distinction is carried in the result ({@link TextProvenance.encodingSource})
|
|
53
|
+
* rather than left to prose, because the two are not the same kind of claim:
|
|
54
|
+
*
|
|
55
|
+
* | input | encoding | encodingSource | why |
|
|
56
|
+
* |---|---|---|---|
|
|
57
|
+
* | leading `ef bb bf` | `utf-8` | `bom` | the bytes say so |
|
|
58
|
+
* | leading `ff fe` (not `ff fe 00 00`) | `utf-16le` | `bom` | the bytes say so |
|
|
59
|
+
* | leading `fe ff` | `utf-16be` | `bom` | the bytes say so |
|
|
60
|
+
* | leading `ff fe 00 00` | `utf-32le` | `bom` | the bytes say so |
|
|
61
|
+
* | leading `00 00 fe ff` | `utf-32be` | `bom` | the bytes say so |
|
|
62
|
+
* | anything else | `utf-8` | `assumed` | the defensible default |
|
|
63
|
+
*
|
|
64
|
+
* ## And what proves the assumption WRONG
|
|
65
|
+
*
|
|
66
|
+
* `encoding` says what was guessed; {@link TextProvenance.replacementCharacters}
|
|
67
|
+
* is what the guess cost. A malformed sequence decodes to U+FFFD instead of
|
|
68
|
+
* throwing, so a mis-decoded document arrives as a well-formed JS string full of
|
|
69
|
+
* garbage — and a byte-level BPE tokenizer has no out-of-vocabulary concept, so
|
|
70
|
+
* it embeds and indexes that garbage without erroring anywhere. Counting the
|
|
71
|
+
* substitutions is what turns "we assumed UTF-8" into "we assumed UTF-8 and were
|
|
72
|
+
* demonstrably wrong 3,200 times in this file".
|
|
73
|
+
*
|
|
74
|
+
* It is counted **without paying for it on the clean path**. Every decode runs
|
|
75
|
+
* first through a `fatal: true` decoder, which throws on the first malformed
|
|
76
|
+
* sequence rather than substituting; a file that decodes cleanly — nearly every
|
|
77
|
+
* file — costs exactly one decode and no scan at all. Only a file that actually
|
|
78
|
+
* threw is decoded a second time in substituting mode and scanned for U+FFFD, so
|
|
79
|
+
* the O(n) scan is charged entirely to broken input.
|
|
80
|
+
*
|
|
81
|
+
* That ordering also buys a correctness property a scan alone cannot have: a
|
|
82
|
+
* document that *legitimately contains* U+FFFD is valid input, so the fatal
|
|
83
|
+
* decoder does not throw and it is reported as **0** replacements rather than
|
|
84
|
+
* accused of a bad decode. A bare scan would count its own content against it.
|
|
85
|
+
*
|
|
86
|
+
* 🪤 **The UTF-32LE BOM starts with the UTF-16LE BOM.** `ff fe 00 00` matches
|
|
87
|
+
* `ff fe`, so a table tested shortest-first decodes every UTF-32LE document as
|
|
88
|
+
* NUL-interleaved UTF-16 — the same bug this module exists to fix, one encoding
|
|
89
|
+
* further down. {@link BOMS} is therefore ordered longest-first and the test
|
|
90
|
+
* suite pins that ordering directly.
|
|
91
|
+
*
|
|
92
|
+
* ## The two limitations, recorded rather than guessed around
|
|
93
|
+
*
|
|
94
|
+
* - **BOM-less UTF-16 is not detected.** It is undecidable from bytes alone: the
|
|
95
|
+
* same byte string is a legal, different UTF-8 document. A NUL-density
|
|
96
|
+
* heuristic would decide it *usually* correctly and silently wrongly the rest
|
|
97
|
+
* of the time, and "silently wrongly" is the failure class this whole module
|
|
98
|
+
* is a reaction to. BOM-less input is UTF-8.
|
|
99
|
+
* - **Latin charsets are not detected either, and there is no windows-1252
|
|
100
|
+
* fallback.** "These bytes are not valid UTF-8" is a fact; "therefore they are
|
|
101
|
+
* latin-1" is a guess, and it is equally consistent with a UTF-8 document
|
|
102
|
+
* carrying one corrupt byte. Malformed input gets U+FFFD. In practice the
|
|
103
|
+
* Latin family costs little: every ASCII byte string is valid UTF-8 and
|
|
104
|
+
* decodes correctly, so only high bytes are affected.
|
|
105
|
+
*
|
|
106
|
+
* Both are pinned as tests that state what is given up, so adding a heuristic
|
|
107
|
+
* later has to edit an assertion rather than quietly widen a claim.
|
|
108
|
+
*
|
|
109
|
+
* ## Why `TextDecoder` for three encodings and hand-rolled code for two
|
|
110
|
+
*
|
|
111
|
+
* `TextDecoder` implements the WHATWG Encoding Standard, which **deliberately
|
|
112
|
+
* omits UTF-32** — no engine offers it. Node's `Buffer` is narrower still:
|
|
113
|
+
* `utf8`, `utf16le`/`ucs2`, `latin1`, `ascii` and the binary-to-text codecs, with
|
|
114
|
+
* **no UTF-16BE at all** (the encoding a round trip through git's
|
|
115
|
+
* `working-tree-encoding=UTF-16` actually produces). So `TextDecoder` carries
|
|
116
|
+
* utf-8/utf-16le/utf-16be, and UTF-32 is converted here.
|
|
117
|
+
*
|
|
118
|
+
* Refusing UTF-32 loudly was the alternative, and it was rejected for one
|
|
119
|
+
* reason: the BOM has to be *recognised* regardless (see the trap above), so the
|
|
120
|
+
* choice was never "detect it or not" — only "having detected it, decode it or
|
|
121
|
+
* throw". Decoding is ~20 lines and leaves no hole.
|
|
122
|
+
*
|
|
123
|
+
* ## The BOM is stripped, and that is load-bearing in both directions
|
|
124
|
+
*
|
|
125
|
+
* A surviving leading U+FEFF stops `# Heading` from parsing as a heading. And
|
|
126
|
+
* every offset downstream is a **character** offset over this decoded string —
|
|
127
|
+
* `lineStartOffsets` in `resources`' `projection/blob-sections.ts` derives them
|
|
128
|
+
* from `content.split('\n')`, and `parseMarkdownContent` takes remark's own
|
|
129
|
+
* character positions — so stripping shifts all of them consistently. What must
|
|
130
|
+
* NOT shift is the content key, which is computed over the raw bytes; see
|
|
131
|
+
* `resources`' `content-key.ts`.
|
|
132
|
+
*/
|
|
133
|
+
/** An encoding this module can decode. */
|
|
134
|
+
export type TextEncoding = 'utf-8' | 'utf-16le' | 'utf-16be' | 'utf-32le' | 'utf-32be';
|
|
135
|
+
/**
|
|
136
|
+
* How the encoding was arrived at.
|
|
137
|
+
*
|
|
138
|
+
* `'bom'` is a fact about the bytes. `'assumed'` is a default, and the honest
|
|
139
|
+
* name for it: BOM-less UTF-16 and BOM-less latin-1 both land here and both
|
|
140
|
+
* decode as UTF-8.
|
|
141
|
+
*/
|
|
142
|
+
export type EncodingSource = 'bom' | 'assumed';
|
|
143
|
+
/**
|
|
144
|
+
* Everything the decode knew, guessed, and lost — the text itself excluded.
|
|
145
|
+
*
|
|
146
|
+
* Split out from {@link DecodedText} so a consumer that must carry the decode's
|
|
147
|
+
* provenance alongside *other* facts about the same bytes — `KeyedContent` in
|
|
148
|
+
* `@vibe-agent-toolkit/resources` carries it beside a content key and a byte
|
|
149
|
+
* length — can hold exactly these three fields without also re-holding the
|
|
150
|
+
* content, and without restating them one by one at every layer they cross.
|
|
151
|
+
*/
|
|
152
|
+
export interface TextProvenance {
|
|
153
|
+
/** The encoding used. */
|
|
154
|
+
readonly encoding: TextEncoding;
|
|
155
|
+
/** Whether {@link encoding} was read off a BOM or assumed. */
|
|
156
|
+
readonly encodingSource: EncodingSource;
|
|
157
|
+
/**
|
|
158
|
+
* How many U+FFFD REPLACEMENT CHARACTERs the decode produced.
|
|
159
|
+
*
|
|
160
|
+
* Zero for a document that decodes cleanly, **including one whose own content
|
|
161
|
+
* legitimately contains U+FFFD** — see the module docstring for why the
|
|
162
|
+
* fatal-first ordering is what makes those two cases distinguishable. A
|
|
163
|
+
* non-zero value is proof, not suspicion: these bytes are not valid in the
|
|
164
|
+
* encoding they were read as.
|
|
165
|
+
*/
|
|
166
|
+
readonly replacementCharacters: number;
|
|
167
|
+
}
|
|
168
|
+
/** Text decoded from bytes, and what it was decoded as. */
|
|
169
|
+
export interface DecodedText extends TextProvenance {
|
|
170
|
+
/** The decoded content, BOM removed, exactly as a parser should receive it. */
|
|
171
|
+
readonly text: string;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* **The canonical content-decoding seam.** Turn file bytes into the text a
|
|
175
|
+
* parser should see.
|
|
176
|
+
*
|
|
177
|
+
* Takes bytes rather than a path so that a caller which must also key, hash or
|
|
178
|
+
* measure the raw bytes reads the file exactly once — `readContentWithKey` in
|
|
179
|
+
* `@vibe-agent-toolkit/resources` is that caller, and its key must stay over the
|
|
180
|
+
* raw byte preimage whatever this function decides the characters are. For the
|
|
181
|
+
* ordinary "read a file, give me its text" case use `readTextContent` from
|
|
182
|
+
* `@vibe-agent-toolkit/utils/fs`.
|
|
183
|
+
*
|
|
184
|
+
* @param bytes - The exact bytes read from disk
|
|
185
|
+
* @returns The decoded text, the encoding used, whether that was a fact, and how
|
|
186
|
+
* many characters the decode had to replace
|
|
187
|
+
*
|
|
188
|
+
* @example
|
|
189
|
+
* ```typescript
|
|
190
|
+
* const bytes = await readFile(path);
|
|
191
|
+
* const { text, encoding, encodingSource, replacementCharacters } = decodeTextContent(bytes);
|
|
192
|
+
* // UTF-16BE file: encoding 'utf-16be', encodingSource 'bom', 0 replacements,
|
|
193
|
+
* // text with no BOM and no NULs
|
|
194
|
+
* ```
|
|
195
|
+
*/
|
|
196
|
+
export declare function decodeTextContent(bytes: Uint8Array): DecodedText;
|
|
197
|
+
//# sourceMappingURL=text-content.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"text-content.d.ts","sourceRoot":"","sources":["../src/text-content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmIG;AAEH,0CAA0C;AAC1C,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAEvF;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,SAAS,CAAC;AAE/C;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,yBAAyB;IACzB,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAChC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC;;;;;;;;OAQG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CAAC;CACxC;AAED,2DAA2D;AAC3D,MAAM,WAAW,WAAY,SAAQ,cAAc;IACjD,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAkLD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,WAAW,CA4BhE"}
|