@withtruing/companion 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -26
- package/dist/anthropic-output-schema.js +41 -0
- package/dist/api-key-caller.js +9 -2
- package/dist/claude-cli-caller.js +107 -8
- package/dist/cli.js +837 -207
- package/dist/codex-binary.js +96 -0
- package/dist/codex-cli-caller.js +90 -16
- package/dist/config.js +177 -2
- package/dist/git-range.js +88 -46
- package/dist/http-client.js +95 -10
- package/dist/investigation-reader.js +12 -6
- package/dist/managed-repo.js +132 -0
- package/dist/observed-sources.js +132 -6
- package/dist/reader-result-contract.js +296 -2
- package/dist/repository-provider.js +608 -0
- package/dist/schema-entities.js +18 -18
- package/dist/service.js +7 -2
- package/dist/surface-mapper.js +53 -11
- package/dist/text-model.js +91 -8
- package/dist/whole-read/archive.js +196 -0
- package/dist/whole-read/checks.js +297 -0
- package/dist/whole-read/claims.js +196 -0
- package/dist/whole-read/coverage-ledger.js +51 -0
- package/dist/whole-read/decode.js +234 -0
- package/dist/whole-read/drafting.js +502 -0
- package/dist/whole-read/generated-rules.js +68 -0
- package/dist/whole-read/go-proof.js +255 -0
- package/dist/whole-read/grouping-prompt.js +19 -0
- package/dist/whole-read/grouping.js +247 -0
- package/dist/whole-read/identity.js +103 -0
- package/dist/whole-read/import-graph.js +293 -0
- package/dist/whole-read/inventory-collector.js +55 -0
- package/dist/whole-read/inventory-rules.js +141 -0
- package/dist/whole-read/lockfiles.js +947 -0
- package/dist/whole-read/notes.js +216 -0
- package/dist/whole-read/npm-proof.js +272 -0
- package/dist/whole-read/owner-decision-detectors.js +270 -0
- package/dist/whole-read/packing.js +612 -0
- package/dist/whole-read/pages.js +429 -0
- package/dist/whole-read/phase-policy.js +54 -0
- package/dist/whole-read/prd-build.js +279 -0
- package/dist/whole-read/prd-fidelity.js +14 -0
- package/dist/whole-read/python-proof.js +489 -0
- package/dist/whole-read/read-prompt.js +67 -0
- package/dist/whole-read/redaction.js +411 -0
- package/dist/whole-read/relationship-mapping.js +98 -0
- package/dist/whole-read/sanitize.js +266 -0
- package/dist/whole-read/secret-rules.js +26 -0
- package/dist/whole-read/structure.js +300 -0
- package/dist/whole-read/types.js +2 -0
- package/dist/whole-read-claims.js +12 -0
- package/dist/whole-read-git.js +340 -0
- package/dist/whole-read-pool.js +80 -0
- package/dist/whole-read-runner.js +945 -0
- package/dist/whole-read-shared-source.js +194 -0
- package/dist/whole-read-temp.js +479 -0
- package/package.json +8 -2
package/dist/schema-entities.js
CHANGED
|
@@ -36,12 +36,12 @@ export function validSchemaProbe(value) {
|
|
|
36
36
|
}
|
|
37
37
|
return new Set(probe.schemaPaths).size === probe.schemaPaths.length;
|
|
38
38
|
}
|
|
39
|
-
function runGit(repoPath, args, outputLimit = MAX_FILE_BYTES + 1) {
|
|
39
|
+
function runGit(repoPath, args, outputLimit = MAX_FILE_BYTES + 1, environment) {
|
|
40
40
|
return new Promise((resolve, reject) => {
|
|
41
41
|
const child = spawn('git', ['--no-replace-objects', ...args], {
|
|
42
42
|
cwd: repoPath,
|
|
43
43
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
44
|
-
env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
|
|
44
|
+
env: environment ?? { ...process.env, GIT_TERMINAL_PROMPT: '0' },
|
|
45
45
|
windowsHide: true,
|
|
46
46
|
});
|
|
47
47
|
const chunks = [];
|
|
@@ -80,8 +80,8 @@ function runGit(repoPath, args, outputLimit = MAX_FILE_BYTES + 1) {
|
|
|
80
80
|
}, GIT_TIMEOUT_MS);
|
|
81
81
|
});
|
|
82
82
|
}
|
|
83
|
-
async function gitText(repoPath, revision, path) {
|
|
84
|
-
const output = await runGit(repoPath, ['show', `${revision}:${path}`]);
|
|
83
|
+
async function gitText(repoPath, revision, path, environment) {
|
|
84
|
+
const output = await runGit(repoPath, ['show', `${revision}:${path}`], undefined, environment);
|
|
85
85
|
if (output.exitCode !== 0)
|
|
86
86
|
return null;
|
|
87
87
|
if (output.stdout.includes(0))
|
|
@@ -93,8 +93,8 @@ async function gitText(repoPath, revision, path) {
|
|
|
93
93
|
return null;
|
|
94
94
|
}
|
|
95
95
|
}
|
|
96
|
-
async function gitPathExists(repoPath, revision, path) {
|
|
97
|
-
return (await runGit(repoPath, ['cat-file', '-e', `${revision}:${path}`], 100)).exitCode === 0;
|
|
96
|
+
async function gitPathExists(repoPath, revision, path, environment) {
|
|
97
|
+
return (await runGit(repoPath, ['cat-file', '-e', `${revision}:${path}`], 100, environment)).exitCode === 0;
|
|
98
98
|
}
|
|
99
99
|
function skipTrivia(source, start) {
|
|
100
100
|
let index = start;
|
|
@@ -617,14 +617,14 @@ function referencesEntity(facts, entity) {
|
|
|
617
617
|
return entity.mappedName !== null
|
|
618
618
|
&& facts.strings.some((value) => value.includes(`"${entity.mappedName}"`));
|
|
619
619
|
}
|
|
620
|
-
async function headFile(repoPath, head, path) {
|
|
621
|
-
const size = await runGit(repoPath, ['cat-file', '-s', `${head}:${path}`], 100);
|
|
620
|
+
async function headFile(repoPath, head, path, environment) {
|
|
621
|
+
const size = await runGit(repoPath, ['cat-file', '-s', `${head}:${path}`], 100, environment);
|
|
622
622
|
if (size.exitCode !== 0)
|
|
623
623
|
return null;
|
|
624
624
|
const parsed = Number(size.stdout.toString('ascii').trim());
|
|
625
625
|
if (!Number.isSafeInteger(parsed) || parsed < 0 || parsed > MAX_FILE_BYTES)
|
|
626
626
|
return null;
|
|
627
|
-
return gitText(repoPath, head, path);
|
|
627
|
+
return gitText(repoPath, head, path, environment);
|
|
628
628
|
}
|
|
629
629
|
export async function detectSchemaImpact(input) {
|
|
630
630
|
if (!validSchemaProbe(input.probe))
|
|
@@ -639,9 +639,9 @@ export async function detectSchemaImpact(input) {
|
|
|
639
639
|
const headSchemas = [];
|
|
640
640
|
const baseSchemas = [];
|
|
641
641
|
for (const path of new Set([...input.probe.schemaPaths, ...changedSchemas])) {
|
|
642
|
-
const text = await gitText(input.repoPath, input.head, path);
|
|
642
|
+
const text = await gitText(input.repoPath, input.head, path, input.environment);
|
|
643
643
|
if (text === null) {
|
|
644
|
-
if (await gitPathExists(input.repoPath, input.head, path)) {
|
|
644
|
+
if (await gitPathExists(input.repoPath, input.head, path, input.environment)) {
|
|
645
645
|
return { entities: null, modulesReferencing: [] };
|
|
646
646
|
}
|
|
647
647
|
continue;
|
|
@@ -653,12 +653,12 @@ export async function detectSchemaImpact(input) {
|
|
|
653
653
|
}
|
|
654
654
|
for (const path of changedSchemas) {
|
|
655
655
|
const [beforeText, afterText] = await Promise.all([
|
|
656
|
-
gitText(input.repoPath, input.base, path),
|
|
657
|
-
gitText(input.repoPath, input.head, path),
|
|
656
|
+
gitText(input.repoPath, input.base, path, input.environment),
|
|
657
|
+
gitText(input.repoPath, input.head, path, input.environment),
|
|
658
658
|
]);
|
|
659
659
|
const [beforeExists, afterExists] = await Promise.all([
|
|
660
|
-
beforeText === null ? gitPathExists(input.repoPath, input.base, path) : false,
|
|
661
|
-
afterText === null ? gitPathExists(input.repoPath, input.head, path) : false,
|
|
660
|
+
beforeText === null ? gitPathExists(input.repoPath, input.base, path, input.environment) : false,
|
|
661
|
+
afterText === null ? gitPathExists(input.repoPath, input.head, path, input.environment) : false,
|
|
662
662
|
]);
|
|
663
663
|
if (beforeExists || afterExists)
|
|
664
664
|
return { entities: null, modulesReferencing: [] };
|
|
@@ -676,9 +676,9 @@ export async function detectSchemaImpact(input) {
|
|
|
676
676
|
}
|
|
677
677
|
}
|
|
678
678
|
for (const path of changedMigrations) {
|
|
679
|
-
const sql = await gitText(input.repoPath, input.head, path);
|
|
679
|
+
const sql = await gitText(input.repoPath, input.head, path, input.environment);
|
|
680
680
|
if (sql === null) {
|
|
681
|
-
if (await gitPathExists(input.repoPath, input.head, path)) {
|
|
681
|
+
if (await gitPathExists(input.repoPath, input.head, path, input.environment)) {
|
|
682
682
|
return { entities: null, modulesReferencing: [] };
|
|
683
683
|
}
|
|
684
684
|
continue;
|
|
@@ -723,7 +723,7 @@ export async function detectSchemaImpact(input) {
|
|
|
723
723
|
const readHeadFile = (path) => {
|
|
724
724
|
let pending = fileCache.get(path);
|
|
725
725
|
if (!pending) {
|
|
726
|
-
pending = headFile(input.repoPath, input.head, path)
|
|
726
|
+
pending = headFile(input.repoPath, input.head, path, input.environment)
|
|
727
727
|
.then((text) => text === null ? null : referenceFacts(path, text));
|
|
728
728
|
fileCache.set(path, pending);
|
|
729
729
|
}
|
package/dist/service.js
CHANGED
|
@@ -17,7 +17,9 @@ const EXIT_WINDOW_MS = 15 * 60_000;
|
|
|
17
17
|
const CRASH_PAUSE_MS = 10 * 60_000;
|
|
18
18
|
const STOP_GRACE_MS = 10_000;
|
|
19
19
|
const BOOT_TIME_TOLERANCE_MS = 2 * 60_000;
|
|
20
|
-
const ALLOWED_ENV = [
|
|
20
|
+
const ALLOWED_ENV = [
|
|
21
|
+
'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'HOME', 'TRUING_PARALLEL_STEPS', 'USERPROFILE',
|
|
22
|
+
];
|
|
21
23
|
const defaultIo = {
|
|
22
24
|
exists: existsSync,
|
|
23
25
|
mkdir: (path) => mkdirSync(path, { recursive: true }),
|
|
@@ -697,10 +699,13 @@ export function serviceStatus(paths = servicePaths(), io = defaultIo) {
|
|
|
697
699
|
pairings: (config?.pairings ?? []).map((pairing) => ({
|
|
698
700
|
url: pairing.url,
|
|
699
701
|
workspaceId: pairing.workspaceId,
|
|
700
|
-
projects: pairing.connections.map(({ connectionId, repoPath, managed }) => ({
|
|
702
|
+
projects: pairing.connections.map(({ connectionId, repoPath, managed, label, repository, repositoryStorage }) => ({
|
|
701
703
|
connectionId,
|
|
702
704
|
repoPath,
|
|
703
705
|
...(managed ? { managed } : {}),
|
|
706
|
+
...(label ? { label } : {}),
|
|
707
|
+
...(repository ? { repository } : {}),
|
|
708
|
+
...(repositoryStorage ? { repositoryStorage } : {}),
|
|
704
709
|
})),
|
|
705
710
|
workspaceTasks: pairing.pollWorkspaceTasks
|
|
706
711
|
? { backend: pairing.workspaceBackend ?? 'api-key' }
|
package/dist/surface-mapper.js
CHANGED
|
@@ -150,6 +150,33 @@ export async function readReuseChangeSet(sandboxPath, candidateCommit, headCommi
|
|
|
150
150
|
return null;
|
|
151
151
|
}
|
|
152
152
|
}
|
|
153
|
+
/** Compare two independently acquired snapshots without assuming either shallow repository has the
|
|
154
|
+
* other's history. Rename/copy detection is deliberately conservative: add+delete declines reuse. */
|
|
155
|
+
export function compareSurfaceSnapshotTrees(base, head) {
|
|
156
|
+
const paths = [...new Set([...base.entries.keys(), ...head.entries.keys()])]
|
|
157
|
+
.sort((left, right) => left.localeCompare(right));
|
|
158
|
+
const changes = [];
|
|
159
|
+
for (const path of paths) {
|
|
160
|
+
const before = base.entries.get(path);
|
|
161
|
+
const after = head.entries.get(path);
|
|
162
|
+
if (!before)
|
|
163
|
+
changes.push({ status: 'A', path });
|
|
164
|
+
else if (!after)
|
|
165
|
+
changes.push({ status: 'D', path });
|
|
166
|
+
else if (before.mode !== after.mode || before.type !== after.type)
|
|
167
|
+
changes.push({ status: 'T', path });
|
|
168
|
+
else if (before.object !== after.object)
|
|
169
|
+
changes.push({ status: 'M', path });
|
|
170
|
+
}
|
|
171
|
+
return changes;
|
|
172
|
+
}
|
|
173
|
+
export async function readSurfaceSnapshotChangeSet(basePath, baseCommit, headPath, headCommit) {
|
|
174
|
+
const [base, head] = await Promise.all([
|
|
175
|
+
enumerateTrackedFiles(basePath, baseCommit),
|
|
176
|
+
enumerateTrackedFiles(headPath, headCommit),
|
|
177
|
+
]);
|
|
178
|
+
return compareSurfaceSnapshotTrees(base, head);
|
|
179
|
+
}
|
|
153
180
|
function isRecord(value) {
|
|
154
181
|
return !!value && typeof value === 'object' && !Array.isArray(value);
|
|
155
182
|
}
|
|
@@ -323,22 +350,37 @@ export async function runSurfaceMapping(input) {
|
|
|
323
350
|
throw new MappingFailure(failure.kind, failure.code, failure.detail);
|
|
324
351
|
}
|
|
325
352
|
try {
|
|
326
|
-
|
|
327
|
-
const candidate = input.claim.reuseCandidate;
|
|
328
|
-
const changeSet = candidate
|
|
329
|
-
? await readReuseChangeSet(sandbox.path, candidate.commit, input.claim.commit)
|
|
330
|
-
: null;
|
|
331
|
-
const expansionFiles = new Set(candidate?.expansionFiles ?? []);
|
|
332
|
-
const reuse = candidate && changeSet
|
|
333
|
-
&& changeSet.every((change) => change.status === 'M' && expansionFiles.has(change.path))
|
|
334
|
-
? { candidate, changeSet }
|
|
335
|
-
: undefined;
|
|
336
|
-
return await mapSurfaceSnapshot({ ...input, sandboxPath: sandbox.path, tree, ...(reuse ? { reuse } : {}) });
|
|
353
|
+
return await runSurfaceMappingCheckout({ ...input, checkoutPath: sandbox.path });
|
|
337
354
|
}
|
|
338
355
|
finally {
|
|
339
356
|
await sandbox.cleanup();
|
|
340
357
|
}
|
|
341
358
|
}
|
|
359
|
+
/** Map an already verified, attempt-owned checkout. The caller owns its repository lease. */
|
|
360
|
+
export async function runSurfaceMappingCheckout(input) {
|
|
361
|
+
input.assertLive?.();
|
|
362
|
+
const tree = await enumerateTrackedFiles(input.checkoutPath, input.claim.commit);
|
|
363
|
+
input.assertLive?.();
|
|
364
|
+
const candidate = input.claim.reuseCandidate;
|
|
365
|
+
const changeSet = candidate
|
|
366
|
+
? input.reuseChangeSet === undefined
|
|
367
|
+
? await readReuseChangeSet(input.checkoutPath, candidate.commit, input.claim.commit)
|
|
368
|
+
: input.reuseChangeSet
|
|
369
|
+
: null;
|
|
370
|
+
const expansionFiles = new Set(candidate?.expansionFiles ?? []);
|
|
371
|
+
const reuse = candidate && changeSet
|
|
372
|
+
&& changeSet.every((change) => change.status === 'M' && expansionFiles.has(change.path))
|
|
373
|
+
? { candidate, changeSet }
|
|
374
|
+
: undefined;
|
|
375
|
+
const manifest = await mapSurfaceSnapshot({
|
|
376
|
+
...input,
|
|
377
|
+
sandboxPath: input.checkoutPath,
|
|
378
|
+
tree,
|
|
379
|
+
...(reuse ? { reuse } : {}),
|
|
380
|
+
});
|
|
381
|
+
input.assertLive?.();
|
|
382
|
+
return manifest;
|
|
383
|
+
}
|
|
342
384
|
export function canonicalManifestJson(manifest) {
|
|
343
385
|
const canonical = (value) => {
|
|
344
386
|
if (Array.isArray(value))
|
package/dist/text-model.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { toAnthropicOutputSchema } from './anthropic-output-schema.js';
|
|
1
2
|
const PURPOSE_ERRORS = {
|
|
2
3
|
'verify_module:file_select': {
|
|
3
4
|
refusal: null,
|
|
@@ -56,6 +57,52 @@ export class ModelRefusalError extends Error {
|
|
|
56
57
|
this.category = category;
|
|
57
58
|
}
|
|
58
59
|
}
|
|
60
|
+
/** Safe by construction: provider output and model text are never attached to this error. */
|
|
61
|
+
export class WholeReadTransportError extends Error {
|
|
62
|
+
code;
|
|
63
|
+
source;
|
|
64
|
+
resetAt;
|
|
65
|
+
retryAfterMs;
|
|
66
|
+
category;
|
|
67
|
+
constructor(code, source, resetAt, retryAfterMs, category) {
|
|
68
|
+
super(code);
|
|
69
|
+
this.code = code;
|
|
70
|
+
this.source = source;
|
|
71
|
+
this.resetAt = resetAt;
|
|
72
|
+
this.retryAfterMs = retryAfterMs;
|
|
73
|
+
this.category = category;
|
|
74
|
+
this.name = 'WholeReadTransportError';
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function retryAfterMs(error) {
|
|
78
|
+
if (!error || typeof error !== 'object')
|
|
79
|
+
return undefined;
|
|
80
|
+
const headers = error.headers;
|
|
81
|
+
const raw = headers?.get?.('retry-after');
|
|
82
|
+
if (!raw)
|
|
83
|
+
return undefined;
|
|
84
|
+
const seconds = Number(raw);
|
|
85
|
+
if (Number.isFinite(seconds) && seconds >= 0)
|
|
86
|
+
return seconds * 1000;
|
|
87
|
+
const at = Date.parse(raw);
|
|
88
|
+
return Number.isFinite(at) ? Math.max(0, at - Date.now()) : undefined;
|
|
89
|
+
}
|
|
90
|
+
function safeAnthropicFailure(error) {
|
|
91
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
92
|
+
return new WholeReadTransportError('cancelled', 'anthropic_api');
|
|
93
|
+
}
|
|
94
|
+
const candidate = error;
|
|
95
|
+
const status = typeof candidate?.status === 'number' ? candidate.status : 0;
|
|
96
|
+
const type = typeof candidate?.error?.type === 'string' ? candidate.error.type : '';
|
|
97
|
+
const message = typeof candidate?.message === 'string' ? candidate.message : '';
|
|
98
|
+
if (status === 429 || type === 'rate_limit_error') {
|
|
99
|
+
return new WholeReadTransportError('provider_rate_limit', 'anthropic_api', undefined, retryAfterMs(error));
|
|
100
|
+
}
|
|
101
|
+
if (status === 402 || /billing|credit balance|payment required/i.test(`${type} ${message}`)) {
|
|
102
|
+
return new WholeReadTransportError('billing_exhausted', 'anthropic_api');
|
|
103
|
+
}
|
|
104
|
+
return new WholeReadTransportError('capability_failure', 'anthropic_api');
|
|
105
|
+
}
|
|
59
106
|
/** The reply text once a refusal is ruled out. A mid-output fallback continues the declined
|
|
60
107
|
* model's partial text in a later block, so every text block is joined; null when there is none. */
|
|
61
108
|
export function replyText(message, refusal) {
|
|
@@ -96,26 +143,62 @@ export function anthropicTextModel(client) {
|
|
|
96
143
|
const content = request.purpose.endsWith(':file_select') && blocks.length === 1 && blocks[0].type === 'text'
|
|
97
144
|
? blocks[0].text
|
|
98
145
|
: blocks;
|
|
99
|
-
const
|
|
146
|
+
const thinking = { type: request.thinking ?? 'adaptive' };
|
|
147
|
+
const params = {
|
|
100
148
|
model: request.model,
|
|
101
149
|
max_tokens: request.maxTokens,
|
|
102
|
-
thinking
|
|
150
|
+
thinking,
|
|
103
151
|
output_config: request.schema
|
|
104
|
-
? { effort: request.effort, format: { type: 'json_schema',
|
|
152
|
+
? { effort: request.effort, format: { type: 'json_schema',
|
|
153
|
+
schema: toAnthropicOutputSchema(request.schema) } }
|
|
105
154
|
: { effort: request.effort },
|
|
106
155
|
system: request.system,
|
|
107
156
|
messages: [{ role: 'user', content }],
|
|
108
|
-
}
|
|
109
|
-
|
|
157
|
+
};
|
|
158
|
+
let message;
|
|
159
|
+
try {
|
|
160
|
+
const stream = request.wholeRead
|
|
161
|
+
? client.beta.messages.stream({ ...params, betas: [FALLBACK_BETA], fallbacks: DEFAULT_FALLBACKS }, {
|
|
162
|
+
signal: request.wholeRead.signal,
|
|
163
|
+
})
|
|
164
|
+
: streamWithFallback(client, params);
|
|
165
|
+
message = await stream.finalMessage();
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
if (request.wholeRead)
|
|
169
|
+
throw safeAnthropicFailure(error);
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
110
172
|
const errors = PURPOSE_ERRORS[request.purpose] ?? {
|
|
111
173
|
refusal: `The model declined the ${request.purpose} call.`,
|
|
112
174
|
empty: `The ${request.purpose} call returned no text.`,
|
|
113
175
|
};
|
|
114
176
|
// A purpose with no refusal wording of its own still fails as a refusal, under its empty-reply
|
|
115
177
|
// wording — never by reading whatever partial text the declined attempt left behind.
|
|
116
|
-
|
|
117
|
-
|
|
178
|
+
let text;
|
|
179
|
+
try {
|
|
180
|
+
text = replyText(message, errors.refusal ?? errors.empty);
|
|
181
|
+
}
|
|
182
|
+
catch (error) {
|
|
183
|
+
if (request.wholeRead && error instanceof ModelRefusalError) {
|
|
184
|
+
throw new WholeReadTransportError('model_declined', 'anthropic_api', undefined, undefined, error.category);
|
|
185
|
+
}
|
|
186
|
+
throw error;
|
|
187
|
+
}
|
|
188
|
+
if (!text) {
|
|
189
|
+
if (request.wholeRead)
|
|
190
|
+
throw new WholeReadTransportError('capability_failure', 'anthropic_api');
|
|
118
191
|
throw new Error(errors.empty);
|
|
119
|
-
|
|
192
|
+
}
|
|
193
|
+
const usage = usageRecord(message.usage);
|
|
194
|
+
return {
|
|
195
|
+
text,
|
|
196
|
+
usage,
|
|
197
|
+
...(request.wholeRead ? { metadata: {
|
|
198
|
+
finishReason: message.stop_reason,
|
|
199
|
+
model: typeof message.model === 'string' ? message.model : null,
|
|
200
|
+
usageAvailable: usage !== null,
|
|
201
|
+
} } : {}),
|
|
202
|
+
};
|
|
120
203
|
};
|
|
121
204
|
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
2
|
+
import { Readable, Transform } from 'node:stream';
|
|
3
|
+
import { pipeline } from 'node:stream/promises';
|
|
4
|
+
import { createGunzip } from 'node:zlib';
|
|
5
|
+
const require = createRequire(import.meta.url);
|
|
6
|
+
const tar = require('tar-stream');
|
|
7
|
+
const DEFAULT_LIMITS = {
|
|
8
|
+
maxCompressedBytes: 64 * 1024 * 1024,
|
|
9
|
+
maxUncompressedBytes: 256 * 1024 * 1024,
|
|
10
|
+
maxMemberBytes: 64 * 1024 * 1024,
|
|
11
|
+
maxMembers: 100_000,
|
|
12
|
+
};
|
|
13
|
+
/** Safe parser error: archive paths and contents are deliberately excluded. */
|
|
14
|
+
export class ArchiveInspectionError extends Error {
|
|
15
|
+
code;
|
|
16
|
+
constructor(code) {
|
|
17
|
+
super(code);
|
|
18
|
+
this.code = code;
|
|
19
|
+
this.name = 'ArchiveInspectionError';
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function fail(code) {
|
|
23
|
+
throw new ArchiveInspectionError(code);
|
|
24
|
+
}
|
|
25
|
+
function resolvedLimits(input) {
|
|
26
|
+
const limits = { ...DEFAULT_LIMITS };
|
|
27
|
+
for (const key of Object.keys(DEFAULT_LIMITS)) {
|
|
28
|
+
const value = input.limits?.[key];
|
|
29
|
+
if (value === undefined)
|
|
30
|
+
continue;
|
|
31
|
+
if (!Number.isSafeInteger(value) || value <= 0 || value > DEFAULT_LIMITS[key]) {
|
|
32
|
+
return fail('archive_invalid');
|
|
33
|
+
}
|
|
34
|
+
limits[key] = value;
|
|
35
|
+
}
|
|
36
|
+
return limits;
|
|
37
|
+
}
|
|
38
|
+
function byteCounter(limit, signal) {
|
|
39
|
+
let total = 0;
|
|
40
|
+
return new Transform({
|
|
41
|
+
transform(chunk, _encoding, callback) {
|
|
42
|
+
if (signal?.aborted) {
|
|
43
|
+
callback(new ArchiveInspectionError('cancelled'));
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
total += chunk.byteLength;
|
|
47
|
+
callback(total > limit ? new ArchiveInspectionError('resource_limit') : null, chunk);
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
function safeParts(name) {
|
|
52
|
+
if (name.length === 0 || name.startsWith('/') || name.includes('\\') || name.includes('\0')) {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
const trimmed = name.endsWith('/') ? name.slice(0, -1) : name;
|
|
56
|
+
if (trimmed.length === 0)
|
|
57
|
+
return null;
|
|
58
|
+
const parts = trimmed.split('/');
|
|
59
|
+
return parts.every((part) => part.length > 0 && part !== '.' && part !== '..') ? parts : null;
|
|
60
|
+
}
|
|
61
|
+
function conflicts(path, kind, seen) {
|
|
62
|
+
if (seen.has(path))
|
|
63
|
+
return true;
|
|
64
|
+
let slash = path.indexOf('/');
|
|
65
|
+
while (slash >= 0) {
|
|
66
|
+
if (seen.get(path.slice(0, slash)) === 'file')
|
|
67
|
+
return true;
|
|
68
|
+
slash = path.indexOf('/', slash + 1);
|
|
69
|
+
}
|
|
70
|
+
if (kind === 'file') {
|
|
71
|
+
for (const existing of seen.keys()) {
|
|
72
|
+
if (existing.startsWith(`${path}/`))
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
async function parseArchive(input) {
|
|
79
|
+
if (input.signal?.aborted)
|
|
80
|
+
return fail('cancelled');
|
|
81
|
+
const limits = resolvedLimits(input);
|
|
82
|
+
const source = Readable.from(input.compressed);
|
|
83
|
+
const extractor = tar.extract();
|
|
84
|
+
const members = [];
|
|
85
|
+
const seen = new Map();
|
|
86
|
+
let prefix = null;
|
|
87
|
+
let rootSeen = false;
|
|
88
|
+
let memberCount = 0;
|
|
89
|
+
const abort = () => {
|
|
90
|
+
source.destroy(new ArchiveInspectionError('cancelled'));
|
|
91
|
+
};
|
|
92
|
+
input.signal?.addEventListener('abort', abort, { once: true });
|
|
93
|
+
extractor.on('entry', (header, stream, next) => {
|
|
94
|
+
memberCount += 1;
|
|
95
|
+
const parts = typeof header.name === 'string' ? safeParts(header.name) : null;
|
|
96
|
+
if (memberCount > limits.maxMembers || parts === null) {
|
|
97
|
+
stream.resume();
|
|
98
|
+
next(new ArchiveInspectionError(memberCount > limits.maxMembers ? 'resource_limit' : 'archive_invalid'));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (prefix === null)
|
|
102
|
+
prefix = parts[0];
|
|
103
|
+
if (parts[0] !== prefix) {
|
|
104
|
+
stream.resume();
|
|
105
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const type = header.type ?? 'file';
|
|
109
|
+
if (type !== 'file' && type !== 'directory') {
|
|
110
|
+
stream.resume();
|
|
111
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (type !== 'directory' && header.name.endsWith('/')) {
|
|
115
|
+
stream.resume();
|
|
116
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
if (parts.length === 1) {
|
|
120
|
+
if (type !== 'directory' || rootSeen || header.size !== 0) {
|
|
121
|
+
stream.resume();
|
|
122
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
rootSeen = true;
|
|
126
|
+
stream.once('end', next);
|
|
127
|
+
stream.resume();
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
const path = parts.slice(1).join('/');
|
|
131
|
+
if (conflicts(path, type, seen)) {
|
|
132
|
+
stream.resume();
|
|
133
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
seen.set(path, type);
|
|
137
|
+
if (type === 'directory') {
|
|
138
|
+
if (header.size !== 0) {
|
|
139
|
+
stream.resume();
|
|
140
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
stream.once('end', next);
|
|
144
|
+
stream.resume();
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
if (!Number.isSafeInteger(header.size) || Number(header.size) < 0
|
|
148
|
+
|| Number(header.size) > limits.maxMemberBytes) {
|
|
149
|
+
stream.resume();
|
|
150
|
+
next(new ArchiveInspectionError(Number(header.size) > limits.maxMemberBytes
|
|
151
|
+
? 'resource_limit'
|
|
152
|
+
: 'archive_invalid'));
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
const chunks = [];
|
|
156
|
+
let size = 0;
|
|
157
|
+
let tooLarge = false;
|
|
158
|
+
stream.on('data', (chunk) => {
|
|
159
|
+
size += chunk.byteLength;
|
|
160
|
+
if (size <= limits.maxMemberBytes)
|
|
161
|
+
chunks.push(Buffer.from(chunk));
|
|
162
|
+
else
|
|
163
|
+
tooLarge = true;
|
|
164
|
+
});
|
|
165
|
+
stream.once('end', () => {
|
|
166
|
+
if (tooLarge) {
|
|
167
|
+
next(new ArchiveInspectionError('resource_limit'));
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
if (size !== header.size) {
|
|
171
|
+
next(new ArchiveInspectionError('archive_invalid'));
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
members.push({ path, size, bytes: Buffer.concat(chunks, size) });
|
|
175
|
+
next();
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
try {
|
|
179
|
+
await pipeline(source, byteCounter(limits.maxCompressedBytes, input.signal), createGunzip(), byteCounter(limits.maxUncompressedBytes, input.signal), extractor);
|
|
180
|
+
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
if (error instanceof ArchiveInspectionError)
|
|
183
|
+
throw error;
|
|
184
|
+
return fail(input.signal?.aborted ? 'cancelled' : 'archive_invalid');
|
|
185
|
+
}
|
|
186
|
+
finally {
|
|
187
|
+
input.signal?.removeEventListener('abort', abort);
|
|
188
|
+
}
|
|
189
|
+
return members;
|
|
190
|
+
}
|
|
191
|
+
/** Inspect a GitHub-style gzip tarball without materialising paths on disk. */
|
|
192
|
+
export async function* inspectArchive(input) {
|
|
193
|
+
const members = await parseArchive(input);
|
|
194
|
+
for (const member of members)
|
|
195
|
+
yield member;
|
|
196
|
+
}
|