@withtruing/companion 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -26
- package/dist/anthropic-output-schema.js +41 -0
- package/dist/api-key-caller.js +9 -2
- package/dist/claude-cli-caller.js +107 -8
- package/dist/cli.js +837 -207
- package/dist/codex-binary.js +96 -0
- package/dist/codex-cli-caller.js +90 -16
- package/dist/config.js +177 -2
- package/dist/git-range.js +88 -46
- package/dist/http-client.js +95 -10
- package/dist/investigation-reader.js +12 -6
- package/dist/managed-repo.js +132 -0
- package/dist/observed-sources.js +132 -6
- package/dist/reader-result-contract.js +296 -2
- package/dist/repository-provider.js +608 -0
- package/dist/schema-entities.js +18 -18
- package/dist/service.js +7 -2
- package/dist/surface-mapper.js +53 -11
- package/dist/text-model.js +91 -8
- package/dist/whole-read/archive.js +196 -0
- package/dist/whole-read/checks.js +297 -0
- package/dist/whole-read/claims.js +196 -0
- package/dist/whole-read/coverage-ledger.js +51 -0
- package/dist/whole-read/decode.js +234 -0
- package/dist/whole-read/drafting.js +502 -0
- package/dist/whole-read/generated-rules.js +68 -0
- package/dist/whole-read/go-proof.js +255 -0
- package/dist/whole-read/grouping-prompt.js +19 -0
- package/dist/whole-read/grouping.js +247 -0
- package/dist/whole-read/identity.js +103 -0
- package/dist/whole-read/import-graph.js +293 -0
- package/dist/whole-read/inventory-collector.js +55 -0
- package/dist/whole-read/inventory-rules.js +141 -0
- package/dist/whole-read/lockfiles.js +947 -0
- package/dist/whole-read/notes.js +216 -0
- package/dist/whole-read/npm-proof.js +272 -0
- package/dist/whole-read/owner-decision-detectors.js +270 -0
- package/dist/whole-read/packing.js +612 -0
- package/dist/whole-read/pages.js +429 -0
- package/dist/whole-read/phase-policy.js +54 -0
- package/dist/whole-read/prd-build.js +279 -0
- package/dist/whole-read/prd-fidelity.js +14 -0
- package/dist/whole-read/python-proof.js +489 -0
- package/dist/whole-read/read-prompt.js +67 -0
- package/dist/whole-read/redaction.js +411 -0
- package/dist/whole-read/relationship-mapping.js +98 -0
- package/dist/whole-read/sanitize.js +266 -0
- package/dist/whole-read/secret-rules.js +26 -0
- package/dist/whole-read/structure.js +300 -0
- package/dist/whole-read/types.js +2 -0
- package/dist/whole-read-claims.js +12 -0
- package/dist/whole-read-git.js +340 -0
- package/dist/whole-read-pool.js +80 -0
- package/dist/whole-read-runner.js +945 -0
- package/dist/whole-read-shared-source.js +194 -0
- package/dist/whole-read-temp.js +479 -0
- package/package.json +8 -2
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { extname, posix } from 'node:path';
|
|
2
|
+
import ts from 'typescript';
|
|
3
|
+
import { decodeCanonicalBlob } from './decode.js';
|
|
4
|
+
const SCRIPT_EXTENSIONS = new Set(['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs']);
|
|
5
|
+
const RESOLUTION_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
6
|
+
const JS_TO_TS = {
|
|
7
|
+
'.js': ['.ts'], '.jsx': ['.tsx'], '.mjs': ['.mts'], '.cjs': ['.cts'],
|
|
8
|
+
};
|
|
9
|
+
const invalid = () => { throw new TypeError('import_graph_invalid'); };
|
|
10
|
+
function rawPath(pathBase64) {
|
|
11
|
+
try {
|
|
12
|
+
return Buffer.from(pathBase64, 'base64');
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return new Uint8Array();
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function compareBytes(left, right) {
|
|
19
|
+
const length = Math.min(left.byteLength, right.byteLength);
|
|
20
|
+
for (let index = 0; index < length; index += 1) {
|
|
21
|
+
if (left[index] !== right[index])
|
|
22
|
+
return left[index] - right[index];
|
|
23
|
+
}
|
|
24
|
+
return left.byteLength - right.byteLength;
|
|
25
|
+
}
|
|
26
|
+
function decodedPath(bytes) {
|
|
27
|
+
try {
|
|
28
|
+
const path = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
29
|
+
const parts = path.split('/');
|
|
30
|
+
if (!path || path.startsWith('/') || path.includes('\0')
|
|
31
|
+
|| parts.some((part) => !part || part === '.' || part === '..'))
|
|
32
|
+
return undefined;
|
|
33
|
+
return path;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function scriptKind(path) {
|
|
40
|
+
const extension = extname(path).toLowerCase();
|
|
41
|
+
if (extension === '.tsx')
|
|
42
|
+
return ts.ScriptKind.TSX;
|
|
43
|
+
if (extension === '.jsx')
|
|
44
|
+
return ts.ScriptKind.JSX;
|
|
45
|
+
if (extension === '.js' || extension === '.mjs' || extension === '.cjs')
|
|
46
|
+
return ts.ScriptKind.JS;
|
|
47
|
+
return ts.ScriptKind.TS;
|
|
48
|
+
}
|
|
49
|
+
function parseImports(path, source) {
|
|
50
|
+
const file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, scriptKind(path));
|
|
51
|
+
const imports = [];
|
|
52
|
+
const add = (kind, expression) => {
|
|
53
|
+
const literal = expression && ts.isStringLiteralLike(expression) ? expression : undefined;
|
|
54
|
+
imports.push({
|
|
55
|
+
kind,
|
|
56
|
+
specifier: literal?.text ?? null,
|
|
57
|
+
startCodeUnit: (literal ?? expression)?.getStart(file) ?? 0,
|
|
58
|
+
endCodeUnit: (literal ?? expression)?.getEnd() ?? 0,
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
const visit = (node) => {
|
|
62
|
+
if (ts.isImportDeclaration(node)) {
|
|
63
|
+
add(node.importClause ? 'import' : 'side_effect', node.moduleSpecifier);
|
|
64
|
+
}
|
|
65
|
+
else if (ts.isExportDeclaration(node) && node.moduleSpecifier) {
|
|
66
|
+
add('reexport', node.moduleSpecifier);
|
|
67
|
+
}
|
|
68
|
+
else if (ts.isImportEqualsDeclaration(node)
|
|
69
|
+
&& ts.isExternalModuleReference(node.moduleReference)) {
|
|
70
|
+
add('require', node.moduleReference.expression);
|
|
71
|
+
}
|
|
72
|
+
else if (ts.isCallExpression(node)) {
|
|
73
|
+
const dynamic = node.expression.kind === ts.SyntaxKind.ImportKeyword;
|
|
74
|
+
const required = ts.isIdentifier(node.expression) && node.expression.text === 'require';
|
|
75
|
+
if (dynamic || required)
|
|
76
|
+
add(dynamic ? 'dynamic' : 'require', node.arguments.length === 1 ? node.arguments[0] : undefined);
|
|
77
|
+
}
|
|
78
|
+
ts.forEachChild(node, visit);
|
|
79
|
+
};
|
|
80
|
+
visit(file);
|
|
81
|
+
const parseFailed = (file.parseDiagnostics?.length ?? 0) > 0;
|
|
82
|
+
return { imports: imports.sort((left, right) => left.startCodeUnit - right.startCodeUnit), parseFailed };
|
|
83
|
+
}
|
|
84
|
+
function characterOffset(codeUnitOffsets, codeUnit) {
|
|
85
|
+
let low = 0;
|
|
86
|
+
let high = codeUnitOffsets.length - 1;
|
|
87
|
+
while (low <= high) {
|
|
88
|
+
const middle = (low + high) >>> 1;
|
|
89
|
+
const value = codeUnitOffsets[middle];
|
|
90
|
+
if (value === codeUnit)
|
|
91
|
+
return middle;
|
|
92
|
+
if (value < codeUnit)
|
|
93
|
+
low = middle + 1;
|
|
94
|
+
else
|
|
95
|
+
high = middle - 1;
|
|
96
|
+
}
|
|
97
|
+
throw new RangeError('import span splits a decoded character');
|
|
98
|
+
}
|
|
99
|
+
function relativeCandidates(importer, specifier) {
|
|
100
|
+
const base = posix.normalize(posix.join(posix.dirname(importer), specifier));
|
|
101
|
+
if (!base || base === '..' || base.startsWith('../') || base.startsWith('/'))
|
|
102
|
+
return [];
|
|
103
|
+
const extension = extname(base).toLowerCase();
|
|
104
|
+
const candidates = [];
|
|
105
|
+
if (JS_TO_TS[extension]) {
|
|
106
|
+
const stem = base.slice(0, -extension.length);
|
|
107
|
+
candidates.push(...JS_TO_TS[extension].map((value) => `${stem}${value}`), base);
|
|
108
|
+
}
|
|
109
|
+
else if (extension)
|
|
110
|
+
candidates.push(base);
|
|
111
|
+
else {
|
|
112
|
+
candidates.push(...RESOLUTION_EXTENSIONS.map((value) => `${base}${value}`));
|
|
113
|
+
candidates.push(...RESOLUTION_EXTENSIONS.map((value) => `${base}/index${value}`));
|
|
114
|
+
}
|
|
115
|
+
return [...new Set(candidates)];
|
|
116
|
+
}
|
|
117
|
+
function packageSpecifier(specifier) {
|
|
118
|
+
const parts = specifier.split('/');
|
|
119
|
+
const count = specifier.startsWith('@') ? 2 : 1;
|
|
120
|
+
return { name: parts.slice(0, count).join('/'), subpath: parts.slice(count).join('/') };
|
|
121
|
+
}
|
|
122
|
+
/** Derives complete-repository ordering hints and exact mechanical import edges from canonical bytes. */
|
|
123
|
+
export function deriveWholeRepoImports(input) {
|
|
124
|
+
const ids = new Set();
|
|
125
|
+
const paths = new Set();
|
|
126
|
+
const indexed = input.entries.map((entry) => {
|
|
127
|
+
if (!entry.id || !entry.sourceId || !entry.pathBase64 || !entry.blobId || ids.has(entry.id))
|
|
128
|
+
invalid();
|
|
129
|
+
ids.add(entry.id);
|
|
130
|
+
const raw = rawPath(entry.pathBase64);
|
|
131
|
+
const pathKey = `${entry.sourceId}\0${Buffer.from(raw).toString('base64')}`;
|
|
132
|
+
if (paths.has(pathKey))
|
|
133
|
+
invalid();
|
|
134
|
+
paths.add(pathKey);
|
|
135
|
+
return { entry, rawPath: raw, path: decodedPath(raw), rank: -1 };
|
|
136
|
+
}).sort((left, right) => left.entry.sourceId.localeCompare(right.entry.sourceId)
|
|
137
|
+
|| compareBytes(left.rawPath, right.rawPath) || left.entry.id.localeCompare(right.entry.id));
|
|
138
|
+
indexed.forEach((value, rank) => { value.rank = rank; });
|
|
139
|
+
const bySourceAndPath = new Map();
|
|
140
|
+
for (const value of indexed) {
|
|
141
|
+
if (!value.path)
|
|
142
|
+
continue;
|
|
143
|
+
const key = `${value.entry.sourceId}\0${value.path}`;
|
|
144
|
+
const matches = bySourceAndPath.get(key) ?? [];
|
|
145
|
+
matches.push(value);
|
|
146
|
+
bySourceAndPath.set(key, matches);
|
|
147
|
+
}
|
|
148
|
+
const packages = new Map();
|
|
149
|
+
for (const value of indexed) {
|
|
150
|
+
if (!value.path || posix.basename(value.path) !== 'package.json' || !['100644', '100755'].includes(value.entry.mode))
|
|
151
|
+
continue;
|
|
152
|
+
const bytes = input.readCanonicalBytes(value.entry);
|
|
153
|
+
if (!bytes)
|
|
154
|
+
continue;
|
|
155
|
+
const decoded = decodeCanonicalBlob(bytes);
|
|
156
|
+
if (decoded.status !== 'decoded')
|
|
157
|
+
continue;
|
|
158
|
+
try {
|
|
159
|
+
const json = JSON.parse(decoded.text);
|
|
160
|
+
if (typeof json.name !== 'string' || !json.name)
|
|
161
|
+
continue;
|
|
162
|
+
const found = packages.get(json.name) ?? [];
|
|
163
|
+
found.push({ sourceId: value.entry.sourceId, root: posix.dirname(value.path) === '.' ? '' : posix.dirname(value.path), json });
|
|
164
|
+
packages.set(json.name, found);
|
|
165
|
+
}
|
|
166
|
+
catch { /* Invalid manifests do not prevent source ordering. */ }
|
|
167
|
+
}
|
|
168
|
+
const diagnostics = [];
|
|
169
|
+
const edges = [];
|
|
170
|
+
const targetFor = (source, specifier) => {
|
|
171
|
+
let candidates = [];
|
|
172
|
+
if (specifier.startsWith('./') || specifier.startsWith('../')) {
|
|
173
|
+
candidates = relativeCandidates(source.path, specifier);
|
|
174
|
+
}
|
|
175
|
+
else if (!specifier.startsWith('/') && !specifier.startsWith('#')) {
|
|
176
|
+
const parsed = packageSpecifier(specifier);
|
|
177
|
+
const matches = (packages.get(parsed.name) ?? []).filter((value) => value.sourceId === source.entry.sourceId);
|
|
178
|
+
if (matches.length === 0)
|
|
179
|
+
return 'external';
|
|
180
|
+
for (const match of matches) {
|
|
181
|
+
let target = parsed.subpath;
|
|
182
|
+
if (!target) {
|
|
183
|
+
target = typeof match.json.module === 'string' ? match.json.module
|
|
184
|
+
: typeof match.json.main === 'string' ? match.json.main : 'index';
|
|
185
|
+
}
|
|
186
|
+
candidates.push(...relativeCandidates(`${match.root ? `${match.root}/` : ''}__package__.ts`, `./${target.replace(/^\.\//, '')}`));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
for (const candidate of candidates) {
|
|
190
|
+
const found = bySourceAndPath.get(`${source.entry.sourceId}\0${candidate}`);
|
|
191
|
+
if (found)
|
|
192
|
+
return found.filter((value) => ['100644', '100755'].includes(value.entry.mode));
|
|
193
|
+
}
|
|
194
|
+
return [];
|
|
195
|
+
};
|
|
196
|
+
for (const source of indexed) {
|
|
197
|
+
if (!source.path) {
|
|
198
|
+
diagnostics.push({ code: 'path_not_utf8', sourceEntryId: source.entry.id });
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (!['100644', '100755'].includes(source.entry.mode))
|
|
202
|
+
continue;
|
|
203
|
+
if (!SCRIPT_EXTENSIONS.has(extname(source.path).toLowerCase())) {
|
|
204
|
+
diagnostics.push({ code: 'unsupported_language', sourceEntryId: source.entry.id });
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
const bytes = input.readCanonicalBytes(source.entry);
|
|
208
|
+
if (!bytes) {
|
|
209
|
+
diagnostics.push({ code: 'source_unavailable', sourceEntryId: source.entry.id });
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
const decoded = decodeCanonicalBlob(bytes);
|
|
213
|
+
if (decoded.status !== 'decoded') {
|
|
214
|
+
diagnostics.push({ code: 'source_not_text', sourceEntryId: source.entry.id });
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
const parsed = parseImports(source.path, decoded.text);
|
|
218
|
+
if (parsed.parseFailed) {
|
|
219
|
+
diagnostics.push({ code: 'parse_failed', sourceEntryId: source.entry.id });
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
for (const imported of parsed.imports) {
|
|
223
|
+
const startCharacter = characterOffset(decoded.mapping.characterCodeUnitOffsets, imported.startCodeUnit);
|
|
224
|
+
const endCharacter = characterOffset(decoded.mapping.characterCodeUnitOffsets, imported.endCodeUnit);
|
|
225
|
+
const sourceSpan = decoded.mapping.canonicalSpanForCharacters(startCharacter, endCharacter);
|
|
226
|
+
if (imported.specifier === null) {
|
|
227
|
+
diagnostics.push({ code: 'unresolved_dynamic', sourceEntryId: source.entry.id, sourceSpan });
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
const targets = targetFor(source, imported.specifier);
|
|
231
|
+
if (targets === 'external') {
|
|
232
|
+
diagnostics.push({ code: 'external_import', sourceEntryId: source.entry.id, specifier: imported.specifier, sourceSpan });
|
|
233
|
+
}
|
|
234
|
+
else if (targets.length === 0) {
|
|
235
|
+
diagnostics.push({ code: 'unresolved_import', sourceEntryId: source.entry.id, specifier: imported.specifier, sourceSpan });
|
|
236
|
+
}
|
|
237
|
+
else if (targets.length > 1) {
|
|
238
|
+
diagnostics.push({ code: 'ambiguous_import', sourceEntryId: source.entry.id, specifier: imported.specifier, sourceSpan });
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
edges.push({
|
|
242
|
+
sourceEntryId: source.entry.id,
|
|
243
|
+
targetEntryId: targets[0].entry.id,
|
|
244
|
+
kind: imported.kind,
|
|
245
|
+
specifier: imported.specifier,
|
|
246
|
+
sourceSpan,
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
const byId = new Map(indexed.map((value) => [value.entry.id, value]));
|
|
252
|
+
edges.sort((left, right) => byId.get(left.sourceEntryId).rank - byId.get(right.sourceEntryId).rank
|
|
253
|
+
|| left.sourceSpan.startByte - right.sourceSpan.startByte || left.targetEntryId.localeCompare(right.targetEntryId));
|
|
254
|
+
diagnostics.sort((left, right) => byId.get(left.sourceEntryId).rank - byId.get(right.sourceEntryId).rank
|
|
255
|
+
|| (left.sourceSpan?.startByte ?? -1) - (right.sourceSpan?.startByte ?? -1)
|
|
256
|
+
|| left.code.localeCompare(right.code) || (left.specifier ?? '').localeCompare(right.specifier ?? ''));
|
|
257
|
+
const dependencies = new Map();
|
|
258
|
+
for (const edge of edges) {
|
|
259
|
+
const found = dependencies.get(edge.sourceEntryId) ?? [];
|
|
260
|
+
if (!found.includes(edge.targetEntryId))
|
|
261
|
+
found.push(edge.targetEntryId);
|
|
262
|
+
dependencies.set(edge.sourceEntryId, found);
|
|
263
|
+
}
|
|
264
|
+
for (const found of dependencies.values())
|
|
265
|
+
found.sort((left, right) => byId.get(left).rank - byId.get(right).rank);
|
|
266
|
+
const state = new Map();
|
|
267
|
+
const orderedEntryIds = [];
|
|
268
|
+
for (const value of indexed) {
|
|
269
|
+
if (state.has(value.entry.id))
|
|
270
|
+
continue;
|
|
271
|
+
const stack = [{ entryId: value.entry.id, expanded: false }];
|
|
272
|
+
while (stack.length > 0) {
|
|
273
|
+
const current = stack.pop();
|
|
274
|
+
if (current.expanded) {
|
|
275
|
+
if (state.get(current.entryId) !== 'visited') {
|
|
276
|
+
state.set(current.entryId, 'visited');
|
|
277
|
+
orderedEntryIds.push(current.entryId);
|
|
278
|
+
}
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (state.has(current.entryId))
|
|
282
|
+
continue;
|
|
283
|
+
state.set(current.entryId, 'visiting');
|
|
284
|
+
stack.push({ entryId: current.entryId, expanded: true });
|
|
285
|
+
const targets = dependencies.get(current.entryId) ?? [];
|
|
286
|
+
for (let index = targets.length - 1; index >= 0; index -= 1) {
|
|
287
|
+
if (!state.has(targets[index]))
|
|
288
|
+
stack.push({ entryId: targets[index], expanded: false });
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return { orderedEntryIds, edges, diagnostics };
|
|
293
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { decodeCanonicalBlob } from './decode.js';
|
|
3
|
+
import { classifyInventoryEntry } from './inventory-rules.js';
|
|
4
|
+
import { collectionDigest, encodeCollection } from './pages.js';
|
|
5
|
+
import { redactCanonicalText } from './redaction.js';
|
|
6
|
+
function entryId(pathBase64) {
|
|
7
|
+
return `entry_${createHash('sha256').update(pathBase64).digest('hex').slice(0, 32)}`;
|
|
8
|
+
}
|
|
9
|
+
/** Pure, source-free inventory metadata construction shared by both content hosts. */
|
|
10
|
+
export function collectInventory(input) {
|
|
11
|
+
const gitmodules = input.entries.find((entry) => entry.pathDisplay === '.gitmodules' && entry.mode !== '160000');
|
|
12
|
+
const gitmodulesBytes = gitmodules ? input.blobs.get(gitmodules.blobId)?.bytes : undefined;
|
|
13
|
+
const records = input.entries.map((entry) => {
|
|
14
|
+
const blob = input.blobs.get(entry.blobId);
|
|
15
|
+
const classification = classifyInventoryEntry({ ...entry, sourceId: input.sourceId,
|
|
16
|
+
bytes: blob?.bytes, gitmodulesBytes,
|
|
17
|
+
...(blob?.acquisitionFailure ? { acquisitionFailure: blob.acquisitionFailure } : {}) });
|
|
18
|
+
const findings = blob?.bytes && classification.encoding
|
|
19
|
+
? (() => {
|
|
20
|
+
const decoded = decodeCanonicalBlob(blob.bytes);
|
|
21
|
+
return decoded.status === 'decoded'
|
|
22
|
+
? redactCanonicalText({ text: decoded.text }).findings : [];
|
|
23
|
+
})()
|
|
24
|
+
: [];
|
|
25
|
+
return {
|
|
26
|
+
collectionId: input.collectionId,
|
|
27
|
+
collectionKind: 'inventory',
|
|
28
|
+
semanticId: entryId(entry.pathBase64),
|
|
29
|
+
recordKind: 'entry',
|
|
30
|
+
payload: {
|
|
31
|
+
sourceId: input.sourceId,
|
|
32
|
+
pathBase64: entry.pathBase64,
|
|
33
|
+
pathDisplay: entry.pathDisplay,
|
|
34
|
+
mode: entry.mode,
|
|
35
|
+
blobId: entry.blobId,
|
|
36
|
+
bytes: classification.bytes,
|
|
37
|
+
encoding: classification.encoding,
|
|
38
|
+
bomBytes: classification.bomBytes,
|
|
39
|
+
lineCount: classification.lineCount,
|
|
40
|
+
laneId: null,
|
|
41
|
+
state: classification.state,
|
|
42
|
+
subState: classification.subState,
|
|
43
|
+
proof: classification.proof,
|
|
44
|
+
ownerDecision: null,
|
|
45
|
+
secretFindings: findings,
|
|
46
|
+
...(classification.generatedCandidate
|
|
47
|
+
? { generatedCandidate: classification.generatedCandidate } : {}),
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
});
|
|
51
|
+
const pages = [...encodeCollection(records)];
|
|
52
|
+
return { records, pages, digest: collectionDigest(pages, {
|
|
53
|
+
collectionId: input.collectionId, kind: 'inventory',
|
|
54
|
+
}) };
|
|
55
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { decodeCanonicalBlob } from './decode.js';
|
|
2
|
+
import { detectGeneratedEntry } from './generated-rules.js';
|
|
3
|
+
import { INVENTORY_RULES_VERSION, } from './types.js';
|
|
4
|
+
export { INVENTORY_RULES_VERSION } from './types.js';
|
|
5
|
+
const UTF8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
6
|
+
const LFS_POINTER = /^version https:\/\/git-lfs\.github\.com\/spec\/v1\noid sha256:[0-9a-f]{64}\nsize (?:0|[1-9][0-9]*)\n$/;
|
|
7
|
+
const startsAt = (bytes, offset, values) => values.every((value, index) => bytes[offset + index] === value);
|
|
8
|
+
const asciiAt = (bytes, offset, value) => startsAt(bytes, offset, [...value].map((character) => character.charCodeAt(0)));
|
|
9
|
+
const u32beAt = (bytes, offset) => offset + 4 <= bytes.length
|
|
10
|
+
? ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0
|
|
11
|
+
: null;
|
|
12
|
+
/** Closed binary header table. Callers must already have observed a raw NUL in the first 8,000 bytes. */
|
|
13
|
+
export const BINARY_SIGNATURES = [
|
|
14
|
+
{ id: 'png', matches: (b) => startsAt(b, 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) },
|
|
15
|
+
{ id: 'jpeg', matches: (b) => startsAt(b, 0, [0xff, 0xd8, 0xff]) },
|
|
16
|
+
{ id: 'gif', matches: (b) => asciiAt(b, 0, 'GIF87a') || asciiAt(b, 0, 'GIF89a') },
|
|
17
|
+
// RIFF size is at offset 4; the mandatory WebP form type is at offset 8.
|
|
18
|
+
{ id: 'webp', matches: (b) => asciiAt(b, 0, 'RIFF') && asciiAt(b, 8, 'WEBP') },
|
|
19
|
+
// ICO/CUR reserved field is 0 at offset 0; type at offset 2 is 1 or 2.
|
|
20
|
+
{ id: 'ico', matches: (b) => startsAt(b, 0, [0, 0, 1, 0]) || startsAt(b, 0, [0, 0, 2, 0]) },
|
|
21
|
+
{ id: 'pdf', matches: (b) => asciiAt(b, 0, '%PDF-') && b[5] !== undefined && b[5] >= 0x31 && b[5] <= 0x39 },
|
|
22
|
+
{ id: 'woff', matches: (b) => asciiAt(b, 0, 'wOFF') },
|
|
23
|
+
{ id: 'woff2', matches: (b) => asciiAt(b, 0, 'wOF2') },
|
|
24
|
+
// TrueType's sfnt version is 0x00010000; OpenType's is the literal OTTO.
|
|
25
|
+
{ id: 'ttf', matches: (b) => startsAt(b, 0, [0, 1, 0, 0]) && b.length >= 12 },
|
|
26
|
+
{ id: 'otf', matches: (b) => asciiAt(b, 0, 'OTTO') && b.length >= 12 },
|
|
27
|
+
// ZIP's local, central-directory, and empty-archive signatures also cover JAR archives.
|
|
28
|
+
{ id: 'zip_or_jar', matches: (b) => asciiAt(b, 0, 'PK\x03\x04') || asciiAt(b, 0, 'PK\x05\x06') || asciiAt(b, 0, 'PK\x07\x08') },
|
|
29
|
+
{ id: 'gzip', matches: (b) => startsAt(b, 0, [0x1f, 0x8b, 0x08]) && (b[3] ?? 0xff) <= 0x1f },
|
|
30
|
+
{ id: 'sqlite', matches: (b) => asciiAt(b, 0, 'SQLite format 3\0') },
|
|
31
|
+
{ id: 'mp3_id3', matches: (b) => asciiAt(b, 0, 'ID3') && (b[3] === 2 || b[3] === 3 || b[3] === 4) },
|
|
32
|
+
{ id: 'mp3_frame', matches: (b) => b[0] === 0xff && ((b[1] ?? 0) & 0xe0) === 0xe0 && ((b[1] ?? 0) & 0x06) !== 0 },
|
|
33
|
+
// ISO base media requires a box length at offset 0 and ftyp at offset 4.
|
|
34
|
+
{ id: 'mp4', matches: (b) => (u32beAt(b, 0) ?? 0) >= 16 && asciiAt(b, 4, 'ftyp') },
|
|
35
|
+
// EBML ID at offset 0; DocType's element ID is 0x4282 and its four-byte value is webm.
|
|
36
|
+
{ id: 'webm', matches: (b) => startsAt(b, 0, [0x1a, 0x45, 0xdf, 0xa3]) && b.some((_, index) => index < 121 && startsAt(b, index, [0x42, 0x82, 0x84]) && asciiAt(b, index + 3, 'webm')) },
|
|
37
|
+
{ id: 'wasm', matches: (b) => startsAt(b, 0, [0, 0x61, 0x73, 0x6d, 1, 0, 0, 0]) },
|
|
38
|
+
];
|
|
39
|
+
function proof(input, ruleId, facts) {
|
|
40
|
+
return { ruleId, rulesVersion: INVENTORY_RULES_VERSION, sourceId: input.sourceId, blobId: input.blobId, facts };
|
|
41
|
+
}
|
|
42
|
+
function unresolved(bytes, subState, input, facts = {}) {
|
|
43
|
+
return {
|
|
44
|
+
state: 'unresolved', subState, bytes, encoding: null, bomBytes: null, lineCount: null,
|
|
45
|
+
proof: proof(input, subState, facts),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function safeGitmodulesUrl(bytes, path) {
|
|
49
|
+
if (!bytes)
|
|
50
|
+
return null;
|
|
51
|
+
let text;
|
|
52
|
+
try {
|
|
53
|
+
text = UTF8.decode(bytes);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
let currentPath = null;
|
|
59
|
+
let currentUrl = null;
|
|
60
|
+
const found = [];
|
|
61
|
+
for (const line of text.split(/\r?\n/)) {
|
|
62
|
+
if (/^\[submodule "[^"\\]+"\]$/.test(line)) {
|
|
63
|
+
if (currentPath === path && currentUrl)
|
|
64
|
+
found.push(currentUrl);
|
|
65
|
+
currentPath = null;
|
|
66
|
+
currentUrl = null;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const field = /^(path|url) = ([^\r\n]+)$/.exec(line);
|
|
70
|
+
if (!field)
|
|
71
|
+
continue;
|
|
72
|
+
if (field[1] === 'path')
|
|
73
|
+
currentPath = field[2];
|
|
74
|
+
else
|
|
75
|
+
currentUrl = field[2];
|
|
76
|
+
}
|
|
77
|
+
if (currentPath === path && currentUrl)
|
|
78
|
+
found.push(currentUrl);
|
|
79
|
+
if (found.length !== 1 || /[\x00-\x1f\s]/.test(found[0]) || /^[a-z][a-z0-9+.-]*:\/\/[^/]*@/i.test(found[0]))
|
|
80
|
+
return null;
|
|
81
|
+
return found[0];
|
|
82
|
+
}
|
|
83
|
+
/** Applies the §3.3 precedence without touching a worktree, filesystem, or network. */
|
|
84
|
+
export function classifyInventoryEntry(input) {
|
|
85
|
+
if (input.mode === '160000') {
|
|
86
|
+
const locator = safeGitmodulesUrl(input.gitmodulesBytes, input.pathDisplay);
|
|
87
|
+
return {
|
|
88
|
+
state: 'set_aside', subState: null, bytes: null, encoding: null, bomBytes: null, lineCount: null,
|
|
89
|
+
proof: proof(input, 'gitlink', { targetCommit: input.blobId, gitmodulesLocator: locator }),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
if (!input.bytes)
|
|
93
|
+
return unresolved(null, 'failed', input, input.acquisitionFailure ?? { code: 'blob_unavailable', source: 'acquisition' });
|
|
94
|
+
if (input.mode === '120000') {
|
|
95
|
+
let targetPath = null;
|
|
96
|
+
try {
|
|
97
|
+
targetPath = UTF8.decode(input.bytes);
|
|
98
|
+
}
|
|
99
|
+
catch { /* metadata remains honestly unavailable */ }
|
|
100
|
+
return {
|
|
101
|
+
state: 'set_aside', subState: null, bytes: input.bytes.length, encoding: targetPath === null ? null : 'utf8',
|
|
102
|
+
bomBytes: targetPath === null ? null : 0, lineCount: null,
|
|
103
|
+
proof: proof(input, 'symlink', { targetPath, targetEncoding: targetPath === null ? 'unsupported' : 'utf8' }),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
const bytes = input.bytes;
|
|
107
|
+
if (bytes.length === 0) {
|
|
108
|
+
return { state: 'set_aside', subState: null, bytes: 0, encoding: null, bomBytes: null,
|
|
109
|
+
lineCount: null, proof: proof(input, 'empty', { byteLength: 0 }) };
|
|
110
|
+
}
|
|
111
|
+
let text = null;
|
|
112
|
+
try {
|
|
113
|
+
text = UTF8.decode(bytes);
|
|
114
|
+
}
|
|
115
|
+
catch { /* decoding below records the actual reason */ }
|
|
116
|
+
if (text !== null && LFS_POINTER.test(text)) {
|
|
117
|
+
return { state: 'set_aside', subState: null, bytes: bytes.length, encoding: 'utf8', bomBytes: 0,
|
|
118
|
+
lineCount: text.split('\n').length, proof: proof(input, 'lfs_pointer', { byteLength: bytes.length }) };
|
|
119
|
+
}
|
|
120
|
+
const decoded = decodeCanonicalBlob(bytes);
|
|
121
|
+
if (decoded.status === 'decoded') {
|
|
122
|
+
const classification = {
|
|
123
|
+
state: 'unresolved', subState: 'pending', bytes: bytes.length, encoding: decoded.encoding,
|
|
124
|
+
bomBytes: decoded.bom?.endByte ?? 0, lineCount: decoded.lineCount, proof: null,
|
|
125
|
+
};
|
|
126
|
+
const generatedCandidate = detectGeneratedEntry({ text: decoded.text });
|
|
127
|
+
return generatedCandidate === null ? classification : { ...classification, generatedCandidate };
|
|
128
|
+
}
|
|
129
|
+
if (decoded.status === 'decode_failed')
|
|
130
|
+
return unresolved(bytes.length, 'decode_failed', input, { encoding: decoded.encoding, bomBytes: decoded.bom.endByte });
|
|
131
|
+
if (decoded.status === 'nul_bytes') {
|
|
132
|
+
const signature = BINARY_SIGNATURES.find((candidate) => candidate.matches(bytes));
|
|
133
|
+
if (signature) {
|
|
134
|
+
return { state: 'set_aside', subState: null, bytes: bytes.length, encoding: null, bomBytes: null,
|
|
135
|
+
lineCount: null, proof: proof(input, 'binary', { signature: signature.id,
|
|
136
|
+
firstNulByte: decoded.firstNulByte, byteLength: bytes.length }) };
|
|
137
|
+
}
|
|
138
|
+
return unresolved(bytes.length, 'unknown_format', input, { firstNulByte: decoded.firstNulByte });
|
|
139
|
+
}
|
|
140
|
+
return unresolved(bytes.length, 'unsupported_encoding', input, { byteLength: bytes.length });
|
|
141
|
+
}
|