@topogram/cli 0.3.85 → 0.3.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/archive/unarchive.js +56 -5
- package/src/catalog/source.js +52 -1
- package/src/cli/commands/emit/snapshot-input.js +111 -0
- package/src/cli/commands/emit.js +11 -1
- package/src/cli/commands/extractor.js +150 -13
- package/src/cli/commands/import/plan.js +101 -4
- package/src/cli/commands/query/workspace.js +2 -67
- package/src/extraction-context.js +79 -0
- package/src/extractor/packages.js +9 -2
- package/src/github-client.js +56 -2
- package/src/import/core/shared/files.js +21 -0
- package/src/remote-payload-limits.js +40 -0
- package/src/template-trust/constants.js +1 -1
- package/src/template-trust/policy.js +3 -4
- package/src/template-trust/status.js +2 -4
- package/src/topogram-config.js +47 -0
- package/src/workflows/reconcile/adoption-plan/outputs.js +31 -4
- package/src/workflows/shared.js +21 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
|
|
6
|
+
import { readExtractionContext } from "../../../extraction-context.js";
|
|
6
7
|
import { runWorkflow } from "../../../workflows.js";
|
|
7
8
|
import {
|
|
8
9
|
countByField,
|
|
@@ -60,6 +61,73 @@ export const BROWNFIELD_BROAD_ADOPT_SELECTORS = [
|
|
|
60
61
|
}
|
|
61
62
|
];
|
|
62
63
|
|
|
64
|
+
/**
|
|
65
|
+
* @param {AnyRecord|null|undefined} extractionContext
|
|
66
|
+
* @param {AnyRecord[]} bundleSurfaces
|
|
67
|
+
* @param {string} bundleSlug
|
|
68
|
+
* @returns {string[]}
|
|
69
|
+
*/
|
|
70
|
+
function tracksForBundle(extractionContext, bundleSurfaces, bundleSlug) {
|
|
71
|
+
const tracks = new Set(bundleSurfaces.map((surface) => surface.track).filter(Boolean));
|
|
72
|
+
if (bundleSlug === "database" || bundleSlug.includes("db")) tracks.add("db");
|
|
73
|
+
if (bundleSlug === "cli") tracks.add("cli");
|
|
74
|
+
if (bundleSlug === "ui") tracks.add("ui");
|
|
75
|
+
if (bundleSlug.includes("api")) tracks.add("api");
|
|
76
|
+
const knownTracks = new Set(Array.isArray(extractionContext?.tracks) ? extractionContext.tracks : []);
|
|
77
|
+
return [...tracks].filter((track) => knownTracks.size === 0 || knownTracks.has(track)).sort((left, right) => left.localeCompare(right));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {AnyRecord} extractor
|
|
82
|
+
* @param {Set<string>} tracks
|
|
83
|
+
* @returns {boolean}
|
|
84
|
+
*/
|
|
85
|
+
function extractorMatchesTracks(extractor, tracks) {
|
|
86
|
+
const extractorTracks = Array.isArray(extractor.tracks) ? extractor.tracks : [];
|
|
87
|
+
return tracks.size === 0 || extractorTracks.length === 0 || extractorTracks.some((track) => tracks.has(track));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* @param {AnyRecord|null|undefined} extractionContext
|
|
92
|
+
* @param {AnyRecord[]} bundleSurfaces
|
|
93
|
+
* @param {string} bundleSlug
|
|
94
|
+
* @returns {AnyRecord|null}
|
|
95
|
+
*/
|
|
96
|
+
function extractorContextForBundle(extractionContext, bundleSurfaces, bundleSlug) {
|
|
97
|
+
if (!extractionContext) {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
const tracks = tracksForBundle(extractionContext, bundleSurfaces, bundleSlug);
|
|
101
|
+
const trackSet = new Set(tracks);
|
|
102
|
+
const packageBackedExtractors = (extractionContext.package_backed_extractors || [])
|
|
103
|
+
.filter((/** @type {AnyRecord} */ extractor) => extractorMatchesTracks(extractor, trackSet))
|
|
104
|
+
.map((/** @type {AnyRecord} */ extractor) => ({
|
|
105
|
+
id: extractor.id || null,
|
|
106
|
+
version: extractor.version || null,
|
|
107
|
+
packageName: extractor.packageName || null,
|
|
108
|
+
extractors: Array.isArray(extractor.extractors) ? extractor.extractors : [],
|
|
109
|
+
tracks: Array.isArray(extractor.tracks) ? extractor.tracks : []
|
|
110
|
+
}));
|
|
111
|
+
const bundledExtractors = (extractionContext.bundled_extractors || [])
|
|
112
|
+
.filter((/** @type {AnyRecord} */ extractor) => extractorMatchesTracks(extractor, trackSet))
|
|
113
|
+
.map((/** @type {AnyRecord} */ extractor) => ({
|
|
114
|
+
id: extractor.id || null,
|
|
115
|
+
version: extractor.version || null,
|
|
116
|
+
extractors: Array.isArray(extractor.extractors) ? extractor.extractors : [],
|
|
117
|
+
tracks: Array.isArray(extractor.tracks) ? extractor.tracks : []
|
|
118
|
+
}));
|
|
119
|
+
if (packageBackedExtractors.length === 0 && bundledExtractors.length === 0) {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
tracks,
|
|
124
|
+
packageBackedExtractors,
|
|
125
|
+
bundledExtractors,
|
|
126
|
+
candidateCounts: extractionContext.candidate_counts || {},
|
|
127
|
+
safetyNotes: extractionContext.safety_notes || []
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
63
131
|
/**
|
|
64
132
|
* @param {string} inputPath
|
|
65
133
|
* @returns {AnyRecord}
|
|
@@ -84,7 +152,8 @@ export function readImportAdoptionArtifacts(inputPath) {
|
|
|
84
152
|
paths,
|
|
85
153
|
adoptionPlan: JSON.parse(fs.readFileSync(paths.adoptionPlanAgent, "utf8")),
|
|
86
154
|
adoptionStatus: readJsonIfExists(paths.adoptionStatus),
|
|
87
|
-
reconcileReport: readJsonIfExists(paths.reconcileReport)
|
|
155
|
+
reconcileReport: readJsonIfExists(paths.reconcileReport),
|
|
156
|
+
extractionContext: readExtractionContext(topogramRoot)
|
|
88
157
|
};
|
|
89
158
|
}
|
|
90
159
|
|
|
@@ -118,9 +187,10 @@ export function buildBrownfieldBroadAdoptSelectors(projectRoot, adoptionPlan) {
|
|
|
118
187
|
* @param {AnyRecord} adoptionPlan
|
|
119
188
|
* @param {AnyRecord} adoptionStatus
|
|
120
189
|
* @param {string} projectRoot
|
|
190
|
+
* @param {AnyRecord|null|undefined} extractionContext
|
|
121
191
|
* @returns {AnyRecord}
|
|
122
192
|
*/
|
|
123
|
-
export function summarizeImportAdoption(adoptionPlan, adoptionStatus, projectRoot) {
|
|
193
|
+
export function summarizeImportAdoption(adoptionPlan, adoptionStatus, projectRoot, extractionContext = null) {
|
|
124
194
|
const surfaces = adoptionPlan.imported_proposal_surfaces || [];
|
|
125
195
|
/** @type {string[]} */
|
|
126
196
|
const slugs = [];
|
|
@@ -162,7 +232,8 @@ export function summarizeImportAdoption(adoptionPlan, adoptionStatus, projectRoo
|
|
|
162
232
|
complete: Boolean(priority?.is_complete) || (pendingItems.length === 0 && blockedItems.length === 0 && appliedItems.length > 0),
|
|
163
233
|
evidenceScore: priority?.evidence_score || 0,
|
|
164
234
|
why: priority?.operator_summary?.whyThisBundle || null,
|
|
165
|
-
nextCommand: importAdoptCommand(projectRoot, `bundle:${slug}`, false)
|
|
235
|
+
nextCommand: importAdoptCommand(projectRoot, `bundle:${slug}`, false),
|
|
236
|
+
extractorContext: extractorContextForBundle(extractionContext, bundleSurfaces, slug)
|
|
166
237
|
};
|
|
167
238
|
});
|
|
168
239
|
const nextBundle = bundles.find((bundle) => !bundle.complete && bundle.pendingItemCount > 0) || bundles.find((bundle) => !bundle.complete) || bundles[0] || null;
|
|
@@ -196,7 +267,7 @@ export function summarizeImportAdoption(adoptionPlan, adoptionStatus, projectRoo
|
|
|
196
267
|
export function buildBrownfieldImportPlanPayload(inputPath) {
|
|
197
268
|
const artifacts = readImportAdoptionArtifacts(inputPath);
|
|
198
269
|
const adoptionStatus = runWorkflow("adoption-status", artifacts.projectRoot).summary || artifacts.adoptionStatus || {};
|
|
199
|
-
const adoption = summarizeImportAdoption(artifacts.adoptionPlan, adoptionStatus, artifacts.projectRoot);
|
|
270
|
+
const adoption = summarizeImportAdoption(artifacts.adoptionPlan, adoptionStatus, artifacts.projectRoot, artifacts.extractionContext);
|
|
200
271
|
return {
|
|
201
272
|
ok: true,
|
|
202
273
|
projectRoot: artifacts.projectRoot,
|
|
@@ -207,6 +278,14 @@ export function buildBrownfieldImportPlanPayload(inputPath) {
|
|
|
207
278
|
adoptionStatus: artifacts.paths.adoptionStatus,
|
|
208
279
|
reconcileReport: artifacts.paths.reconcileReport
|
|
209
280
|
},
|
|
281
|
+
extractorContext: artifacts.extractionContext ? {
|
|
282
|
+
provenancePath: artifacts.extractionContext.provenance_path,
|
|
283
|
+
packageBackedExtractors: artifacts.extractionContext.package_backed_extractors,
|
|
284
|
+
bundledExtractors: artifacts.extractionContext.bundled_extractors,
|
|
285
|
+
candidateCounts: artifacts.extractionContext.candidate_counts,
|
|
286
|
+
safetyNotes: artifacts.extractionContext.safety_notes,
|
|
287
|
+
summary: artifacts.extractionContext.summary
|
|
288
|
+
} : null,
|
|
210
289
|
...adoption,
|
|
211
290
|
commands: {
|
|
212
291
|
check: `topogram extract check ${importProjectCommandPath(artifacts.projectRoot)}`,
|
|
@@ -229,6 +308,14 @@ export function printBrownfieldImportPlan(payload) {
|
|
|
229
308
|
if (bundle.why) {
|
|
230
309
|
console.log(` ${bundle.why}`);
|
|
231
310
|
}
|
|
311
|
+
if (bundle.extractorContext?.packageBackedExtractors?.length > 0) {
|
|
312
|
+
const names = bundle.extractorContext.packageBackedExtractors
|
|
313
|
+
.map((/** @type {AnyRecord} */ extractor) => extractor.packageName || extractor.id)
|
|
314
|
+
.filter(Boolean)
|
|
315
|
+
.join(", ");
|
|
316
|
+
console.log(` Extractors: ${names}`);
|
|
317
|
+
console.log(" Safety: package-backed extractor candidates are review-only; run dry-run adoption before --write.");
|
|
318
|
+
}
|
|
232
319
|
console.log(` Preview: ${bundle.nextCommand}`);
|
|
233
320
|
}
|
|
234
321
|
if (payload.risks.length > 0) {
|
|
@@ -257,6 +344,7 @@ export function buildBrownfieldImportAdoptListPayload(inputPath) {
|
|
|
257
344
|
appliedItemCount: bundle.appliedItemCount,
|
|
258
345
|
blockedItemCount: bundle.blockedItemCount,
|
|
259
346
|
complete: bundle.complete,
|
|
347
|
+
extractorContext: bundle.extractorContext || null,
|
|
260
348
|
previewCommand: importAdoptCommand(plan.projectRoot, `bundle:${bundle.bundle}`, false),
|
|
261
349
|
writeCommand: importAdoptCommand(plan.projectRoot, `bundle:${bundle.bundle}`, true)
|
|
262
350
|
}));
|
|
@@ -270,6 +358,7 @@ export function buildBrownfieldImportAdoptListPayload(inputPath) {
|
|
|
270
358
|
selectors,
|
|
271
359
|
broadSelectorCount: broadSelectors.length,
|
|
272
360
|
broadSelectors,
|
|
361
|
+
extractorContext: plan.extractorContext,
|
|
273
362
|
nextCommand: selectors.find((/** @type {AnyRecord} */ selector) => !selector.complete)?.previewCommand || plan.commands.status
|
|
274
363
|
};
|
|
275
364
|
}
|
|
@@ -286,6 +375,14 @@ export function printBrownfieldImportAdoptList(payload) {
|
|
|
286
375
|
}
|
|
287
376
|
for (const selector of payload.selectors) {
|
|
288
377
|
console.log(`- ${selector.selector}: ${selector.itemCount} item(s), ${selector.pendingItemCount} pending, ${selector.appliedItemCount} applied`);
|
|
378
|
+
if (selector.extractorContext?.packageBackedExtractors?.length > 0) {
|
|
379
|
+
const names = selector.extractorContext.packageBackedExtractors
|
|
380
|
+
.map((/** @type {AnyRecord} */ extractor) => extractor.packageName || extractor.id)
|
|
381
|
+
.filter(Boolean)
|
|
382
|
+
.join(", ");
|
|
383
|
+
console.log(` Extractors: ${names}`);
|
|
384
|
+
console.log(" Safety: package-backed extractor candidates are review-only; run dry-run adoption before --write.");
|
|
385
|
+
}
|
|
289
386
|
console.log(` Preview: ${selector.previewCommand}`);
|
|
290
387
|
console.log(` Write: ${selector.writeCommand}`);
|
|
291
388
|
}
|
|
@@ -4,9 +4,9 @@ import fs from "node:fs";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
|
|
6
6
|
import { generateWorkspace } from "../../../generator.js";
|
|
7
|
-
import { TOPOGRAM_IMPORT_FILE } from "../../../import/provenance.js";
|
|
8
7
|
import { formatValidationErrors } from "../../../validator.js";
|
|
9
8
|
import { buildChangePlanPayload } from "../../../agent-ops/query-builders.js";
|
|
9
|
+
import { buildExtractionContext, readExtractionContext } from "../../../extraction-context.js";
|
|
10
10
|
import { resolveTopoRoot } from "../../../workspace-paths.js";
|
|
11
11
|
|
|
12
12
|
/**
|
|
@@ -197,72 +197,7 @@ export function readJson(filePath) {
|
|
|
197
197
|
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
-
|
|
201
|
-
* @param {AnyRecord} record
|
|
202
|
-
* @param {string} provenancePath
|
|
203
|
-
* @returns {AnyRecord}
|
|
204
|
-
*/
|
|
205
|
-
export function buildExtractionContext(record, provenancePath) {
|
|
206
|
-
const extractorPackages = /** @type {AnyRecord[]} */ (Array.isArray(record.extract?.extractorPackages)
|
|
207
|
-
? record.extract.extractorPackages
|
|
208
|
-
: []);
|
|
209
|
-
const packageBackedExtractors = extractorPackages
|
|
210
|
-
.filter((entry) => entry?.source === "package")
|
|
211
|
-
.map((entry) => ({
|
|
212
|
-
id: entry.id || null,
|
|
213
|
-
version: entry.version || null,
|
|
214
|
-
packageName: entry.packageName || null,
|
|
215
|
-
extractors: Array.isArray(entry.extractors) ? entry.extractors : [],
|
|
216
|
-
manifestPath: entry.manifestPath || null
|
|
217
|
-
}));
|
|
218
|
-
const bundledExtractors = extractorPackages
|
|
219
|
-
.filter((entry) => entry?.source === "bundled")
|
|
220
|
-
.map((entry) => ({
|
|
221
|
-
id: entry.id || null,
|
|
222
|
-
version: entry.version || null,
|
|
223
|
-
extractors: Array.isArray(entry.extractors) ? entry.extractors : []
|
|
224
|
-
}));
|
|
225
|
-
return {
|
|
226
|
-
type: "extraction_context",
|
|
227
|
-
provenance_path: provenancePath,
|
|
228
|
-
kind: record.kind || null,
|
|
229
|
-
extracted_at: record.extractedAt || null,
|
|
230
|
-
refreshed_at: record.refreshedAt || null,
|
|
231
|
-
source_path: record.source?.path || null,
|
|
232
|
-
tracks: Array.isArray(record.extract?.tracks) ? record.extract.tracks : [],
|
|
233
|
-
findings_count: record.extract?.findingsCount || 0,
|
|
234
|
-
candidate_counts: record.extract?.candidateCounts || {},
|
|
235
|
-
package_backed_extractors: packageBackedExtractors,
|
|
236
|
-
bundled_extractors: bundledExtractors,
|
|
237
|
-
summary: {
|
|
238
|
-
package_backed_extractor_count: packageBackedExtractors.length,
|
|
239
|
-
bundled_extractor_count: bundledExtractors.length,
|
|
240
|
-
source_file_count: Array.isArray(record.files) ? record.files.length : 0
|
|
241
|
-
},
|
|
242
|
-
next_commands: [
|
|
243
|
-
"topogram extract check",
|
|
244
|
-
"topogram extract plan",
|
|
245
|
-
"topogram adopt --list",
|
|
246
|
-
"topogram adopt <selector> --dry-run"
|
|
247
|
-
],
|
|
248
|
-
safety_notes: [
|
|
249
|
-
"Extractor packages are evidence producers only; review candidates before canonical adoption.",
|
|
250
|
-
"Use dry-run adoption before --write, especially when package-backed extractors contributed candidates."
|
|
251
|
-
]
|
|
252
|
-
};
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* @param {string} topogramRoot
|
|
257
|
-
* @returns {AnyRecord|null}
|
|
258
|
-
*/
|
|
259
|
-
export function readExtractionContext(topogramRoot) {
|
|
260
|
-
const provenancePath = path.join(path.dirname(topogramRoot), TOPOGRAM_IMPORT_FILE);
|
|
261
|
-
if (!fs.existsSync(provenancePath)) {
|
|
262
|
-
return null;
|
|
263
|
-
}
|
|
264
|
-
return buildExtractionContext(readJson(provenancePath), provenancePath);
|
|
265
|
-
}
|
|
200
|
+
export { buildExtractionContext, readExtractionContext };
|
|
266
201
|
|
|
267
202
|
/**
|
|
268
203
|
* @param {AnyRecord} options
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { TOPOGRAM_IMPORT_FILE } from "./import/provenance.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* @typedef {Record<string, any>} AnyRecord
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @param {AnyRecord} record
|
|
14
|
+
* @param {string} provenancePath
|
|
15
|
+
* @returns {AnyRecord}
|
|
16
|
+
*/
|
|
17
|
+
export function buildExtractionContext(record, provenancePath) {
|
|
18
|
+
const extractorPackages = /** @type {AnyRecord[]} */ (Array.isArray(record.extract?.extractorPackages)
|
|
19
|
+
? record.extract.extractorPackages
|
|
20
|
+
: []);
|
|
21
|
+
const packageBackedExtractors = extractorPackages
|
|
22
|
+
.filter((entry) => entry?.source === "package")
|
|
23
|
+
.map((entry) => ({
|
|
24
|
+
id: entry.id || null,
|
|
25
|
+
version: entry.version || null,
|
|
26
|
+
packageName: entry.packageName || null,
|
|
27
|
+
extractors: Array.isArray(entry.extractors) ? entry.extractors : [],
|
|
28
|
+
tracks: Array.isArray(entry.tracks) ? entry.tracks : [],
|
|
29
|
+
manifestPath: entry.manifestPath || null
|
|
30
|
+
}));
|
|
31
|
+
const bundledExtractors = extractorPackages
|
|
32
|
+
.filter((entry) => entry?.source === "bundled")
|
|
33
|
+
.map((entry) => ({
|
|
34
|
+
id: entry.id || null,
|
|
35
|
+
version: entry.version || null,
|
|
36
|
+
extractors: Array.isArray(entry.extractors) ? entry.extractors : [],
|
|
37
|
+
tracks: Array.isArray(entry.tracks) ? entry.tracks : []
|
|
38
|
+
}));
|
|
39
|
+
return {
|
|
40
|
+
type: "extraction_context",
|
|
41
|
+
provenance_path: provenancePath,
|
|
42
|
+
kind: record.kind || null,
|
|
43
|
+
extracted_at: record.extractedAt || null,
|
|
44
|
+
refreshed_at: record.refreshedAt || null,
|
|
45
|
+
source_path: record.source?.path || null,
|
|
46
|
+
tracks: Array.isArray(record.extract?.tracks) ? record.extract.tracks : [],
|
|
47
|
+
findings_count: record.extract?.findingsCount || 0,
|
|
48
|
+
candidate_counts: record.extract?.candidateCounts || {},
|
|
49
|
+
package_backed_extractors: packageBackedExtractors,
|
|
50
|
+
bundled_extractors: bundledExtractors,
|
|
51
|
+
summary: {
|
|
52
|
+
package_backed_extractor_count: packageBackedExtractors.length,
|
|
53
|
+
bundled_extractor_count: bundledExtractors.length,
|
|
54
|
+
source_file_count: Array.isArray(record.files) ? record.files.length : 0
|
|
55
|
+
},
|
|
56
|
+
next_commands: [
|
|
57
|
+
"topogram extract check",
|
|
58
|
+
"topogram extract plan",
|
|
59
|
+
"topogram adopt --list",
|
|
60
|
+
"topogram adopt <selector> --dry-run"
|
|
61
|
+
],
|
|
62
|
+
safety_notes: [
|
|
63
|
+
"Extractor packages are evidence producers only; review candidates before canonical adoption.",
|
|
64
|
+
"Use dry-run adoption before --write, especially when package-backed extractors contributed candidates."
|
|
65
|
+
]
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* @param {string} topogramRoot
|
|
71
|
+
* @returns {AnyRecord|null}
|
|
72
|
+
*/
|
|
73
|
+
export function readExtractionContext(topogramRoot) {
|
|
74
|
+
const provenancePath = path.join(path.dirname(topogramRoot), TOPOGRAM_IMPORT_FILE);
|
|
75
|
+
if (!fs.existsSync(provenancePath)) {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
return buildExtractionContext(JSON.parse(fs.readFileSync(provenancePath, "utf8")), provenancePath);
|
|
79
|
+
}
|
|
@@ -228,13 +228,19 @@ export function packageExtractorsForContext(context) {
|
|
|
228
228
|
const bundledPack = getBundledExtractorPack(spec);
|
|
229
229
|
if (bundledPack) {
|
|
230
230
|
extractors.push(...bundledPack.extractors);
|
|
231
|
-
provenance.push({
|
|
231
|
+
provenance.push({
|
|
232
|
+
source: "bundled",
|
|
233
|
+
id: bundledPack.manifest.id,
|
|
234
|
+
version: bundledPack.manifest.version,
|
|
235
|
+
tracks: bundledPack.manifest.tracks || [],
|
|
236
|
+
extractors: bundledPack.manifest.extractors
|
|
237
|
+
});
|
|
232
238
|
continue;
|
|
233
239
|
}
|
|
234
240
|
const bundledExtractor = getBundledExtractorById(spec);
|
|
235
241
|
if (bundledExtractor) {
|
|
236
242
|
extractors.push(bundledExtractor);
|
|
237
|
-
provenance.push({ source: "bundled", id: bundledExtractor.id, version: "1", extractors: [bundledExtractor.id] });
|
|
243
|
+
provenance.push({ source: "bundled", id: bundledExtractor.id, version: "1", tracks: bundledExtractor.track ? [bundledExtractor.track] : [], extractors: [bundledExtractor.id] });
|
|
238
244
|
continue;
|
|
239
245
|
}
|
|
240
246
|
const packageManifest = loadExtractorPackageManifestForSpec(spec, { cwd });
|
|
@@ -273,6 +279,7 @@ export function packageExtractorsForContext(context) {
|
|
|
273
279
|
id: packageManifest.manifest.id,
|
|
274
280
|
version: packageManifest.manifest.version,
|
|
275
281
|
packageName,
|
|
282
|
+
tracks: packageManifest.manifest.tracks || [],
|
|
276
283
|
manifestPath: packageManifest.manifestPath,
|
|
277
284
|
packageRoot: packageManifest.packageRoot,
|
|
278
285
|
extractors: packageManifest.manifest.extractors
|
package/src/github-client.js
CHANGED
|
@@ -2,9 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
import childProcess from "node:child_process";
|
|
4
4
|
|
|
5
|
+
import { remotePayloadMaxBytes } from "./remote-payload-limits.js";
|
|
6
|
+
|
|
5
7
|
const DEFAULT_GITHUB_API_BASE_URL = "https://api.github.com";
|
|
6
8
|
const GITHUB_REST_SCRIPT = `
|
|
7
9
|
const request = JSON.parse(process.argv[1]);
|
|
10
|
+
const maxBytes = Number.parseInt(String(request.maxBytes || ""), 10) || 5242880;
|
|
8
11
|
const base = String(request.baseUrl || "https://api.github.com").replace(/\\/+$/, "") + "/";
|
|
9
12
|
const path = String(request.path || "").replace(/^\\/+/, "");
|
|
10
13
|
const url = new URL(path, base);
|
|
@@ -24,6 +27,39 @@ const headers = {
|
|
|
24
27
|
if (request.token && canAttachToken(url)) {
|
|
25
28
|
headers.authorization = "Bearer " + request.token;
|
|
26
29
|
}
|
|
30
|
+
async function readResponseText(response) {
|
|
31
|
+
const declaredLength = Number.parseInt(response.headers.get("content-length") || "", 10);
|
|
32
|
+
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
|
33
|
+
throw new Error("GitHub REST response exceeded " + maxBytes + " byte limit.");
|
|
34
|
+
}
|
|
35
|
+
if (!response.body) {
|
|
36
|
+
const text = await response.text();
|
|
37
|
+
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
38
|
+
throw new Error("GitHub REST response exceeded " + maxBytes + " byte limit.");
|
|
39
|
+
}
|
|
40
|
+
return text;
|
|
41
|
+
}
|
|
42
|
+
const reader = response.body.getReader();
|
|
43
|
+
const decoder = new TextDecoder();
|
|
44
|
+
const chunks = [];
|
|
45
|
+
let total = 0;
|
|
46
|
+
while (true) {
|
|
47
|
+
const { value, done } = await reader.read();
|
|
48
|
+
if (done) {
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
total += value.byteLength;
|
|
52
|
+
if (total > maxBytes) {
|
|
53
|
+
try {
|
|
54
|
+
await reader.cancel();
|
|
55
|
+
} catch {}
|
|
56
|
+
throw new Error("GitHub REST response exceeded " + maxBytes + " byte limit.");
|
|
57
|
+
}
|
|
58
|
+
chunks.push(decoder.decode(value, { stream: true }));
|
|
59
|
+
}
|
|
60
|
+
chunks.push(decoder.decode());
|
|
61
|
+
return chunks.join("");
|
|
62
|
+
}
|
|
27
63
|
if (process.env.TOPOGRAM_GITHUB_API_FIXTURE_ROOT) {
|
|
28
64
|
const fs = await import("node:fs");
|
|
29
65
|
const pathModule = await import("node:path");
|
|
@@ -50,6 +86,11 @@ if (process.env.TOPOGRAM_GITHUB_API_FIXTURE_ROOT) {
|
|
|
50
86
|
}));
|
|
51
87
|
process.exit(2);
|
|
52
88
|
}
|
|
89
|
+
const fixtureSize = fs.statSync(fixturePath).size;
|
|
90
|
+
if (fixtureSize > maxBytes) {
|
|
91
|
+
process.stderr.write("GitHub REST fixture response exceeded " + maxBytes + " byte limit.");
|
|
92
|
+
process.exit(1);
|
|
93
|
+
}
|
|
53
94
|
process.stdout.write(JSON.stringify({
|
|
54
95
|
status: 200,
|
|
55
96
|
body: fs.readFileSync(fixturePath, "utf8"),
|
|
@@ -59,7 +100,7 @@ if (process.env.TOPOGRAM_GITHUB_API_FIXTURE_ROOT) {
|
|
|
59
100
|
}
|
|
60
101
|
try {
|
|
61
102
|
const response = await fetch(url, { headers });
|
|
62
|
-
const text = await response
|
|
103
|
+
const text = await readResponseText(response);
|
|
63
104
|
if (!response.ok) {
|
|
64
105
|
process.stderr.write(JSON.stringify({
|
|
65
106
|
status: response.status,
|
|
@@ -150,6 +191,11 @@ function shouldUseRestApi() {
|
|
|
150
191
|
* @returns {any}
|
|
151
192
|
*/
|
|
152
193
|
function githubRequestJson(path, options = {}) {
|
|
194
|
+
const maxBytes = remotePayloadMaxBytes(
|
|
195
|
+
["TOPOGRAM_GITHUB_FETCH_MAX_BYTES", "TOPOGRAM_REMOTE_FETCH_MAX_BYTES"],
|
|
196
|
+
undefined,
|
|
197
|
+
["githubFetchMaxBytes", "remoteFetchMaxBytes"]
|
|
198
|
+
);
|
|
153
199
|
const result = childProcess.spawnSync(process.execPath, [
|
|
154
200
|
"--input-type=module",
|
|
155
201
|
"-e",
|
|
@@ -158,10 +204,12 @@ function githubRequestJson(path, options = {}) {
|
|
|
158
204
|
baseUrl: githubApiBaseUrl(),
|
|
159
205
|
path,
|
|
160
206
|
query: options.query || {},
|
|
161
|
-
token: githubTokenFromEnv() || ""
|
|
207
|
+
token: githubTokenFromEnv() || "",
|
|
208
|
+
maxBytes
|
|
162
209
|
})
|
|
163
210
|
], {
|
|
164
211
|
encoding: "utf8",
|
|
212
|
+
maxBuffer: (maxBytes * 2) + 8192,
|
|
165
213
|
env: {
|
|
166
214
|
...process.env,
|
|
167
215
|
PATH: process.env.PATH || ""
|
|
@@ -479,9 +527,15 @@ function normalizeWorkflowJob(job) {
|
|
|
479
527
|
* @returns {ReturnType<typeof childProcess.spawnSync>}
|
|
480
528
|
*/
|
|
481
529
|
function runGh(args, cwd = process.cwd()) {
|
|
530
|
+
const maxBytes = remotePayloadMaxBytes(
|
|
531
|
+
["TOPOGRAM_GITHUB_FETCH_MAX_BYTES", "TOPOGRAM_REMOTE_FETCH_MAX_BYTES"],
|
|
532
|
+
undefined,
|
|
533
|
+
["githubFetchMaxBytes", "remoteFetchMaxBytes"]
|
|
534
|
+
);
|
|
482
535
|
return childProcess.spawnSync("gh", args, {
|
|
483
536
|
cwd,
|
|
484
537
|
encoding: "utf8",
|
|
538
|
+
maxBuffer: maxBytes + 4096,
|
|
485
539
|
env: {
|
|
486
540
|
...process.env,
|
|
487
541
|
GH_TOKEN: process.env.GH_TOKEN || process.env.GITHUB_TOKEN || "",
|
|
@@ -50,14 +50,35 @@ export function listFilesRecursive(rootDir, predicate = () => true, options = {}
|
|
|
50
50
|
return [];
|
|
51
51
|
}
|
|
52
52
|
const ignoredDirs = options.ignoredDirs || DEFAULT_IGNORED_DIRS;
|
|
53
|
+
let rootRealPath;
|
|
54
|
+
try {
|
|
55
|
+
rootRealPath = fs.realpathSync(rootDir);
|
|
56
|
+
} catch {
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
const visitedDirs = new Set([rootRealPath]);
|
|
53
60
|
const files = /** @type {any[]} */ ([]);
|
|
54
61
|
const walk = /** @param {any} currentDir */ (currentDir) => {
|
|
55
62
|
for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) {
|
|
56
63
|
const childPath = path.join(currentDir, entry.name);
|
|
64
|
+
if (entry.isSymbolicLink()) {
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
57
67
|
if (entry.isDirectory()) {
|
|
58
68
|
if (ignoredDirs.has(entry.name)) {
|
|
59
69
|
continue;
|
|
60
70
|
}
|
|
71
|
+
let childRealPath;
|
|
72
|
+
try {
|
|
73
|
+
childRealPath = fs.realpathSync(childPath);
|
|
74
|
+
} catch {
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
const relativeToRoot = path.relative(rootRealPath, childRealPath);
|
|
78
|
+
if (relativeToRoot.startsWith("..") || path.isAbsolute(relativeToRoot) || visitedDirs.has(childRealPath)) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
visitedDirs.add(childRealPath);
|
|
61
82
|
walk(childPath);
|
|
62
83
|
continue;
|
|
63
84
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import { topogramRuntimeConfig } from "./topogram-config.js";
|
|
4
|
+
|
|
5
|
+
export const DEFAULT_REMOTE_FETCH_MAX_BYTES = 5 * 1024 * 1024;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @param {string|null|undefined} value
|
|
9
|
+
* @returns {number|null}
|
|
10
|
+
*/
|
|
11
|
+
function parsePositiveInteger(value) {
|
|
12
|
+
if (!value) {
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
const parsed = Number.parseInt(String(value), 10);
|
|
16
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {string[]} envNames
|
|
21
|
+
* @param {number} [fallback]
|
|
22
|
+
* @param {Array<"remoteFetchMaxBytes"|"catalogFetchMaxBytes"|"githubFetchMaxBytes">} [configKeys]
|
|
23
|
+
* @returns {number}
|
|
24
|
+
*/
|
|
25
|
+
export function remotePayloadMaxBytes(envNames, fallback = DEFAULT_REMOTE_FETCH_MAX_BYTES, configKeys = []) {
|
|
26
|
+
for (const envName of envNames) {
|
|
27
|
+
const parsed = parsePositiveInteger(process.env[envName]);
|
|
28
|
+
if (parsed) {
|
|
29
|
+
return parsed;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const limits = topogramRuntimeConfig(process.cwd()).limits;
|
|
33
|
+
for (const configKey of configKeys) {
|
|
34
|
+
const parsed = parsePositiveInteger(String(limits[configKey] || ""));
|
|
35
|
+
if (parsed) {
|
|
36
|
+
return parsed;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return fallback;
|
|
40
|
+
}
|
|
@@ -35,7 +35,7 @@ export function unsupportedImplementationSymlinkMessage(relativePath) {
|
|
|
35
35
|
* @returns {string}
|
|
36
36
|
*/
|
|
37
37
|
export function implementationOutsideRootMessage(modulePath) {
|
|
38
|
-
return `Template implementation module '${modulePath}' must be under implementation
|
|
38
|
+
return `Template implementation module '${modulePath}' must be under implementation/. Keep executable template code inside implementation/ so the trust record covers what topogram generate may load. Move the module back under implementation/, then run ${TRUST_REVIEW_COMMANDS} after review.`;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
/**
|
|
@@ -43,10 +43,9 @@ export function projectHasTemplateAttachment(projectConfig) {
|
|
|
43
43
|
* @returns {boolean}
|
|
44
44
|
*/
|
|
45
45
|
export function implementationRequiresTrust(implementationInfo, projectConfig = null) {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
return isSameOrInside(implementationRoot, modulePath) || projectHasTemplateAttachment(projectConfig);
|
|
46
|
+
void projectConfig;
|
|
47
|
+
implementationTrustFingerprint(implementationInfo.config);
|
|
48
|
+
return true;
|
|
50
49
|
}
|
|
51
50
|
|
|
52
51
|
/**
|
|
@@ -16,8 +16,7 @@ import {
|
|
|
16
16
|
import {
|
|
17
17
|
implementationModuleIsUnderRoot,
|
|
18
18
|
implementationRequiresTrust,
|
|
19
|
-
implementationTrustFingerprint
|
|
20
|
-
projectHasTemplateAttachment
|
|
19
|
+
implementationTrustFingerprint
|
|
21
20
|
} from "./policy.js";
|
|
22
21
|
import { readTemplateTrustRecord } from "./record.js";
|
|
23
22
|
|
|
@@ -44,7 +43,6 @@ export function assertTrustedImplementation(implementationInfo, projectConfig =
|
|
|
44
43
|
* @returns {{ ok: boolean, requiresTrust: boolean, trustPath: string, trustRecord: import("./record.js").TemplateTrustRecord|null, template: { id: string|null, version: string|null, source: string|null, sourceSpec: string|null, requested: string|null, sourceRoot: string|null, catalog?: Record<string, any>|null, includesExecutableImplementation: boolean|null }, implementation: { id: string|null, module: string|null, export: string|null }, content: { trustedDigest: string|null, currentDigest: string|null, added: string[], removed: string[], changed: string[] }, issues: string[] }}
|
|
45
44
|
*/
|
|
46
45
|
export function getTemplateTrustStatus(implementationInfo, projectConfig = null) {
|
|
47
|
-
const templateAttached = projectHasTemplateAttachment(projectConfig);
|
|
48
46
|
if (!implementationRequiresTrust(implementationInfo, projectConfig)) {
|
|
49
47
|
return {
|
|
50
48
|
ok: true,
|
|
@@ -68,7 +66,7 @@ export function getTemplateTrustStatus(implementationInfo, projectConfig = null)
|
|
|
68
66
|
/** @type {{ trustedDigest: string|null, currentDigest: string|null, added: string[], removed: string[], changed: string[] }} */
|
|
69
67
|
const contentStatus = { trustedDigest: null, currentDigest: null, added: [], removed: [], changed: [] };
|
|
70
68
|
|
|
71
|
-
if (
|
|
69
|
+
if (!moduleInsideImplementation) {
|
|
72
70
|
issues.push(implementationOutsideRootMessage(fingerprint.module));
|
|
73
71
|
}
|
|
74
72
|
|