@visulima/task-runner 1.0.0-alpha.5 → 1.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/archive.d.ts +38 -0
- package/dist/cache.d.ts +31 -3
- package/dist/chrome-trace.d.ts +53 -0
- package/dist/file-access-tracker.d.ts +7 -1
- package/dist/fingerprint.d.ts +9 -0
- package/dist/incremental-hasher.d.ts +18 -0
- package/dist/index.d.ts +8 -3
- package/dist/index.js +22 -19
- package/dist/life-cycle.d.ts +2 -0
- package/dist/log-reporter.d.ts +34 -0
- package/dist/output-resolver.d.ts +20 -0
- package/dist/packem_shared/{Cache-iAjRMV2d.js → Cache-CWaX_c8U.js} +135 -45
- package/dist/packem_shared/{CompositeLifeCycle-7AtYw1dv.js → CompositeLifeCycle-CSVbRC_5.js} +10 -0
- package/dist/packem_shared/{FileAccessTracker-CrtBAt5D.js → FileAccessTracker-CQ5Ot7Hd.js} +68 -16
- package/dist/packem_shared/{FingerprintManager-Cu-ta9ee.js → FingerprintManager-CV7U4f4f.js} +22 -1
- package/dist/packem_shared/{IncrementalFileHasher-Cm_kJY5V.js → IncrementalFileHasher-BRS76-mb.js} +26 -0
- package/dist/packem_shared/LogReporter-BDt52HLu.js +44 -0
- package/dist/packem_shared/{RemoteCache-BFceSe4a.js → RemoteCache-DSU3lc87.js} +77 -37
- package/dist/packem_shared/{TaskOrchestrator-lLn-PH1m.js → TaskOrchestrator-rf45vW5c.js} +94 -15
- package/dist/packem_shared/{TerminalBuffer-CnPyFgPB.js → TerminalBuffer-qVJvbRQZ.js} +1 -1
- package/dist/packem_shared/{TrackedTaskExecutor-BGUKFE-7.js → TrackedTaskExecutor-CFPpQfXF.js} +1 -1
- package/dist/packem_shared/archive-UQHAnZUa.js +102 -0
- package/dist/packem_shared/{buildForwardDependencyMap-0BJFMMPv.js → buildForwardDependencyMap-DLPgKEto.js} +2 -1
- package/dist/packem_shared/{computeTaskHash-B2SVZqgp.js → computeTaskHash-DYqfrDGq.js} +122 -6
- package/dist/packem_shared/{createTaskGraph-CcsFaSrz.js → createTaskGraph-B7nH0kY_.js} +2 -2
- package/dist/packem_shared/{defaultTaskRunner-BdFTifsh.js → defaultTaskRunner-Cp7jCmIl.js} +28 -6
- package/dist/packem_shared/{extractPackageName-CbVNW-dr.js → extractPackageName-BllKetnz.js} +2 -1
- package/dist/packem_shared/{generateRunSummary-qn-_jKwt.js → generateRunSummary-BE1jnQ3H.js} +19 -1
- package/dist/packem_shared/{parsePartition-C4-P5RjK.js → parsePartition-BfLbHGAx.js} +18 -0
- package/dist/packem_shared/{projectGraphToDot-C8uYeaPo.js → projectGraphToDot-DU1lSe-c.js} +1 -1
- package/dist/packem_shared/resolveOutputs-n6MCKoTe.js +111 -0
- package/dist/packem_shared/{runConcurrentFallback-CGHz_f-Q.js → runConcurrentFallback-BTmgGV1H.js} +1 -1
- package/dist/packem_shared/{runConcurrently-qrkWyzXW.js → runConcurrently-CmfC4r-f.js} +1 -1
- package/dist/packem_shared/toChromeTrace-B2tZoJ-7.js +121 -0
- package/dist/remote-cache.d.ts +45 -0
- package/dist/run-summary.d.ts +26 -4
- package/dist/task-hasher.d.ts +37 -0
- package/dist/task-orchestrator.d.ts +2 -2
- package/dist/types.d.ts +137 -3
- package/index.js +52 -52
- package/package.json +12 -12
package/dist/packem_shared/{CompositeLifeCycle-7AtYw1dv.js → CompositeLifeCycle-CSVbRC_5.js}
RENAMED
|
@@ -59,6 +59,16 @@ class CompositeLifeCycle {
|
|
|
59
59
|
lc.printCacheMiss?.(task, reasons);
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
|
+
onTaskStdout(task, chunk) {
|
|
63
|
+
for (const lc of this.#lifeCycles) {
|
|
64
|
+
lc.onTaskStdout?.(task, chunk);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
onTaskStderr(task, chunk) {
|
|
68
|
+
for (const lc of this.#lifeCycles) {
|
|
69
|
+
lc.onTaskStderr?.(task, chunk);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
62
72
|
}
|
|
63
73
|
class ConsoleLifeCycle {
|
|
64
74
|
#verbose;
|
|
@@ -47,11 +47,19 @@ const isStraceAvailable = () => {
|
|
|
47
47
|
return straceAvailable;
|
|
48
48
|
};
|
|
49
49
|
const STRACE_PATTERNS = [
|
|
50
|
-
{ pattern: /openat\(AT_FDCWD,\s*"([^"]+)"
|
|
51
|
-
{ pattern: /^(?:\d+\s+)?open\("([^"]+)"
|
|
52
|
-
{ pattern: /(
|
|
53
|
-
{ pattern: /
|
|
50
|
+
{ kind: "open", pattern: /openat\(AT_FDCWD,\s*"([^"]+)"/ },
|
|
51
|
+
{ kind: "open", pattern: /^(?:\d+\s+)?open\("([^"]+)"/ },
|
|
52
|
+
{ kind: "creat", pattern: /(?:^|\s)creat\("([^"]+)"/ },
|
|
53
|
+
{ kind: "stat", pattern: /(?:stat|lstat|newfstatat)\((?:AT_FDCWD,\s*)?"([^"]+)"/ },
|
|
54
|
+
{ kind: "stat", pattern: /access\("([^"]+)"/ },
|
|
55
|
+
{ kind: "getdents", pattern: /getdents(?:64)?\(\d+/ }
|
|
54
56
|
];
|
|
57
|
+
const parseOpenAccessType = (line) => {
|
|
58
|
+
if (/O_WRONLY|O_RDWR|O_CREAT|O_TRUNC|O_APPEND/.test(line)) {
|
|
59
|
+
return "write";
|
|
60
|
+
}
|
|
61
|
+
return "read";
|
|
62
|
+
};
|
|
55
63
|
class FileAccessTracker {
|
|
56
64
|
#workspaceRoot;
|
|
57
65
|
#excludePatterns;
|
|
@@ -93,7 +101,7 @@ class FileAccessTracker {
|
|
|
93
101
|
const traceDirectory = join(this.#workspaceRoot, "node_modules", ".cache", "task-runner");
|
|
94
102
|
await mkdir(traceDirectory, { recursive: true });
|
|
95
103
|
const traceFile = join(traceDirectory, `strace-${uniqueId()}.log`);
|
|
96
|
-
const straceCommand = `strace -f -qq -e trace=open,openat,stat,lstat,newfstatat,access,getdents,getdents64 -o ${traceFile} -- ${command}`;
|
|
104
|
+
const straceCommand = `strace -f -qq -e trace=open,openat,creat,stat,lstat,newfstatat,access,getdents,getdents64,unlink,unlinkat,rename,renameat,renameat2 -o ${traceFile} -- ${command}`;
|
|
97
105
|
return new Promise((_resolve) => {
|
|
98
106
|
const child = exec(
|
|
99
107
|
straceCommand,
|
|
@@ -125,14 +133,23 @@ class FileAccessTracker {
|
|
|
125
133
|
}
|
|
126
134
|
/**
|
|
127
135
|
* Parses strace output to extract file accesses.
|
|
136
|
+
*
|
|
137
|
+
* A single path may appear with both `read` and `write` types when a
|
|
138
|
+
* task reads a file and later rewrites it — self-modifying detection
|
|
139
|
+
* relies on seeing both, so we dedupe per `(path, type)` rather than
|
|
140
|
+
* by path alone.
|
|
128
141
|
*/
|
|
129
142
|
#parseStraceOutput(traceContent, cwd) {
|
|
130
143
|
const accesses = [];
|
|
131
|
-
const
|
|
144
|
+
const seen = /* @__PURE__ */ new Set();
|
|
132
145
|
for (const line of traceContent.split("\n")) {
|
|
133
146
|
const parsed = this.#parseStraceLine(line, cwd);
|
|
134
|
-
if (
|
|
135
|
-
|
|
147
|
+
if (!parsed) {
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const key = `${parsed.type}:${parsed.path}`;
|
|
151
|
+
if (!seen.has(key)) {
|
|
152
|
+
seen.add(key);
|
|
136
153
|
accesses.push(parsed);
|
|
137
154
|
}
|
|
138
155
|
}
|
|
@@ -140,23 +157,46 @@ class FileAccessTracker {
|
|
|
140
157
|
}
|
|
141
158
|
/**
|
|
142
159
|
* Parses a single strace output line.
|
|
143
|
-
* Each entry maps a regex to the file access
|
|
160
|
+
* Each entry maps a regex to the file access kind; the actual type
|
|
161
|
+
* (read vs write) depends on flag inspection for `open`/`openat`.
|
|
144
162
|
*/
|
|
145
163
|
#parseStraceLine(line, cwd) {
|
|
146
164
|
const isMissing = line.includes("ENOENT");
|
|
147
|
-
for (const {
|
|
165
|
+
for (const { kind, pattern } of STRACE_PATTERNS) {
|
|
148
166
|
const match = pattern.exec(line);
|
|
149
|
-
if (!match
|
|
167
|
+
if (!match) {
|
|
150
168
|
continue;
|
|
151
169
|
}
|
|
152
|
-
|
|
170
|
+
if (kind === "getdents") {
|
|
171
|
+
return void 0;
|
|
172
|
+
}
|
|
173
|
+
const capturedPath = match[1];
|
|
174
|
+
if (!capturedPath) {
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
let path = capturedPath;
|
|
153
178
|
if (!path.startsWith("/")) {
|
|
154
179
|
path = resolve(cwd, path);
|
|
155
180
|
}
|
|
156
181
|
if (this.#shouldExclude(path) || !path.startsWith(this.#workspaceRoot)) {
|
|
157
182
|
return void 0;
|
|
158
183
|
}
|
|
159
|
-
|
|
184
|
+
if (isMissing) {
|
|
185
|
+
return { path, type: "missing" };
|
|
186
|
+
}
|
|
187
|
+
const type = kind === "open" ? parseOpenAccessType(line) : kind === "creat" ? "write" : "stat";
|
|
188
|
+
return { path, type };
|
|
189
|
+
}
|
|
190
|
+
const destructive = /(?:^|\s)(?:unlink|unlinkat|rename|renameat|renameat2)\((?:AT_FDCWD,\s*)?"([^"]+)"/.exec(line);
|
|
191
|
+
if (destructive?.[1]) {
|
|
192
|
+
let path = destructive[1];
|
|
193
|
+
if (!path.startsWith("/")) {
|
|
194
|
+
path = resolve(cwd, path);
|
|
195
|
+
}
|
|
196
|
+
if (this.#shouldExclude(path) || !path.startsWith(this.#workspaceRoot)) {
|
|
197
|
+
return void 0;
|
|
198
|
+
}
|
|
199
|
+
return { path, type: "write" };
|
|
160
200
|
}
|
|
161
201
|
return void 0;
|
|
162
202
|
}
|
|
@@ -220,19 +260,31 @@ const patch = (obj, method, type) => {
|
|
|
220
260
|
};
|
|
221
261
|
};
|
|
222
262
|
|
|
223
|
-
//
|
|
263
|
+
// Reads / stats / directory listing
|
|
224
264
|
patch(fs, "readFileSync", "read");
|
|
225
265
|
patch(fs, "statSync", "stat");
|
|
226
266
|
patch(fs, "readdirSync", "readdir");
|
|
227
|
-
// Async (callback)
|
|
228
267
|
patch(fs, "readFile", "read");
|
|
229
268
|
patch(fs, "stat", "stat");
|
|
230
269
|
patch(fs, "readdir", "readdir");
|
|
231
|
-
// Async (promises)
|
|
232
270
|
patch(fsp, "readFile", "read");
|
|
233
271
|
patch(fsp, "stat", "stat");
|
|
234
272
|
patch(fsp, "readdir", "readdir");
|
|
235
273
|
|
|
274
|
+
// Writes and path-mutating ops — needed for self-modifying task detection
|
|
275
|
+
patch(fs, "writeFileSync", "write");
|
|
276
|
+
patch(fs, "appendFileSync", "write");
|
|
277
|
+
patch(fs, "unlinkSync", "write");
|
|
278
|
+
patch(fs, "renameSync", "write");
|
|
279
|
+
patch(fs, "writeFile", "write");
|
|
280
|
+
patch(fs, "appendFile", "write");
|
|
281
|
+
patch(fs, "unlink", "write");
|
|
282
|
+
patch(fs, "rename", "write");
|
|
283
|
+
patch(fsp, "writeFile", "write");
|
|
284
|
+
patch(fsp, "appendFile", "write");
|
|
285
|
+
patch(fsp, "unlink", "write");
|
|
286
|
+
patch(fsp, "rename", "write");
|
|
287
|
+
|
|
236
288
|
process.on("beforeExit", () => { logStream.end(); });
|
|
237
289
|
`;
|
|
238
290
|
|
package/dist/packem_shared/{FingerprintManager-Cu-ta9ee.js → FingerprintManager-CV7U4f4f.js}
RENAMED
|
@@ -34,6 +34,8 @@ class FingerprintManager {
|
|
|
34
34
|
const fileHashes = {};
|
|
35
35
|
const missingPaths = /* @__PURE__ */ new Set();
|
|
36
36
|
const directoryListings = {};
|
|
37
|
+
const readPaths = /* @__PURE__ */ new Set();
|
|
38
|
+
const writePaths = /* @__PURE__ */ new Set();
|
|
37
39
|
for (const access of accesses) {
|
|
38
40
|
const relativePath = relative(this.#workspaceRoot, access.path);
|
|
39
41
|
switch (access.type) {
|
|
@@ -43,6 +45,7 @@ class FingerprintManager {
|
|
|
43
45
|
}
|
|
44
46
|
case "read":
|
|
45
47
|
case "stat": {
|
|
48
|
+
readPaths.add(relativePath);
|
|
46
49
|
if (!fileHashes[relativePath]) {
|
|
47
50
|
const hash = await this.#hashFileWithCache(access.path);
|
|
48
51
|
if (hash) {
|
|
@@ -62,8 +65,19 @@ class FingerprintManager {
|
|
|
62
65
|
}
|
|
63
66
|
break;
|
|
64
67
|
}
|
|
68
|
+
case "write": {
|
|
69
|
+
writePaths.add(relativePath);
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const modifiedInputs = [];
|
|
75
|
+
for (const path of writePaths) {
|
|
76
|
+
if (readPaths.has(path)) {
|
|
77
|
+
modifiedInputs.push(path);
|
|
65
78
|
}
|
|
66
79
|
}
|
|
80
|
+
modifiedInputs.sort();
|
|
67
81
|
const commandHash = hashStrings(`${command}:${JSON.stringify(sortObjectKeys(args))}`);
|
|
68
82
|
const envHashes = {};
|
|
69
83
|
const matchedEnvVariables = FingerprintManager.#resolveEnvPatterns(envPatterns, envVariables);
|
|
@@ -75,7 +89,14 @@ class FingerprintManager {
|
|
|
75
89
|
envHashes[key] = hashStrings(`${key}=${value ?? ""}`);
|
|
76
90
|
}
|
|
77
91
|
const missingFiles = [...missingPaths].toSorted();
|
|
78
|
-
return {
|
|
92
|
+
return {
|
|
93
|
+
commandHash,
|
|
94
|
+
directoryListings,
|
|
95
|
+
envHashes,
|
|
96
|
+
fileHashes,
|
|
97
|
+
missingFiles,
|
|
98
|
+
...modifiedInputs.length > 0 ? { modifiedInputs } : {}
|
|
99
|
+
};
|
|
79
100
|
}
|
|
80
101
|
/**
|
|
81
102
|
* Validates a stored fingerprint against the current state.
|
package/dist/packem_shared/{IncrementalFileHasher-Cm_kJY5V.js → IncrementalFileHasher-BRS76-mb.js}
RENAMED
|
@@ -146,6 +146,32 @@ class IncrementalFileHasher {
|
|
|
146
146
|
clear() {
|
|
147
147
|
this.#snapshot.clear();
|
|
148
148
|
}
|
|
149
|
+
/**
|
|
150
|
+
* Looks up the cached hash for `absolutePath` when its mtime and
|
|
151
|
+
* size match the snapshot; returns `undefined` otherwise. Callers
|
|
152
|
+
* that already have a `stat` result (e.g. `InProcessTaskHasher`)
|
|
153
|
+
* skip an extra syscall by passing it through directly.
|
|
154
|
+
*
|
|
155
|
+
* The snapshot is considered loaded on first call — lazy-load is
|
|
156
|
+
* synchronous-friendly here because the caller already performed
|
|
157
|
+
* an async `stat` before calling this method.
|
|
158
|
+
*/
|
|
159
|
+
getSnapshotHash(absolutePath, mtimeMs, size) {
|
|
160
|
+
const entry = this.#snapshot.get(absolutePath);
|
|
161
|
+
if (!entry) {
|
|
162
|
+
return void 0;
|
|
163
|
+
}
|
|
164
|
+
return entry.mtimeMs === mtimeMs && entry.size === size ? entry.hash : void 0;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Writes a fresh snapshot entry after the caller has computed the
|
|
168
|
+
* hash. Pairs with {@link IncrementalFileHasher.getSnapshotHash}
|
|
169
|
+
* — after a miss, the caller hashes the file and records the
|
|
170
|
+
* result here so the next run can reuse it.
|
|
171
|
+
*/
|
|
172
|
+
recordSnapshot(absolutePath, hash, mtimeMs, size) {
|
|
173
|
+
this.#snapshot.set(absolutePath, { hash, mtimeMs, size });
|
|
174
|
+
}
|
|
149
175
|
}
|
|
150
176
|
|
|
151
177
|
export { IncrementalFileHasher };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
const formatTaskLabel = (task) => `${task.target.project}#${task.target.target}`;
|
|
2
|
+
const prefixLines = (text, label) => {
|
|
3
|
+
const trailing = text.endsWith("\n") ? "\n" : "";
|
|
4
|
+
const body = trailing ? text.slice(0, -1) : text;
|
|
5
|
+
if (body.length === 0) {
|
|
6
|
+
return trailing;
|
|
7
|
+
}
|
|
8
|
+
return `${body.split("\n").map((line) => `[${label}] ${line}`).join("\n")}${trailing}`;
|
|
9
|
+
};
|
|
10
|
+
class LogReporter {
|
|
11
|
+
#mode;
|
|
12
|
+
#write;
|
|
13
|
+
constructor(mode, write = (chunk) => process.stdout.write(chunk)) {
|
|
14
|
+
this.#mode = mode;
|
|
15
|
+
this.#write = write;
|
|
16
|
+
}
|
|
17
|
+
printTaskTerminalOutput(task, _status, terminalOutput) {
|
|
18
|
+
if (terminalOutput.length === 0) {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
if (this.#mode === "interleaved") {
|
|
22
|
+
this.#write(terminalOutput.endsWith("\n") ? terminalOutput : `${terminalOutput}
|
|
23
|
+
`);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const label = formatTaskLabel(task);
|
|
27
|
+
if (this.#mode === "labeled") {
|
|
28
|
+
this.#write(prefixLines(terminalOutput.endsWith("\n") ? terminalOutput : `${terminalOutput}
|
|
29
|
+
`, label));
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const body = terminalOutput.endsWith("\n") ? terminalOutput : `${terminalOutput}
|
|
33
|
+
`;
|
|
34
|
+
this.#write(`── ${label} ──
|
|
35
|
+
${body}
|
|
36
|
+
`);
|
|
37
|
+
}
|
|
38
|
+
// eslint-disable-next-line class-methods-use-this
|
|
39
|
+
endTasks(_taskResults) {
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const createLogReporter = (mode, write) => new LogReporter(mode, write);
|
|
43
|
+
|
|
44
|
+
export { LogReporter, createLogReporter };
|
|
@@ -18,40 +18,45 @@ const __cjs_getBuiltinModule = (module) => {
|
|
|
18
18
|
};
|
|
19
19
|
|
|
20
20
|
const {
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
createHmac,
|
|
22
|
+
timingSafeEqual
|
|
23
|
+
} = __cjs_getBuiltinModule("node:crypto");
|
|
23
24
|
const {
|
|
24
|
-
createWriteStream
|
|
25
|
+
createWriteStream,
|
|
26
|
+
createReadStream
|
|
25
27
|
} = __cjs_getBuiltinModule("node:fs");
|
|
26
28
|
const {
|
|
27
29
|
mkdir,
|
|
28
30
|
rm,
|
|
29
|
-
stat
|
|
30
|
-
readFile
|
|
31
|
+
stat
|
|
31
32
|
} = __cjs_getBuiltinModule("node:fs/promises");
|
|
32
33
|
const {
|
|
33
34
|
pipeline
|
|
34
35
|
} = __cjs_getBuiltinModule("node:stream/promises");
|
|
35
36
|
import { join } from '@visulima/path';
|
|
37
|
+
import { e as extractTarBrotli, a as extractTarGz, c as createTarBrotli, b as createTarGz } from './archive-UQHAnZUa.js';
|
|
36
38
|
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}
|
|
39
|
+
const SIGNATURE_HEADER = "X-Artifact-Signature";
|
|
40
|
+
const MIN_SECRET_LENGTH = 16;
|
|
41
|
+
const computeArtifactSignatureStream = async (secret, hash, archivePath) => {
|
|
42
|
+
const hmac = createHmac("sha256", secret);
|
|
43
|
+
hmac.update(hash);
|
|
44
|
+
const source = createReadStream(archivePath);
|
|
45
|
+
for await (const chunk of source) {
|
|
46
|
+
hmac.update(chunk);
|
|
47
|
+
}
|
|
48
|
+
return hmac.digest("hex");
|
|
49
|
+
};
|
|
50
|
+
const signaturesMatch = (a, b) => {
|
|
51
|
+
if (a.length !== b.length) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
try {
|
|
55
|
+
return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
56
|
+
} catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
55
60
|
class RemoteCache {
|
|
56
61
|
#url;
|
|
57
62
|
#token;
|
|
@@ -60,6 +65,9 @@ class RemoteCache {
|
|
|
60
65
|
#read;
|
|
61
66
|
#write;
|
|
62
67
|
#onUploadError;
|
|
68
|
+
#compression;
|
|
69
|
+
#signingSecret;
|
|
70
|
+
#verifyOnDownload;
|
|
63
71
|
constructor(options) {
|
|
64
72
|
this.#url = options.url.replace(/\/$/, "");
|
|
65
73
|
this.#token = options.token;
|
|
@@ -68,6 +76,17 @@ class RemoteCache {
|
|
|
68
76
|
this.#read = options.read ?? true;
|
|
69
77
|
this.#write = options.write ?? true;
|
|
70
78
|
this.#onUploadError = options.onUploadError;
|
|
79
|
+
this.#compression = options.compression ?? "gzip";
|
|
80
|
+
if (options.signing) {
|
|
81
|
+
if (options.signing.secret.length < MIN_SECRET_LENGTH) {
|
|
82
|
+
throw new Error(`Remote cache signing secret must be at least ${String(MIN_SECRET_LENGTH)} characters.`);
|
|
83
|
+
}
|
|
84
|
+
this.#signingSecret = options.signing.secret;
|
|
85
|
+
this.#verifyOnDownload = options.signing.verifyOnDownload ?? false;
|
|
86
|
+
} else {
|
|
87
|
+
this.#signingSecret = void 0;
|
|
88
|
+
this.#verifyOnDownload = false;
|
|
89
|
+
}
|
|
71
90
|
}
|
|
72
91
|
/**
|
|
73
92
|
* Retrieves a cached artifact from the remote cache.
|
|
@@ -89,15 +108,23 @@ class RemoteCache {
|
|
|
89
108
|
if (!response.ok) {
|
|
90
109
|
return false;
|
|
91
110
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
111
|
+
const receivedSignature = this.#signingSecret ? response.headers.get(SIGNATURE_HEADER.toLowerCase()) ?? response.headers.get(SIGNATURE_HEADER) : null;
|
|
112
|
+
if (this.#signingSecret && !receivedSignature && this.#verifyOnDownload) {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
if (!response.body) {
|
|
95
116
|
return false;
|
|
96
117
|
}
|
|
97
|
-
|
|
98
|
-
await pipeline(body,
|
|
118
|
+
await mkdir(localCacheDirectory, { recursive: true });
|
|
119
|
+
await pipeline(response.body, createWriteStream(archivePath));
|
|
120
|
+
if (this.#signingSecret && receivedSignature) {
|
|
121
|
+
const expected = await computeArtifactSignatureStream(this.#signingSecret, hash, archivePath);
|
|
122
|
+
if (!signaturesMatch(receivedSignature, expected)) {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
99
126
|
await mkdir(entryDirectory, { recursive: true });
|
|
100
|
-
await extractTarGz(archivePath, entryDirectory);
|
|
127
|
+
await (this.#compression === "brotli" ? extractTarBrotli(archivePath, entryDirectory) : extractTarGz(archivePath, entryDirectory));
|
|
101
128
|
await rm(archivePath, { force: true });
|
|
102
129
|
return true;
|
|
103
130
|
} catch {
|
|
@@ -119,16 +146,29 @@ class RemoteCache {
|
|
|
119
146
|
const archivePath = join(localCacheDirectory, `.upload-${hash}.tar.gz`);
|
|
120
147
|
try {
|
|
121
148
|
await stat(join(entryDirectory, ".commit"));
|
|
122
|
-
await createTarGz(entryDirectory, archivePath);
|
|
149
|
+
await (this.#compression === "brotli" ? createTarBrotli(entryDirectory, archivePath) : createTarGz(entryDirectory, archivePath));
|
|
123
150
|
const artifactUrl = this.#buildUrl(`/v8/artifacts/${hash}`);
|
|
124
|
-
const
|
|
151
|
+
const { size } = await stat(archivePath);
|
|
152
|
+
const uploadHeaders = {
|
|
153
|
+
...this.#buildHeaders(),
|
|
154
|
+
"Content-Length": String(size),
|
|
155
|
+
// Advertise the compression format in a custom header so
|
|
156
|
+
// spec-compatible servers (and the matching download side)
|
|
157
|
+
// can branch if needed. The body is still an opaque blob
|
|
158
|
+
// from the server's perspective.
|
|
159
|
+
"Content-Type": "application/octet-stream",
|
|
160
|
+
"X-Artifact-Compression": this.#compression
|
|
161
|
+
};
|
|
162
|
+
if (this.#signingSecret) {
|
|
163
|
+
uploadHeaders[SIGNATURE_HEADER] = await computeArtifactSignatureStream(this.#signingSecret, hash, archivePath);
|
|
164
|
+
}
|
|
125
165
|
const response = await fetch(artifactUrl, {
|
|
126
|
-
body:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
166
|
+
body: createReadStream(archivePath),
|
|
167
|
+
// @ts-expect-error — `duplex` is a Node-specific fetch
|
|
168
|
+
// option required when the body is a stream. The DOM
|
|
169
|
+
// `RequestInit` type doesn't include it yet.
|
|
170
|
+
duplex: "half",
|
|
171
|
+
headers: uploadHeaders,
|
|
132
172
|
method: "PUT",
|
|
133
173
|
signal: AbortSignal.timeout(this.#timeout)
|
|
134
174
|
});
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { d as resolveTaskCwd, a as createFailureResult, e as createXxh3Hasher } from './utils-zO0ZRgtf.js';
|
|
2
|
-
import { join } from '@visulima/path';
|
|
3
|
-
import { FingerprintManager } from './FingerprintManager-
|
|
4
|
-
import { generateRunSummary, writeRunSummary } from './generateRunSummary-
|
|
5
|
-
import { computeTaskHash } from './computeTaskHash-
|
|
6
|
-
import { TrackedTaskExecutor } from './TrackedTaskExecutor-
|
|
2
|
+
import { resolve, join } from '@visulima/path';
|
|
3
|
+
import { FingerprintManager } from './FingerprintManager-CV7U4f4f.js';
|
|
4
|
+
import { generateRunSummary, writeLastRunSummary, writeRunSummary } from './generateRunSummary-BE1jnQ3H.js';
|
|
5
|
+
import { computeTaskHash } from './computeTaskHash-DYqfrDGq.js';
|
|
6
|
+
import { TrackedTaskExecutor } from './TrackedTaskExecutor-CFPpQfXF.js';
|
|
7
7
|
|
|
8
8
|
const hashFingerprint = (fingerprint) => {
|
|
9
9
|
const hash = createXxh3Hasher();
|
|
@@ -26,11 +26,11 @@ const hashFingerprint = (fingerprint) => {
|
|
|
26
26
|
return hash.digest();
|
|
27
27
|
};
|
|
28
28
|
const createDeferred = () => {
|
|
29
|
-
let
|
|
29
|
+
let resolve2;
|
|
30
30
|
const promise = new Promise((r) => {
|
|
31
|
-
|
|
31
|
+
resolve2 = r;
|
|
32
32
|
});
|
|
33
|
-
return { promise, resolve };
|
|
33
|
+
return { promise, resolve: resolve2 };
|
|
34
34
|
};
|
|
35
35
|
class TaskOrchestrator {
|
|
36
36
|
#taskHasher;
|
|
@@ -101,9 +101,12 @@ class TaskOrchestrator {
|
|
|
101
101
|
process.removeListener("SIGTERM", signalHandler);
|
|
102
102
|
this.#lifeCycle.endCommand?.();
|
|
103
103
|
}
|
|
104
|
-
if (this.#
|
|
104
|
+
if (this.#taskGraph && !this.#aborted) {
|
|
105
105
|
const summary = generateRunSummary(this.#results, this.#taskGraph, this.#startTime);
|
|
106
|
-
await
|
|
106
|
+
await writeLastRunSummary(summary, this.#workspaceRoot);
|
|
107
|
+
if (this.#summarize) {
|
|
108
|
+
await writeRunSummary(summary, this.#workspaceRoot);
|
|
109
|
+
}
|
|
107
110
|
}
|
|
108
111
|
return this.#results;
|
|
109
112
|
}
|
|
@@ -243,7 +246,13 @@ class TaskOrchestrator {
|
|
|
243
246
|
};
|
|
244
247
|
this.#results.set(task.id, result);
|
|
245
248
|
if (code === 0 && task.cache !== false && task.hash) {
|
|
246
|
-
await this.#
|
|
249
|
+
const modified = await this.#detectSelfModifiedInputs(task);
|
|
250
|
+
if (modified.length > 0) {
|
|
251
|
+
result.selfModified = true;
|
|
252
|
+
this.#lifeCycle.printSelfModifyingSkip?.(task, modified);
|
|
253
|
+
} else {
|
|
254
|
+
await this.#cache.put(task.hash, terminalOutput, task.outputs, code);
|
|
255
|
+
}
|
|
247
256
|
}
|
|
248
257
|
return result;
|
|
249
258
|
} catch (error) {
|
|
@@ -252,6 +261,28 @@ class TaskOrchestrator {
|
|
|
252
261
|
return result;
|
|
253
262
|
}
|
|
254
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* Re-hashes every file recorded in `task.hashDetails.nodes` and returns
|
|
266
|
+
* the workspace-relative paths whose post-execution hash differs from the
|
|
267
|
+
* pre-execution hash. A non-empty result means the task wrote to its own
|
|
268
|
+
* tracked inputs and the cache entry would be unsafe to persist.
|
|
269
|
+
*/
|
|
270
|
+
async #detectSelfModifiedInputs(task) {
|
|
271
|
+
const nodes = task.hashDetails?.nodes;
|
|
272
|
+
if (!nodes || typeof this.#taskHasher.rehashFile !== "function") {
|
|
273
|
+
return [];
|
|
274
|
+
}
|
|
275
|
+
const rehash = this.#taskHasher.rehashFile.bind(this.#taskHasher);
|
|
276
|
+
const entries = Object.entries(nodes);
|
|
277
|
+
const checks = await Promise.all(
|
|
278
|
+
entries.map(async ([path, priorHash]) => {
|
|
279
|
+
const absolute = resolve(this.#workspaceRoot, path);
|
|
280
|
+
const fresh = await rehash(absolute);
|
|
281
|
+
return fresh !== void 0 && fresh !== priorHash ? path : void 0;
|
|
282
|
+
})
|
|
283
|
+
);
|
|
284
|
+
return checks.filter((path) => path !== void 0);
|
|
285
|
+
}
|
|
255
286
|
async #executeTaskWithTracking(task, startTime) {
|
|
256
287
|
if (!this.#fingerprintManager) {
|
|
257
288
|
return this.#executeTask(task, startTime);
|
|
@@ -262,12 +293,18 @@ class TaskOrchestrator {
|
|
|
262
293
|
let code;
|
|
263
294
|
let terminalOutput;
|
|
264
295
|
let fingerprint;
|
|
296
|
+
let trackerAccessCount = 0;
|
|
297
|
+
let usedRealTracker = false;
|
|
298
|
+
let autoWrites;
|
|
265
299
|
const shellCommand = this.#resolveCommand?.(task);
|
|
266
300
|
const canTrack = shellCommand && this.#trackedExecutor?.isTrackingSupported;
|
|
267
301
|
if (canTrack && this.#trackedExecutor) {
|
|
268
302
|
const trackedResult = await this.#trackedExecutor.execute(task, { captureOutput: this.#captureOutput, cwd }, shellCommand);
|
|
269
303
|
code = trackedResult.code;
|
|
270
304
|
terminalOutput = trackedResult.terminalOutput;
|
|
305
|
+
trackerAccessCount = trackedResult.accesses.length;
|
|
306
|
+
usedRealTracker = true;
|
|
307
|
+
autoWrites = trackedResult.accesses.filter((a) => a.type === "write").map((a) => a.path);
|
|
271
308
|
fingerprint = await this.#fingerprintManager.createFingerprint(
|
|
272
309
|
trackedResult.accesses,
|
|
273
310
|
taskCommand,
|
|
@@ -309,10 +346,20 @@ class TaskOrchestrator {
|
|
|
309
346
|
};
|
|
310
347
|
this.#results.set(task.id, result);
|
|
311
348
|
if (code === 0 && task.cache !== false && fingerprint) {
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
349
|
+
const modified = this.#detectSelfModifiedFingerprint(fingerprint);
|
|
350
|
+
const emptyFingerprintReason = this.#describeEmptyFingerprint(fingerprint, usedRealTracker, trackerAccessCount);
|
|
351
|
+
if (modified.length > 0) {
|
|
352
|
+
result.selfModified = true;
|
|
353
|
+
this.#lifeCycle.printSelfModifyingSkip?.(task, modified);
|
|
354
|
+
} else if (emptyFingerprintReason) {
|
|
355
|
+
result.emptyFingerprint = true;
|
|
356
|
+
this.#lifeCycle.printEmptyFingerprintWarning?.(task, emptyFingerprintReason);
|
|
357
|
+
} else {
|
|
358
|
+
const hash = hashFingerprint(fingerprint);
|
|
359
|
+
Object.assign(task, { hash });
|
|
360
|
+
await this.#cache.put(hash, terminalOutput, task.outputs, code, fingerprint, autoWrites);
|
|
361
|
+
await this.#cache.setTaskIndex(task.id, hash);
|
|
362
|
+
}
|
|
316
363
|
}
|
|
317
364
|
return result;
|
|
318
365
|
} catch (error) {
|
|
@@ -321,6 +368,38 @@ class TaskOrchestrator {
|
|
|
321
368
|
return result;
|
|
322
369
|
}
|
|
323
370
|
}
|
|
371
|
+
/**
|
|
372
|
+
* In auto-fingerprint mode, returns the workspace-relative paths the
|
|
373
|
+
* task both read *and* wrote during execution. {@link FingerprintManager}
|
|
374
|
+
* populates `modifiedInputs` from the tracker's `"write"`-typed accesses;
|
|
375
|
+
* backends that don't yet emit write accesses leave it empty.
|
|
376
|
+
*/
|
|
377
|
+
// eslint-disable-next-line class-methods-use-this
|
|
378
|
+
#detectSelfModifiedFingerprint(fingerprint) {
|
|
379
|
+
return fingerprint.modifiedInputs ?? [];
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Returns a human-readable reason string when a fingerprint looks
|
|
383
|
+
* suspiciously empty (tracker ran but observed no workspace files),
|
|
384
|
+
* or `undefined` when the fingerprint is trustworthy. Caching is
|
|
385
|
+
* skipped for empty fingerprints — silently persisting one would
|
|
386
|
+
* guarantee false cache hits on every subsequent run.
|
|
387
|
+
*
|
|
388
|
+
* Only flags results from the real tracker; the glob-based fallback
|
|
389
|
+
* path legitimately produces wide fingerprints and isn't at risk.
|
|
390
|
+
*/
|
|
391
|
+
// eslint-disable-next-line class-methods-use-this
|
|
392
|
+
#describeEmptyFingerprint(fingerprint, usedRealTracker, trackerAccessCount) {
|
|
393
|
+
if (!usedRealTracker) {
|
|
394
|
+
return void 0;
|
|
395
|
+
}
|
|
396
|
+
const hasAnyAccess = Object.keys(fingerprint.fileHashes).length > 0 || Object.keys(fingerprint.directoryListings).length > 0 || fingerprint.missingFiles.length > 0;
|
|
397
|
+
if (hasAnyAccess) {
|
|
398
|
+
return void 0;
|
|
399
|
+
}
|
|
400
|
+
const zeroAccesses = trackerAccessCount === 0;
|
|
401
|
+
return zeroAccesses ? "Tracker observed no workspace file accesses — likely a static binary on a platform without strace. Caching skipped." : "Tracker returned accesses but none fell inside the workspace. Caching skipped to avoid false cache hits.";
|
|
402
|
+
}
|
|
324
403
|
#dryRunResult(task, startTime) {
|
|
325
404
|
const cacheStatus = task.hash ? `[hash: ${task.hash.slice(0, 12)}...]` : "[no hash]";
|
|
326
405
|
const result = {
|
package/dist/packem_shared/{TrackedTaskExecutor-BGUKFE-7.js → TrackedTaskExecutor-CFPpQfXF.js}
RENAMED
|
@@ -27,7 +27,7 @@ const {
|
|
|
27
27
|
rm
|
|
28
28
|
} = __cjs_getBuiltinModule("node:fs/promises");
|
|
29
29
|
import { join } from '@visulima/path';
|
|
30
|
-
import { FileAccessTracker, generatePreloadScript } from './FileAccessTracker-
|
|
30
|
+
import { FileAccessTracker, generatePreloadScript } from './FileAccessTracker-CQ5Ot7Hd.js';
|
|
31
31
|
import { u as uniqueId } from './utils-zO0ZRgtf.js';
|
|
32
32
|
|
|
33
33
|
class TrackedTaskExecutor {
|