@visulima/task-runner 1.0.0-alpha.7 → 1.0.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +3 -1
- package/dist/index.d.ts +3093 -49
- package/dist/index.js +29 -19
- package/dist/packem_chunks/index.js +5593 -0
- package/dist/packem_shared/{Cache-CWaX_c8U.js → Cache-CbhoA268.js} +151 -10
- package/dist/packem_shared/{FileAccessTracker-CQ5Ot7Hd.js → FileAccessTracker-D8zIURPU.js} +1 -1
- package/dist/packem_shared/{FingerprintManager-CV7U4f4f.js → FingerprintManager-78DjwWQ4.js} +1 -1
- package/dist/packem_shared/HttpRemoteCache-BTXUBH7t.js +290 -0
- package/dist/packem_shared/INPUT_URI_SCHEMES-DRm76YI5.js +69 -0
- package/dist/packem_shared/{IncrementalFileHasher-BRS76-mb.js → IncrementalFileHasher-BBhVK491.js} +1 -1
- package/dist/packem_shared/ReapiRemoteCache-vgRxDMmu.js +1012 -0
- package/dist/packem_shared/{TaskOrchestrator-rf45vW5c.js → TaskOrchestrator-CdRaQhTO.js} +100 -11
- package/dist/packem_shared/{TrackedTaskExecutor-CFPpQfXF.js → TrackedTaskExecutor-CWSMfHAW.js} +2 -2
- package/dist/packem_shared/V2_ROOT-DKBLxKo4.js +14 -0
- package/dist/packem_shared/actionDigestForTaskHash-BRE-9MT6.js +121 -0
- package/dist/packem_shared/archive-CnggHWb-.js +152 -0
- package/dist/packem_shared/{buildForwardDependencyMap-DLPgKEto.js → buildForwardDependencyMap-0BJFMMPv.js} +1 -2
- package/dist/packem_shared/{collectFiles-ClXHnHhg.js → collectFiles-cc1gokGU.js} +2 -1
- package/dist/packem_shared/{computeTaskHash-DYqfrDGq.js → computeTaskHash-DHoBJ_-V.js} +10 -4
- package/dist/packem_shared/containsBlob-CwGB0a_q.js +125 -0
- package/dist/packem_shared/{createTaskGraph-B7nH0kY_.js → createTaskGraph-Bwl4hwAf.js} +23 -2
- package/dist/packem_shared/{defaultTaskRunner-Cp7jCmIl.js → defaultTaskRunner-BaX4ZbFv.js} +58 -15
- package/dist/packem_shared/{detectFrameworks-CeFzKE6J.js → detectFrameworks-D7nyTc-o.js} +1 -1
- package/dist/packem_shared/{detectScriptShell-CR-xXKA4.js → detectScriptShell-CzxCM9-t.js} +1 -1
- package/dist/packem_shared/digestBuffer-CPdI2E1d.js +48 -0
- package/dist/packem_shared/{expandArguments-0AwD2BIA.js → expandArguments-Ba-hHYff.js} +2 -1
- package/dist/packem_shared/expandTokensInString-Bb7nYehP.js +47 -0
- package/dist/packem_shared/{extractPackageName-BllKetnz.js → extractPackageName-CMHjqGj_.js} +2 -3
- package/dist/packem_shared/{generateRunSummary-BE1jnQ3H.js → generateRunSummary-Bah7CFay.js} +1 -1
- package/dist/packem_shared/getCurrentBranch-DVNikt0P.js +156 -0
- package/dist/packem_shared/getMainWorktreeRoot-iBqToQJ4.js +114 -0
- package/dist/packem_shared/{parseCommands-D-IgF8Zh.js → parseCommands-DDdIxaH5.js} +8 -3
- package/dist/packem_shared/resolveCacheMode-CsmHT_0o.js +21 -0
- package/dist/packem_shared/{runConcurrently-CmfC4r-f.js → runConcurrently-BCGQ9fJl.js} +1 -1
- package/dist/packem_shared/shell-quote-DWJJbt21.js +3 -0
- package/dist/packem_shared/{utils-zO0ZRgtf.js → utils-Bmnj-H2J.js} +4 -1
- package/index.js +556 -723
- package/package.json +26 -13
- package/dist/affected.d.ts +0 -82
- package/dist/archive.d.ts +0 -38
- package/dist/cache.d.ts +0 -138
- package/dist/chrome-trace.d.ts +0 -53
- package/dist/command-parser/expand-arguments.d.ts +0 -11
- package/dist/command-parser/expand-shortcut.d.ts +0 -15
- package/dist/command-parser/expand-wildcard.d.ts +0 -13
- package/dist/command-parser/index.d.ts +0 -18
- package/dist/command-parser/strip-quotes.d.ts +0 -6
- package/dist/concurrent-fallback.d.ts +0 -16
- package/dist/concurrent.d.ts +0 -23
- package/dist/default-task-runner.d.ts +0 -44
- package/dist/detect-shell.d.ts +0 -19
- package/dist/file-access-tracker.d.ts +0 -59
- package/dist/fingerprint.d.ts +0 -54
- package/dist/flow-controllers/index.d.ts +0 -7
- package/dist/flow-controllers/input-handler.d.ts +0 -44
- package/dist/flow-controllers/log-timings.d.ts +0 -18
- package/dist/flow-controllers/restart-process.d.ts +0 -21
- package/dist/flow-controllers/teardown.d.ts +0 -22
- package/dist/framework-inference.d.ts +0 -35
- package/dist/graph-visualizer.d.ts +0 -74
- package/dist/incremental-hasher.d.ts +0 -76
- package/dist/life-cycle.d.ts +0 -38
- package/dist/lockfile-hasher.d.ts +0 -73
- package/dist/log-reporter.d.ts +0 -34
- package/dist/native-binding.d.ts +0 -106
- package/dist/output-resolver.d.ts +0 -20
- package/dist/packem_shared/RemoteCache-DSU3lc87.js +0 -219
- package/dist/packem_shared/archive-UQHAnZUa.js +0 -102
- package/dist/project-constraints.d.ts +0 -9
- package/dist/remote-cache.d.ts +0 -100
- package/dist/run-summary.d.ts +0 -111
- package/dist/task-graph-utils.d.ts +0 -39
- package/dist/task-graph.d.ts +0 -22
- package/dist/task-hasher.d.ts +0 -104
- package/dist/task-orchestrator.d.ts +0 -38
- package/dist/task-scheduler.d.ts +0 -41
- package/dist/terminal-buffer.d.ts +0 -29
- package/dist/tracked-executor.d.ts +0 -46
- package/dist/types.d.ts +0 -757
- package/dist/utils.d.ts +0 -39
|
@@ -17,8 +17,12 @@ const __cjs_getBuiltinModule = (module) => {
|
|
|
17
17
|
return __cjs_require(module);
|
|
18
18
|
};
|
|
19
19
|
|
|
20
|
+
const {
|
|
21
|
+
createWriteStream
|
|
22
|
+
} = __cjs_getBuiltinModule("node:fs");
|
|
20
23
|
const {
|
|
21
24
|
readFile,
|
|
25
|
+
utimes,
|
|
22
26
|
mkdir,
|
|
23
27
|
writeFile,
|
|
24
28
|
rename,
|
|
@@ -27,11 +31,71 @@ const {
|
|
|
27
31
|
stat,
|
|
28
32
|
cp
|
|
29
33
|
} = __cjs_getBuiltinModule("node:fs/promises");
|
|
34
|
+
const {
|
|
35
|
+
pipeline
|
|
36
|
+
} = __cjs_getBuiltinModule("node:stream/promises");
|
|
30
37
|
import { formatBytes, parseBytes } from '@visulima/humanizer';
|
|
31
|
-
import { join, resolve
|
|
32
|
-
import {
|
|
38
|
+
import { dirname, join, resolve } from '@visulima/path';
|
|
39
|
+
import { a as extractTarBrotli, b as createTarBrotli } from './archive-CnggHWb-.js';
|
|
40
|
+
import { u as uniqueId } from './utils-Bmnj-H2J.js';
|
|
41
|
+
import { acEntryPath, taskHashIndexPath, tmpDirectory, V2_ROOT } from './V2_ROOT-DKBLxKo4.js';
|
|
42
|
+
import { fetchBlobToFile, containsBlob, putBlobFromFile } from './containsBlob-CwGB0a_q.js';
|
|
33
43
|
import { resolveOutputs } from './resolveOutputs-n6MCKoTe.js';
|
|
34
|
-
|
|
44
|
+
|
|
45
|
+
const writeActionEntry = async (root, actionHash, result) => {
|
|
46
|
+
const finalPath = acEntryPath(root, actionHash);
|
|
47
|
+
const tmpDirectoryPath = tmpDirectory(root);
|
|
48
|
+
const stagingPath = `${tmpDirectoryPath}/${uniqueId()}`;
|
|
49
|
+
await mkdir(tmpDirectoryPath, { recursive: true });
|
|
50
|
+
await mkdir(dirname(finalPath), { recursive: true });
|
|
51
|
+
const payload = `${JSON.stringify(result, null, 2)}
|
|
52
|
+
`;
|
|
53
|
+
await writeFile(stagingPath, payload);
|
|
54
|
+
try {
|
|
55
|
+
await rename(stagingPath, finalPath);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
await rm(stagingPath, { force: true }).catch(() => {
|
|
58
|
+
});
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
const readActionEntry = async (root, actionHash) => {
|
|
63
|
+
const path = acEntryPath(root, actionHash);
|
|
64
|
+
try {
|
|
65
|
+
const content = await readFile(path, "utf8");
|
|
66
|
+
const parsed = JSON.parse(content);
|
|
67
|
+
const now = /* @__PURE__ */ new Date();
|
|
68
|
+
await utimes(path, now, now).catch(() => {
|
|
69
|
+
});
|
|
70
|
+
return parsed;
|
|
71
|
+
} catch {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
const writeTaskHashIndex = async (root, taskHash, actionHash) => {
|
|
76
|
+
const finalPath = taskHashIndexPath(root, taskHash);
|
|
77
|
+
const tmpDirectoryPath = tmpDirectory(root);
|
|
78
|
+
const stagingPath = `${tmpDirectoryPath}/${uniqueId()}`;
|
|
79
|
+
await mkdir(tmpDirectoryPath, { recursive: true });
|
|
80
|
+
await mkdir(dirname(finalPath), { recursive: true });
|
|
81
|
+
await writeFile(stagingPath, actionHash);
|
|
82
|
+
try {
|
|
83
|
+
await rename(stagingPath, finalPath);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
await rm(stagingPath, { force: true }).catch(() => {
|
|
86
|
+
});
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
const readTaskHashIndex = async (root, taskHash) => {
|
|
91
|
+
try {
|
|
92
|
+
const content = await readFile(taskHashIndexPath(root, taskHash), "utf8");
|
|
93
|
+
const trimmed = content.trim();
|
|
94
|
+
return trimmed.length > 0 ? trimmed : null;
|
|
95
|
+
} catch {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
};
|
|
35
99
|
|
|
36
100
|
const DEFAULT_MAX_CACHE_AGE = 7 * 24 * 60 * 60 * 1e3;
|
|
37
101
|
const DEFAULT_CACHE_DIRECTORY_NAME = ".task-runner-cache";
|
|
@@ -100,6 +164,82 @@ class Cache {
|
|
|
100
164
|
get cacheDirectory() {
|
|
101
165
|
return this.#cacheDirectory;
|
|
102
166
|
}
|
|
167
|
+
/**
|
|
168
|
+
* Root for v2 CAS-shaped reads/writes. Backend implementations
|
|
169
|
+
* (HTTP today, REAPI gRPC next) take this path so they can
|
|
170
|
+
* hydrate fetched blobs into the same CAS the local cache reads
|
|
171
|
+
* from. Equal to {@link cacheDirectory} — `v2/` lives under that.
|
|
172
|
+
*/
|
|
173
|
+
get casRoot() {
|
|
174
|
+
return this.#cacheDirectory;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Read a v2 {@link ActionResult} by action digest. Resolves to
|
|
178
|
+
* `null` on miss. The orchestrator is expected to follow up with
|
|
179
|
+
* {@link materializeOutputs} to place the referenced blobs into
|
|
180
|
+
* the workspace.
|
|
181
|
+
*/
|
|
182
|
+
async getActionResult(actionDigest) {
|
|
183
|
+
return readActionEntry(this.#cacheDirectory, actionDigest.hash);
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Look up an action digest by the legacy task hash. Returns
|
|
187
|
+
* `null` when the bridge file isn't present — caller falls
|
|
188
|
+
* through to the legacy `<hash>/` layout (or executes the task).
|
|
189
|
+
*/
|
|
190
|
+
async resolveActionDigestForTaskHash(taskHash) {
|
|
191
|
+
return readTaskHashIndex(this.#cacheDirectory, taskHash);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Persist a v2 entry: writes the AC JSON, copies referenced
|
|
195
|
+
* blobs into the CAS via the lazy {@link BlobSource} handles,
|
|
196
|
+
* and binds the task hash → action digest redirect last so a
|
|
197
|
+
* partial failure can't surface a half-written entry.
|
|
198
|
+
*/
|
|
199
|
+
async putActionResult(taskHash, actionDigest, result, blobs) {
|
|
200
|
+
await this.#ensureBlobs(blobs);
|
|
201
|
+
await writeActionEntry(this.#cacheDirectory, actionDigest.hash, result);
|
|
202
|
+
await writeTaskHashIndex(this.#cacheDirectory, taskHash, actionDigest.hash);
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Materialize an action's outputs into the workspace. Streams
|
|
206
|
+
* each referenced blob from the local CAS to its workspace path.
|
|
207
|
+
* Returns `false` when any blob is missing — caller treats that
|
|
208
|
+
* as a cache miss and re-executes.
|
|
209
|
+
*/
|
|
210
|
+
async materializeOutputs(result, workspaceRoot) {
|
|
211
|
+
for (const file of result.outputFiles) {
|
|
212
|
+
const destination = resolve(workspaceRoot, file.path);
|
|
213
|
+
const ok = await fetchBlobToFile(this.#cacheDirectory, file.digest, destination);
|
|
214
|
+
if (!ok) {
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return true;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Sequentially materializes each {@link BlobSource} into the local
|
|
222
|
+
* CAS. Serial on purpose: a single Action typically references a
|
|
223
|
+
* handful of blobs, and parallelizing would multiply RSS by the
|
|
224
|
+
* largest payload's size while gaining nothing on disk-bound IO.
|
|
225
|
+
* The small per-blob loop also keeps tmp-file naming and rename
|
|
226
|
+
* pressure predictable instead of fanning out a burst of writers.
|
|
227
|
+
*/
|
|
228
|
+
async #ensureBlobs(blobs) {
|
|
229
|
+
for (const blob of blobs) {
|
|
230
|
+
const present = await containsBlob(this.#cacheDirectory, blob.digest);
|
|
231
|
+
if (present) {
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
const tmpPath = join(this.#cacheDirectory, V2_ROOT, "tmp", `.put-${uniqueId()}`);
|
|
235
|
+
await mkdir(dirname(tmpPath), { recursive: true });
|
|
236
|
+
const source = await blob.open();
|
|
237
|
+
await pipeline(source, createWriteStream(tmpPath));
|
|
238
|
+
await putBlobFromFile(this.#cacheDirectory, blob.digest, tmpPath);
|
|
239
|
+
await rm(tmpPath, { force: true }).catch(() => {
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
103
243
|
/**
|
|
104
244
|
* Retrieves a cached result for the given task hash.
|
|
105
245
|
* Returns undefined if no valid cache entry exists.
|
|
@@ -175,9 +315,9 @@ class Cache {
|
|
|
175
315
|
* extracted staging become the set of swap roots. Still accepted
|
|
176
316
|
* for backward compat.
|
|
177
317
|
*/
|
|
178
|
-
async restoreOutputs(hash, _outputs) {
|
|
318
|
+
async restoreOutputs(hash, _outputs, options) {
|
|
179
319
|
const cacheEntryDirectory = join(this.#cacheDirectory, hash);
|
|
180
|
-
return restoreOutputsCompressed(cacheEntryDirectory, this.#workspaceRoot);
|
|
320
|
+
return restoreOutputsCompressed(cacheEntryDirectory, this.#workspaceRoot, options);
|
|
181
321
|
}
|
|
182
322
|
/**
|
|
183
323
|
* Retrieves the most recent cached result for a task by its ID.
|
|
@@ -311,7 +451,7 @@ const archiveOutputsCompressed = async (workspaceRoot, cacheEntryDirectory, outp
|
|
|
311
451
|
const stagedOutput = join(stagingDirectory, relativePath);
|
|
312
452
|
try {
|
|
313
453
|
await mkdir(dirname(stagedOutput), { recursive: true });
|
|
314
|
-
await cp(absoluteOutput, stagedOutput, { recursive: true });
|
|
454
|
+
await cp(absoluteOutput, stagedOutput, { preserveTimestamps: true, recursive: true });
|
|
315
455
|
return true;
|
|
316
456
|
} catch {
|
|
317
457
|
return false;
|
|
@@ -342,7 +482,7 @@ const runBounded = async (limit, items, fn) => {
|
|
|
342
482
|
await Promise.all(workers);
|
|
343
483
|
return results;
|
|
344
484
|
};
|
|
345
|
-
const restoreOutputsCompressed = async (cacheEntryDirectory, workspaceRoot) => {
|
|
485
|
+
const restoreOutputsCompressed = async (cacheEntryDirectory, workspaceRoot, options) => {
|
|
346
486
|
const archivePath = join(cacheEntryDirectory, OUTPUTS_ARCHIVE_FILENAME);
|
|
347
487
|
try {
|
|
348
488
|
await stat(archivePath);
|
|
@@ -351,10 +491,11 @@ const restoreOutputsCompressed = async (cacheEntryDirectory, workspaceRoot) => {
|
|
|
351
491
|
}
|
|
352
492
|
const stagingDirectory = join(cacheEntryDirectory, `.restore-${uniqueId()}`);
|
|
353
493
|
const backups = [];
|
|
494
|
+
const preserveMtime = options?.preserveMtime ?? true;
|
|
354
495
|
try {
|
|
355
496
|
await mkdir(stagingDirectory, { recursive: true });
|
|
356
|
-
await extractTarBrotli(archivePath, stagingDirectory);
|
|
357
|
-
const topLevel = await readdir(stagingDirectory);
|
|
497
|
+
await extractTarBrotli(archivePath, stagingDirectory, options);
|
|
498
|
+
const topLevel = (await readdir(stagingDirectory)).sort();
|
|
358
499
|
await runBounded(STAGE_CONCURRENCY, topLevel, async (entry) => {
|
|
359
500
|
const absoluteOutput = resolve(workspaceRoot, entry);
|
|
360
501
|
const stagedOutput = join(stagingDirectory, entry);
|
|
@@ -366,7 +507,7 @@ const restoreOutputsCompressed = async (cacheEntryDirectory, workspaceRoot) => {
|
|
|
366
507
|
backups.push({ backup, original: absoluteOutput });
|
|
367
508
|
} catch {
|
|
368
509
|
}
|
|
369
|
-
await cp(stagedOutput, absoluteOutput, { recursive: true });
|
|
510
|
+
await cp(stagedOutput, absoluteOutput, { preserveTimestamps: preserveMtime, recursive: true });
|
|
370
511
|
});
|
|
371
512
|
await Promise.all(backups.map(({ backup }) => rm(backup, { force: true, recursive: true }).catch(() => {
|
|
372
513
|
})));
|
|
@@ -30,7 +30,7 @@ const {
|
|
|
30
30
|
platform
|
|
31
31
|
} = __cjs_getBuiltinModule("node:os");
|
|
32
32
|
import { resolve, join } from '@visulima/path';
|
|
33
|
-
import { u as uniqueId } from './utils-
|
|
33
|
+
import { u as uniqueId } from './utils-Bmnj-H2J.js';
|
|
34
34
|
|
|
35
35
|
import __cjs_mod__ from "node:module"; // -- packem CommonJS require shim --
|
|
36
36
|
const require = __cjs_mod__.createRequire(import.meta.url);
|
package/dist/packem_shared/{FingerprintManager-CV7U4f4f.js → FingerprintManager-78DjwWQ4.js}
RENAMED
|
@@ -22,7 +22,7 @@ const {
|
|
|
22
22
|
stat
|
|
23
23
|
} = __cjs_getBuiltinModule("node:fs/promises");
|
|
24
24
|
import { resolve, relative } from '@visulima/path';
|
|
25
|
-
import { b as hashStrings, s as sortObjectKeys, h as hashFile } from './utils-
|
|
25
|
+
import { b as hashStrings, s as sortObjectKeys, h as hashFile } from './utils-Bmnj-H2J.js';
|
|
26
26
|
|
|
27
27
|
class FingerprintManager {
|
|
28
28
|
#workspaceRoot;
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { createRequire as __cjs_createRequire } from "node:module";
|
|
2
|
+
|
|
3
|
+
const __cjs_require = __cjs_createRequire(import.meta.url);
|
|
4
|
+
|
|
5
|
+
const __cjs_getProcess = typeof globalThis !== "undefined" && typeof globalThis.process !== "undefined" ? globalThis.process : process;
|
|
6
|
+
|
|
7
|
+
const __cjs_getBuiltinModule = (module) => {
|
|
8
|
+
// Check if we're in Node.js and version supports getBuiltinModule
|
|
9
|
+
if (typeof __cjs_getProcess !== "undefined" && __cjs_getProcess.versions && __cjs_getProcess.versions.node) {
|
|
10
|
+
const [major, minor] = __cjs_getProcess.versions.node.split(".").map(Number);
|
|
11
|
+
// Node.js 20.16.0+ and 22.3.0+
|
|
12
|
+
if (major > 22 || (major === 22 && minor >= 3) || (major === 20 && minor >= 16)) {
|
|
13
|
+
return __cjs_getProcess.getBuiltinModule(module);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
// Fallback to createRequire
|
|
17
|
+
return __cjs_require(module);
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const {
|
|
21
|
+
createHmac,
|
|
22
|
+
timingSafeEqual,
|
|
23
|
+
createHash
|
|
24
|
+
} = __cjs_getBuiltinModule("node:crypto");
|
|
25
|
+
const {
|
|
26
|
+
createWriteStream,
|
|
27
|
+
createReadStream
|
|
28
|
+
} = __cjs_getBuiltinModule("node:fs");
|
|
29
|
+
const {
|
|
30
|
+
mkdir,
|
|
31
|
+
rm,
|
|
32
|
+
stat
|
|
33
|
+
} = __cjs_getBuiltinModule("node:fs/promises");
|
|
34
|
+
const {
|
|
35
|
+
tmpdir
|
|
36
|
+
} = __cjs_getBuiltinModule("node:os");
|
|
37
|
+
const {
|
|
38
|
+
pipeline
|
|
39
|
+
} = __cjs_getBuiltinModule("node:stream/promises");
|
|
40
|
+
import { join } from '@visulima/path';
|
|
41
|
+
import { fetchBlobToFile, putBlobFromFile } from './containsBlob-CwGB0a_q.js';
|
|
42
|
+
import { u as uniqueId } from './utils-Bmnj-H2J.js';
|
|
43
|
+
|
|
44
|
+
const SIGNATURE_HEADER = "X-Artifact-Signature";
|
|
45
|
+
const MIN_SECRET_LENGTH = 16;
|
|
46
|
+
const BLOB_OUTPUT_PATH = "vis-entry.tar.gz";
|
|
47
|
+
const digestFile = async (path) => {
|
|
48
|
+
const hash = createHash("sha256");
|
|
49
|
+
let sizeBytes = 0;
|
|
50
|
+
for await (const chunk of createReadStream(path)) {
|
|
51
|
+
const buffer = chunk;
|
|
52
|
+
hash.update(buffer);
|
|
53
|
+
sizeBytes += buffer.byteLength;
|
|
54
|
+
}
|
|
55
|
+
return { hash: hash.digest("hex"), sizeBytes };
|
|
56
|
+
};
|
|
57
|
+
const computeArtifactSignatureStream = async (secret, hash, archivePath) => {
|
|
58
|
+
const hmac = createHmac("sha256", secret);
|
|
59
|
+
hmac.update(hash);
|
|
60
|
+
const source = createReadStream(archivePath);
|
|
61
|
+
for await (const chunk of source) {
|
|
62
|
+
hmac.update(chunk);
|
|
63
|
+
}
|
|
64
|
+
return hmac.digest("hex");
|
|
65
|
+
};
|
|
66
|
+
const signaturesMatch = (a, b) => {
|
|
67
|
+
if (a.length !== b.length) {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
return timingSafeEqual(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
72
|
+
} catch {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
class HttpRemoteCache {
|
|
77
|
+
#url;
|
|
78
|
+
#token;
|
|
79
|
+
#teamId;
|
|
80
|
+
#timeout;
|
|
81
|
+
#read;
|
|
82
|
+
#write;
|
|
83
|
+
#onUploadError;
|
|
84
|
+
#compression;
|
|
85
|
+
#signingSecret;
|
|
86
|
+
#verifyOnDownload;
|
|
87
|
+
#localCasRoot;
|
|
88
|
+
/**
|
|
89
|
+
* Per-digest in-flight upload dedup. Two concurrent `storeAction`
|
|
90
|
+
* calls referencing the same blob upload it once per process
|
|
91
|
+
* lifetime, matching REAPI's `FindMissingBlobs` elision.
|
|
92
|
+
*/
|
|
93
|
+
#inflightUploads = /* @__PURE__ */ new Map();
|
|
94
|
+
constructor(options) {
|
|
95
|
+
this.#url = options.url.replace(/\/$/, "");
|
|
96
|
+
this.#token = options.token;
|
|
97
|
+
this.#teamId = options.teamId;
|
|
98
|
+
this.#timeout = options.timeout ?? 3e4;
|
|
99
|
+
const mode = options.mode ?? "readwrite";
|
|
100
|
+
this.#read = mode === "read" || mode === "readwrite";
|
|
101
|
+
this.#write = mode === "write" || mode === "readwrite";
|
|
102
|
+
this.#onUploadError = options.onUploadError;
|
|
103
|
+
this.#compression = options.compression ?? "gzip";
|
|
104
|
+
this.#localCasRoot = options.localCasRoot;
|
|
105
|
+
if (options.signing) {
|
|
106
|
+
if (options.signing.secret.length < MIN_SECRET_LENGTH) {
|
|
107
|
+
throw new Error(`Remote cache signing secret must be at least ${String(MIN_SECRET_LENGTH)} characters.`);
|
|
108
|
+
}
|
|
109
|
+
this.#signingSecret = options.signing.secret;
|
|
110
|
+
this.#verifyOnDownload = options.signing.verifyOnDownload ?? false;
|
|
111
|
+
} else {
|
|
112
|
+
this.#signingSecret = void 0;
|
|
113
|
+
this.#verifyOnDownload = false;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* No-op. The HTTP backend uses Node's global `fetch`, which has no
|
|
118
|
+
* persistent connection to release. Implemented to satisfy the
|
|
119
|
+
* {@link RemoteCacheBackend.close} contract uniformly.
|
|
120
|
+
*/
|
|
121
|
+
// eslint-disable-next-line class-methods-use-this
|
|
122
|
+
async close() {
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* {@link RemoteCacheBackend.containsAction}: HEAD on the artifact URL.
|
|
126
|
+
* Resolves `false` on any wire failure — existence checks are best
|
|
127
|
+
* effort and never block the caller.
|
|
128
|
+
*/
|
|
129
|
+
async containsAction(actionDigest) {
|
|
130
|
+
if (!this.#read) {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
const artifactUrl = this.#buildUrl(`/v8/artifacts/${actionDigest.hash}`);
|
|
135
|
+
const response = await fetch(artifactUrl, {
|
|
136
|
+
headers: this.#buildHeaders(),
|
|
137
|
+
method: "HEAD",
|
|
138
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
139
|
+
});
|
|
140
|
+
return response.ok;
|
|
141
|
+
} catch {
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* {@link RemoteCacheBackend.fetchBlob}: streams a blob out of the
|
|
147
|
+
* local CAS that was hydrated by a previous {@link retrieveAction}
|
|
148
|
+
* call. The HTTP wire ships one tarball per action, so per-blob
|
|
149
|
+
* fetches are local-only — there's no remote endpoint to call.
|
|
150
|
+
*/
|
|
151
|
+
async fetchBlob(digest, destinationPath) {
|
|
152
|
+
if (!this.#localCasRoot) {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
return fetchBlobToFile(this.#localCasRoot, digest, destinationPath);
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* {@link RemoteCacheBackend.retrieveAction}: GETs the artifact at
|
|
159
|
+
* `/v8/artifacts/{actionDigest.hash}`, ingests the response bytes
|
|
160
|
+
* as a single CAS blob in the local store, and synthesises an
|
|
161
|
+
* {@link ActionResult} that points at that blob. Resolves to `null`
|
|
162
|
+
* on a 404 / signature mismatch / missing local CAS root.
|
|
163
|
+
*/
|
|
164
|
+
async retrieveAction(actionDigest) {
|
|
165
|
+
if (!this.#read || !this.#localCasRoot) {
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
const stagingRoot = join(this.#localCasRoot, "v2", "tmp");
|
|
169
|
+
const stagingPath = join(stagingRoot, `.remote-${actionDigest.hash}-${uniqueId()}`);
|
|
170
|
+
try {
|
|
171
|
+
await mkdir(stagingRoot, { recursive: true });
|
|
172
|
+
const artifactUrl = this.#buildUrl(`/v8/artifacts/${actionDigest.hash}`);
|
|
173
|
+
const response = await fetch(artifactUrl, {
|
|
174
|
+
headers: this.#buildHeaders(),
|
|
175
|
+
method: "GET",
|
|
176
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
177
|
+
});
|
|
178
|
+
if (!response.ok || !response.body) {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
const receivedSignature = this.#signingSecret ? response.headers.get(SIGNATURE_HEADER.toLowerCase()) ?? response.headers.get(SIGNATURE_HEADER) : null;
|
|
182
|
+
if (this.#signingSecret && !receivedSignature && this.#verifyOnDownload) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
await pipeline(response.body, createWriteStream(stagingPath));
|
|
186
|
+
if (this.#signingSecret && receivedSignature) {
|
|
187
|
+
const expected = await computeArtifactSignatureStream(this.#signingSecret, actionDigest.hash, stagingPath);
|
|
188
|
+
if (!signaturesMatch(receivedSignature, expected)) {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
const blobDigest = await digestFile(stagingPath);
|
|
193
|
+
await putBlobFromFile(this.#localCasRoot, blobDigest, stagingPath);
|
|
194
|
+
return {
|
|
195
|
+
exitCode: 0,
|
|
196
|
+
outputDirectories: [],
|
|
197
|
+
outputFiles: [{ digest: blobDigest, isExecutable: false, path: BLOB_OUTPUT_PATH }]
|
|
198
|
+
};
|
|
199
|
+
} catch {
|
|
200
|
+
return null;
|
|
201
|
+
} finally {
|
|
202
|
+
await rm(stagingPath, { force: true }).catch(() => {
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* {@link RemoteCacheBackend.storeAction}: takes the single blob
|
|
208
|
+
* referenced by `result.outputFiles[0]`, streams its bytes as the
|
|
209
|
+
* PUT body, and signs the body when a signing secret is configured.
|
|
210
|
+
* Per-digest in-flight dedup means parallel writers racing on the
|
|
211
|
+
* same action upload exactly once.
|
|
212
|
+
*/
|
|
213
|
+
async storeAction(actionDigest, result, blobs) {
|
|
214
|
+
if (!this.#write) {
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
if (result.outputFiles.length !== 1 || blobs.length === 0) {
|
|
218
|
+
return false;
|
|
219
|
+
}
|
|
220
|
+
const tarballEntry = result.outputFiles[0];
|
|
221
|
+
if (!tarballEntry) {
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
const blob = blobs.find((candidate) => candidate.digest.hash === tarballEntry.digest.hash);
|
|
225
|
+
if (!blob) {
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
const existing = this.#inflightUploads.get(actionDigest.hash);
|
|
229
|
+
if (existing) {
|
|
230
|
+
return existing;
|
|
231
|
+
}
|
|
232
|
+
const upload = this.#uploadAction(actionDigest, blob).finally(() => {
|
|
233
|
+
this.#inflightUploads.delete(actionDigest.hash);
|
|
234
|
+
});
|
|
235
|
+
this.#inflightUploads.set(actionDigest.hash, upload);
|
|
236
|
+
return upload;
|
|
237
|
+
}
|
|
238
|
+
async #uploadAction(actionDigest, blob) {
|
|
239
|
+
const stagingRoot = this.#localCasRoot ? join(this.#localCasRoot, "v2", "tmp") : join(tmpdir(), "visulima-task-runner-uploads");
|
|
240
|
+
const stagingPath = join(stagingRoot, `.upload-${actionDigest.hash}-${uniqueId()}`);
|
|
241
|
+
try {
|
|
242
|
+
await mkdir(stagingRoot, { recursive: true });
|
|
243
|
+
const source = await blob.open();
|
|
244
|
+
await pipeline(source, createWriteStream(stagingPath));
|
|
245
|
+
const artifactUrl = this.#buildUrl(`/v8/artifacts/${actionDigest.hash}`);
|
|
246
|
+
const { size } = await stat(stagingPath);
|
|
247
|
+
const uploadHeaders = {
|
|
248
|
+
...this.#buildHeaders(),
|
|
249
|
+
"Content-Length": String(size),
|
|
250
|
+
"Content-Type": "application/octet-stream",
|
|
251
|
+
"X-Artifact-Compression": this.#compression
|
|
252
|
+
};
|
|
253
|
+
if (this.#signingSecret) {
|
|
254
|
+
uploadHeaders[SIGNATURE_HEADER] = await computeArtifactSignatureStream(this.#signingSecret, actionDigest.hash, stagingPath);
|
|
255
|
+
}
|
|
256
|
+
const response = await fetch(artifactUrl, {
|
|
257
|
+
body: createReadStream(stagingPath),
|
|
258
|
+
// @ts-expect-error — `duplex` is a Node-specific fetch
|
|
259
|
+
// option required when the body is a stream.
|
|
260
|
+
duplex: "half",
|
|
261
|
+
headers: uploadHeaders,
|
|
262
|
+
method: "PUT",
|
|
263
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
264
|
+
});
|
|
265
|
+
return response.ok;
|
|
266
|
+
} catch (error) {
|
|
267
|
+
this.#onUploadError?.(actionDigest.hash, error);
|
|
268
|
+
return false;
|
|
269
|
+
} finally {
|
|
270
|
+
await rm(stagingPath, { force: true }).catch(() => {
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
#buildUrl(path) {
|
|
275
|
+
const url = `${this.#url}${path}`;
|
|
276
|
+
if (this.#teamId) {
|
|
277
|
+
return `${url}?teamId=${encodeURIComponent(this.#teamId)}`;
|
|
278
|
+
}
|
|
279
|
+
return url;
|
|
280
|
+
}
|
|
281
|
+
#buildHeaders() {
|
|
282
|
+
const headers = {};
|
|
283
|
+
if (this.#token) {
|
|
284
|
+
headers["Authorization"] = `Bearer ${this.#token}`;
|
|
285
|
+
}
|
|
286
|
+
return headers;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
export { HttpRemoteCache };
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
const INPUT_URI_SCHEMES = ["file", "glob", "env", "func", "dep"];
|
|
2
|
+
const URI_SCHEME_RE = /^([a-z][a-z0-9+.-]*):\/\//i;
|
|
3
|
+
class InvalidInputUriError extends Error {
|
|
4
|
+
constructor(message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "InvalidInputUriError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
const parseInputUri = (input) => {
|
|
10
|
+
const negated = input.startsWith("!");
|
|
11
|
+
const body = negated ? input.slice(1) : input;
|
|
12
|
+
const match = URI_SCHEME_RE.exec(body);
|
|
13
|
+
if (!match) {
|
|
14
|
+
return void 0;
|
|
15
|
+
}
|
|
16
|
+
const scheme = match[1].toLowerCase();
|
|
17
|
+
const rest = body.slice(match[0].length);
|
|
18
|
+
switch (scheme) {
|
|
19
|
+
case "file":
|
|
20
|
+
case "glob": {
|
|
21
|
+
if (rest.length === 0) {
|
|
22
|
+
throw new InvalidInputUriError(`${scheme}:// input requires a path or pattern (got "${input}").`);
|
|
23
|
+
}
|
|
24
|
+
return { fileset: negated ? `!${rest}` : rest };
|
|
25
|
+
}
|
|
26
|
+
case "env": {
|
|
27
|
+
if (negated) {
|
|
28
|
+
throw new InvalidInputUriError(`Negation is not supported for env:// inputs (got "${input}"). Drop env vars from the named-input set instead.`);
|
|
29
|
+
}
|
|
30
|
+
if (rest.length === 0) {
|
|
31
|
+
throw new InvalidInputUriError(`env:// input requires a variable name (got "${input}").`);
|
|
32
|
+
}
|
|
33
|
+
return { env: rest };
|
|
34
|
+
}
|
|
35
|
+
case "func": {
|
|
36
|
+
if (negated) {
|
|
37
|
+
throw new InvalidInputUriError(`Negation is not supported for func:// inputs (got "${input}").`);
|
|
38
|
+
}
|
|
39
|
+
if (rest.length === 0) {
|
|
40
|
+
throw new InvalidInputUriError(`func:// input requires a command (got "${input}").`);
|
|
41
|
+
}
|
|
42
|
+
return { runtime: rest };
|
|
43
|
+
}
|
|
44
|
+
case "dep": {
|
|
45
|
+
if (negated) {
|
|
46
|
+
throw new InvalidInputUriError(`Negation is not supported for dep:// inputs (got "${input}").`);
|
|
47
|
+
}
|
|
48
|
+
if (rest.length === 0) {
|
|
49
|
+
throw new InvalidInputUriError(`dep:// input requires at least one dependency name (got "${input}").`);
|
|
50
|
+
}
|
|
51
|
+
const names = rest.split(",").map((s) => s.trim());
|
|
52
|
+
if (names.some((s) => s.length === 0)) {
|
|
53
|
+
throw new InvalidInputUriError(`dep:// input contains an empty dependency segment (got "${input}"). Remove trailing or repeated commas.`);
|
|
54
|
+
}
|
|
55
|
+
return { externalDependencies: names };
|
|
56
|
+
}
|
|
57
|
+
default: {
|
|
58
|
+
throw new InvalidInputUriError(
|
|
59
|
+
`Unknown input URI scheme "${scheme}://" in "${input}". Recognized schemes: ${INPUT_URI_SCHEMES.map((s) => `${s}://`).join(", ")}.`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const looksLikeInputUri = (input) => {
|
|
65
|
+
const body = input.startsWith("!") ? input.slice(1) : input;
|
|
66
|
+
return URI_SCHEME_RE.test(body);
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
export { INPUT_URI_SCHEMES, InvalidInputUriError, looksLikeInputUri, parseInputUri };
|
package/dist/packem_shared/{IncrementalFileHasher-BRS76-mb.js → IncrementalFileHasher-BBhVK491.js}
RENAMED
|
@@ -23,7 +23,7 @@ const {
|
|
|
23
23
|
writeFile,
|
|
24
24
|
stat
|
|
25
25
|
} = __cjs_getBuiltinModule("node:fs/promises");
|
|
26
|
-
import { c as collectFiles, x as xxh3Hash } from './utils-
|
|
26
|
+
import { c as collectFiles, x as xxh3Hash } from './utils-Bmnj-H2J.js';
|
|
27
27
|
import { join, dirname, relative } from '@visulima/path';
|
|
28
28
|
|
|
29
29
|
const DEFAULT_IGNORED_DIRS = /* @__PURE__ */ new Set([".cache", ".git", ".task-runner-cache", "coverage", "dist", "node_modules"]);
|