@nowcrew/daemon 0.6.88-beta.15 → 0.6.88-beta.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/daemon-update-eligibility.js +1 -1
- package/dist/local-executor.js +4 -2
- package/dist/machine-info.js +48 -15
- package/dist/main.js +4 -69
- package/dist/project-skills/execution-adapter.js +3 -2
- package/dist/project-skills/runtime-launch.js +14 -0
- package/dist/provider-env.js +3 -4
- package/dist/runtime-probe.js +34 -5
- package/dist/runtimes/deepseek-harness-profile.js +92 -0
- package/dist/runtimes/deepseek-harness-runner.js +224 -0
- package/dist/runtimes/deepseek-harness.js +61 -10
- package/dist/runtimes/kimi-acp-runner.js +37 -6
- package/dist/serve.js +36 -68
- package/dist/storage-identity.js +0 -3
- package/dist/storage-serve-adapter.js +6 -72
- package/dist/supervised-runtime.js +17 -2
- package/package.json +1 -1
- package/dist/daemon-migration-controller.js +0 -211
- package/dist/daemon-migration-wiring.js +0 -22
- package/dist/storage-control.js +0 -212
- package/dist/storage-copy.js +0 -628
- package/dist/storage-credential-guard.js +0 -21
- package/dist/storage-doctor.js +0 -228
- package/dist/storage-manifest.js +0 -1027
- package/dist/storage-migration-controller.js +0 -440
- package/dist/storage-migration-manifest.js +0 -105
- package/dist/storage-migration-policy.js +0 -100
- package/dist/storage-runtime-controller.js +0 -381
- package/dist/storage-runtime-readiness.js +0 -240
- package/dist/storage-safe-fs.js +0 -302
- package/dist/storage-status-delivery.js +0 -224
- package/dist/storage-status-replay.js +0 -140
- package/dist/storage-target-confirmation.js +0 -129
- package/dist/storage-worker-command.js +0 -37
- package/dist/storage-worker-executor.js +0 -449
- package/dist/storage-worker-handoff.js +0 -151
- package/dist/storage-worker-operation.js +0 -294
- package/dist/storage-worker-status.js +0 -259
package/dist/storage-copy.js
DELETED
|
@@ -1,628 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import { constants } from "node:fs";
|
|
3
|
-
import { lstat, realpath, rename, } from "node:fs/promises";
|
|
4
|
-
import { dirname, isAbsolute, relative, resolve } from "node:path";
|
|
5
|
-
import { readStorageManifestChunks, storageManifestChunkSequenceSeed, storageManifestChunkSequenceStep, } from "./storage-manifest.js";
|
|
6
|
-
import { containsStorageCredentialMaterial } from "./storage-credential-guard.js";
|
|
7
|
-
import { chmodAnchoredDirectory, createAnchoredFile, createAnchoredSymlink, ensureAnchoredDirectory, inspectAnchoredEntry, openAnchoredFile, readAnchoredDirectory, removeAnchoredEntry, } from "./storage-safe-fs.js";
|
|
8
|
-
export const REDACTED_STORAGE_TOKEN = "[REDACTED: historical generic_sk credential removed during NOWWORK_HOME migration]";
|
|
9
|
-
const WORKSPACE_DIRECTORIES = ["agent-items", "daemon", "repos", "suite-tools"];
|
|
10
|
-
const LIVE_LIKE_TOKEN = /(?:bearer[ \t]+[a-z0-9._~-]{20,}|sk-[a-z0-9_-]{20,}|AKIA[0-9A-Z]{16})/giu;
|
|
11
|
-
const MAX_TRANSFORM_FILE_BYTES = 8 * 1024 * 1024;
|
|
12
|
-
const CREDENTIAL_SCAN_OVERLAP_BYTES = 4 * 1024;
|
|
13
|
-
const FILE_OPERATION_CONCURRENCY = 8;
|
|
14
|
-
const MAX_DESTINATION_INDEX_ENTRIES = 2_000_000;
|
|
15
|
-
function containedPath(root, logicalPath) {
|
|
16
|
-
if (logicalPath.length === 0 || isAbsolute(logicalPath) || logicalPath.includes("\0")
|
|
17
|
-
|| logicalPath.includes("\n") || logicalPath.includes("\r")) {
|
|
18
|
-
throw new Error(`unsafe logical migration path: ${logicalPath}`);
|
|
19
|
-
}
|
|
20
|
-
const target = resolve(root, logicalPath);
|
|
21
|
-
const child = relative(root, target);
|
|
22
|
-
if (child === "" || child.startsWith("..") || isAbsolute(child)) {
|
|
23
|
-
throw new Error(`logical migration path escapes staging: ${logicalPath}`);
|
|
24
|
-
}
|
|
25
|
-
return target;
|
|
26
|
-
}
|
|
27
|
-
function sourcePath(input) {
|
|
28
|
-
const root = input.sourceRoots[input.entry.sourceKind];
|
|
29
|
-
if (root === undefined)
|
|
30
|
-
throw new Error(`missing source root: ${input.entry.sourceKind}`);
|
|
31
|
-
return containedPath(root, input.entry.sourcePath);
|
|
32
|
-
}
|
|
33
|
-
/**
|
|
34
|
-
* Chunk-backed manifests intentionally carry an empty `entries` array. Read and
|
|
35
|
-
* authenticate their persisted chunks before any target mutation so all copy,
|
|
36
|
-
* verify, and reconcile paths observe the same complete manifest.
|
|
37
|
-
*/
|
|
38
|
-
async function* iterateManifestEntries(manifest) {
|
|
39
|
-
if (manifest.chunkStore === undefined) {
|
|
40
|
-
yield* manifest.entries;
|
|
41
|
-
return;
|
|
42
|
-
}
|
|
43
|
-
if (manifest.entries.length !== 0 || manifest.chunkStore.migrationId !== manifest.migrationId) {
|
|
44
|
-
throw new Error("storage manifest chunk store identity is inconsistent");
|
|
45
|
-
}
|
|
46
|
-
let sequenceDigest = storageManifestChunkSequenceSeed({
|
|
47
|
-
migrationId: manifest.migrationId,
|
|
48
|
-
environment: manifest.environment,
|
|
49
|
-
workspaceId: manifest.workspaceId,
|
|
50
|
-
targetWorkspaceRoot: manifest.targetWorkspaceRoot,
|
|
51
|
-
sourceKinds: manifest.sourceKinds,
|
|
52
|
-
});
|
|
53
|
-
for await (const chunk of readStorageManifestChunks(manifest.chunkStore)) {
|
|
54
|
-
sequenceDigest = storageManifestChunkSequenceStep(sequenceDigest, chunk.digest);
|
|
55
|
-
for (const entry of chunk.entries)
|
|
56
|
-
yield entry;
|
|
57
|
-
}
|
|
58
|
-
if (sequenceDigest !== manifest.chunkStore.sequenceDigest) {
|
|
59
|
-
throw new Error("storage manifest chunk sequence digest mismatch");
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
async function exists(path) {
|
|
63
|
-
try {
|
|
64
|
-
await lstat(path);
|
|
65
|
-
return true;
|
|
66
|
-
}
|
|
67
|
-
catch (error) {
|
|
68
|
-
if (error.code === "ENOENT")
|
|
69
|
-
return false;
|
|
70
|
-
throw error;
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
async function copyVerifiedFile(source, target, stagingWorkspaceRoot, entry, replaceExisting, afterSourceLstat, afterTargetParentOpen, assertHealthy) {
|
|
74
|
-
assertHealthy?.();
|
|
75
|
-
if (entry.sha256 === undefined)
|
|
76
|
-
throw new Error(`manifest file digest is missing: ${source}`);
|
|
77
|
-
const sourceHandle = await openAnchoredFile(source, constants.O_RDONLY).catch((error) => {
|
|
78
|
-
throw new Error(`migration source cannot be opened without following links: ${entry.sourcePath}`, { cause: error });
|
|
79
|
-
});
|
|
80
|
-
try {
|
|
81
|
-
const opened = await sourceHandle.stat();
|
|
82
|
-
if (!opened.isFile())
|
|
83
|
-
throw new Error(`migration source is no longer a file: ${entry.sourcePath}`);
|
|
84
|
-
await afterSourceLstat?.(source);
|
|
85
|
-
let sourceAfterHook;
|
|
86
|
-
try {
|
|
87
|
-
sourceAfterHook = inspectAnchoredEntry(dirname(source), source);
|
|
88
|
-
}
|
|
89
|
-
catch (error) {
|
|
90
|
-
throw new Error(`migration source changed before copy: ${entry.sourcePath}`, { cause: error });
|
|
91
|
-
}
|
|
92
|
-
if (sourceAfterHook.kind !== "file" || sourceAfterHook.stats.dev !== opened.dev
|
|
93
|
-
|| sourceAfterHook.stats.ino !== opened.ino) {
|
|
94
|
-
throw new Error(`migration source changed before copy: ${entry.sourcePath}`);
|
|
95
|
-
}
|
|
96
|
-
if (replaceExisting)
|
|
97
|
-
removeReplaceableTarget(stagingWorkspaceRoot, target, entry.logicalDestination ?? entry.sourcePath);
|
|
98
|
-
const targetHandle = await createAnchoredFile(stagingWorkspaceRoot, target, 0o600, afterTargetParentOpen).catch((error) => {
|
|
99
|
-
throw new Error(`migration target cannot be opened without following links: ${entry.logicalDestination}`, { cause: error });
|
|
100
|
-
});
|
|
101
|
-
const targetOpened = await targetHandle.stat();
|
|
102
|
-
const digest = createHash("sha256");
|
|
103
|
-
const sourceStream = sourceHandle.createReadStream({ autoClose: false });
|
|
104
|
-
try {
|
|
105
|
-
for await (const rawChunk of sourceStream) {
|
|
106
|
-
assertHealthy?.();
|
|
107
|
-
const chunk = Buffer.from(rawChunk);
|
|
108
|
-
digest.update(chunk);
|
|
109
|
-
await targetHandle.write(chunk, 0, chunk.length, null);
|
|
110
|
-
}
|
|
111
|
-
assertHealthy?.();
|
|
112
|
-
await targetHandle.sync();
|
|
113
|
-
await targetHandle.chmod(Number.parseInt(entry.mode, 8));
|
|
114
|
-
}
|
|
115
|
-
finally {
|
|
116
|
-
await targetHandle.close();
|
|
117
|
-
}
|
|
118
|
-
const after = await sourceHandle.stat();
|
|
119
|
-
const sourceCurrent = inspectAnchoredEntry(dirname(source), source);
|
|
120
|
-
if (opened.dev !== after.dev || opened.ino !== after.ino || opened.size !== after.size
|
|
121
|
-
|| opened.mtimeMs !== after.mtimeMs || digest.digest("hex") !== entry.sha256
|
|
122
|
-
|| sourceCurrent.kind !== "file" || sourceCurrent.stats.dev !== opened.dev
|
|
123
|
-
|| sourceCurrent.stats.ino !== opened.ino) {
|
|
124
|
-
throw new Error(`source digest mismatch: ${source}`);
|
|
125
|
-
}
|
|
126
|
-
let targetCurrent;
|
|
127
|
-
try {
|
|
128
|
-
targetCurrent = inspectAnchoredEntry(stagingWorkspaceRoot, target);
|
|
129
|
-
}
|
|
130
|
-
catch (error) {
|
|
131
|
-
throw new Error(`migration target changed while copying: ${entry.logicalDestination}`, { cause: error });
|
|
132
|
-
}
|
|
133
|
-
if (targetCurrent.kind !== "file" || targetCurrent.stats.dev !== targetOpened.dev
|
|
134
|
-
|| targetCurrent.stats.ino !== targetOpened.ino) {
|
|
135
|
-
throw new Error(`migration target changed while copying: ${entry.logicalDestination}`);
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
finally {
|
|
139
|
-
await sourceHandle.close();
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
async function createSafeDirectory(root, target) {
|
|
143
|
-
try {
|
|
144
|
-
ensureAnchoredDirectory(root, target, 0o700);
|
|
145
|
-
}
|
|
146
|
-
catch (error) {
|
|
147
|
-
throw new Error(`migration target ancestor is a symbolic link or non-directory: ${target}`, { cause: error });
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
async function createSafeParent(root, path) {
|
|
151
|
-
await createSafeDirectory(root, dirname(path));
|
|
152
|
-
}
|
|
153
|
-
function removeReplaceableTarget(root, path, logicalDestination) {
|
|
154
|
-
try {
|
|
155
|
-
const existing = inspectAnchoredEntry(root, path);
|
|
156
|
-
if (existing.kind === "symlink") {
|
|
157
|
-
throw new Error(`migration target is a symbolic link: ${logicalDestination}`);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
catch (error) {
|
|
161
|
-
if (error.code === "ENOENT")
|
|
162
|
-
return;
|
|
163
|
-
throw new Error(`migration target ancestor is a symbolic link or non-directory: ${logicalDestination}`, { cause: error });
|
|
164
|
-
}
|
|
165
|
-
removeAnchoredEntry(root, path);
|
|
166
|
-
}
|
|
167
|
-
async function assertRealpathWithin(root, candidate, message) {
|
|
168
|
-
const [canonicalRoot, canonicalCandidate] = await Promise.all([realpath(root), realpath(candidate)]);
|
|
169
|
-
const child = relative(canonicalRoot, canonicalCandidate);
|
|
170
|
-
if (child.startsWith("..") || isAbsolute(child))
|
|
171
|
-
throw new Error(message);
|
|
172
|
-
}
|
|
173
|
-
async function mapInBatches(items, concurrency, operation) {
|
|
174
|
-
const results = [];
|
|
175
|
-
for (let offset = 0; offset < items.length; offset += concurrency) {
|
|
176
|
-
results.push(...await Promise.all(items.slice(offset, offset + concurrency).map(operation)));
|
|
177
|
-
}
|
|
178
|
-
return results;
|
|
179
|
-
}
|
|
180
|
-
async function transformFile(source, target, stagingWorkspaceRoot, entry, replaceExisting, afterSourceLstat, afterTargetParentOpen, assertHealthy) {
|
|
181
|
-
assertHealthy?.();
|
|
182
|
-
if (entry.sha256 === undefined)
|
|
183
|
-
throw new Error(`manifest file digest is missing: ${source}`);
|
|
184
|
-
const sourceHandle = await openAnchoredFile(source, constants.O_RDONLY).catch((error) => {
|
|
185
|
-
throw new Error(`migration source cannot be opened without following links: ${entry.sourcePath}`, { cause: error });
|
|
186
|
-
});
|
|
187
|
-
let input;
|
|
188
|
-
try {
|
|
189
|
-
const opened = await sourceHandle.stat();
|
|
190
|
-
if (!opened.isFile())
|
|
191
|
-
throw new Error(`migration source is no longer a file: ${entry.sourcePath}`);
|
|
192
|
-
await afterSourceLstat?.(source);
|
|
193
|
-
let sourceAfterHook;
|
|
194
|
-
try {
|
|
195
|
-
sourceAfterHook = inspectAnchoredEntry(dirname(source), source);
|
|
196
|
-
}
|
|
197
|
-
catch (error) {
|
|
198
|
-
throw new Error(`migration source changed before transform: ${entry.sourcePath}`, { cause: error });
|
|
199
|
-
}
|
|
200
|
-
if (sourceAfterHook.kind !== "file" || sourceAfterHook.stats.dev !== opened.dev
|
|
201
|
-
|| sourceAfterHook.stats.ino !== opened.ino) {
|
|
202
|
-
throw new Error(`migration source changed before transform: ${entry.sourcePath}`);
|
|
203
|
-
}
|
|
204
|
-
if (opened.size > MAX_TRANSFORM_FILE_BYTES) {
|
|
205
|
-
throw new Error(`target transformation source exceeds bounded size: ${entry.sourcePath}`);
|
|
206
|
-
}
|
|
207
|
-
input = await sourceHandle.readFile("utf8");
|
|
208
|
-
const after = await sourceHandle.stat();
|
|
209
|
-
const sourceCurrent = inspectAnchoredEntry(dirname(source), source);
|
|
210
|
-
if (opened.dev !== after.dev || opened.ino !== after.ino || opened.size !== after.size
|
|
211
|
-
|| opened.mtimeMs !== after.mtimeMs || createHash("sha256").update(input).digest("hex") !== entry.sha256
|
|
212
|
-
|| sourceCurrent.kind !== "file" || sourceCurrent.stats.dev !== opened.dev
|
|
213
|
-
|| sourceCurrent.stats.ino !== opened.ino) {
|
|
214
|
-
throw new Error(`source digest mismatch: ${source}`);
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
finally {
|
|
218
|
-
await sourceHandle.close();
|
|
219
|
-
}
|
|
220
|
-
const output = input.replaceAll(LIVE_LIKE_TOKEN, REDACTED_STORAGE_TOKEN);
|
|
221
|
-
if (output === input || LIVE_LIKE_TOKEN.test(output))
|
|
222
|
-
throw new Error(`target transformation did not remove live-like tokens: ${entry.sourcePath}`);
|
|
223
|
-
LIVE_LIKE_TOKEN.lastIndex = 0;
|
|
224
|
-
if (entry.targetSha256 === undefined || createHash("sha256").update(output).digest("hex") !== entry.targetSha256) {
|
|
225
|
-
throw new Error(`transformed target digest mismatch: ${entry.sourcePath}`);
|
|
226
|
-
}
|
|
227
|
-
if (replaceExisting)
|
|
228
|
-
removeReplaceableTarget(stagingWorkspaceRoot, target, entry.logicalDestination ?? entry.sourcePath);
|
|
229
|
-
const targetHandle = await createAnchoredFile(stagingWorkspaceRoot, target, 0o600, afterTargetParentOpen).catch((error) => {
|
|
230
|
-
throw new Error(`migration target cannot be opened without following links: ${entry.logicalDestination}`, { cause: error });
|
|
231
|
-
});
|
|
232
|
-
const targetOpened = await targetHandle.stat();
|
|
233
|
-
try {
|
|
234
|
-
assertHealthy?.();
|
|
235
|
-
await targetHandle.writeFile(output, "utf8");
|
|
236
|
-
await targetHandle.sync();
|
|
237
|
-
await targetHandle.chmod(Number.parseInt(entry.targetMode ?? "0600", 8));
|
|
238
|
-
}
|
|
239
|
-
finally {
|
|
240
|
-
await targetHandle.close();
|
|
241
|
-
}
|
|
242
|
-
let targetCurrent;
|
|
243
|
-
try {
|
|
244
|
-
targetCurrent = inspectAnchoredEntry(stagingWorkspaceRoot, target);
|
|
245
|
-
}
|
|
246
|
-
catch (error) {
|
|
247
|
-
throw new Error(`migration target changed while transforming: ${entry.logicalDestination}`, { cause: error });
|
|
248
|
-
}
|
|
249
|
-
if (targetCurrent.kind !== "file" || targetCurrent.stats.dev !== targetOpened.dev
|
|
250
|
-
|| targetCurrent.stats.ino !== targetOpened.ino) {
|
|
251
|
-
throw new Error(`migration target changed while transforming: ${entry.logicalDestination}`);
|
|
252
|
-
}
|
|
253
|
-
return Buffer.byteLength(output);
|
|
254
|
-
}
|
|
255
|
-
async function materializeChunkedManifest(input, injected) {
|
|
256
|
-
injected.assertHealthy?.();
|
|
257
|
-
if (await exists(input.stagingWorkspaceRoot)) {
|
|
258
|
-
if (!(await lstat(input.stagingWorkspaceRoot)).isDirectory()
|
|
259
|
-
|| (!input.mergeIntoExisting
|
|
260
|
-
&& (await readAnchoredDirectory(input.stagingWorkspaceRoot, input.stagingWorkspaceRoot)).length > 0)) {
|
|
261
|
-
throw new Error("staging collision: staging workspace root is not empty");
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
else {
|
|
265
|
-
ensureAnchoredDirectory(dirname(input.stagingWorkspaceRoot), input.stagingWorkspaceRoot, 0o700);
|
|
266
|
-
}
|
|
267
|
-
let createdDirectories = 0;
|
|
268
|
-
for (const directory of WORKSPACE_DIRECTORIES) {
|
|
269
|
-
await createSafeDirectory(input.stagingWorkspaceRoot, containedPath(input.stagingWorkspaceRoot, directory));
|
|
270
|
-
createdDirectories += 1;
|
|
271
|
-
}
|
|
272
|
-
let copiedFiles = 0;
|
|
273
|
-
let transformedFiles = 0;
|
|
274
|
-
let copiedSymlinks = 0;
|
|
275
|
-
let bytesWritten = 0;
|
|
276
|
-
let pending = [];
|
|
277
|
-
const flushFiles = async () => {
|
|
278
|
-
if (pending.length === 0)
|
|
279
|
-
return;
|
|
280
|
-
const batch = pending;
|
|
281
|
-
pending = [];
|
|
282
|
-
injected.assertHealthy?.();
|
|
283
|
-
const results = await mapInBatches(batch, FILE_OPERATION_CONCURRENCY, async (entry) => {
|
|
284
|
-
injected.assertHealthy?.();
|
|
285
|
-
const target = containedPath(input.stagingWorkspaceRoot, entry.logicalDestination);
|
|
286
|
-
const source = sourcePath({ sourceRoots: input.sourceRoots, entry });
|
|
287
|
-
if (entry.disposition === "transform") {
|
|
288
|
-
return { transformed: true, bytesWritten: await transformFile(source, target, input.stagingWorkspaceRoot, entry, input.mergeIntoExisting === true, injected.afterSourceLstat, injected.afterTargetParentOpen, injected.assertHealthy) };
|
|
289
|
-
}
|
|
290
|
-
await copyVerifiedFile(source, target, input.stagingWorkspaceRoot, entry, input.mergeIntoExisting === true, injected.afterSourceLstat, injected.afterTargetParentOpen, injected.assertHealthy);
|
|
291
|
-
return { transformed: false, bytesWritten: entry.sizeBytes };
|
|
292
|
-
});
|
|
293
|
-
for (const result of results) {
|
|
294
|
-
if (result.transformed)
|
|
295
|
-
transformedFiles += 1;
|
|
296
|
-
else
|
|
297
|
-
copiedFiles += 1;
|
|
298
|
-
bytesWritten += result.bytesWritten;
|
|
299
|
-
}
|
|
300
|
-
};
|
|
301
|
-
for await (const entry of iterateManifestEntries(input.manifest)) {
|
|
302
|
-
injected.assertHealthy?.();
|
|
303
|
-
if (entry.disposition === "skip" || entry.logicalDestination === null)
|
|
304
|
-
continue;
|
|
305
|
-
const target = containedPath(input.stagingWorkspaceRoot, entry.logicalDestination);
|
|
306
|
-
if (entry.type === "directory") {
|
|
307
|
-
await createSafeDirectory(input.stagingWorkspaceRoot, target);
|
|
308
|
-
createdDirectories += 1;
|
|
309
|
-
}
|
|
310
|
-
else if (entry.type === "file") {
|
|
311
|
-
pending.push(entry);
|
|
312
|
-
if (pending.length >= 64)
|
|
313
|
-
await flushFiles();
|
|
314
|
-
}
|
|
315
|
-
else if (entry.type !== "symlink") {
|
|
316
|
-
throw new Error(`unsupported copied manifest entry: ${entry.sourcePath}`);
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
await flushFiles();
|
|
320
|
-
for await (const entry of iterateManifestEntries(input.manifest)) {
|
|
321
|
-
injected.assertHealthy?.();
|
|
322
|
-
if (entry.disposition === "skip" || entry.logicalDestination === null || entry.type !== "symlink")
|
|
323
|
-
continue;
|
|
324
|
-
if (entry.link?.classification !== "internal" || !entry.link.portable || entry.link.relativeTarget === undefined)
|
|
325
|
-
throw new Error(`manifest contains a non-portable copied symlink: ${entry.sourcePath}`);
|
|
326
|
-
const target = containedPath(input.stagingWorkspaceRoot, entry.logicalDestination);
|
|
327
|
-
await createSafeParent(input.stagingWorkspaceRoot, target);
|
|
328
|
-
if (input.mergeIntoExisting)
|
|
329
|
-
removeReplaceableTarget(input.stagingWorkspaceRoot, target, entry.logicalDestination);
|
|
330
|
-
createAnchoredSymlink(input.stagingWorkspaceRoot, target, entry.link.relativeTarget);
|
|
331
|
-
await assertRealpathWithin(input.stagingWorkspaceRoot, target, `staged symlink escapes workspace: ${entry.sourcePath}`);
|
|
332
|
-
copiedSymlinks += 1;
|
|
333
|
-
}
|
|
334
|
-
for await (const entry of iterateManifestEntries(input.manifest)) {
|
|
335
|
-
injected.assertHealthy?.();
|
|
336
|
-
if (entry.disposition === "skip" || entry.logicalDestination === null || entry.type !== "directory")
|
|
337
|
-
continue;
|
|
338
|
-
chmodAnchoredDirectory(input.stagingWorkspaceRoot, containedPath(input.stagingWorkspaceRoot, entry.logicalDestination), Number.parseInt(entry.mode, 8));
|
|
339
|
-
}
|
|
340
|
-
return Object.freeze({ copiedFiles, transformedFiles, copiedSymlinks, createdDirectories, bytesWritten });
|
|
341
|
-
}
|
|
342
|
-
export async function materializeStorageManifest(input, injected = {}) {
|
|
343
|
-
injected.assertHealthy?.();
|
|
344
|
-
if (input.manifest.chunkStore !== undefined)
|
|
345
|
-
return materializeChunkedManifest(input, injected);
|
|
346
|
-
const manifest = input.manifest;
|
|
347
|
-
if (await exists(input.stagingWorkspaceRoot)) {
|
|
348
|
-
if (!(await lstat(input.stagingWorkspaceRoot)).isDirectory()
|
|
349
|
-
|| (!input.mergeIntoExisting
|
|
350
|
-
&& (await readAnchoredDirectory(input.stagingWorkspaceRoot, input.stagingWorkspaceRoot)).length > 0)) {
|
|
351
|
-
throw new Error("staging collision: staging workspace root is not empty");
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
else {
|
|
355
|
-
ensureAnchoredDirectory(dirname(input.stagingWorkspaceRoot), input.stagingWorkspaceRoot, 0o700);
|
|
356
|
-
}
|
|
357
|
-
let createdDirectories = 0;
|
|
358
|
-
for (const directory of WORKSPACE_DIRECTORIES) {
|
|
359
|
-
const path = containedPath(input.stagingWorkspaceRoot, directory);
|
|
360
|
-
await createSafeDirectory(input.stagingWorkspaceRoot, path);
|
|
361
|
-
createdDirectories += 1;
|
|
362
|
-
}
|
|
363
|
-
const directoryModes = [];
|
|
364
|
-
const symlinks = [];
|
|
365
|
-
const files = [];
|
|
366
|
-
for (const entry of manifest.entries) {
|
|
367
|
-
if (entry.disposition === "skip" || entry.logicalDestination === null)
|
|
368
|
-
continue;
|
|
369
|
-
const target = containedPath(input.stagingWorkspaceRoot, entry.logicalDestination);
|
|
370
|
-
if (entry.type === "directory") {
|
|
371
|
-
await createSafeDirectory(input.stagingWorkspaceRoot, target);
|
|
372
|
-
directoryModes.push({ path: target, mode: Number.parseInt(entry.mode, 8) });
|
|
373
|
-
createdDirectories += 1;
|
|
374
|
-
continue;
|
|
375
|
-
}
|
|
376
|
-
if (entry.type === "file") {
|
|
377
|
-
files.push(entry);
|
|
378
|
-
continue;
|
|
379
|
-
}
|
|
380
|
-
if (entry.type === "symlink") {
|
|
381
|
-
if (entry.link?.classification !== "internal" || !entry.link.portable || entry.link.relativeTarget === undefined) {
|
|
382
|
-
throw new Error(`manifest contains a non-portable copied symlink: ${entry.sourcePath}`);
|
|
383
|
-
}
|
|
384
|
-
symlinks.push({ entry, target });
|
|
385
|
-
continue;
|
|
386
|
-
}
|
|
387
|
-
throw new Error(`unsupported copied manifest entry: ${entry.sourcePath}`);
|
|
388
|
-
}
|
|
389
|
-
const fileResults = await mapInBatches(files, FILE_OPERATION_CONCURRENCY, async (entry) => {
|
|
390
|
-
injected.assertHealthy?.();
|
|
391
|
-
const target = containedPath(input.stagingWorkspaceRoot, entry.logicalDestination);
|
|
392
|
-
const source = sourcePath({ sourceRoots: input.sourceRoots, entry });
|
|
393
|
-
if (entry.disposition === "transform") {
|
|
394
|
-
return {
|
|
395
|
-
transformed: true,
|
|
396
|
-
bytesWritten: await transformFile(source, target, input.stagingWorkspaceRoot, entry, input.mergeIntoExisting === true, injected.afterSourceLstat, injected.afterTargetParentOpen, injected.assertHealthy),
|
|
397
|
-
};
|
|
398
|
-
}
|
|
399
|
-
await copyVerifiedFile(source, target, input.stagingWorkspaceRoot, entry, input.mergeIntoExisting === true, injected.afterSourceLstat, injected.afterTargetParentOpen, injected.assertHealthy);
|
|
400
|
-
return { transformed: false, bytesWritten: entry.sizeBytes };
|
|
401
|
-
});
|
|
402
|
-
for (const { entry, target } of symlinks) {
|
|
403
|
-
injected.assertHealthy?.();
|
|
404
|
-
await createSafeParent(input.stagingWorkspaceRoot, target);
|
|
405
|
-
if (input.mergeIntoExisting)
|
|
406
|
-
removeReplaceableTarget(input.stagingWorkspaceRoot, target, entry.logicalDestination ?? entry.sourcePath);
|
|
407
|
-
createAnchoredSymlink(input.stagingWorkspaceRoot, target, entry.link.relativeTarget);
|
|
408
|
-
}
|
|
409
|
-
for (const { entry, target } of symlinks) {
|
|
410
|
-
injected.assertHealthy?.();
|
|
411
|
-
await assertRealpathWithin(input.stagingWorkspaceRoot, target, `staged symlink escapes workspace: ${entry.sourcePath}`);
|
|
412
|
-
}
|
|
413
|
-
directoryModes.sort((left, right) => right.path.length - left.path.length);
|
|
414
|
-
for (const directory of directoryModes) {
|
|
415
|
-
injected.assertHealthy?.();
|
|
416
|
-
chmodAnchoredDirectory(input.stagingWorkspaceRoot, directory.path, directory.mode);
|
|
417
|
-
}
|
|
418
|
-
const transformedFiles = fileResults.filter((result) => result.transformed).length;
|
|
419
|
-
return Object.freeze({
|
|
420
|
-
copiedFiles: fileResults.length - transformedFiles,
|
|
421
|
-
transformedFiles,
|
|
422
|
-
copiedSymlinks: symlinks.length,
|
|
423
|
-
createdDirectories,
|
|
424
|
-
bytesWritten: fileResults.reduce((total, result) => total + result.bytesWritten, 0),
|
|
425
|
-
});
|
|
426
|
-
}
|
|
427
|
-
function copiedEntries(manifest) {
|
|
428
|
-
return new Map(manifest.entries
|
|
429
|
-
.filter((entry) => entry.disposition !== "skip" && entry.logicalDestination !== null)
|
|
430
|
-
.map((entry) => [entry.logicalDestination, entry]));
|
|
431
|
-
}
|
|
432
|
-
async function collectChunkDestinations(manifest) {
|
|
433
|
-
const destinations = new Set();
|
|
434
|
-
for await (const entry of iterateManifestEntries(manifest)) {
|
|
435
|
-
if (entry.disposition === "skip" || entry.logicalDestination === null)
|
|
436
|
-
continue;
|
|
437
|
-
destinations.add(entry.logicalDestination);
|
|
438
|
-
if (destinations.size > MAX_DESTINATION_INDEX_ENTRIES)
|
|
439
|
-
throw new Error("storage destination index entry limit exceeded");
|
|
440
|
-
}
|
|
441
|
-
return destinations;
|
|
442
|
-
}
|
|
443
|
-
async function pruneStaleChunkPaths(root, destinations) {
|
|
444
|
-
const walk = async (directory) => {
|
|
445
|
-
let retained = false;
|
|
446
|
-
for (const name of await readAnchoredDirectory(root, directory)) {
|
|
447
|
-
const child = resolve(directory, name);
|
|
448
|
-
const logical = relative(root, child).replaceAll("\\", "/");
|
|
449
|
-
const current = inspectAnchoredEntry(root, child);
|
|
450
|
-
if (current.kind === "directory") {
|
|
451
|
-
const childRetained = await walk(child);
|
|
452
|
-
if (!childRetained && !destinations.has(logical))
|
|
453
|
-
removeAnchoredEntry(root, child);
|
|
454
|
-
else
|
|
455
|
-
retained = true;
|
|
456
|
-
}
|
|
457
|
-
else if (destinations.has(logical)) {
|
|
458
|
-
retained = true;
|
|
459
|
-
}
|
|
460
|
-
else {
|
|
461
|
-
removeAnchoredEntry(root, child);
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
return retained || destinations.has(relative(root, directory).replaceAll("\\", "/"));
|
|
465
|
-
};
|
|
466
|
-
for (const directory of WORKSPACE_DIRECTORIES) {
|
|
467
|
-
const path = containedPath(root, directory);
|
|
468
|
-
if (inspectAnchoredEntry(root, path).kind === "directory")
|
|
469
|
-
await walk(path);
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
export async function reconcileStorageManifest(input) {
|
|
473
|
-
input.assertHealthy?.();
|
|
474
|
-
if (input.previousManifest.migrationId !== input.manifest.migrationId
|
|
475
|
-
|| input.previousManifest.workspaceId !== input.manifest.workspaceId
|
|
476
|
-
|| input.previousManifest.targetWorkspaceRoot !== input.manifest.targetWorkspaceRoot) {
|
|
477
|
-
throw new Error("storage reconciliation manifest identity mismatch");
|
|
478
|
-
}
|
|
479
|
-
const stagingRoot = await lstat(input.stagingWorkspaceRoot);
|
|
480
|
-
if (!stagingRoot.isDirectory() || stagingRoot.isSymbolicLink()) {
|
|
481
|
-
throw new Error("storage reconciliation requires existing staging");
|
|
482
|
-
}
|
|
483
|
-
if (input.previousManifest.chunkStore !== undefined || input.manifest.chunkStore !== undefined) {
|
|
484
|
-
// Build a bounded destination index, prune stale staging paths, then replay
|
|
485
|
-
// the authenticated final snapshot without retaining manifest entries.
|
|
486
|
-
const destinations = await collectChunkDestinations(input.manifest);
|
|
487
|
-
input.assertHealthy?.();
|
|
488
|
-
await pruneStaleChunkPaths(input.stagingWorkspaceRoot, destinations);
|
|
489
|
-
return materializeChunkedManifest({
|
|
490
|
-
manifest: input.manifest,
|
|
491
|
-
sourceRoots: input.sourceRoots,
|
|
492
|
-
stagingWorkspaceRoot: input.stagingWorkspaceRoot,
|
|
493
|
-
mergeIntoExisting: true,
|
|
494
|
-
}, input.assertHealthy === undefined ? {} : { assertHealthy: input.assertHealthy });
|
|
495
|
-
}
|
|
496
|
-
const previous = copiedEntries(input.previousManifest);
|
|
497
|
-
const next = copiedEntries(input.manifest);
|
|
498
|
-
const removals = [...previous.entries()].filter(([logical, entry]) => {
|
|
499
|
-
const current = next.get(logical);
|
|
500
|
-
return current === undefined || current.type !== entry.type;
|
|
501
|
-
}).sort(([left], [right]) => right.length - left.length);
|
|
502
|
-
for (const [logical] of removals) {
|
|
503
|
-
input.assertHealthy?.();
|
|
504
|
-
const target = containedPath(input.stagingWorkspaceRoot, logical);
|
|
505
|
-
try {
|
|
506
|
-
removeAnchoredEntry(input.stagingWorkspaceRoot, target);
|
|
507
|
-
}
|
|
508
|
-
catch (error) {
|
|
509
|
-
throw new Error(`migration target ancestor is a symbolic link or non-directory: ${logical}`, { cause: error });
|
|
510
|
-
}
|
|
511
|
-
}
|
|
512
|
-
const changed = input.manifest.entries.filter((entry) => {
|
|
513
|
-
if (entry.logicalDestination === null || entry.disposition === "skip")
|
|
514
|
-
return false;
|
|
515
|
-
const prior = previous.get(entry.logicalDestination);
|
|
516
|
-
return prior === undefined || JSON.stringify(prior) !== JSON.stringify(entry);
|
|
517
|
-
});
|
|
518
|
-
return materializeStorageManifest({
|
|
519
|
-
manifest: { ...input.manifest, entries: changed },
|
|
520
|
-
sourceRoots: input.sourceRoots,
|
|
521
|
-
stagingWorkspaceRoot: input.stagingWorkspaceRoot,
|
|
522
|
-
mergeIntoExisting: true,
|
|
523
|
-
}, input.assertHealthy === undefined ? {} : { assertHealthy: input.assertHealthy });
|
|
524
|
-
}
|
|
525
|
-
async function inspectStorageFile(root, path, afterTargetLstat) {
|
|
526
|
-
const handle = await openAnchoredFile(path, constants.O_RDONLY).catch((error) => {
|
|
527
|
-
throw new Error(`staged file cannot be opened without following links: ${path}`, { cause: error });
|
|
528
|
-
});
|
|
529
|
-
try {
|
|
530
|
-
const opened = await handle.stat();
|
|
531
|
-
if (!opened.isFile())
|
|
532
|
-
throw new Error(`staged file is no longer a regular file: ${path}`);
|
|
533
|
-
await afterTargetLstat?.(path);
|
|
534
|
-
const digest = createHash("sha256");
|
|
535
|
-
let tail = "";
|
|
536
|
-
let credentialMaterial = false;
|
|
537
|
-
const stream = handle.createReadStream({ autoClose: false });
|
|
538
|
-
for await (const chunk of stream) {
|
|
539
|
-
digest.update(chunk);
|
|
540
|
-
const text = tail + Buffer.from(chunk).toString("latin1");
|
|
541
|
-
if (containsStorageCredentialMaterial(text))
|
|
542
|
-
credentialMaterial = true;
|
|
543
|
-
tail = text.slice(-CREDENTIAL_SCAN_OVERLAP_BYTES);
|
|
544
|
-
}
|
|
545
|
-
const after = await handle.stat();
|
|
546
|
-
const pathAfter = inspectAnchoredEntry(root, path);
|
|
547
|
-
if (pathAfter.kind !== "file" || opened.dev !== after.dev || opened.ino !== after.ino
|
|
548
|
-
|| opened.size !== after.size || opened.mtimeMs !== after.mtimeMs
|
|
549
|
-
|| opened.dev !== pathAfter.stats.dev || opened.ino !== pathAfter.stats.ino) {
|
|
550
|
-
throw new Error(`staged file changed while verifying: ${path}`);
|
|
551
|
-
}
|
|
552
|
-
return Object.freeze({ digest: digest.digest("hex"), credentialMaterial });
|
|
553
|
-
}
|
|
554
|
-
finally {
|
|
555
|
-
await handle.close();
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
async function scanLiveLikeMatches(root, alreadyScanned) {
|
|
559
|
-
let matches = 0;
|
|
560
|
-
const pending = [root];
|
|
561
|
-
while (pending.length > 0) {
|
|
562
|
-
const directory = pending.pop();
|
|
563
|
-
for (const name of await readAnchoredDirectory(root, directory)) {
|
|
564
|
-
const path = resolve(directory, name);
|
|
565
|
-
const entry = inspectAnchoredEntry(root, path);
|
|
566
|
-
if (entry.kind === "directory")
|
|
567
|
-
pending.push(path);
|
|
568
|
-
else if (entry.kind === "file" && !alreadyScanned.has(path)
|
|
569
|
-
&& (await inspectStorageFile(root, path)).credentialMaterial)
|
|
570
|
-
matches += 1;
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
return matches;
|
|
574
|
-
}
|
|
575
|
-
export async function verifyStagedStorage(input, injected = {}) {
|
|
576
|
-
let checkedEntries = 0;
|
|
577
|
-
let liveLikeMatches = 0;
|
|
578
|
-
for (const directory of WORKSPACE_DIRECTORIES) {
|
|
579
|
-
const target = containedPath(input.stagingWorkspaceRoot, directory);
|
|
580
|
-
const targetEntry = inspectAnchoredEntry(input.stagingWorkspaceRoot, target);
|
|
581
|
-
if (targetEntry.kind !== "directory" || (targetEntry.stats.mode & 0o777) !== 0o700) {
|
|
582
|
-
throw new Error(`required workspace directory is invalid: ${directory}`);
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
for await (const entry of iterateManifestEntries(input.manifest)) {
|
|
586
|
-
const logical = entry.logicalDestination;
|
|
587
|
-
if (entry.disposition === "skip" || logical === null)
|
|
588
|
-
continue;
|
|
589
|
-
const target = containedPath(input.stagingWorkspaceRoot, logical);
|
|
590
|
-
const targetEntry = inspectAnchoredEntry(input.stagingWorkspaceRoot, target);
|
|
591
|
-
if (entry.type === "directory" && targetEntry.kind !== "directory")
|
|
592
|
-
throw new Error(`staged type mismatch: ${logical}`);
|
|
593
|
-
if (entry.type === "file") {
|
|
594
|
-
if (targetEntry.kind !== "file")
|
|
595
|
-
throw new Error(`staged type mismatch: ${logical}`);
|
|
596
|
-
const expected = entry.disposition === "transform" ? entry.targetSha256 : entry.sha256;
|
|
597
|
-
const inspected = await inspectStorageFile(input.stagingWorkspaceRoot, target, injected.afterTargetLstat);
|
|
598
|
-
if (expected === undefined || inspected.digest !== expected)
|
|
599
|
-
throw new Error(`staged digest mismatch: ${logical}`);
|
|
600
|
-
if (inspected.credentialMaterial)
|
|
601
|
-
liveLikeMatches += 1;
|
|
602
|
-
const expectedMode = Number.parseInt(entry.disposition === "transform" ? entry.targetMode ?? "0600" : entry.mode, 8);
|
|
603
|
-
if ((targetEntry.stats.mode & 0o7777) !== expectedMode)
|
|
604
|
-
throw new Error(`staged mode mismatch: ${logical}`);
|
|
605
|
-
}
|
|
606
|
-
if (entry.type === "symlink") {
|
|
607
|
-
if (targetEntry.kind !== "symlink" || targetEntry.linkTarget !== entry.link?.relativeTarget) {
|
|
608
|
-
throw new Error(`staged symlink mismatch: ${logical}`);
|
|
609
|
-
}
|
|
610
|
-
await assertRealpathWithin(input.stagingWorkspaceRoot, target, `staged symlink escapes workspace: ${logical}`);
|
|
611
|
-
}
|
|
612
|
-
checkedEntries += 1;
|
|
613
|
-
}
|
|
614
|
-
liveLikeMatches += await scanLiveLikeMatches(input.stagingWorkspaceRoot, new Set());
|
|
615
|
-
if (liveLikeMatches !== 0)
|
|
616
|
-
throw new Error(`staging contains credential material in ${liveLikeMatches} file(s)`);
|
|
617
|
-
return Object.freeze({ valid: true, checkedEntries, liveLikeMatches: 0 });
|
|
618
|
-
}
|
|
619
|
-
export async function publishStagedStorage(input, dependencies = {}) {
|
|
620
|
-
if (dirname(input.stagingWorkspaceRoot) !== dirname(input.targetWorkspaceRoot)) {
|
|
621
|
-
throw new Error("staging and target workspace roots must be siblings on one filesystem");
|
|
622
|
-
}
|
|
623
|
-
if (await exists(input.targetWorkspaceRoot))
|
|
624
|
-
throw new Error("target collision: target workspace root already exists");
|
|
625
|
-
if (!(await lstat(input.stagingWorkspaceRoot)).isDirectory())
|
|
626
|
-
throw new Error("staging workspace root is not a directory");
|
|
627
|
-
await (dependencies.rename ?? rename)(input.stagingWorkspaceRoot, input.targetWorkspaceRoot);
|
|
628
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
const CREDENTIAL_PATTERNS = Object.freeze([
|
|
2
|
-
/-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----/iu,
|
|
3
|
-
/\bAuthorization\s*:\s*(?:Bearer|Basic)\s+\S{8,}/iu,
|
|
4
|
-
/\bCookie\s*:\s*[^\n]*(?:session|token|auth)[^\n]{8,}/iu,
|
|
5
|
-
/\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis):\/\/[^\s:/]+:[^\s@]+@/iu,
|
|
6
|
-
/\bsk_(?:machine|user|agent)_[A-Za-z0-9_-]{4,}\b/u,
|
|
7
|
-
/\bAKIA[0-9A-Z]{16}\b/u,
|
|
8
|
-
/\b(?:sk|gh[oprsu]|xox[baprs])-[-A-Za-z0-9_]{12,}\b/u,
|
|
9
|
-
/(?:^|[^A-Za-z0-9])(?:[A-Z0-9]+_)*(?:PASSWORD|PASSWD|SECRET|TOKEN|API_KEY|ACCESS_KEY|PRIVATE_KEY)["']?\s*[:=]\s*["']?[^\s"';&,}]{8,}/imu,
|
|
10
|
-
/\b(?:AWS_SECRET_ACCESS_KEY|TENCENTCLOUD_SECRET_KEY|SECRET_KEY|API_KEY|ACCESS_TOKEN|AUTH_TOKEN|SESSION_TOKEN)\s*[:=]\s*\S{8,}/iu,
|
|
11
|
-
/\b(?:api[-_]?key|access[-_]?token|refresh[-_]?token|password|passwd|secret|credential|authorization|session|token)\s*[:=]\s*["']?[^\s"';&]{8,}/iu,
|
|
12
|
-
/\bbearer[ \t]+[A-Za-z0-9._~+/=-]{12,}\b/iu,
|
|
13
|
-
]);
|
|
14
|
-
export function containsStorageCredentialMaterial(value) {
|
|
15
|
-
return CREDENTIAL_PATTERNS.some((pattern) => pattern.test(value));
|
|
16
|
-
}
|
|
17
|
-
export function assertStorageControlMaterialTokenFree(value, source) {
|
|
18
|
-
if (containsStorageCredentialMaterial(value)) {
|
|
19
|
-
throw new Error(`credential material detected in ${source}`);
|
|
20
|
-
}
|
|
21
|
-
}
|