@nowcrew/daemon 0.6.70 → 0.6.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/main.js +0 -0
- package/dist/provider-error.js +2 -3
- package/dist/runtime-health-handler.js +5 -0
- package/dist/runtime-health.js +93 -50
- package/dist/serve.js +9 -4
- package/dist/slog.js +3 -1
- package/dist/storage-manifest.js +214 -51
- package/package.json +10 -9
package/dist/main.js
CHANGED
|
File without changes
|
package/dist/provider-error.js
CHANGED
|
@@ -3,14 +3,13 @@ export function providerErrorFromExit(exitCode, errorText) {
|
|
|
3
3
|
return undefined;
|
|
4
4
|
return errorText ? classifyProviderError(errorText) : { category: "cli", code: "process_exit" };
|
|
5
5
|
}
|
|
6
|
-
const STATUS_PATTERN = /\b(?:http(?:\/\d(?:\.\d)?)?\s
|
|
7
|
-
const KNOWN_STATUS_PATTERN = /\b(401|403|408|429|500|502|503|504)\b/u;
|
|
6
|
+
const STATUS_PATTERN = /\b(?:http(?:\/\d(?:\.\d)?)?\s+|status(?:_code)?\s*[:=]?\s*|response\s+status\s+|error\s+|failed\s*[:=]?\s*|returned\s+|responded\s+)([1-5]\d{2})\b/iu;
|
|
8
7
|
const CODE_BY_STATUS = Object.freeze({
|
|
9
8
|
401: "unauthorized", 403: "forbidden", 408: "request_timeout", 429: "rate_limited",
|
|
10
9
|
500: "internal_server_error", 502: "bad_gateway", 503: "service_unavailable", 504: "gateway_timeout",
|
|
11
10
|
});
|
|
12
11
|
function statusFromText(text) {
|
|
13
|
-
const explicit = STATUS_PATTERN.exec(text)?.[1]
|
|
12
|
+
const explicit = STATUS_PATTERN.exec(text)?.[1];
|
|
14
13
|
if (explicit === undefined)
|
|
15
14
|
return undefined;
|
|
16
15
|
const value = Number(explicit);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { handleRuntimeProbeFrame, probeRuntimeHealth, } from "./runtime-health.js";
|
|
2
|
+
export const runtimeHealthProbeHandler = (configuredProbe, executionRuntimesKnown, detectedExecutionRuntimes) => configuredProbe ?? ((request) => probeRuntimeHealth(request, {
|
|
3
|
+
...(executionRuntimesKnown ? { availableRuntimes: detectedExecutionRuntimes } : {}),
|
|
4
|
+
}));
|
|
5
|
+
export const handleRuntimeHealthProbeFrame = (decoded, configuredProbe, executionRuntimesKnown, detectedExecutionRuntimes) => handleRuntimeProbeFrame(decoded, runtimeHealthProbeHandler(configuredProbe, executionRuntimesKnown, detectedExecutionRuntimes));
|
package/dist/runtime-health.js
CHANGED
|
@@ -13,6 +13,12 @@ const RuntimeProbeTargetSchema = z.object({
|
|
|
13
13
|
export const RuntimeProbeRequestSchema = z.object({
|
|
14
14
|
runtimes: z.array(RuntimeProbeTargetSchema).min(1).max(RUNTIME_HEALTH_NAMES.length),
|
|
15
15
|
timeoutMs: z.number().int().min(10).max(60_000),
|
|
16
|
+
probeMode: z.enum(["inventory", "active"]).optional(),
|
|
17
|
+
/**
|
|
18
|
+
* Optional daemon-side execution adapter inventory. Older callers omit this
|
|
19
|
+
* field and retain the historical executable probing behavior.
|
|
20
|
+
*/
|
|
21
|
+
availableRuntimes: z.array(z.enum(RUNTIME_HEALTH_NAMES)).max(RUNTIME_HEALTH_NAMES.length).optional(),
|
|
16
22
|
}).strict().superRefine((value, context) => {
|
|
17
23
|
const seen = new Set();
|
|
18
24
|
for (let index = 0; index < value.runtimes.length; index += 1) {
|
|
@@ -34,13 +40,15 @@ export const RuntimeProbeFrameSchema = z.object({
|
|
|
34
40
|
reqId: z.string().min(1).max(128),
|
|
35
41
|
runtimes: z.array(RuntimeProbeTargetSchema).min(1).max(RUNTIME_HEALTH_NAMES.length),
|
|
36
42
|
timeoutMs: z.number().int().min(10).max(60_000),
|
|
43
|
+
probeMode: z.enum(["inventory", "active"]).optional(),
|
|
44
|
+
availableRuntimes: z.array(z.enum(RUNTIME_HEALTH_NAMES)).max(RUNTIME_HEALTH_NAMES.length).optional(),
|
|
37
45
|
}).strict();
|
|
38
46
|
const execFileAsync = promisify(execFile);
|
|
39
47
|
const defaultExecutor = async (file, args, options) => {
|
|
40
48
|
const result = await execFileAsync(file, [...args], options);
|
|
41
49
|
return { stdout: String(result.stdout), stderr: String(result.stderr) };
|
|
42
50
|
};
|
|
43
|
-
const adapterOf = (
|
|
51
|
+
const adapterOf = (runtime, availableRuntimes) => (availableRuntimes === undefined || availableRuntimes.includes(runtime) ? "executable" : "unsupported");
|
|
44
52
|
function modelArgs(runtime, modelId) {
|
|
45
53
|
const model = modelId === null ? [] : ["--model", modelId];
|
|
46
54
|
if (runtime === "claude")
|
|
@@ -72,9 +80,16 @@ function failureCode(error) {
|
|
|
72
80
|
return "mapping_invalid";
|
|
73
81
|
if (/\b429\b|rate.?limit|too many requests/.test(text))
|
|
74
82
|
return "rate_limited";
|
|
75
|
-
|
|
83
|
+
const providerError = classifyProviderError(text);
|
|
84
|
+
// A local diagnostic may contain the word "upstream" or an exit code without
|
|
85
|
+
// proving a Provider 5xx. Only explicit HTTP context or well-known gateway
|
|
86
|
+
// messages are strong enough here.
|
|
87
|
+
if (providerError.category === "upstream" || /bad gateway|service unavailable|gateway timeout/.test(text)) {
|
|
76
88
|
return "upstream_error";
|
|
77
|
-
|
|
89
|
+
}
|
|
90
|
+
if (providerError.category === "timeout")
|
|
91
|
+
return "timeout";
|
|
92
|
+
return "unknown_failure";
|
|
78
93
|
}
|
|
79
94
|
const isMissing = (error) => typeof error === "object" && error !== null && error.code === "ENOENT";
|
|
80
95
|
const isAbort = (error) => typeof error === "object" && error !== null && error.name === "AbortError";
|
|
@@ -109,60 +124,88 @@ export async function probeRuntimeHealth(request, dependencies = {}) {
|
|
|
109
124
|
const monotonicNow = dependencies.monotonicNow ?? (() => performance.now());
|
|
110
125
|
const sourceEnv = dependencies.env ?? process.env;
|
|
111
126
|
const env = { ...sourceEnv, PATH: augmentedPath(sourceEnv) };
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
await runBounded(execute, target.runtime, ["--version"], parsed.timeoutMs, env);
|
|
125
|
-
}
|
|
126
|
-
catch (error) {
|
|
127
|
-
return {
|
|
128
|
-
...base,
|
|
129
|
-
installation: isMissing(error) ? "missing" : "timeout",
|
|
130
|
-
model: "not_checked",
|
|
131
|
-
latencyMs: latencyMs(),
|
|
132
|
-
detailCode: isMissing(error) ? "executable_missing" : "version_timeout",
|
|
127
|
+
const availableRuntimes = dependencies.availableRuntimes ?? parsed.availableRuntimes;
|
|
128
|
+
const results = [];
|
|
129
|
+
for (const target of parsed.runtimes) {
|
|
130
|
+
results.push(await (async () => {
|
|
131
|
+
const started = monotonicNow();
|
|
132
|
+
const checkedAt = now().toISOString();
|
|
133
|
+
const base = {
|
|
134
|
+
runtime: target.runtime,
|
|
135
|
+
adapter: adapterOf(target.runtime, availableRuntimes),
|
|
136
|
+
modelId: target.modelId,
|
|
137
|
+
configFingerprint: target.configFingerprint,
|
|
138
|
+
checkedAt,
|
|
133
139
|
};
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
140
|
+
const latencyMs = () => Math.max(0, Math.round(monotonicNow() - started));
|
|
141
|
+
if (availableRuntimes !== undefined && !availableRuntimes.includes(target.runtime)) {
|
|
142
|
+
return {
|
|
143
|
+
...base,
|
|
144
|
+
installation: "available",
|
|
145
|
+
adapter: "unsupported",
|
|
146
|
+
model: "not_checked",
|
|
147
|
+
latencyMs: latencyMs(),
|
|
148
|
+
detailCode: "health_adapter_unavailable",
|
|
149
|
+
startupStage: "unknown",
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
await runBounded(execute, target.runtime, ["--version"], parsed.timeoutMs, env);
|
|
154
|
+
}
|
|
155
|
+
catch (error) {
|
|
156
|
+
const missing = isMissing(error);
|
|
157
|
+
const timedOut = isAbort(error);
|
|
158
|
+
return {
|
|
159
|
+
...base,
|
|
160
|
+
installation: missing || timedOut ? (missing ? "missing" : "timeout") : "available",
|
|
161
|
+
model: "not_checked",
|
|
162
|
+
latencyMs: latencyMs(),
|
|
163
|
+
detailCode: missing ? "executable_missing" : timedOut ? "version_timeout" : "version_check_failed",
|
|
164
|
+
startupStage: missing ? "spawn" : "version",
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
if (base.adapter !== "executable") {
|
|
168
|
+
return { ...base, installation: "available", model: "not_checked", latencyMs: latencyMs(), detailCode: null, startupStage: "version" };
|
|
169
|
+
}
|
|
170
|
+
if ((parsed.probeMode ?? "active") === "inventory") {
|
|
171
|
+
return { ...base, installation: "available", model: "not_checked", latencyMs: latencyMs(), detailCode: "inventory_only", startupStage: "version" };
|
|
172
|
+
}
|
|
173
|
+
try {
|
|
174
|
+
await runBounded(execute, target.runtime, modelArgs(target.runtime, target.modelId), parsed.timeoutMs, env);
|
|
175
|
+
return { ...base, installation: "available", model: "ok", latencyMs: latencyMs(), detailCode: null, startupStage: "ready" };
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
const code = isAbort(error) ? "timeout" : failureCode(error);
|
|
179
|
+
const providerError = isAbort(error)
|
|
180
|
+
? { category: "timeout", code: "timeout" }
|
|
181
|
+
: classifyProviderError(failureText(error));
|
|
182
|
+
return {
|
|
183
|
+
...base,
|
|
184
|
+
installation: "available",
|
|
185
|
+
model: code,
|
|
186
|
+
latencyMs: latencyMs(),
|
|
187
|
+
detailCode: code,
|
|
188
|
+
startupStage: "model_probe",
|
|
189
|
+
providerErrorCategory: providerError.category,
|
|
190
|
+
...(providerError.httpStatus === undefined ? {} : { providerHttpStatus: providerError.httpStatus }),
|
|
191
|
+
...(providerError.code === undefined ? {} : { providerErrorCode: providerError.code }),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
})());
|
|
195
|
+
}
|
|
196
|
+
return results;
|
|
159
197
|
}
|
|
160
198
|
export async function handleRuntimeProbeFrame(decoded, probe = probeRuntimeHealth) {
|
|
161
199
|
const frame = RuntimeProbeFrameSchema.safeParse(decoded);
|
|
162
200
|
if (!frame.success)
|
|
163
201
|
return null;
|
|
164
202
|
try {
|
|
165
|
-
const data = await probe({
|
|
203
|
+
const data = await probe({
|
|
204
|
+
runtimes: frame.data.runtimes,
|
|
205
|
+
timeoutMs: frame.data.timeoutMs,
|
|
206
|
+
...(frame.data.probeMode === undefined ? {} : { probeMode: frame.data.probeMode }),
|
|
207
|
+
...(frame.data.availableRuntimes === undefined ? {} : { availableRuntimes: frame.data.availableRuntimes }),
|
|
208
|
+
});
|
|
166
209
|
return { reqId: frame.data.reqId, ok: true, data };
|
|
167
210
|
}
|
|
168
211
|
catch {
|
package/dist/serve.js
CHANGED
|
@@ -47,7 +47,8 @@ import { createAgentAbilityRuntime } from "./agent-ability/runtime.js";
|
|
|
47
47
|
import { createProjectSkillsReconciler, } from "./project-skills/reconciler.js";
|
|
48
48
|
import { initializedProjectSkillsReconciler } from "./project-skills/initialized-reconciler.js";
|
|
49
49
|
import { parseMemoryPruneTraceId } from "./memory-prune-diagnostics.js";
|
|
50
|
-
import {
|
|
50
|
+
import { probeRuntimeHealth } from "./runtime-health.js";
|
|
51
|
+
import { handleRuntimeHealthProbeFrame } from "./runtime-health-handler.js";
|
|
51
52
|
import { buildControlPlaneUrl } from "./control-plane-url.js";
|
|
52
53
|
import { createServeMigrationController, handleMigrationControlMessage } from "./daemon-migration-wiring.js";
|
|
53
54
|
import { createCodexHomeModeController } from "./codex-home-mode-controller.js";
|
|
@@ -78,7 +79,7 @@ export function serve(config, opts = {}) {
|
|
|
78
79
|
?? (config.agentMemory === null ? undefined : createAgentMemoryBridge(config.agentMemory));
|
|
79
80
|
const executionTelemetry = createExecutionTelemetryJournal(config.agentsRoot);
|
|
80
81
|
const executeProtocol = opts.execution?.runExecution ?? runExecution;
|
|
81
|
-
let detectedExecutionRuntimes = [];
|
|
82
|
+
let detectedExecutionRuntimes = [], executionRuntimesKnown = false;
|
|
82
83
|
let runtimeFacts = null;
|
|
83
84
|
const runtimeProbe = createRuntimeProbeCoordinator();
|
|
84
85
|
const hostCoordinator = opts.execution?.hostCoordinator
|
|
@@ -356,9 +357,12 @@ export function serve(config, opts = {}) {
|
|
|
356
357
|
});
|
|
357
358
|
runtimeFacts = helloPromise
|
|
358
359
|
.then(async (hello) => {
|
|
359
|
-
if (hello.executionProtocol === undefined)
|
|
360
|
+
if (hello.executionProtocol === undefined) {
|
|
361
|
+
executionRuntimesKnown = true;
|
|
360
362
|
return [];
|
|
363
|
+
}
|
|
361
364
|
const detected = await runtimeProbe.detect(hello.runtimes, detectExecutable);
|
|
365
|
+
executionRuntimesKnown = true;
|
|
362
366
|
await storageReplay;
|
|
363
367
|
if (ws === openedSocket && openedSocket.readyState === WebSocket.OPEN
|
|
364
368
|
&& detected.some((runtime) => !hello.executionRuntimes.includes(runtime))) {
|
|
@@ -368,6 +372,7 @@ export function serve(config, opts = {}) {
|
|
|
368
372
|
return detected;
|
|
369
373
|
})
|
|
370
374
|
.catch((error) => {
|
|
375
|
+
executionRuntimesKnown = true;
|
|
371
376
|
dslog("execution.runtime_detection_failed", "runtime 探测失败", {
|
|
372
377
|
level: "ERROR", error_message: error.message,
|
|
373
378
|
});
|
|
@@ -872,7 +877,7 @@ export function serve(config, opts = {}) {
|
|
|
872
877
|
return;
|
|
873
878
|
}
|
|
874
879
|
if (msg.type === "runtime-health:probe") {
|
|
875
|
-
const result = await
|
|
880
|
+
const result = await handleRuntimeHealthProbeFrame(msg, opts.runtimeHealth?.probe, executionRuntimesKnown, detectedExecutionRuntimes);
|
|
876
881
|
if (result !== null) {
|
|
877
882
|
try {
|
|
878
883
|
ws?.send(JSON.stringify({ type: "fs:result", ...result }));
|
package/dist/slog.js
CHANGED
|
@@ -32,10 +32,12 @@ function deploymentEnvironment(serverHost) {
|
|
|
32
32
|
return "prod";
|
|
33
33
|
if (serverHost === "nowwork-dev.nowcoder.com")
|
|
34
34
|
return "dev";
|
|
35
|
+
if (serverHost === "nowwork-pre.nowcoder.com")
|
|
36
|
+
return "pre";
|
|
35
37
|
if (serverHost === "localhost" || serverHost === "127.0.0.1"
|
|
36
38
|
|| serverHost === "::1" || serverHost === "[::1]")
|
|
37
39
|
return "local";
|
|
38
|
-
return
|
|
40
|
+
return "unknown";
|
|
39
41
|
}
|
|
40
42
|
function spoolDir() {
|
|
41
43
|
return process.env.CREW_SLS_SPOOL_DIR ?? join(homedir(), ".crew", "logs", "sls-spool");
|
package/dist/storage-manifest.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { createReadStream } from "node:fs";
|
|
3
|
-
import { chmod, lstat, mkdir, open, readFile, readdir, readlink, realpath, rename
|
|
3
|
+
import { chmod, lstat, mkdir, open, readFile, readdir, readlink, realpath, rename } from "node:fs/promises";
|
|
4
4
|
import { createInterface } from "node:readline";
|
|
5
5
|
import { basename, dirname, isAbsolute, join, posix, relative, resolve, win32 } from "node:path";
|
|
6
6
|
import { z } from "zod";
|
|
@@ -111,15 +111,30 @@ function pathReason(sourceKind, sourcePath) {
|
|
|
111
111
|
return "local_runtime";
|
|
112
112
|
return null;
|
|
113
113
|
}
|
|
114
|
-
|
|
114
|
+
function abortedManifestScanError() {
|
|
115
|
+
return Object.assign(new Error("storage manifest scan aborted"), { code: "preflight_aborted" });
|
|
116
|
+
}
|
|
117
|
+
async function fileDigest(path, signal) {
|
|
118
|
+
if (signal?.aborted)
|
|
119
|
+
throw abortedManifestScanError();
|
|
115
120
|
const before = await lstat(path);
|
|
116
121
|
const digest = createHash("sha256");
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
122
|
+
try {
|
|
123
|
+
await new Promise((resolvePromise, reject) => {
|
|
124
|
+
const stream = createReadStream(path, signal === undefined ? {} : { signal });
|
|
125
|
+
stream.on("data", (chunk) => digest.update(chunk));
|
|
126
|
+
stream.once("end", resolvePromise);
|
|
127
|
+
stream.once("error", reject);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
if (signal?.aborted || (error instanceof Error && error.name === "AbortError")) {
|
|
132
|
+
throw abortedManifestScanError();
|
|
133
|
+
}
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
136
|
+
if (signal?.aborted)
|
|
137
|
+
throw abortedManifestScanError();
|
|
123
138
|
const after = await lstat(path);
|
|
124
139
|
if (before.ino !== after.ino || before.size !== after.size || before.mtimeMs !== after.mtimeMs) {
|
|
125
140
|
throw new Error(`source changed while hashing: ${path}`);
|
|
@@ -329,7 +344,7 @@ async function* scanSourceEntries(input) {
|
|
|
329
344
|
if (transformation !== undefined) {
|
|
330
345
|
if (type !== "file")
|
|
331
346
|
throw new Error(`transformation source is not a file: ${sourcePath}`);
|
|
332
|
-
const digest = await fileDigest(path);
|
|
347
|
+
const digest = await fileDigest(path, input.signal);
|
|
333
348
|
if (digest !== transformation.sourceSha256)
|
|
334
349
|
throw new Error(`transformation source drift: ${sourcePath}`);
|
|
335
350
|
const entry = emit(ManifestEntrySchema.parse({
|
|
@@ -394,7 +409,7 @@ async function* scanSourceEntries(input) {
|
|
|
394
409
|
reason: "durable_workspace_data",
|
|
395
410
|
mode: modeString(stat.mode),
|
|
396
411
|
sizeBytes: stat.size,
|
|
397
|
-
...(type === "file" ? { sha256: await fileDigest(path) } : {}),
|
|
412
|
+
...(type === "file" ? { sha256: await fileDigest(path, input.signal) } : {}),
|
|
398
413
|
}));
|
|
399
414
|
input.onEntry?.(entry.sourceKind, entry.sourcePath);
|
|
400
415
|
yield entry;
|
|
@@ -663,6 +678,55 @@ const StoredChunkSchema = z.object({
|
|
|
663
678
|
digest: z.string().regex(CHUNK_DIGEST),
|
|
664
679
|
entries: z.array(ManifestEntrySchema).max(50_000),
|
|
665
680
|
}).strict();
|
|
681
|
+
// A timeout aborts the scan asynchronously. Keep a process-local single-flight
|
|
682
|
+
// guard so a retry cannot append to the same partial JSONL while the timed-out
|
|
683
|
+
// invocation is still unwinding. A daemon restart clears this guard; the
|
|
684
|
+
// persisted checkpoint and chunk digests remain the cross-process authority.
|
|
685
|
+
const activeManifestPersists = new Set();
|
|
686
|
+
const StorageManifestChunkCheckpointSchema = z.object({
|
|
687
|
+
version: z.literal(1),
|
|
688
|
+
migrationId: z.string().uuid(),
|
|
689
|
+
nextChunkIndex: z.number().int().nonnegative(),
|
|
690
|
+
emittedEntries: z.number().int().nonnegative(),
|
|
691
|
+
sourceIndex: z.number().int().nonnegative(),
|
|
692
|
+
sourcePath: z.string().min(1).nullable(),
|
|
693
|
+
identityDigest: z.string().regex(CHUNK_DIGEST),
|
|
694
|
+
sequenceDigest: z.string().regex(CHUNK_DIGEST),
|
|
695
|
+
}).strict();
|
|
696
|
+
function emptyManifestSummary() {
|
|
697
|
+
return { records: 0, files: 0, directories: 0, symlinks: 0, copied: 0, skipped: 0, transformations: 0, externalSymlinks: 0, bytesToCopy: 0, errors: 0 };
|
|
698
|
+
}
|
|
699
|
+
function emptyManifestWarnings() {
|
|
700
|
+
return { credentialLinks: 0, unapprovedLinks: 0, brokenLinks: 0, unsupportedItems: 0 };
|
|
701
|
+
}
|
|
702
|
+
function addManifestEntryToSummary(summary, warnings, entry) {
|
|
703
|
+
summary.records += 1;
|
|
704
|
+
if (entry.type === "file")
|
|
705
|
+
summary.files += 1;
|
|
706
|
+
if (entry.type === "directory")
|
|
707
|
+
summary.directories += 1;
|
|
708
|
+
if (entry.type === "symlink")
|
|
709
|
+
summary.symlinks += 1;
|
|
710
|
+
if (entry.disposition === "copy")
|
|
711
|
+
summary.copied += 1;
|
|
712
|
+
if (entry.disposition === "skip")
|
|
713
|
+
summary.skipped += 1;
|
|
714
|
+
if (entry.disposition === "transform")
|
|
715
|
+
summary.transformations += 1;
|
|
716
|
+
if (entry.type === "file" && entry.disposition !== "skip")
|
|
717
|
+
summary.bytesToCopy += entry.sizeBytes;
|
|
718
|
+
if (entry.link?.classification === "external") {
|
|
719
|
+
summary.externalSymlinks += 1;
|
|
720
|
+
if (entry.reason === "credential_material")
|
|
721
|
+
warnings.credentialLinks += 1;
|
|
722
|
+
else if (entry.reason !== "local_runtime")
|
|
723
|
+
warnings.unapprovedLinks += 1;
|
|
724
|
+
}
|
|
725
|
+
if (entry.link?.classification === "broken")
|
|
726
|
+
warnings.brokenLinks += 1;
|
|
727
|
+
if (entry.type === "unsupported")
|
|
728
|
+
warnings.unsupportedItems += 1;
|
|
729
|
+
}
|
|
666
730
|
function chunkStorePaths(root, migrationId) {
|
|
667
731
|
if (!isAbsolute(root) || CONTROL.test(root))
|
|
668
732
|
throw new Error("manifest chunk store root must be an absolute single-line path");
|
|
@@ -670,6 +734,82 @@ function chunkStorePaths(root, migrationId) {
|
|
|
670
734
|
const directory = resolve(root, "migrations", migrationId);
|
|
671
735
|
return Object.freeze({ directory, path: join(directory, "manifest.jsonl"), checkpointPath: join(directory, "checkpoint.json") });
|
|
672
736
|
}
|
|
737
|
+
async function pathExists(path) {
|
|
738
|
+
try {
|
|
739
|
+
await lstat(path);
|
|
740
|
+
return true;
|
|
741
|
+
}
|
|
742
|
+
catch (error) {
|
|
743
|
+
if (error.code === "ENOENT")
|
|
744
|
+
return false;
|
|
745
|
+
throw error;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
async function readManifestCheckpoint(path) {
|
|
749
|
+
try {
|
|
750
|
+
return StorageManifestChunkCheckpointSchema.parse(JSON.parse(await readFile(path, "utf8")));
|
|
751
|
+
}
|
|
752
|
+
catch (error) {
|
|
753
|
+
throw new Error("Invalid storage manifest checkpoint", { cause: error });
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
async function readPartialManifestChunkState(path, input, prepared) {
|
|
757
|
+
if (input.signal?.aborted)
|
|
758
|
+
throw abortedManifestScanError();
|
|
759
|
+
const stream = createReadStream(path, {
|
|
760
|
+
encoding: "utf8",
|
|
761
|
+
...(input.signal === undefined ? {} : { signal: input.signal }),
|
|
762
|
+
});
|
|
763
|
+
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
|
764
|
+
let expected = 0;
|
|
765
|
+
let entryCount = 0;
|
|
766
|
+
let sequenceDigest = storageManifestChunkSequenceSeed({
|
|
767
|
+
migrationId: input.migrationId,
|
|
768
|
+
environment: prepared.environment,
|
|
769
|
+
workspaceId: input.workspaceId,
|
|
770
|
+
targetWorkspaceRoot: input.targetWorkspaceRoot,
|
|
771
|
+
sourceKinds: prepared.sourceKinds,
|
|
772
|
+
});
|
|
773
|
+
const summary = emptyManifestSummary();
|
|
774
|
+
const warnings = emptyManifestWarnings();
|
|
775
|
+
try {
|
|
776
|
+
for await (const line of lines) {
|
|
777
|
+
if (input.signal?.aborted)
|
|
778
|
+
throw abortedManifestScanError();
|
|
779
|
+
if (line.trim().length === 0)
|
|
780
|
+
continue;
|
|
781
|
+
let parsed;
|
|
782
|
+
try {
|
|
783
|
+
parsed = StoredChunkSchema.parse(JSON.parse(line));
|
|
784
|
+
}
|
|
785
|
+
catch (error) {
|
|
786
|
+
throw new Error("Invalid partial storage manifest chunk", { cause: error });
|
|
787
|
+
}
|
|
788
|
+
if (parsed.index !== expected)
|
|
789
|
+
throw new Error("storage manifest partial chunk sequence is not contiguous");
|
|
790
|
+
if (storageManifestChunkDigest(parsed.entries) !== parsed.digest) {
|
|
791
|
+
throw new Error("storage manifest partial chunk digest mismatch");
|
|
792
|
+
}
|
|
793
|
+
for (const entry of parsed.entries)
|
|
794
|
+
addManifestEntryToSummary(summary, warnings, entry);
|
|
795
|
+
expected += 1;
|
|
796
|
+
entryCount += parsed.entries.length;
|
|
797
|
+
sequenceDigest = storageManifestChunkSequenceStep(sequenceDigest, parsed.digest);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
catch (error) {
|
|
801
|
+
if (input.signal?.aborted || (error instanceof Error && error.name === "AbortError")) {
|
|
802
|
+
throw abortedManifestScanError();
|
|
803
|
+
}
|
|
804
|
+
throw error;
|
|
805
|
+
}
|
|
806
|
+
finally {
|
|
807
|
+
lines.close();
|
|
808
|
+
stream.destroy();
|
|
809
|
+
}
|
|
810
|
+
assertStorageManifestEntryLimit(entryCount);
|
|
811
|
+
return { chunkCount: expected, entryCount, sequenceDigest, summary, warnings };
|
|
812
|
+
}
|
|
673
813
|
/** Iterate a persisted manifest without materializing its entries array. */
|
|
674
814
|
export async function* readStorageManifestChunks(store) {
|
|
675
815
|
const record = z.object({
|
|
@@ -760,29 +900,63 @@ function manifestMigrationIdFromStorePath(path) {
|
|
|
760
900
|
return z.string().uuid().parse(id);
|
|
761
901
|
}
|
|
762
902
|
/** Persist a chunk stream and return a compact manifest reference. */
|
|
763
|
-
|
|
764
|
-
if (options.resumeFrom !== undefined) {
|
|
765
|
-
throw new Error("persisting a resumed manifest requires the original chunk store; resume the stream before persistence");
|
|
766
|
-
}
|
|
903
|
+
async function persistStorageManifestChunksLocked(input, options) {
|
|
767
904
|
const prepared = prepareManifestScan(input);
|
|
768
905
|
const chunkSize = z.number().int().positive().max(50_000).parse(options.chunkSize ?? 10_000);
|
|
769
906
|
const paths = chunkStorePaths(options.root, input.migrationId);
|
|
770
907
|
await mkdir(paths.directory, { recursive: true, mode: 0o700 });
|
|
771
908
|
await chmod(paths.directory, 0o700);
|
|
772
909
|
const temporary = `${paths.path}.tmp`;
|
|
773
|
-
await
|
|
910
|
+
const temporaryExists = await pathExists(temporary);
|
|
911
|
+
const checkpointExists = await pathExists(paths.checkpointPath);
|
|
912
|
+
if (temporaryExists && !checkpointExists) {
|
|
913
|
+
throw new Error("storage manifest partial state is incomplete");
|
|
914
|
+
}
|
|
915
|
+
const resumeFrom = temporaryExists
|
|
916
|
+
? await readManifestCheckpoint(paths.checkpointPath)
|
|
917
|
+
: options.resumeFrom;
|
|
918
|
+
if (options.resumeFrom !== undefined && resumeFrom !== undefined
|
|
919
|
+
&& JSON.stringify(options.resumeFrom) !== JSON.stringify(resumeFrom)) {
|
|
920
|
+
throw new Error("storage manifest checkpoint does not match persisted state");
|
|
921
|
+
}
|
|
922
|
+
if (resumeFrom !== undefined && !temporaryExists) {
|
|
923
|
+
throw new Error("storage manifest checkpoint has no partial chunk store");
|
|
924
|
+
}
|
|
925
|
+
const partial = temporaryExists
|
|
926
|
+
? await readPartialManifestChunkState(temporary, input, prepared)
|
|
927
|
+
: null;
|
|
928
|
+
if (resumeFrom !== undefined && partial !== null) {
|
|
929
|
+
const expectedIdentityDigest = storageManifestChunkSequenceSeed({
|
|
930
|
+
migrationId: input.migrationId,
|
|
931
|
+
environment: prepared.environment,
|
|
932
|
+
workspaceId: input.workspaceId,
|
|
933
|
+
targetWorkspaceRoot: input.targetWorkspaceRoot,
|
|
934
|
+
sourceKinds: prepared.sourceKinds,
|
|
935
|
+
});
|
|
936
|
+
if (resumeFrom.migrationId !== input.migrationId
|
|
937
|
+
|| resumeFrom.identityDigest !== expectedIdentityDigest
|
|
938
|
+
|| resumeFrom.nextChunkIndex !== partial.chunkCount
|
|
939
|
+
|| resumeFrom.emittedEntries !== partial.entryCount
|
|
940
|
+
|| resumeFrom.sequenceDigest !== partial.sequenceDigest
|
|
941
|
+
|| resumeFrom.sourceIndex > input.sources.length) {
|
|
942
|
+
throw new Error("storage manifest checkpoint identity or digest mismatch");
|
|
943
|
+
}
|
|
944
|
+
}
|
|
774
945
|
let handle = null;
|
|
775
|
-
let chunkCount = 0;
|
|
776
|
-
let entryCount = 0;
|
|
777
|
-
let sequenceDigest = storageManifestChunkSequenceDigest({
|
|
946
|
+
let chunkCount = partial?.chunkCount ?? 0;
|
|
947
|
+
let entryCount = partial?.entryCount ?? 0;
|
|
948
|
+
let sequenceDigest = partial?.sequenceDigest ?? storageManifestChunkSequenceDigest({
|
|
778
949
|
migrationId: input.migrationId, environment: prepared.environment, workspaceId: input.workspaceId,
|
|
779
950
|
targetWorkspaceRoot: input.targetWorkspaceRoot, sourceKinds: prepared.sourceKinds, chunkDigests: [],
|
|
780
951
|
});
|
|
781
|
-
const summary =
|
|
782
|
-
const warnings =
|
|
952
|
+
const summary = partial?.summary ?? emptyManifestSummary();
|
|
953
|
+
const warnings = partial?.warnings ?? emptyManifestWarnings();
|
|
783
954
|
try {
|
|
784
|
-
handle = await open(temporary, "wx", 0o600);
|
|
785
|
-
for await (const chunk of streamStorageManifestChunks(input, {
|
|
955
|
+
handle = await open(temporary, temporaryExists ? "a" : "wx", 0o600);
|
|
956
|
+
for await (const chunk of streamStorageManifestChunks(input, {
|
|
957
|
+
chunkSize,
|
|
958
|
+
...(resumeFrom === undefined ? {} : { resumeFrom }),
|
|
959
|
+
})) {
|
|
786
960
|
const line = `${JSON.stringify({ version: 1, index: chunk.index, digest: chunk.digest, entries: chunk.entries })}\n`;
|
|
787
961
|
await handle.writeFile(line, "utf8");
|
|
788
962
|
await handle.sync();
|
|
@@ -790,34 +964,8 @@ export async function persistStorageManifestChunks(input, options) {
|
|
|
790
964
|
chunkCount += 1;
|
|
791
965
|
entryCount += chunk.entries.length;
|
|
792
966
|
sequenceDigest = chunk.checkpoint.sequenceDigest;
|
|
793
|
-
for (const entry of chunk.entries)
|
|
794
|
-
summary
|
|
795
|
-
if (entry.type === "file")
|
|
796
|
-
summary.files += 1;
|
|
797
|
-
if (entry.type === "directory")
|
|
798
|
-
summary.directories += 1;
|
|
799
|
-
if (entry.type === "symlink")
|
|
800
|
-
summary.symlinks += 1;
|
|
801
|
-
if (entry.disposition === "copy")
|
|
802
|
-
summary.copied += 1;
|
|
803
|
-
if (entry.disposition === "skip")
|
|
804
|
-
summary.skipped += 1;
|
|
805
|
-
if (entry.disposition === "transform")
|
|
806
|
-
summary.transformations += 1;
|
|
807
|
-
if (entry.type === "file" && entry.disposition !== "skip")
|
|
808
|
-
summary.bytesToCopy += entry.sizeBytes;
|
|
809
|
-
if (entry.link?.classification === "external") {
|
|
810
|
-
summary.externalSymlinks += 1;
|
|
811
|
-
if (entry.reason === "credential_material")
|
|
812
|
-
warnings.credentialLinks += 1;
|
|
813
|
-
else if (entry.reason !== "local_runtime")
|
|
814
|
-
warnings.unapprovedLinks += 1;
|
|
815
|
-
}
|
|
816
|
-
if (entry.link?.classification === "broken")
|
|
817
|
-
warnings.brokenLinks += 1;
|
|
818
|
-
if (entry.type === "unsupported")
|
|
819
|
-
warnings.unsupportedItems += 1;
|
|
820
|
-
}
|
|
967
|
+
for (const entry of chunk.entries)
|
|
968
|
+
addManifestEntryToSummary(summary, warnings, entry);
|
|
821
969
|
input.onProgress?.({ scannedEntries: entryCount, sourceKind: chunk.sourceKind === "mixed" ? prepared.sourceKinds[0] : chunk.sourceKind, sourcePath: chunk.entries.at(-1)?.sourcePath ?? "" });
|
|
822
970
|
}
|
|
823
971
|
await handle.close();
|
|
@@ -833,10 +981,25 @@ export async function persistStorageManifestChunks(input, options) {
|
|
|
833
981
|
}
|
|
834
982
|
catch (error) {
|
|
835
983
|
await handle?.close().catch(() => undefined);
|
|
836
|
-
|
|
984
|
+
// Keep both the partial JSONL and its last authenticated checkpoint. A
|
|
985
|
+
// subsequent preflight can resume this exact migration, while a mismatched
|
|
986
|
+
// or incomplete pair fails closed instead of being silently overwritten.
|
|
837
987
|
throw error;
|
|
838
988
|
}
|
|
839
989
|
}
|
|
990
|
+
export async function persistStorageManifestChunks(input, options) {
|
|
991
|
+
const paths = chunkStorePaths(options.root, input.migrationId);
|
|
992
|
+
if (activeManifestPersists.has(paths.path)) {
|
|
993
|
+
throw new Error("storage manifest persistence is already running for this migration");
|
|
994
|
+
}
|
|
995
|
+
activeManifestPersists.add(paths.path);
|
|
996
|
+
try {
|
|
997
|
+
return await persistStorageManifestChunksLocked(input, options);
|
|
998
|
+
}
|
|
999
|
+
finally {
|
|
1000
|
+
activeManifestPersists.delete(paths.path);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
840
1003
|
export function parseStorageManifest(raw) {
|
|
841
1004
|
try {
|
|
842
1005
|
return StorageManifestSchema.parse(JSON.parse(raw));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nowcrew/daemon",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.71",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "crew daemon — 运行在用户机器:拉起/管理 agent 进程,注入 crew CLI,归一化 runtime 事件",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -16,6 +16,14 @@
|
|
|
16
16
|
"publishConfig": {
|
|
17
17
|
"access": "public"
|
|
18
18
|
},
|
|
19
|
+
"scripts": {
|
|
20
|
+
"daemon": "pnpm --filter @nowcrew/cli build && tsx src/main.ts",
|
|
21
|
+
"build": "tsc -p tsconfig.json",
|
|
22
|
+
"codex-home:migrate": "tsx src/runtimes/codex-home-migration-cli.ts",
|
|
23
|
+
"prepublishOnly": "pnpm build && node ../scripts/daemon-release-artifact.mjs --strict-registry",
|
|
24
|
+
"test": "vitest run",
|
|
25
|
+
"typecheck": "tsc --noEmit"
|
|
26
|
+
},
|
|
19
27
|
"dependencies": {
|
|
20
28
|
"@agentclientprotocol/sdk": "1.2.1",
|
|
21
29
|
"@nowcrew/cli": "^0.4.13",
|
|
@@ -34,12 +42,5 @@
|
|
|
34
42
|
"tsx": "^4.19.0",
|
|
35
43
|
"typescript": "^5.6.0",
|
|
36
44
|
"vitest": "^2.1.0"
|
|
37
|
-
},
|
|
38
|
-
"scripts": {
|
|
39
|
-
"daemon": "pnpm --filter @nowcrew/cli build && tsx src/main.ts",
|
|
40
|
-
"build": "tsc -p tsconfig.json",
|
|
41
|
-
"codex-home:migrate": "tsx src/runtimes/codex-home-migration-cli.ts",
|
|
42
|
-
"test": "vitest run",
|
|
43
|
-
"typecheck": "tsc --noEmit"
|
|
44
45
|
}
|
|
45
|
-
}
|
|
46
|
+
}
|