@ricsam/r5d-worker 0.0.108 → 0.0.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/main.cjs +336 -125
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/recovery-output-window.cjs +77 -0
- package/dist/cjs/recovery-protocol.cjs +31 -0
- package/dist/cjs/recovery-store.cjs +229 -0
- package/dist/cjs/supervisor.cjs +11 -4
- package/dist/mjs/main.mjs +339 -125
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/recovery-output-window.mjs +53 -0
- package/dist/mjs/recovery-protocol.mjs +6 -0
- package/dist/mjs/recovery-store.mjs +194 -0
- package/dist/mjs/supervisor.mjs +10 -4
- package/dist/types/main.d.ts +9 -2
- package/dist/types/recovery-output-window.d.ts +19 -0
- package/dist/types/recovery-protocol.d.ts +59 -0
- package/dist/types/recovery-store.d.ts +38 -0
- package/dist/types/supervisor.d.ts +2 -1
- package/package.json +1 -1
package/dist/mjs/package.json
CHANGED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
const keyOf = (runId, stream) => `${runId}:${stream}`;
|
|
2
|
+
class WorkerOutputWindow {
|
|
3
|
+
constructor(windowBytes = 1024 * 1024) {
|
|
4
|
+
this.windowBytes = windowBytes;
|
|
5
|
+
}
|
|
6
|
+
windowBytes;
|
|
7
|
+
sent = /* @__PURE__ */ new Map();
|
|
8
|
+
acknowledged = /* @__PURE__ */ new Map();
|
|
9
|
+
outstanding = 0;
|
|
10
|
+
reset(cursors) {
|
|
11
|
+
this.sent.clear();
|
|
12
|
+
this.acknowledged.clear();
|
|
13
|
+
this.outstanding = 0;
|
|
14
|
+
for (const cursor of cursors) {
|
|
15
|
+
for (const stream of ["stdout", "stderr"]) {
|
|
16
|
+
const offset = stream === "stdout" ? cursor.stdoutBytes : cursor.stderrBytes;
|
|
17
|
+
this.sent.set(keyOf(cursor.runId, stream), offset);
|
|
18
|
+
this.acknowledged.set(keyOf(cursor.runId, stream), offset);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
send(frame, send, bufferedAmount = 0) {
|
|
23
|
+
const key = keyOf(frame.runId, frame.stream);
|
|
24
|
+
const sent = this.sent.get(key) ?? 0;
|
|
25
|
+
const bytes = Buffer.from(frame.data, "utf8");
|
|
26
|
+
const end = frame.offset + bytes.length;
|
|
27
|
+
if (end <= sent) return true;
|
|
28
|
+
if (frame.offset > sent) return false;
|
|
29
|
+
let cursor = sent;
|
|
30
|
+
while (cursor < end) {
|
|
31
|
+
let count = Math.min(end - cursor, 64 * 1024, this.windowBytes - this.outstanding);
|
|
32
|
+
while (count > 0 && ((bytes[cursor - frame.offset + count] ?? 0) & 192) === 128) count -= 1;
|
|
33
|
+
if (count <= 0 || bufferedAmount >= this.windowBytes) return false;
|
|
34
|
+
const data = bytes.subarray(cursor - frame.offset, cursor - frame.offset + count).toString("utf8");
|
|
35
|
+
if (!send({ ...frame, offset: cursor, data })) return false;
|
|
36
|
+
cursor += count;
|
|
37
|
+
this.sent.set(key, cursor);
|
|
38
|
+
this.outstanding += count;
|
|
39
|
+
}
|
|
40
|
+
return true;
|
|
41
|
+
}
|
|
42
|
+
acknowledge(runId, stream, offset) {
|
|
43
|
+
if (!Number.isSafeInteger(offset) || offset < 0) return;
|
|
44
|
+
const key = keyOf(runId, stream);
|
|
45
|
+
const previous = this.acknowledged.get(key) ?? 0;
|
|
46
|
+
const next = Math.max(previous, Math.min(this.sent.get(key) ?? 0, offset));
|
|
47
|
+
this.acknowledged.set(key, next);
|
|
48
|
+
this.outstanding = Math.max(0, this.outstanding - (next - previous));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export {
|
|
52
|
+
WorkerOutputWindow
|
|
53
|
+
};
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
function workerOperationDigest(message) {
|
|
6
|
+
const canonical = (value) => {
|
|
7
|
+
if (Array.isArray(value)) return value.map(canonical);
|
|
8
|
+
if (value && typeof value === "object")
|
|
9
|
+
return Object.fromEntries(
|
|
10
|
+
Object.entries(value).filter(([, item]) => item !== void 0).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, canonical(item)])
|
|
11
|
+
);
|
|
12
|
+
return value;
|
|
13
|
+
};
|
|
14
|
+
const { requestId: _requestId, ...identity } = message;
|
|
15
|
+
return createHash("sha256").update(JSON.stringify(canonical(identity))).digest("hex");
|
|
16
|
+
}
|
|
17
|
+
class WorkerRecoveryStore {
|
|
18
|
+
db;
|
|
19
|
+
ledgerId;
|
|
20
|
+
constructor(filename) {
|
|
21
|
+
fs.mkdirSync(path.dirname(filename), { recursive: true, mode: 448 });
|
|
22
|
+
this.db = new Database(filename, { create: true });
|
|
23
|
+
fs.chmodSync(filename, 384);
|
|
24
|
+
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
|
|
25
|
+
CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
26
|
+
CREATE TABLE IF NOT EXISTS operations (request_id TEXT PRIMARY KEY, run_id TEXT, session_id TEXT, request_type TEXT NOT NULL, digest TEXT NOT NULL, state TEXT NOT NULL, response TEXT);
|
|
27
|
+
CREATE UNIQUE INDEX IF NOT EXISTS operation_run_start ON operations(run_id) WHERE request_type='exec_start';
|
|
28
|
+
CREATE TABLE IF NOT EXISTS aliases (request_id TEXT PRIMARY KEY, operation_id TEXT NOT NULL);
|
|
29
|
+
CREATE TABLE IF NOT EXISTS streams (run_id TEXT NOT NULL, stream TEXT NOT NULL, bytes INTEGER NOT NULL DEFAULT 0, acknowledged INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(run_id, stream));
|
|
30
|
+
CREATE TABLE IF NOT EXISTS output (run_id TEXT NOT NULL, stream TEXT NOT NULL, offset INTEGER NOT NULL, bytes INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(run_id, stream, offset));
|
|
31
|
+
CREATE TABLE IF NOT EXISTS terminals (run_id TEXT PRIMARY KEY, message TEXT NOT NULL);
|
|
32
|
+
CREATE TABLE IF NOT EXISTS cancelled_requests (request_id TEXT PRIMARY KEY);
|
|
33
|
+
`);
|
|
34
|
+
this.db.run("INSERT OR IGNORE INTO metadata(key,value) VALUES ('ledgerId',?)", [crypto.randomUUID()]);
|
|
35
|
+
this.ledgerId = this.db.query("SELECT value FROM metadata WHERE key='ledgerId'").get().value;
|
|
36
|
+
this.db.run("UPDATE operations SET state='unknown' WHERE state IN ('accepted','preparing','running')");
|
|
37
|
+
}
|
|
38
|
+
close() {
|
|
39
|
+
this.db.close();
|
|
40
|
+
}
|
|
41
|
+
row(requestId) {
|
|
42
|
+
return this.db.query("SELECT * FROM operations WHERE request_id=? OR request_id=(SELECT operation_id FROM aliases WHERE request_id=?)").get(requestId, requestId);
|
|
43
|
+
}
|
|
44
|
+
admit(message) {
|
|
45
|
+
const digest = workerOperationDigest(message);
|
|
46
|
+
let existing = this.row(message.requestId);
|
|
47
|
+
if (!existing && message.type === "exec_start" && typeof message.runId === "string")
|
|
48
|
+
existing = this.db.query("SELECT * FROM operations WHERE run_id=? AND request_type='exec_start'").get(message.runId);
|
|
49
|
+
if (existing) {
|
|
50
|
+
if (existing.digest !== digest) return "conflict";
|
|
51
|
+
this.db.run("INSERT OR IGNORE INTO aliases(request_id, operation_id) VALUES (?,?)", [message.requestId, existing.request_id]);
|
|
52
|
+
return existing.state === "unknown" ? "unknown" : "duplicate";
|
|
53
|
+
}
|
|
54
|
+
this.db.run("INSERT INTO operations(request_id, run_id, session_id, request_type, digest, state) VALUES (?,?,?,?,?,'accepted')", [
|
|
55
|
+
message.requestId,
|
|
56
|
+
typeof message.runId === "string" ? message.runId : null,
|
|
57
|
+
typeof message.sessionId === "string" ? message.sessionId : null,
|
|
58
|
+
message.type,
|
|
59
|
+
digest
|
|
60
|
+
]);
|
|
61
|
+
return "new";
|
|
62
|
+
}
|
|
63
|
+
isUnknown(requestId) {
|
|
64
|
+
return this.row(requestId)?.state === "unknown";
|
|
65
|
+
}
|
|
66
|
+
unknown(requestId) {
|
|
67
|
+
const row = this.row(requestId);
|
|
68
|
+
if (row) this.db.run("UPDATE operations SET state='unknown' WHERE request_id=?", [row.request_id]);
|
|
69
|
+
}
|
|
70
|
+
unknownRun(runId) {
|
|
71
|
+
this.db.run("UPDATE operations SET state='unknown' WHERE run_id=? AND request_type IN ('exec_start','exec')", [runId]);
|
|
72
|
+
}
|
|
73
|
+
record(message) {
|
|
74
|
+
if (message.type === "exec_output" && typeof message.runId === "string" && (message.stream === "stdout" || message.stream === "stderr") && typeof message.data === "string") {
|
|
75
|
+
const runId = message.runId, stream = message.stream, data = message.data;
|
|
76
|
+
const offset = this.db.transaction(() => {
|
|
77
|
+
this.db.run("INSERT OR IGNORE INTO streams(run_id, stream) VALUES (?,?)", [runId, stream]);
|
|
78
|
+
const offset2 = this.db.query("SELECT bytes FROM streams WHERE run_id=? AND stream=?").get(runId, stream).bytes;
|
|
79
|
+
const bytes = Buffer.byteLength(data);
|
|
80
|
+
if (bytes) {
|
|
81
|
+
this.db.run("INSERT INTO output(run_id, stream, offset, bytes, data) VALUES (?,?,?,?,?)", [runId, stream, offset2, bytes, data]);
|
|
82
|
+
this.db.run("UPDATE streams SET bytes=bytes+? WHERE run_id=? AND stream=?", [bytes, runId, stream]);
|
|
83
|
+
}
|
|
84
|
+
return offset2;
|
|
85
|
+
})();
|
|
86
|
+
return { ...message, offset };
|
|
87
|
+
}
|
|
88
|
+
if ((message.type === "exec_exit" || message.type === "exec_error") && typeof message.runId === "string") {
|
|
89
|
+
const terminal = { ...message, ...this.lengths(message.runId) };
|
|
90
|
+
this.db.transaction(() => {
|
|
91
|
+
this.db.run("INSERT OR REPLACE INTO terminals(run_id,message) VALUES (?,?)", [message.runId, JSON.stringify(terminal)]);
|
|
92
|
+
this.db.run("UPDATE operations SET state='completed' WHERE run_id=? AND request_type='exec_start'", [message.runId]);
|
|
93
|
+
})();
|
|
94
|
+
return terminal;
|
|
95
|
+
}
|
|
96
|
+
if (typeof message.requestId === "string") {
|
|
97
|
+
const row = this.row(message.requestId);
|
|
98
|
+
if (row) {
|
|
99
|
+
const state = message.type === "exec_accepted" || message.type === "operation_queued" ? "preparing" : message.type === "exec_started" ? "running" : message.type === "operation_received" ? "accepted" : "completed";
|
|
100
|
+
if (message.type !== "operation_received")
|
|
101
|
+
this.db.run("UPDATE operations SET state=?, response=? WHERE request_id=?", [state, JSON.stringify(message), row.request_id]);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return message;
|
|
105
|
+
}
|
|
106
|
+
operations() {
|
|
107
|
+
return this.db.query(
|
|
108
|
+
"SELECT * FROM operations WHERE response IS NOT NULL OR state!='completed' UNION ALL SELECT aliases.request_id,operations.run_id,operations.session_id,operations.request_type,operations.digest,operations.state,operations.response FROM aliases JOIN operations ON operations.request_id=aliases.operation_id WHERE aliases.request_id!=operations.request_id AND (response IS NOT NULL OR state!='completed')"
|
|
109
|
+
).all().map((row) => ({
|
|
110
|
+
requestId: row.request_id,
|
|
111
|
+
requestType: row.request_type,
|
|
112
|
+
state: row.state,
|
|
113
|
+
...row.run_id ? { runId: row.run_id } : {},
|
|
114
|
+
...row.session_id ? { sessionId: row.session_id } : {}
|
|
115
|
+
}));
|
|
116
|
+
}
|
|
117
|
+
response(requestId) {
|
|
118
|
+
const row = this.row(requestId);
|
|
119
|
+
return row?.response && row.state !== "unknown" ? { ...JSON.parse(row.response), requestId } : void 0;
|
|
120
|
+
}
|
|
121
|
+
responses(requestIds) {
|
|
122
|
+
const ids = /* @__PURE__ */ new Set([...requestIds, ...this.operations().map((operation) => operation.requestId)]);
|
|
123
|
+
return [...ids].flatMap((requestId) => {
|
|
124
|
+
const response = this.response(requestId);
|
|
125
|
+
return response ? [response] : [];
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
lengths(runId) {
|
|
129
|
+
const result = { stdoutBytes: 0, stderrBytes: 0 };
|
|
130
|
+
for (const row of this.db.query("SELECT stream,bytes FROM streams WHERE run_id=?").all(runId))
|
|
131
|
+
result[row.stream === "stdout" ? "stdoutBytes" : "stderrBytes"] = row.bytes;
|
|
132
|
+
return result;
|
|
133
|
+
}
|
|
134
|
+
outputCursors() {
|
|
135
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
136
|
+
for (const stream of this.db.query("SELECT run_id,stream,acknowledged FROM streams").all()) {
|
|
137
|
+
const cursor = cursors.get(stream.run_id) ?? { runId: stream.run_id, stdoutBytes: 0, stderrBytes: 0 };
|
|
138
|
+
cursor[stream.stream === "stdout" ? "stdoutBytes" : "stderrBytes"] = stream.acknowledged;
|
|
139
|
+
cursors.set(stream.run_id, cursor);
|
|
140
|
+
}
|
|
141
|
+
return [...cursors.values()];
|
|
142
|
+
}
|
|
143
|
+
*output(offsets = []) {
|
|
144
|
+
const cursors = new Map(offsets.map((offset) => [offset.runId, offset]));
|
|
145
|
+
for (const stream of this.db.query("SELECT * FROM streams WHERE NOT EXISTS (SELECT 1 FROM operations WHERE operations.run_id=streams.run_id AND operations.state='unknown') ORDER BY rowid").all()) {
|
|
146
|
+
const supplied = cursors.get(stream.run_id);
|
|
147
|
+
const cursor = supplied ? stream.stream === "stdout" ? supplied.stdoutBytes : supplied.stderrBytes : stream.acknowledged;
|
|
148
|
+
if (cursor >= stream.bytes) continue;
|
|
149
|
+
const firstOffset = this.db.query("SELECT MAX(offset) AS offset FROM output WHERE run_id=? AND stream=? AND offset<=?").get(stream.run_id, stream.stream, cursor)?.offset ?? 0;
|
|
150
|
+
let nextChunk = firstOffset;
|
|
151
|
+
while (true) {
|
|
152
|
+
const row = this.db.query("SELECT offset,data FROM output WHERE run_id=? AND stream=? AND offset>=? AND offset<? AND offset+bytes>? ORDER BY offset LIMIT 1").get(stream.run_id, stream.stream, nextChunk, stream.bytes, cursor);
|
|
153
|
+
if (!row) break;
|
|
154
|
+
nextChunk = row.offset + 1;
|
|
155
|
+
yield { type: "exec_output", runId: stream.run_id, stream: stream.stream, offset: row.offset, data: row.data };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
terminals() {
|
|
160
|
+
return this.db.query(
|
|
161
|
+
"SELECT message FROM terminals WHERE NOT EXISTS (SELECT 1 FROM operations WHERE operations.run_id=terminals.run_id AND operations.state='unknown')"
|
|
162
|
+
).all().map((row) => JSON.parse(row.message));
|
|
163
|
+
}
|
|
164
|
+
acknowledgeOutput(runId, stream, offset) {
|
|
165
|
+
if (!Number.isSafeInteger(offset) || offset < 0) return;
|
|
166
|
+
this.db.run("UPDATE streams SET acknowledged=MAX(acknowledged,MIN(bytes,?)) WHERE run_id=? AND stream=?", [offset, runId, stream]);
|
|
167
|
+
}
|
|
168
|
+
acknowledgeResult(requestId) {
|
|
169
|
+
const row = this.row(requestId);
|
|
170
|
+
if (row && row.request_type !== "exec_start")
|
|
171
|
+
this.db.run("UPDATE operations SET response=NULL WHERE request_id=? AND state='completed'", [row.request_id]);
|
|
172
|
+
}
|
|
173
|
+
acknowledgeTerminal(runId) {
|
|
174
|
+
this.db.transaction(() => {
|
|
175
|
+
this.db.run("DELETE FROM terminals WHERE run_id=?", [runId]);
|
|
176
|
+
this.db.run("DELETE FROM output WHERE run_id=?", [runId]);
|
|
177
|
+
this.db.run("UPDATE streams SET acknowledged=bytes WHERE run_id=?", [runId]);
|
|
178
|
+
this.db.run("UPDATE operations SET response=NULL WHERE run_id=? AND request_type='exec_start' AND state='completed'", [runId]);
|
|
179
|
+
})();
|
|
180
|
+
}
|
|
181
|
+
cancelSession(sessionId) {
|
|
182
|
+
this.db.run("INSERT OR IGNORE INTO cancelled_requests SELECT request_id FROM operations WHERE session_id=? AND state!='completed'", [
|
|
183
|
+
sessionId
|
|
184
|
+
]);
|
|
185
|
+
}
|
|
186
|
+
isRequestCancelled(requestId) {
|
|
187
|
+
const row = this.row(requestId);
|
|
188
|
+
return !!row && this.db.query("SELECT request_id FROM cancelled_requests WHERE request_id=?").get(row.request_id) !== null;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
export {
|
|
192
|
+
WorkerRecoveryStore,
|
|
193
|
+
workerOperationDigest
|
|
194
|
+
};
|
package/dist/mjs/supervisor.mjs
CHANGED
|
@@ -6,6 +6,7 @@ const WORKER_RECONNECT_EXIT_CODE = 76;
|
|
|
6
6
|
const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
|
|
7
7
|
const WORKER_RECONNECT_DELAY_MS = 2e3;
|
|
8
8
|
const WORKER_RECONNECT_JITTER_MS = 3e3;
|
|
9
|
+
const WORKER_RECONNECT_MAX_DELAY_MS = 3e4;
|
|
9
10
|
const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
|
|
10
11
|
function defaultRuntimeSpawner(argv, options) {
|
|
11
12
|
return Bun.spawn(argv, options);
|
|
@@ -13,9 +14,12 @@ function defaultRuntimeSpawner(argv, options) {
|
|
|
13
14
|
function defaultRuntimeDelay(delayMs) {
|
|
14
15
|
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
15
16
|
}
|
|
16
|
-
function workerReconnectDelayMs(random = Math.random) {
|
|
17
|
+
function workerReconnectDelayMs(random = Math.random, attempt = 0) {
|
|
17
18
|
const sample = Math.max(0, Math.min(1, random()));
|
|
18
|
-
|
|
19
|
+
const exponent = Math.min(20, Math.max(0, Math.floor(attempt)));
|
|
20
|
+
const ceiling = Math.min(WORKER_RECONNECT_MAX_DELAY_MS, (WORKER_RECONNECT_DELAY_MS + WORKER_RECONNECT_JITTER_MS) * 2 ** exponent);
|
|
21
|
+
const floor = Math.min(WORKER_RECONNECT_DELAY_MS * 2 ** exponent, ceiling * 0.4);
|
|
22
|
+
return floor + Math.floor(sample * (ceiling - floor));
|
|
19
23
|
}
|
|
20
24
|
async function defaultRuntimeReaper(runtime) {
|
|
21
25
|
await terminateProcessTree(runtime);
|
|
@@ -29,10 +33,11 @@ function signalExitCode(signal) {
|
|
|
29
33
|
function osSignalNumber(signal) {
|
|
30
34
|
return osConstants.signals[signal];
|
|
31
35
|
}
|
|
32
|
-
async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay
|
|
36
|
+
async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay) {
|
|
33
37
|
if (!process.argv[1]) {
|
|
34
38
|
throw new Error("Cannot locate the r5d-worker entrypoint");
|
|
35
39
|
}
|
|
40
|
+
let reconnectAttempt = 0;
|
|
36
41
|
while (true) {
|
|
37
42
|
const runtime = spawnRuntime([process.execPath, process.argv[1], ...argv], {
|
|
38
43
|
stdin: "inherit",
|
|
@@ -81,7 +86,7 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
|
|
|
81
86
|
continue;
|
|
82
87
|
}
|
|
83
88
|
if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
|
|
84
|
-
const reconnectDelayMs = reconnectDelay();
|
|
89
|
+
const reconnectDelayMs = reconnectDelay?.() ?? workerReconnectDelayMs(Math.random, reconnectAttempt++);
|
|
85
90
|
process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
|
|
86
91
|
`);
|
|
87
92
|
await waitBeforeReconnect(reconnectDelayMs);
|
|
@@ -100,6 +105,7 @@ export {
|
|
|
100
105
|
WORKER_RECONNECT_DELAY_MS,
|
|
101
106
|
WORKER_RECONNECT_EXIT_CODE,
|
|
102
107
|
WORKER_RECONNECT_JITTER_MS,
|
|
108
|
+
WORKER_RECONNECT_MAX_DELAY_MS,
|
|
103
109
|
WORKER_RELOAD_EXIT_CODE,
|
|
104
110
|
WORKER_RUNTIME_ENV,
|
|
105
111
|
isRetryableWorkerServerStatus,
|
package/dist/types/main.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import type { WorkerLintPlansResult, WorkerUpdatePlanTaskResult } from "./plan-paths";
|
|
3
|
+
import { WORKER_RESUMABLE_PROTOCOL, type WorkerRecoveryClientMessage, type WorkerRecoveryServerMessage } from "./recovery-protocol";
|
|
3
4
|
import { Database } from "bun:sqlite";
|
|
4
5
|
import type { WorkerGitIdentity } from "./git-identity";
|
|
5
6
|
import { configureGitHubRegistryAuthFiles, type PreparedPrivateAuthFileGeneration } from "./registry-auth";
|
|
@@ -135,8 +136,11 @@ export type WorkerSessionTarget = {
|
|
|
135
136
|
ownerUserId: string;
|
|
136
137
|
rootProfile: "visible_projects" | "canonical_sync";
|
|
137
138
|
};
|
|
138
|
-
type WorkerClientMessage = {
|
|
139
|
+
type WorkerClientMessage = WorkerRecoveryClientMessage | {
|
|
139
140
|
type: "hello";
|
|
141
|
+
resumableProtocol: typeof WORKER_RESUMABLE_PROTOCOL;
|
|
142
|
+
runtimeId: string;
|
|
143
|
+
ledgerId: string;
|
|
140
144
|
hostInfo?: Record<string, unknown>;
|
|
141
145
|
startupReady?: boolean;
|
|
142
146
|
} | {
|
|
@@ -278,7 +282,7 @@ type WorkerClientMessage = {
|
|
|
278
282
|
installedR5dctlVersion?: string;
|
|
279
283
|
error?: string;
|
|
280
284
|
};
|
|
281
|
-
type WorkerServerMessage = {
|
|
285
|
+
type WorkerServerMessage = WorkerRecoveryServerMessage | {
|
|
282
286
|
type: "connected";
|
|
283
287
|
label: string;
|
|
284
288
|
workerId: string;
|
|
@@ -415,15 +419,18 @@ type WorkerServerMessage = {
|
|
|
415
419
|
}>;
|
|
416
420
|
} | {
|
|
417
421
|
type: "pty_input";
|
|
422
|
+
requestId: string;
|
|
418
423
|
ptyId: string;
|
|
419
424
|
data: string;
|
|
420
425
|
} | {
|
|
421
426
|
type: "pty_resize";
|
|
427
|
+
requestId: string;
|
|
422
428
|
ptyId: string;
|
|
423
429
|
cols: number;
|
|
424
430
|
rows: number;
|
|
425
431
|
} | {
|
|
426
432
|
type: "pty_close";
|
|
433
|
+
requestId: string;
|
|
427
434
|
ptyId: string;
|
|
428
435
|
} | {
|
|
429
436
|
type: "read_plan";
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { WorkerOutputOffsets } from "./recovery-protocol";
|
|
2
|
+
export type WorkerOutputFrame = {
|
|
3
|
+
type: "exec_output";
|
|
4
|
+
runId: string;
|
|
5
|
+
stream: "stdout" | "stderr";
|
|
6
|
+
offset: number;
|
|
7
|
+
data: string;
|
|
8
|
+
};
|
|
9
|
+
/** Bound outstanding application bytes, including data already buffered by the server's socket. */
|
|
10
|
+
export declare class WorkerOutputWindow {
|
|
11
|
+
private readonly windowBytes;
|
|
12
|
+
private sent;
|
|
13
|
+
private acknowledged;
|
|
14
|
+
private outstanding;
|
|
15
|
+
constructor(windowBytes?: number);
|
|
16
|
+
reset(cursors: WorkerOutputOffsets[]): void;
|
|
17
|
+
send(frame: WorkerOutputFrame, send: (frame: WorkerOutputFrame) => boolean, bufferedAmount?: number): boolean;
|
|
18
|
+
acknowledge(runId: string, stream: "stdout" | "stderr", offset: number): void;
|
|
19
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/** The application protocol survives replacement of its WebSocket transport. */
|
|
2
|
+
export declare const WORKER_RESUMABLE_PROTOCOL: 1;
|
|
3
|
+
export declare const WORKER_RECOVERY_TIMEOUT_MS: number;
|
|
4
|
+
export type WorkerOperationRecovery = {
|
|
5
|
+
requestId: string;
|
|
6
|
+
runId?: string;
|
|
7
|
+
sessionId?: string;
|
|
8
|
+
requestType?: string;
|
|
9
|
+
state: "accepted" | "preparing" | "running" | "completed" | "unknown";
|
|
10
|
+
};
|
|
11
|
+
export type WorkerOutputOffsets = {
|
|
12
|
+
runId: string;
|
|
13
|
+
stdoutBytes: number;
|
|
14
|
+
stderrBytes: number;
|
|
15
|
+
};
|
|
16
|
+
export type WorkerRecoveryServerMessage = {
|
|
17
|
+
type: "recovery_request";
|
|
18
|
+
recoveryId: string;
|
|
19
|
+
requests: string[];
|
|
20
|
+
outputOffsets: WorkerOutputOffsets[];
|
|
21
|
+
} | {
|
|
22
|
+
type: "recovery_ready";
|
|
23
|
+
recoveryId: string;
|
|
24
|
+
leaseMs: number;
|
|
25
|
+
} | {
|
|
26
|
+
type: "operation_result_ack";
|
|
27
|
+
requestId: string;
|
|
28
|
+
} | {
|
|
29
|
+
type: "exec_output_ack";
|
|
30
|
+
runId: string;
|
|
31
|
+
stream: "stdout" | "stderr";
|
|
32
|
+
nextOffset: number;
|
|
33
|
+
} | {
|
|
34
|
+
type: "worker_execution_lease";
|
|
35
|
+
leaseMs: number;
|
|
36
|
+
} | {
|
|
37
|
+
type: "cancel_session";
|
|
38
|
+
sessionId: string;
|
|
39
|
+
cancellationId: string;
|
|
40
|
+
};
|
|
41
|
+
export type WorkerRecoveryClientMessage = {
|
|
42
|
+
type: "recovery_state";
|
|
43
|
+
recoveryId: string;
|
|
44
|
+
runtimeId: string;
|
|
45
|
+
ledgerId: string;
|
|
46
|
+
operations: WorkerOperationRecovery[];
|
|
47
|
+
} | {
|
|
48
|
+
type: "recovery_complete";
|
|
49
|
+
recoveryId: string;
|
|
50
|
+
} | {
|
|
51
|
+
type: "operation_received";
|
|
52
|
+
requestId: string;
|
|
53
|
+
} | {
|
|
54
|
+
type: "operation_completed";
|
|
55
|
+
requestId: string;
|
|
56
|
+
} | {
|
|
57
|
+
type: "cancel_session_ack";
|
|
58
|
+
cancellationId: string;
|
|
59
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { WorkerOperationRecovery, WorkerOutputOffsets } from "./recovery-protocol";
|
|
2
|
+
type Message = {
|
|
3
|
+
type: string;
|
|
4
|
+
[key: string]: unknown;
|
|
5
|
+
};
|
|
6
|
+
/** Stable semantic identity, independent of JSON key order and transport request aliases. */
|
|
7
|
+
export declare function workerOperationDigest(message: Message): string;
|
|
8
|
+
/** The receipt is committed before a handler can mutate anything. Unknown crash outcomes never replay effects. */
|
|
9
|
+
export declare class WorkerRecoveryStore {
|
|
10
|
+
private readonly db;
|
|
11
|
+
readonly ledgerId: string;
|
|
12
|
+
constructor(filename: string);
|
|
13
|
+
close(): void;
|
|
14
|
+
private row;
|
|
15
|
+
admit(message: Message & {
|
|
16
|
+
requestId: string;
|
|
17
|
+
}): "new" | "duplicate" | "unknown" | "conflict";
|
|
18
|
+
isUnknown(requestId: string): boolean;
|
|
19
|
+
unknown(requestId: string): void;
|
|
20
|
+
unknownRun(runId: string): void;
|
|
21
|
+
record(message: Message): Message;
|
|
22
|
+
operations(): WorkerOperationRecovery[];
|
|
23
|
+
response(requestId: string): Message | undefined;
|
|
24
|
+
responses(requestIds: string[]): Message[];
|
|
25
|
+
lengths(runId: string): {
|
|
26
|
+
stdoutBytes: number;
|
|
27
|
+
stderrBytes: number;
|
|
28
|
+
};
|
|
29
|
+
outputCursors(): WorkerOutputOffsets[];
|
|
30
|
+
output(offsets?: WorkerOutputOffsets[]): Generator<Message>;
|
|
31
|
+
terminals(): Message[];
|
|
32
|
+
acknowledgeOutput(runId: string, stream: "stdout" | "stderr", offset: number): void;
|
|
33
|
+
acknowledgeResult(requestId: string): void;
|
|
34
|
+
acknowledgeTerminal(runId: string): void;
|
|
35
|
+
cancelSession(sessionId: string): void;
|
|
36
|
+
isRequestCancelled(requestId: string): boolean;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
@@ -5,6 +5,7 @@ export declare const WORKER_RECONNECT_EXIT_CODE = 76;
|
|
|
5
5
|
export declare const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
|
|
6
6
|
export declare const WORKER_RECONNECT_DELAY_MS = 2000;
|
|
7
7
|
export declare const WORKER_RECONNECT_JITTER_MS = 3000;
|
|
8
|
+
export declare const WORKER_RECONNECT_MAX_DELAY_MS = 30000;
|
|
8
9
|
type RuntimeProcess = KillableSubprocess;
|
|
9
10
|
export type RuntimeSpawner = (argv: string[], options: {
|
|
10
11
|
stdin: "inherit";
|
|
@@ -16,7 +17,7 @@ export type RuntimeSpawner = (argv: string[], options: {
|
|
|
16
17
|
export type RuntimeDelay = (delayMs: number) => Promise<void>;
|
|
17
18
|
export type RuntimeReaper = (runtime: RuntimeProcess) => Promise<void>;
|
|
18
19
|
export type RuntimeReconnectDelay = () => number;
|
|
19
|
-
export declare function workerReconnectDelayMs(random?: () => number): number;
|
|
20
|
+
export declare function workerReconnectDelayMs(random?: () => number, attempt?: number): number;
|
|
20
21
|
export declare function isRetryableWorkerServerStatus(status: number): boolean;
|
|
21
22
|
export declare function superviseWorkerRuntime(argv: string[], env?: Record<string, string | undefined>, spawnRuntime?: RuntimeSpawner, waitBeforeReconnect?: RuntimeDelay, reapCredentialRuntime?: RuntimeReaper, reconnectDelay?: RuntimeReconnectDelay): Promise<number>;
|
|
22
23
|
export {};
|