@moinulmoin/eden-world-cloudflare 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +46 -4
- package/dist/core/index.d.ts +41 -1
- package/dist/core/index.js +419 -33
- package/package.json +11 -5
package/dist/client/index.js
CHANGED
|
@@ -9,6 +9,12 @@ export function createWorld(options = {}) {
|
|
|
9
9
|
throw new Error("EDEN_WORLD_URL must be set to the Eden World RPC URL");
|
|
10
10
|
const endpoint = new URL(url).toString();
|
|
11
11
|
const pollIntervalMs = options.pollIntervalMs ?? 200;
|
|
12
|
+
function deploymentId() {
|
|
13
|
+
const id = process.env.EDEN_EVE_DEPLOYMENT_ID;
|
|
14
|
+
if (!id)
|
|
15
|
+
throw new Error("EDEN_EVE_DEPLOYMENT_ID must be set to the producer deployment identity");
|
|
16
|
+
return id;
|
|
17
|
+
}
|
|
12
18
|
const local = createLocalWorld({ recoverActiveRuns: false });
|
|
13
19
|
async function rpc(op, args) {
|
|
14
20
|
if (op === "events.create" && args[2] && typeof args[2] === "object") {
|
|
@@ -21,7 +27,10 @@ export function createWorld(options = {}) {
|
|
|
21
27
|
const body = new Uint8Array(encode({ op, args }));
|
|
22
28
|
const response = await fetch(endpoint, {
|
|
23
29
|
method: "POST",
|
|
24
|
-
headers: {
|
|
30
|
+
headers: {
|
|
31
|
+
"content-type": "application/cbor",
|
|
32
|
+
"x-eden-world-token": process.env.EDEN_WORLD_TOKEN ?? "",
|
|
33
|
+
},
|
|
25
34
|
body,
|
|
26
35
|
});
|
|
27
36
|
if (!response.ok)
|
|
@@ -81,14 +90,47 @@ export function createWorld(options = {}) {
|
|
|
81
90
|
};
|
|
82
91
|
return {
|
|
83
92
|
specVersion: SPEC_VERSION_CURRENT,
|
|
93
|
+
capabilities: { deploymentAffinity: true, hookForceClaim: true, hookRetention: { active: true } },
|
|
84
94
|
runs: methods("runs", ["get", "getMany", "list", "experimentalSetAttributes"]),
|
|
85
95
|
steps: methods("steps", ["get", "list"]),
|
|
86
96
|
events: methods("events", ["create", "get", "list", "listByCorrelationId"]),
|
|
87
97
|
hooks: methods("hooks", ["get", "getByToken", "list"]),
|
|
88
98
|
streams,
|
|
89
|
-
getDeploymentId: async () =>
|
|
90
|
-
queue: (
|
|
91
|
-
|
|
99
|
+
getDeploymentId: async () => deploymentId(),
|
|
100
|
+
queue: (queueName, message, opts) => {
|
|
101
|
+
const sourceDeploymentId = deploymentId();
|
|
102
|
+
// Run invocations resolve their durable pin in World. Control messages
|
|
103
|
+
// such as health probes have no run and address this producer explicitly.
|
|
104
|
+
const hasRun = message && typeof message === "object" && "runId" in message && typeof message.runId === "string";
|
|
105
|
+
return rpc("queue.enqueue", [
|
|
106
|
+
queueName, message, { ...opts, deploymentId: opts?.deploymentId ?? (hasRun ? undefined : sourceDeploymentId), sourceDeploymentId },
|
|
107
|
+
]);
|
|
108
|
+
},
|
|
109
|
+
createQueueHandler: (prefix, handler) => local.createQueueHandler(prefix, async (message, metadata) => {
|
|
110
|
+
let handlerFailed = false;
|
|
111
|
+
let handlerError;
|
|
112
|
+
let result;
|
|
113
|
+
try {
|
|
114
|
+
result = await handler(message, metadata);
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
handlerFailed = true;
|
|
118
|
+
handlerError = error;
|
|
119
|
+
}
|
|
120
|
+
// This runs in the container process even if the HTTP consumer was
|
|
121
|
+
// replaced or disconnected. Response EOF remains a redundant ack.
|
|
122
|
+
try {
|
|
123
|
+
await rpc("queue.deliverySettled", [metadata.messageId, metadata.attempt]);
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
if (handlerFailed)
|
|
127
|
+
throw new AggregateError([handlerError, error], "Queue handler and delivery settlement failed", { cause: handlerError });
|
|
128
|
+
throw error;
|
|
129
|
+
}
|
|
130
|
+
if (handlerFailed)
|
|
131
|
+
throw handlerError;
|
|
132
|
+
return result;
|
|
133
|
+
}),
|
|
92
134
|
start: async () => { },
|
|
93
135
|
close: async () => { await local.close?.(); },
|
|
94
136
|
};
|
package/dist/core/index.d.ts
CHANGED
|
@@ -12,27 +12,65 @@ export interface QueueDelivery {
|
|
|
12
12
|
body: Uint8Array;
|
|
13
13
|
path: 'flow' | 'step';
|
|
14
14
|
headers?: Record<string, string>;
|
|
15
|
+
deploymentId?: string;
|
|
16
|
+
sourceDeploymentId?: string;
|
|
17
|
+
sandboxSessionId?: string;
|
|
15
18
|
}): Promise<{
|
|
16
19
|
ok: boolean;
|
|
17
20
|
retryAfterMs?: number;
|
|
21
|
+
sandboxTransferCompleted?: boolean;
|
|
22
|
+
sandboxTransferFailed?: boolean;
|
|
18
23
|
}>;
|
|
19
24
|
}
|
|
25
|
+
export type DeploymentDrainState = {
|
|
26
|
+
liveRuns: number;
|
|
27
|
+
liveSessions: number;
|
|
28
|
+
pendingMessages: number;
|
|
29
|
+
unfinishedTransfers: number;
|
|
30
|
+
drained: boolean;
|
|
31
|
+
};
|
|
32
|
+
export declare function callDeploymentControlRpc(options: {
|
|
33
|
+
origin: string;
|
|
34
|
+
token: string;
|
|
35
|
+
operation: 'deployments.drainState' | 'deployments.retire';
|
|
36
|
+
args: unknown[];
|
|
37
|
+
signal?: AbortSignal;
|
|
38
|
+
}): Promise<unknown>;
|
|
20
39
|
export declare class WorldCore {
|
|
21
40
|
private readonly sql;
|
|
22
41
|
private readonly delivery;
|
|
23
42
|
private readonly now;
|
|
24
43
|
private readonly scheduleAlarm;
|
|
25
|
-
private
|
|
44
|
+
private readonly inFlight;
|
|
45
|
+
private readonly waitUntil;
|
|
46
|
+
private readonly retireDeployment;
|
|
47
|
+
private readonly currentDeploymentId;
|
|
26
48
|
constructor(opts: {
|
|
27
49
|
sql: SqlExec;
|
|
28
50
|
delivery: QueueDelivery;
|
|
29
51
|
now?: () => number;
|
|
30
52
|
scheduleAlarm: (atMs: number | null) => void | Promise<void>;
|
|
53
|
+
waitUntil?: (promise: Promise<void>) => void;
|
|
54
|
+
retireDeployment?: (deploymentId: string) => Promise<void>;
|
|
55
|
+
currentDeploymentId?: string;
|
|
31
56
|
});
|
|
32
57
|
private rows;
|
|
33
58
|
private atomic;
|
|
34
59
|
migrate(): void;
|
|
35
60
|
private run;
|
|
61
|
+
private assertDeploymentActive;
|
|
62
|
+
deliverySettled(messageId: string, attempt: number): void;
|
|
63
|
+
private reconcileDeliveryClaims;
|
|
64
|
+
deploymentActivity(deploymentId: string): {
|
|
65
|
+
activeDeliveries: number;
|
|
66
|
+
unfinishedTransfers: number;
|
|
67
|
+
};
|
|
68
|
+
private queueMessage;
|
|
69
|
+
private queueTarget;
|
|
70
|
+
private sessionId;
|
|
71
|
+
private successorSandbox;
|
|
72
|
+
private reconcileTerminalMessages;
|
|
73
|
+
private drainState;
|
|
36
74
|
private step;
|
|
37
75
|
private saveRun;
|
|
38
76
|
private saveStep;
|
|
@@ -61,9 +99,11 @@ export declare class WorldCore {
|
|
|
61
99
|
private readChunks;
|
|
62
100
|
private getChunks;
|
|
63
101
|
private enqueue;
|
|
102
|
+
private failSandboxTransfer;
|
|
64
103
|
private reschedule;
|
|
65
104
|
runAlarm(): Promise<void>;
|
|
66
105
|
handleRpc(body: Uint8Array): Promise<Uint8Array>;
|
|
106
|
+
handleControlRpc(body: Uint8Array): Promise<Uint8Array>;
|
|
67
107
|
private dispatch;
|
|
68
108
|
}
|
|
69
109
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/core/index.js
CHANGED
|
@@ -3,6 +3,20 @@ import { decode, encode } from 'cbor-x';
|
|
|
3
3
|
import { monotonicFactory } from 'ulid';
|
|
4
4
|
import { EntityConflictError, HookForceClaimedError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from '@workflow/errors';
|
|
5
5
|
import { EventSchema, HookSchema, StepSchema, WorkflowRunSchema, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, SPEC_VERSION_SUPPORTS_HOOK_FORCE_CLAIM, slotToEventId, eventIdToSlot, stripEventDataRefs, validateAttributeChanges, validateUlidTimestamp, isTerminalWorkflowRunStatus, isTerminalStepStatus, isChildEntityCreationEvent, isTerminalRunEventType, requiresNewerWorld, isLegacySpecVersion } from '@workflow/world';
|
|
6
|
+
export async function callDeploymentControlRpc(options) {
|
|
7
|
+
const response = await fetch(new URL('/__eden/world/rpc', options.origin), {
|
|
8
|
+
method: 'POST',
|
|
9
|
+
headers: { 'content-type': 'application/cbor', authorization: `Bearer ${options.token}` },
|
|
10
|
+
body: new Uint8Array(encode({ op: options.operation, args: options.args })),
|
|
11
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
12
|
+
});
|
|
13
|
+
if (!response.ok)
|
|
14
|
+
throw Object.assign(new Error(`Eden deployment control RPC ${options.operation} failed: HTTP ${response.status}`), { status: response.status });
|
|
15
|
+
const envelope = decode(new Uint8Array(await response.arrayBuffer()));
|
|
16
|
+
if (!envelope.ok)
|
|
17
|
+
throw new Error(envelope.error?.message ?? `Eden deployment control RPC ${options.operation} failed`);
|
|
18
|
+
return envelope.result;
|
|
19
|
+
}
|
|
6
20
|
const ulid = monotonicFactory();
|
|
7
21
|
// Same default as @workflow/world when WORKFLOW_MAX_EVENTS is unset.
|
|
8
22
|
const DEFAULT_MAX_EVENTS_PER_RUN = 25000;
|
|
@@ -25,12 +39,18 @@ export class WorldCore {
|
|
|
25
39
|
delivery;
|
|
26
40
|
now;
|
|
27
41
|
scheduleAlarm;
|
|
28
|
-
|
|
42
|
+
inFlight = new Set();
|
|
43
|
+
waitUntil;
|
|
44
|
+
retireDeployment;
|
|
45
|
+
currentDeploymentId;
|
|
29
46
|
constructor(opts) {
|
|
30
47
|
this.sql = opts.sql;
|
|
31
48
|
this.delivery = opts.delivery;
|
|
32
49
|
this.now = opts.now ?? Date.now;
|
|
33
50
|
this.scheduleAlarm = opts.scheduleAlarm;
|
|
51
|
+
this.waitUntil = opts.waitUntil ?? (() => { });
|
|
52
|
+
this.retireDeployment = opts.retireDeployment;
|
|
53
|
+
this.currentDeploymentId = opts.currentDeploymentId;
|
|
34
54
|
}
|
|
35
55
|
rows(query, ...args) { return this.sql.exec(query, ...args).toArray(); }
|
|
36
56
|
atomic(fn) { return this.sql.transactionSync ? this.sql.transactionSync(fn) : fn(); }
|
|
@@ -49,6 +69,8 @@ export class WorldCore {
|
|
|
49
69
|
'CREATE TABLE IF NOT EXISTS workflow_hooks (hook_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, token TEXT NOT NULL, data BLOB NOT NULL)',
|
|
50
70
|
'CREATE INDEX IF NOT EXISTS workflow_hooks_run ON workflow_hooks(run_id)',
|
|
51
71
|
'CREATE INDEX IF NOT EXISTS workflow_hooks_token ON workflow_hooks(token)',
|
|
72
|
+
// Routing tombstones outlive invocation-data retention; never store payloads here.
|
|
73
|
+
'CREATE TABLE IF NOT EXISTS workflow_hook_disposals (run_id TEXT NOT NULL, hook_id TEXT NOT NULL, token TEXT NOT NULL, claimed_run_id TEXT NOT NULL, claimed_hook_id TEXT NOT NULL, PRIMARY KEY(run_id,hook_id))',
|
|
52
74
|
'CREATE TABLE IF NOT EXISTS workflow_waits (wait_id TEXT PRIMARY KEY, run_id TEXT NOT NULL, data BLOB NOT NULL)',
|
|
53
75
|
'CREATE TABLE IF NOT EXISTS workflow_stream_chunks (sequence INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL, stream_id TEXT NOT NULL, run_id TEXT NOT NULL, data BLOB NOT NULL, eof INTEGER NOT NULL, created_at INTEGER NOT NULL, UNIQUE(stream_id,id))',
|
|
54
76
|
'CREATE INDEX IF NOT EXISTS workflow_stream_chunks_name ON workflow_stream_chunks(stream_id,sequence)',
|
|
@@ -56,13 +78,221 @@ export class WorldCore {
|
|
|
56
78
|
'CREATE TABLE IF NOT EXISTS workflow_queue_messages (id TEXT PRIMARY KEY, queue_name TEXT NOT NULL, body BLOB NOT NULL, path TEXT NOT NULL, due_at INTEGER NOT NULL, attempt INTEGER NOT NULL DEFAULT 1, completed INTEGER NOT NULL DEFAULT 0, idempotency_key TEXT)',
|
|
57
79
|
'CREATE UNIQUE INDEX IF NOT EXISTS workflow_queue_idempotency ON workflow_queue_messages(idempotency_key) WHERE idempotency_key IS NOT NULL',
|
|
58
80
|
'CREATE INDEX IF NOT EXISTS workflow_queue_due ON workflow_queue_messages(completed,due_at)',
|
|
81
|
+
// completed: 0 = pending, 1 = imported, -1 = FAILED (never an import acknowledgement).
|
|
82
|
+
'CREATE TABLE IF NOT EXISTS workflow_sandbox_transfers (message_id TEXT PRIMARY KEY, source_deployment_id TEXT NOT NULL, deployment_id TEXT NOT NULL, session_id TEXT NOT NULL, completed INTEGER NOT NULL DEFAULT 0)',
|
|
83
|
+
'CREATE TABLE IF NOT EXISTS workflow_retired_deployments (deployment_id TEXT PRIMARY KEY)',
|
|
84
|
+
'CREATE TABLE IF NOT EXISTS workflow_delivery_claims (message_id TEXT NOT NULL, attempt INTEGER NOT NULL, deployment_id TEXT NOT NULL, settled INTEGER NOT NULL DEFAULT 0, legacy INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(message_id,attempt))',
|
|
85
|
+
'CREATE INDEX IF NOT EXISTS workflow_delivery_claims_active ON workflow_delivery_claims(deployment_id,settled)',
|
|
59
86
|
])
|
|
60
87
|
this.sql.exec(statement);
|
|
61
88
|
if (!this.rows('PRAGMA table_info(workflow_queue_messages)').some(row => row.name === 'headers'))
|
|
62
89
|
this.sql.exec('ALTER TABLE workflow_queue_messages ADD COLUMN headers BLOB');
|
|
90
|
+
for (const column of ['deployment_id', 'source_deployment_id'])
|
|
91
|
+
if (!this.rows('PRAGMA table_info(workflow_queue_messages)').some(row => row.name === column))
|
|
92
|
+
this.sql.exec(`ALTER TABLE workflow_queue_messages ADD COLUMN ${column} TEXT`);
|
|
93
|
+
if (!this.rows('PRAGMA table_info(workflow_queue_messages)').some(row => row.name === 'lease_until'))
|
|
94
|
+
this.sql.exec('ALTER TABLE workflow_queue_messages ADD COLUMN lease_until INTEGER');
|
|
95
|
+
if (!this.rows('PRAGMA table_info(workflow_delivery_claims)').some(row => row.name === 'legacy'))
|
|
96
|
+
this.sql.exec('ALTER TABLE workflow_delivery_claims ADD COLUMN legacy INTEGER NOT NULL DEFAULT 0');
|
|
97
|
+
this.atomic(() => {
|
|
98
|
+
// Pre-protocol outstanding attempts cannot be declared stopped by a
|
|
99
|
+
// replacement actor. Preserve their exact identity, and reserve a new
|
|
100
|
+
// attempt number for recovery rather than reusing that old claim.
|
|
101
|
+
for (const row of this.rows('SELECT * FROM workflow_queue_messages q WHERE completed=0 AND (lease_until IS NULL OR lease_until>0) AND NOT EXISTS (SELECT 1 FROM workflow_delivery_claims c WHERE c.message_id=q.id)')) {
|
|
102
|
+
const deploymentId = this.queueTarget(this.queueMessage(sqlBytes(row.body)), row.deployment_id == null ? undefined : String(row.deployment_id));
|
|
103
|
+
this.sql.exec('INSERT INTO workflow_delivery_claims(message_id,attempt,deployment_id,legacy) VALUES(?,?,?,1)', row.id, row.attempt, deploymentId);
|
|
104
|
+
this.sql.exec('UPDATE workflow_queue_messages SET attempt=attempt+1 WHERE id=?', row.id);
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
// A previously drained affinity key can become current again on rollback.
|
|
108
|
+
// Only Worker publication metadata, never a caller's requested target,
|
|
109
|
+
// authorizes reopening that key's admission fence.
|
|
110
|
+
if (this.currentDeploymentId)
|
|
111
|
+
this.sql.exec('DELETE FROM workflow_retired_deployments WHERE deployment_id=?', this.currentDeploymentId);
|
|
63
112
|
}
|
|
64
113
|
run(id) { const value = unpack(this.rows('SELECT data FROM workflow_runs WHERE id = ?', id)[0]); if (!value)
|
|
65
114
|
throw new WorkflowRunNotFoundError(id); return WorkflowRunSchema.parse(value); }
|
|
115
|
+
assertDeploymentActive(deploymentId) {
|
|
116
|
+
if (this.rows('SELECT deployment_id FROM workflow_retired_deployments WHERE deployment_id=?', deploymentId).length)
|
|
117
|
+
throw new WorkflowWorldError(`Deployment ${deploymentId} has been retired`, { status: 409 });
|
|
118
|
+
}
|
|
119
|
+
deliverySettled(messageId, attempt) {
|
|
120
|
+
// Called by the receiving Container DO through the current World actor.
|
|
121
|
+
// A different attempt's success never reconciles this execution.
|
|
122
|
+
this.sql.exec('UPDATE workflow_delivery_claims SET settled=1 WHERE message_id=? AND attempt=?', messageId, attempt);
|
|
123
|
+
}
|
|
124
|
+
reconcileDeliveryClaims() {
|
|
125
|
+
for (const row of this.rows('SELECT c.message_id,c.attempt,c.legacy,q.body FROM workflow_delivery_claims c JOIN workflow_queue_messages q ON q.id=c.message_id WHERE c.settled=0')) {
|
|
126
|
+
const message = this.queueMessage(sqlBytes(row.body));
|
|
127
|
+
if (typeof message?.runId !== 'string')
|
|
128
|
+
continue;
|
|
129
|
+
if (typeof message.stepId !== 'string' && !Number(row.legacy))
|
|
130
|
+
continue;
|
|
131
|
+
try {
|
|
132
|
+
// Terminal step evidence proves a tool finished, even when its HTTP
|
|
133
|
+
// response was lost. Legacy flows additionally use terminal-run
|
|
134
|
+
// evidence; a running anchor alone never reconciles an unknown flow.
|
|
135
|
+
const terminal = typeof message.stepId === 'string'
|
|
136
|
+
? isTerminalStepStatus(this.step(message.runId, message.stepId).status)
|
|
137
|
+
: isTerminalWorkflowRunStatus(this.run(message.runId).status);
|
|
138
|
+
if (terminal)
|
|
139
|
+
this.deliverySettled(String(row.message_id), Number(row.attempt));
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
if (!(error instanceof WorkflowRunNotFoundError) && !(error instanceof WorkflowWorldError))
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
deploymentActivity(deploymentId) {
|
|
148
|
+
this.reconcileDeliveryClaims();
|
|
149
|
+
const activeDeliveries = Number(this.rows('SELECT COUNT(*) AS count FROM workflow_delivery_claims WHERE deployment_id=? AND settled=0', deploymentId)[0]?.count ?? 0);
|
|
150
|
+
const unfinishedTransfers = Number(this.rows('SELECT COUNT(*) AS count FROM workflow_sandbox_transfers WHERE completed=0 AND (deployment_id=? OR source_deployment_id=?)', deploymentId, deploymentId)[0]?.count ?? 0);
|
|
151
|
+
return { activeDeliveries, unfinishedTransfers };
|
|
152
|
+
}
|
|
153
|
+
queueMessage(body) {
|
|
154
|
+
return JSON.parse(new TextDecoder().decode(body), (_key, value) => {
|
|
155
|
+
if (value && typeof value === 'object' && '__type' in value && value.__type === 'Uint8Array' && 'data' in value && typeof value.data === 'string') {
|
|
156
|
+
return Uint8Array.from(atob(value.data), character => character.charCodeAt(0));
|
|
157
|
+
}
|
|
158
|
+
return value;
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
queueTarget(message, explicit) {
|
|
162
|
+
if (explicit)
|
|
163
|
+
return explicit;
|
|
164
|
+
if (typeof message?.runId === 'string') {
|
|
165
|
+
try {
|
|
166
|
+
return this.run(message.runId).deploymentId;
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
if (!(error instanceof WorkflowRunNotFoundError))
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (typeof message?.runInput?.deploymentId === 'string')
|
|
174
|
+
return message.runInput.deploymentId;
|
|
175
|
+
// Old clients never supplied routing metadata. Their unpinned probes and
|
|
176
|
+
// pre-affinity messages belong to the original, unsuffixed container.
|
|
177
|
+
return 'eden-cloudflare';
|
|
178
|
+
}
|
|
179
|
+
sessionId(run) {
|
|
180
|
+
const visited = new Set();
|
|
181
|
+
let current = run;
|
|
182
|
+
while (current && !visited.has(current.runId)) {
|
|
183
|
+
visited.add(current.runId);
|
|
184
|
+
const sessionId = current.attributes?.['$eve.session'];
|
|
185
|
+
if (typeof sessionId === 'string' && sessionId)
|
|
186
|
+
return sessionId;
|
|
187
|
+
if (current.attributes?.['$eve.type'] === 'session')
|
|
188
|
+
return current.runId;
|
|
189
|
+
const parentId = current.attributes?.['$parentRunId'];
|
|
190
|
+
if (typeof parentId !== 'string' || !parentId)
|
|
191
|
+
return;
|
|
192
|
+
try {
|
|
193
|
+
current = this.run(parentId);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (!(error instanceof WorkflowRunNotFoundError))
|
|
197
|
+
throw error;
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
successorSandbox(message) {
|
|
203
|
+
if (!message?.runInput || typeof message.runId !== 'string')
|
|
204
|
+
return;
|
|
205
|
+
let run;
|
|
206
|
+
try {
|
|
207
|
+
run = this.run(message.runId);
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
if (!(error instanceof WorkflowRunNotFoundError))
|
|
211
|
+
throw error;
|
|
212
|
+
// run_created races the initial queue write. Rebuild exactly the same
|
|
213
|
+
// pending creation state instead of treating a missing row as no import.
|
|
214
|
+
const now = new Date(this.now());
|
|
215
|
+
run = WorkflowRunSchema.parse({ ...message.runInput, runId: message.runId, status: 'pending', attributes: message.runInput.attributes ?? {}, createdAt: now, updatedAt: now, specVersion: message.runInput.specVersion ?? message.specVersion ?? SPEC_VERSION_CURRENT });
|
|
216
|
+
}
|
|
217
|
+
if (run.status !== 'pending' || run.workflowName !== 'workflow//eve//workflowEntry' || run.attributes?.['$eve.session'])
|
|
218
|
+
return;
|
|
219
|
+
const parentId = run.attributes?.['$parentRunId'];
|
|
220
|
+
if (typeof parentId !== 'string' || !parentId)
|
|
221
|
+
return;
|
|
222
|
+
// Missing parent metadata is retryable, never permission to start empty.
|
|
223
|
+
const parent = this.run(parentId);
|
|
224
|
+
if (parent.workflowName !== 'workflow//eve//workflowEntry')
|
|
225
|
+
return;
|
|
226
|
+
const sessionId = this.sessionId(parent);
|
|
227
|
+
if (!sessionId)
|
|
228
|
+
throw new WorkflowWorldError(`Cannot resolve sandbox session for successor ${run.runId}`);
|
|
229
|
+
return { sessionId, sourceDeploymentId: parent.deploymentId };
|
|
230
|
+
}
|
|
231
|
+
reconcileTerminalMessages(runId) {
|
|
232
|
+
const terminalRuns = new Set(this.rows(`SELECT id FROM workflow_runs WHERE status IN ('completed','failed','cancelled')${runId === undefined ? '' : ' AND id=?'}`, ...(runId === undefined ? [] : [runId])).map(row => String(row.id)));
|
|
233
|
+
if (!terminalRuns.size)
|
|
234
|
+
return;
|
|
235
|
+
// A terminal run makes undelivered wakes obsolete, not active executions
|
|
236
|
+
// or sandbox imports. Keep their durable ownership evidence untouched.
|
|
237
|
+
for (const row of this.rows('SELECT q.id,q.body FROM workflow_queue_messages q WHERE q.completed=0 AND COALESCE(q.lease_until,0)<=? AND NOT EXISTS (SELECT 1 FROM workflow_delivery_claims c WHERE c.message_id=q.id AND c.settled=0) AND NOT EXISTS (SELECT 1 FROM workflow_sandbox_transfers t WHERE t.message_id=q.id AND t.completed=0)', this.now())) {
|
|
238
|
+
if (this.inFlight.has(String(row.id)))
|
|
239
|
+
continue;
|
|
240
|
+
const message = this.queueMessage(sqlBytes(row.body));
|
|
241
|
+
if (typeof message?.runId !== 'string' || !terminalRuns.has(message.runId))
|
|
242
|
+
continue;
|
|
243
|
+
this.sql.exec('UPDATE workflow_queue_messages SET completed=1,idempotency_key=NULL,lease_until=0 WHERE id=?', row.id);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
drainState(deploymentIds) {
|
|
247
|
+
this.reconcileDeliveryClaims();
|
|
248
|
+
this.reconcileTerminalMessages();
|
|
249
|
+
const result = Object.fromEntries(deploymentIds.map(id => [id, { liveRuns: 0, liveSessions: 0, pendingMessages: 0, unfinishedTransfers: 0, drained: false }]));
|
|
250
|
+
const sessions = new Map(deploymentIds.map(id => [id, new Set()]));
|
|
251
|
+
for (const row of this.rows('SELECT data FROM workflow_runs')) {
|
|
252
|
+
const run = WorkflowRunSchema.parse(unpack(row));
|
|
253
|
+
const state = result[run.deploymentId];
|
|
254
|
+
if (!state || isTerminalWorkflowRunStatus(run.status))
|
|
255
|
+
continue;
|
|
256
|
+
state.liveRuns++;
|
|
257
|
+
const sessionId = this.sessionId(run);
|
|
258
|
+
if (sessionId)
|
|
259
|
+
sessions.get(run.deploymentId).add(sessionId);
|
|
260
|
+
}
|
|
261
|
+
for (const row of this.rows('SELECT * FROM workflow_queue_messages WHERE completed=0')) {
|
|
262
|
+
const message = this.queueMessage(sqlBytes(row.body));
|
|
263
|
+
const target = this.queueTarget(message, row.deployment_id == null ? undefined : String(row.deployment_id));
|
|
264
|
+
if (result[target])
|
|
265
|
+
result[target].pendingMessages++;
|
|
266
|
+
// An admitted successor retains its source even before its first alarm
|
|
267
|
+
// has persisted a transfer journal entry.
|
|
268
|
+
const sandbox = this.successorSandbox(message);
|
|
269
|
+
const source = sandbox?.sourceDeploymentId;
|
|
270
|
+
if (source && source !== target && !this.rows('SELECT message_id FROM workflow_sandbox_transfers WHERE message_id=?', row.id).length) {
|
|
271
|
+
for (const id of [source, target])
|
|
272
|
+
if (result[id])
|
|
273
|
+
result[id].unfinishedTransfers++;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
for (const row of this.rows('SELECT source_deployment_id,deployment_id FROM workflow_sandbox_transfers WHERE completed=0')) {
|
|
277
|
+
for (const id of new Set([String(row.source_deployment_id), String(row.deployment_id)]))
|
|
278
|
+
if (result[id])
|
|
279
|
+
result[id].unfinishedTransfers++;
|
|
280
|
+
}
|
|
281
|
+
// A later retry completing the queue message does not settle an earlier
|
|
282
|
+
// container execution whose response was lost. Keep those deliveries in
|
|
283
|
+
// retirement accounting without double-counting still-pending messages.
|
|
284
|
+
for (const row of this.rows('SELECT DISTINCT c.message_id,c.deployment_id FROM workflow_delivery_claims c LEFT JOIN workflow_queue_messages q ON q.id=c.message_id WHERE c.settled=0 AND (q.id IS NULL OR q.completed=1)')) {
|
|
285
|
+
const state = result[String(row.deployment_id)];
|
|
286
|
+
if (state)
|
|
287
|
+
state.pendingMessages++;
|
|
288
|
+
}
|
|
289
|
+
for (const id of deploymentIds) {
|
|
290
|
+
const state = result[id];
|
|
291
|
+
state.liveSessions = sessions.get(id).size;
|
|
292
|
+
state.drained = state.liveRuns === 0 && state.pendingMessages === 0 && state.unfinishedTransfers === 0;
|
|
293
|
+
}
|
|
294
|
+
return result;
|
|
295
|
+
}
|
|
66
296
|
step(runId, id) { const value = unpack(this.rows('SELECT data FROM workflow_steps WHERE run_id = ? AND step_id = ?', runId, id)[0]); if (!value)
|
|
67
297
|
throw new WorkflowWorldError(`Step not found: ${id}`); return StepSchema.parse(value); }
|
|
68
298
|
saveRun(run) { this.sql.exec('INSERT INTO workflow_runs(id,name,status,deployment_id,data) VALUES(?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET name=excluded.name,status=excluded.status,deployment_id=excluded.deployment_id,data=excluded.data', run.runId, run.workflowName, run.status, run.deploymentId, encode(compact(run))); }
|
|
@@ -128,19 +358,26 @@ export class WorldCore {
|
|
|
128
358
|
return updated;
|
|
129
359
|
}
|
|
130
360
|
forceRefusal(runId, hookId, token) {
|
|
131
|
-
const
|
|
361
|
+
const marker = this.rows('SELECT token,claimed_run_id,claimed_hook_id FROM workflow_hook_disposals WHERE run_id=? AND hook_id=?', runId, hookId)[0];
|
|
362
|
+
// Older deployments journaled the routing evidence only in the event log.
|
|
363
|
+
const disposal = marker ? undefined : this.prior(runId, hookId, 'hook_disposed');
|
|
132
364
|
const data = disposal?.eventType === 'hook_disposed' ? disposal.eventData : undefined;
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
365
|
+
const claimedBy = marker ? { runId: String(marker.claimed_run_id), hookId: String(marker.claimed_hook_id) } : data?.forceClaimedBy;
|
|
366
|
+
if (claimedBy) {
|
|
367
|
+
const routeToken = token ?? (marker ? String(marker.token) : data?.token) ?? '';
|
|
368
|
+
const successor = this.findHook(routeToken, true);
|
|
369
|
+
throw new HookForceClaimedError(routeToken, successor?.runId ?? claimedBy.runId, successor?.hookId ?? claimedBy.hookId);
|
|
136
370
|
}
|
|
137
371
|
throw new HookNotFoundError(hookId);
|
|
138
372
|
}
|
|
139
373
|
createRun(runId, data) {
|
|
374
|
+
if ((data.specVersion ?? SPEC_VERSION_CURRENT) > SPEC_VERSION_CURRENT)
|
|
375
|
+
throw new RunNotSupportedError(data.specVersion, SPEC_VERSION_CURRENT);
|
|
140
376
|
if (this.rows('SELECT id FROM workflow_runs WHERE id=?', runId).length)
|
|
141
377
|
throw new EntityConflictError(`Workflow run "${runId}" already exists`);
|
|
142
378
|
const d = data.eventData;
|
|
143
379
|
validateAttributeChanges(Object.entries(d.attributes ?? {}).map(([key, value]) => ({ key, value })), { allowReservedAttributes: d.allowReservedAttributes === true });
|
|
380
|
+
this.assertDeploymentActive(d.deploymentId);
|
|
144
381
|
const now = new Date(this.now());
|
|
145
382
|
const run = WorkflowRunSchema.parse(compact({ runId, ...d, status: 'pending', attributes: d.attributes ?? {}, createdAt: now, updatedAt: now, specVersion: data.specVersion ?? SPEC_VERSION_CURRENT }));
|
|
146
383
|
this.saveRun(run);
|
|
@@ -199,7 +436,7 @@ export class WorldCore {
|
|
|
199
436
|
if (!current && ['run_started', 'attr_set', 'hook_received'].includes(type))
|
|
200
437
|
throw new WorkflowRunNotFoundError(id);
|
|
201
438
|
const validatesRunVersion = type !== 'step_completed' && type !== 'step_retrying';
|
|
202
|
-
if (current && validatesRunVersion && requiresNewerWorld(current.specVersion))
|
|
439
|
+
if (current && validatesRunVersion && (requiresNewerWorld(current.specVersion) || (current.specVersion ?? SPEC_VERSION_LEGACY) > SPEC_VERSION_CURRENT))
|
|
203
440
|
throw new RunNotSupportedError(current.specVersion, SPEC_VERSION_CURRENT);
|
|
204
441
|
if (current && validatesRunVersion && isLegacySpecVersion(current.specVersion)) {
|
|
205
442
|
if (type === 'run_cancelled') {
|
|
@@ -207,6 +444,7 @@ export class WorldCore {
|
|
|
207
444
|
this.saveRun(run);
|
|
208
445
|
this.sql.exec('DELETE FROM workflow_hooks WHERE run_id=?', id);
|
|
209
446
|
this.sql.exec('DELETE FROM workflow_waits WHERE run_id=?', id);
|
|
447
|
+
this.reconcileTerminalMessages(id);
|
|
210
448
|
return { run };
|
|
211
449
|
}
|
|
212
450
|
if (!['wait_completed', 'hook_received'].includes(type))
|
|
@@ -215,6 +453,19 @@ export class WorldCore {
|
|
|
215
453
|
throw new RunExpiredError(`Workflow run "${id}" is already in terminal state "${current.status}"`);
|
|
216
454
|
return { event: stripEventDataRefs(this.insertEvent(id, data), params?.resolveData ?? 'all') };
|
|
217
455
|
}
|
|
456
|
+
// A stale resolved delivery must follow a takeover even after the victim
|
|
457
|
+
// finishes or purges its invocation data. Successful resumes still dedup below.
|
|
458
|
+
if (type === 'hook_received' && !this.findHook(correlationId) && !(params?.resumeId && this.rows("SELECT id FROM workflow_events WHERE run_id=? AND resume_id=? AND type='hook_received'", id, params.resumeId).length)) {
|
|
459
|
+
try {
|
|
460
|
+
this.forceRefusal(id, correlationId, data.eventData.token);
|
|
461
|
+
}
|
|
462
|
+
catch (error) {
|
|
463
|
+
if (error instanceof HookForceClaimedError)
|
|
464
|
+
throw error;
|
|
465
|
+
if (!(error instanceof HookNotFoundError))
|
|
466
|
+
throw error;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
218
469
|
if (type === 'hook_received' && params?.resumeId) {
|
|
219
470
|
if (current?.expiredAt || (current && isTerminalWorkflowRunStatus(current.status) && current.attributes['$retention'] === '0'))
|
|
220
471
|
throw new WorkflowWorldError('Invocation data has expired under the run retention policy', { status: 410, code: 'INVOCATION_DATA_EXPIRED' });
|
|
@@ -230,8 +481,10 @@ export class WorldCore {
|
|
|
230
481
|
const lazy = isChildEntityCreationEvent(data) && type === 'step_started';
|
|
231
482
|
const terminal = current && isTerminalWorkflowRunStatus(current.status);
|
|
232
483
|
if (terminal && current) {
|
|
233
|
-
if (type === 'run_cancelled' && current.status === 'cancelled')
|
|
484
|
+
if (type === 'run_cancelled' && current.status === 'cancelled') {
|
|
485
|
+
this.reconcileTerminalMessages(id);
|
|
234
486
|
return { event: stripEventDataRefs(this.insertEvent(id, data), params?.resolveData ?? 'all'), run: current };
|
|
487
|
+
}
|
|
235
488
|
if (type === 'run_started' || type === 'hook_received')
|
|
236
489
|
throw new RunExpiredError(`Workflow run "${id}" is already in terminal state "${current.status}"`);
|
|
237
490
|
if (isTerminalRunEventType(type) && !(type === 'run_cancelled' && current.status === 'cancelled'))
|
|
@@ -271,6 +524,7 @@ export class WorldCore {
|
|
|
271
524
|
this.sql.exec('DELETE FROM workflow_hooks WHERE hook_id=?', h.hookId);
|
|
272
525
|
}
|
|
273
526
|
this.sql.exec('DELETE FROM workflow_waits WHERE run_id=?', id);
|
|
527
|
+
this.reconcileTerminalMessages(id);
|
|
274
528
|
}
|
|
275
529
|
}
|
|
276
530
|
if (data.eventType === 'attr_set') {
|
|
@@ -344,6 +598,7 @@ export class WorldCore {
|
|
|
344
598
|
claimedFrom = { runId: existing.runId, hookId: existing.hookId, ...(victimRunning ? { workflowName: victim.workflowName, deploymentId: victim.deploymentId, runSpecVersion: victim.specVersion } : {}) };
|
|
345
599
|
if (victimRunning)
|
|
346
600
|
this.insertEvent(existing.runId, { eventType: 'hook_disposed', correlationId: existing.hookId, eventData: { token: d.token, forceClaimedBy: { runId: id, hookId: correlationId } }, specVersion: victim.specVersion ?? specVersion });
|
|
601
|
+
this.sql.exec('INSERT INTO workflow_hook_disposals(run_id,hook_id,token,claimed_run_id,claimed_hook_id) VALUES(?,?,?,?,?)', existing.runId, existing.hookId, d.token, id, correlationId);
|
|
347
602
|
this.sql.exec('DELETE FROM workflow_hooks WHERE hook_id=?', existing.hookId);
|
|
348
603
|
data = { ...data, eventData: { ...d, forceClaimedFrom: claimedFrom } };
|
|
349
604
|
}
|
|
@@ -355,7 +610,7 @@ export class WorldCore {
|
|
|
355
610
|
hook = this.findHook(correlationId);
|
|
356
611
|
if (!hook)
|
|
357
612
|
this.forceRefusal(id, correlationId, type === 'hook_received' ? data.eventData.token : undefined);
|
|
358
|
-
if (type === 'hook_received'
|
|
613
|
+
if (type === 'hook_received') {
|
|
359
614
|
const token = data.eventData.token;
|
|
360
615
|
if (hook.runId !== id || (token !== undefined && token !== hook.token))
|
|
361
616
|
throw new HookNotFoundError(correlationId);
|
|
@@ -417,6 +672,11 @@ export class WorldCore {
|
|
|
417
672
|
}
|
|
418
673
|
for (const row of this.rows('SELECT data FROM workflow_events WHERE run_id=?', id)) {
|
|
419
674
|
const event = unpack(row);
|
|
675
|
+
// Promote pre-upgrade routing evidence before erasing event payloads.
|
|
676
|
+
if (event.eventType === 'hook_disposed' && event.eventData?.forceClaimedBy) {
|
|
677
|
+
const { token, forceClaimedBy } = event.eventData;
|
|
678
|
+
this.sql.exec('INSERT OR IGNORE INTO workflow_hook_disposals(run_id,hook_id,token,claimed_run_id,claimed_hook_id) VALUES(?,?,?,?,?)', id, event.correlationId, token ?? '', forceClaimedBy.runId, forceClaimedBy.hookId);
|
|
679
|
+
}
|
|
420
680
|
this.sql.exec('UPDATE workflow_events SET data=?,resume_payload_digest=NULL WHERE run_id=? AND id=?', encode({ ...event, eventData: undefined }), id, event.eventId);
|
|
421
681
|
}
|
|
422
682
|
for (const row of this.rows('SELECT data FROM workflow_hooks WHERE run_id=?', id)) {
|
|
@@ -424,6 +684,7 @@ export class WorldCore {
|
|
|
424
684
|
this.saveHook({ ...hook, metadata: undefined, resumeContext: undefined });
|
|
425
685
|
}
|
|
426
686
|
this.sql.exec('UPDATE workflow_stream_chunks SET data=? WHERE run_id=?', new Uint8Array(), id);
|
|
687
|
+
this.reconcileTerminalMessages(id);
|
|
427
688
|
}
|
|
428
689
|
listRuns(params) {
|
|
429
690
|
const args = [];
|
|
@@ -513,44 +774,134 @@ export class WorldCore {
|
|
|
513
774
|
}
|
|
514
775
|
return value;
|
|
515
776
|
});
|
|
516
|
-
|
|
777
|
+
const deploymentId = this.queueTarget(message, opts?.deploymentId);
|
|
778
|
+
this.assertDeploymentActive(deploymentId);
|
|
779
|
+
const sandbox = this.successorSandbox(message);
|
|
780
|
+
const sourceDeploymentId = sandbox?.sourceDeploymentId ?? opts?.sourceDeploymentId;
|
|
781
|
+
if (sourceDeploymentId && sourceDeploymentId !== deploymentId)
|
|
782
|
+
this.assertDeploymentActive(sourceDeploymentId);
|
|
783
|
+
this.atomic(() => {
|
|
784
|
+
this.sql.exec('INSERT INTO workflow_queue_messages(id,queue_name,body,path,due_at,idempotency_key,headers,deployment_id,source_deployment_id,lease_until) VALUES(?,?,?,?,?,?,?,?,?,0)', id, queueName, new TextEncoder().encode(json), path, this.now() + Math.max(0, opts?.delaySeconds ?? 0) * 1000, opts?.idempotencyKey ?? null, opts?.headers ? encode(opts.headers) : null, deploymentId, sourceDeploymentId ?? null);
|
|
785
|
+
if (sandbox && sourceDeploymentId && sourceDeploymentId !== deploymentId)
|
|
786
|
+
this.sql.exec('INSERT INTO workflow_sandbox_transfers(message_id,source_deployment_id,deployment_id,session_id) VALUES(?,?,?,?)', id, sourceDeploymentId, deploymentId, sandbox.sessionId);
|
|
787
|
+
});
|
|
517
788
|
await this.reschedule();
|
|
518
789
|
return { messageId: id };
|
|
519
790
|
}
|
|
520
|
-
|
|
791
|
+
failSandboxTransfer(id, attempt, message) {
|
|
792
|
+
this.atomic(() => {
|
|
793
|
+
if (typeof message.runId === 'string') {
|
|
794
|
+
let run;
|
|
795
|
+
try {
|
|
796
|
+
run = this.run(message.runId);
|
|
797
|
+
}
|
|
798
|
+
catch (error) {
|
|
799
|
+
if (!(error instanceof WorkflowRunNotFoundError) || !message.runInput)
|
|
800
|
+
throw error;
|
|
801
|
+
run = this.createSync(message.runId, { eventType: 'run_created', eventData: message.runInput, specVersion: message.runInput.specVersion ?? message.specVersion ?? SPEC_VERSION_CURRENT }).run;
|
|
802
|
+
}
|
|
803
|
+
// Import precedes workflow dispatch: fail only the unstarted successor,
|
|
804
|
+
// never its source session or a successor that already took ownership.
|
|
805
|
+
if (run.status === 'pending')
|
|
806
|
+
this.createSync(message.runId, { eventType: 'run_failed', specVersion: run.specVersion, eventData: { error: new TextEncoder().encode(JSON.stringify(['Sandbox export failed; session remains on its current deployment.'])), errorCode: 'SANDBOX_EXPORT_FAILED' } });
|
|
807
|
+
}
|
|
808
|
+
this.sql.exec('UPDATE workflow_sandbox_transfers SET completed=-1 WHERE message_id=? AND completed=0', id);
|
|
809
|
+
this.deliverySettled(id, attempt);
|
|
810
|
+
this.sql.exec('UPDATE workflow_queue_messages SET completed=1,idempotency_key=NULL,lease_until=0 WHERE id=?', id);
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
async reschedule() { const row = this.rows('SELECT MIN(due_at) AS due FROM workflow_queue_messages WHERE completed=0')[0]; const due = row?.due === null || row?.due === undefined ? null : Number(row.due); await this.scheduleAlarm(this.inFlight.size ? Math.min(due ?? Infinity, this.now() + 30000) : due); }
|
|
521
814
|
async runAlarm() {
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
815
|
+
// Alarms cannot overlap on Cloudflare. Never keep one open on a delivery:
|
|
816
|
+
// it would block every subsequent wake, and has a 15 minute wall-time limit.
|
|
817
|
+
// Renew live deliveries before selecting due work. Their durable leases
|
|
818
|
+
// recover a lost instance; the in-memory set, not elapsed time, owns a live
|
|
819
|
+
// delivery. The 30 second heartbeat also keeps older compatibility dates
|
|
820
|
+
// active, where pending RPCs/waitUntil alone do not prevent idle eviction.
|
|
821
|
+
const leaseUntil = this.now() + 60000;
|
|
822
|
+
for (const id of this.inFlight)
|
|
823
|
+
this.sql.exec('UPDATE workflow_queue_messages SET due_at=?,lease_until=? WHERE id=? AND completed=0', leaseUntil, leaseUntil, id);
|
|
824
|
+
const due = this.rows('SELECT * FROM workflow_queue_messages WHERE completed=0 AND due_at<=? ORDER BY due_at,id LIMIT 50', this.now()).filter(row => !this.inFlight.has(String(row.id)));
|
|
825
|
+
for (const row of due) {
|
|
826
|
+
const id = String(row.id);
|
|
827
|
+
const deploymentId = this.queueTarget(this.queueMessage(sqlBytes(row.body)), row.deployment_id == null ? undefined : String(row.deployment_id));
|
|
828
|
+
this.atomic(() => {
|
|
829
|
+
this.sql.exec('INSERT INTO workflow_delivery_claims(message_id,attempt,deployment_id) VALUES(?,?,?)', id, row.attempt, deploymentId);
|
|
830
|
+
this.sql.exec('UPDATE workflow_queue_messages SET due_at=?,lease_until=?,attempt=attempt+1,deployment_id=? WHERE id=?', leaseUntil, leaseUntil, deploymentId, id);
|
|
831
|
+
});
|
|
832
|
+
this.inFlight.add(id);
|
|
833
|
+
}
|
|
834
|
+
// Persist recovery before making external effects. A crash here leaves
|
|
835
|
+
// every claimed message eligible again after its lease expires.
|
|
525
836
|
try {
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
this.
|
|
532
|
-
|
|
837
|
+
await this.reschedule();
|
|
838
|
+
}
|
|
839
|
+
catch (error) {
|
|
840
|
+
for (const row of due) {
|
|
841
|
+
this.inFlight.delete(String(row.id));
|
|
842
|
+
this.deliverySettled(String(row.id), Number(row.attempt));
|
|
843
|
+
}
|
|
844
|
+
throw error;
|
|
845
|
+
}
|
|
846
|
+
for (const row of due) {
|
|
847
|
+
const id = String(row.id);
|
|
848
|
+
const attempt = Number(row.attempt);
|
|
849
|
+
this.waitUntil((async () => {
|
|
533
850
|
try {
|
|
534
|
-
|
|
851
|
+
let dispatched = false;
|
|
852
|
+
let result;
|
|
853
|
+
try {
|
|
854
|
+
const body = sqlBytes(row.body);
|
|
855
|
+
const message = this.queueMessage(body);
|
|
856
|
+
const deploymentId = this.queueTarget(message, row.deployment_id == null ? undefined : String(row.deployment_id));
|
|
857
|
+
this.assertDeploymentActive(deploymentId);
|
|
858
|
+
this.sql.exec('UPDATE workflow_queue_messages SET deployment_id=? WHERE id=?', deploymentId, id);
|
|
859
|
+
let transfer = this.rows('SELECT * FROM workflow_sandbox_transfers WHERE message_id=?', id)[0];
|
|
860
|
+
const sandbox = transfer ? undefined : this.successorSandbox(message);
|
|
861
|
+
const sourceDeploymentId = transfer ? String(transfer.source_deployment_id) : sandbox?.sourceDeploymentId ?? (row.source_deployment_id == null ? undefined : String(row.source_deployment_id));
|
|
862
|
+
if (sandbox && sourceDeploymentId && sourceDeploymentId !== deploymentId) {
|
|
863
|
+
this.assertDeploymentActive(sourceDeploymentId);
|
|
864
|
+
this.sql.exec('UPDATE workflow_queue_messages SET source_deployment_id=? WHERE id=?', sourceDeploymentId, id);
|
|
865
|
+
this.sql.exec('INSERT INTO workflow_sandbox_transfers(message_id,source_deployment_id,deployment_id,session_id) VALUES(?,?,?,?)', id, sourceDeploymentId, deploymentId, sandbox.sessionId);
|
|
866
|
+
transfer = this.rows('SELECT * FROM workflow_sandbox_transfers WHERE message_id=?', id)[0];
|
|
867
|
+
}
|
|
868
|
+
const sandboxSessionId = transfer && !Number(transfer.completed) ? String(transfer.session_id) : undefined;
|
|
869
|
+
dispatched = true;
|
|
870
|
+
result = await this.delivery.deliver({ queueName: String(row.queue_name), messageId: id, attempt, body, path: row.path, ...(row.headers ? { headers: decode(sqlBytes(row.headers)) } : {}), deploymentId, ...(sourceDeploymentId === undefined ? {} : { sourceDeploymentId }), ...(sandboxSessionId === undefined ? {} : { sandboxSessionId }) });
|
|
871
|
+
if (sandboxSessionId) {
|
|
872
|
+
if (result.sandboxTransferFailed) {
|
|
873
|
+
this.failSandboxTransfer(id, attempt, message);
|
|
874
|
+
return;
|
|
875
|
+
}
|
|
876
|
+
if (result.sandboxTransferCompleted)
|
|
877
|
+
this.sql.exec('UPDATE workflow_sandbox_transfers SET completed=1 WHERE message_id=?', id);
|
|
878
|
+
else
|
|
879
|
+
result = { ok: false, ...(result.retryAfterMs === undefined ? {} : { retryAfterMs: result.retryAfterMs }) };
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
catch {
|
|
883
|
+
if (!dispatched)
|
|
884
|
+
this.deliverySettled(id, attempt);
|
|
885
|
+
result = { ok: false };
|
|
886
|
+
}
|
|
887
|
+
// Like world-local, deduplicate outstanding deliveries, not future wakes.
|
|
888
|
+
if (result.ok && result.retryAfterMs === undefined)
|
|
889
|
+
this.sql.exec('UPDATE workflow_queue_messages SET completed=1,idempotency_key=NULL,lease_until=0 WHERE id=?', id);
|
|
890
|
+
else
|
|
891
|
+
this.sql.exec('UPDATE workflow_queue_messages SET due_at=? WHERE id=?', this.now() + Math.max(1, result.retryAfterMs ?? Math.min(60000, 1000 * 2 ** Math.min(attempt - 1, 6))), id);
|
|
535
892
|
}
|
|
536
|
-
|
|
537
|
-
|
|
893
|
+
finally {
|
|
894
|
+
this.inFlight.delete(id);
|
|
895
|
+
await this.reschedule();
|
|
538
896
|
}
|
|
539
|
-
|
|
540
|
-
if (result.ok && result.retryAfterMs === undefined)
|
|
541
|
-
this.sql.exec('UPDATE workflow_queue_messages SET completed=1,idempotency_key=NULL WHERE id=?', id);
|
|
542
|
-
else
|
|
543
|
-
this.sql.exec('UPDATE workflow_queue_messages SET attempt=?,due_at=? WHERE id=?', attempt + 1, this.now() + Math.max(1, result.retryAfterMs ?? Math.min(60000, 1000 * 2 ** Math.min(attempt - 1, 6))), id);
|
|
544
|
-
}));
|
|
545
|
-
}
|
|
546
|
-
finally {
|
|
547
|
-
this.alarmRunning = false;
|
|
548
|
-
await this.reschedule();
|
|
897
|
+
})());
|
|
549
898
|
}
|
|
550
899
|
}
|
|
551
900
|
async handleRpc(body) {
|
|
552
901
|
try {
|
|
553
902
|
const { op, args } = decode(body);
|
|
903
|
+
if (op.startsWith('deployments.'))
|
|
904
|
+
throw new WorkflowWorldError('Deployment operations require deployment control RPC', { status: 403 });
|
|
554
905
|
const result = await this.dispatch(op, args);
|
|
555
906
|
return encode({ ok: true, result });
|
|
556
907
|
}
|
|
@@ -559,8 +910,42 @@ export class WorldCore {
|
|
|
559
910
|
return encode({ ok: false, error: { ...Object.fromEntries(Object.entries(e)), name: e.name, message: e.message } });
|
|
560
911
|
}
|
|
561
912
|
}
|
|
913
|
+
async handleControlRpc(body) {
|
|
914
|
+
try {
|
|
915
|
+
const request = decode(body);
|
|
916
|
+
if (request.op !== 'deployments.drainState' && request.op !== 'deployments.retire')
|
|
917
|
+
throw new WorkflowWorldError('Operation is not available through deployment control RPC', { status: 403 });
|
|
918
|
+
return encode({ ok: true, result: await this.dispatch(request.op, request.args) });
|
|
919
|
+
}
|
|
920
|
+
catch (error) {
|
|
921
|
+
const e = error instanceof Error ? error : new Error(String(error));
|
|
922
|
+
return encode({ ok: false, error: { ...Object.fromEntries(Object.entries(e)), name: e.name, message: e.message } });
|
|
923
|
+
}
|
|
924
|
+
}
|
|
562
925
|
async dispatch(op, args) {
|
|
563
926
|
switch (op) {
|
|
927
|
+
case 'deployments.drainState': {
|
|
928
|
+
const request = args[0];
|
|
929
|
+
return this.drainState(request.deploymentIds);
|
|
930
|
+
}
|
|
931
|
+
case 'deployments.retire': {
|
|
932
|
+
const request = args[0];
|
|
933
|
+
const ids = request.deploymentIds;
|
|
934
|
+
if (this.currentDeploymentId && ids.includes(this.currentDeploymentId))
|
|
935
|
+
throw new WorkflowWorldError('Cannot retire the current deployment', { status: 409 });
|
|
936
|
+
const states = this.drainState(ids);
|
|
937
|
+
if (ids.some(id => !states[id].drained))
|
|
938
|
+
throw new WorkflowWorldError('Cannot retire a deployment with live runs, queued messages, or unfinished transfers', { status: 409 });
|
|
939
|
+
if (!this.retireDeployment)
|
|
940
|
+
throw new WorkflowWorldError('Deployment retirement is unavailable', { status: 501 });
|
|
941
|
+
// Drain and admission fencing are synchronous in the same DO turn.
|
|
942
|
+
// Keep fences durable even if external deletion must be retried.
|
|
943
|
+
this.atomic(() => { for (const id of ids)
|
|
944
|
+
this.sql.exec('INSERT OR IGNORE INTO workflow_retired_deployments(deployment_id) VALUES(?)', id); });
|
|
945
|
+
for (const id of ids)
|
|
946
|
+
await this.retireDeployment(id);
|
|
947
|
+
return { retired: ids };
|
|
948
|
+
}
|
|
564
949
|
case 'runs.get': return filterData(this.run(args[0]), args[1]);
|
|
565
950
|
case 'runs.getMany': return args[0].map(id => { try {
|
|
566
951
|
return filterData(this.run(id), args[1]);
|
|
@@ -589,6 +974,7 @@ export class WorldCore {
|
|
|
589
974
|
case 'streams.getChunks': return this.getChunks(args[1], args[2]);
|
|
590
975
|
case 'streams.getInfo': return { tailIndex: this.streamCount(args[1]) - 1, done: this.streamDone(args[1]) };
|
|
591
976
|
case 'queue.enqueue': return this.enqueue(args[0], args[1], args[2]);
|
|
977
|
+
case 'queue.deliverySettled': return this.deliverySettled(args[0], args[1]);
|
|
592
978
|
case 'queue.enqueueBatch': return Promise.all(args[1].map(async (item) => { try {
|
|
593
979
|
return await this.enqueue(args[0], item.message, item.opts);
|
|
594
980
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@moinulmoin/eden-world-cloudflare",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Cloudflare Durable Object SQLite World for Eve workflows.",
|
|
5
|
+
"eden": {
|
|
6
|
+
"capabilities": {
|
|
7
|
+
"deploymentAffinity": true,
|
|
8
|
+
"worldRpcAuth": true
|
|
9
|
+
}
|
|
10
|
+
},
|
|
5
11
|
"type": "module",
|
|
6
12
|
"license": "Apache-2.0",
|
|
7
13
|
"types": "./dist/index.d.ts",
|
|
@@ -23,15 +29,15 @@
|
|
|
23
29
|
"NOTICE"
|
|
24
30
|
],
|
|
25
31
|
"dependencies": {
|
|
26
|
-
"@workflow/errors": "5.
|
|
27
|
-
"@workflow/world": "5.
|
|
28
|
-
"@workflow/world-local": "5.
|
|
32
|
+
"@workflow/errors": "5.1.0",
|
|
33
|
+
"@workflow/world": "5.1.0",
|
|
34
|
+
"@workflow/world-local": "5.2.0",
|
|
29
35
|
"cbor-x": "1.6.4",
|
|
30
36
|
"ulid": "3.0.2"
|
|
31
37
|
},
|
|
32
38
|
"devDependencies": {
|
|
33
39
|
"@types/node": "24.0.0",
|
|
34
|
-
"@workflow/world-testing": "5.0.
|
|
40
|
+
"@workflow/world-testing": "5.0.7"
|
|
35
41
|
},
|
|
36
42
|
"repository": {
|
|
37
43
|
"type": "git",
|