@porulle/jobs-cloudflare 0.29.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/coordinator.d.ts +27 -12
- package/dist/coordinator.js +114 -51
- package/dist/index.d.ts +8 -0
- package/dist/index.js +25 -0
- package/package.json +2 -2
- package/src/coordinator.ts +143 -59
- package/src/index.ts +25 -0
package/dist/coordinator.d.ts
CHANGED
|
@@ -10,17 +10,28 @@ export interface CoordinatorStorage {
|
|
|
10
10
|
*/
|
|
11
11
|
export declare class JobCoordinatorLogic {
|
|
12
12
|
private readonly storage;
|
|
13
|
-
|
|
14
|
-
constructor(storage: CoordinatorStorage, isStale: (instanceId: string) => Promise<boolean>);
|
|
13
|
+
constructor(storage: CoordinatorStorage);
|
|
15
14
|
/** Registers `instanceId` as pending for `key` before the caller creates it, so
|
|
16
15
|
* a later supersede can see it even if it has not started running yet. When
|
|
17
16
|
* `supersedes` is set, the previously pending ids are cleared and returned for
|
|
18
17
|
* the caller to terminate. Never touches the currently running instance —
|
|
19
18
|
* matching the drizzle adapter, supersede only drops jobs that have not started. */
|
|
20
|
-
enqueue(key: string, supersedes: boolean, instanceId: string): Promise<{
|
|
19
|
+
enqueue(key: string, supersedes: boolean, instanceId: string, inputHash?: string): Promise<{
|
|
21
20
|
terminated: string[];
|
|
21
|
+
coalescedInto?: string;
|
|
22
22
|
}>;
|
|
23
|
-
|
|
23
|
+
/** First gate phase: storage only. Returns `needsStaleCheck` when another instance
|
|
24
|
+
* holds the key so the Durable Object can ask the Workflow binding outside the gate. */
|
|
25
|
+
acquireRead(key: string, instanceId: string): Promise<"granted" | {
|
|
26
|
+
needsStaleCheck: string;
|
|
27
|
+
}>;
|
|
28
|
+
/** Re-enter after a live holder was confirmed outside the gate. */
|
|
29
|
+
acquireWhenHolderLive(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
30
|
+
/** Re-enter after the holder was stale outside the gate — only grants when the
|
|
31
|
+
* same id still holds the key, so a concurrent acquirer cannot be raced. */
|
|
32
|
+
acquireAfterStale(key: string, instanceId: string, checkedHolderId: string): Promise<"granted" | "pending">;
|
|
33
|
+
private grantKey;
|
|
34
|
+
private enqueuePending;
|
|
24
35
|
/** Releases the lock if `instanceId` holds it and hands it to the next
|
|
25
36
|
* pending instance (if any), returning that instance's id so the caller can
|
|
26
37
|
* wake it. A release from an instance that does not hold the lock is a no-op. */
|
|
@@ -66,32 +77,36 @@ type DurableObjectConstructor = abstract new (...args: any[]) => object;
|
|
|
66
77
|
* export class PorulleJobCoordinator extends porulleJobCoordinator(DurableObject) {}
|
|
67
78
|
* ```
|
|
68
79
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
80
|
+
* State mutations run under `blockConcurrencyWhile`; Workflow binding calls
|
|
81
|
+
* (stale-holder checks and turn events) run outside it so a long release loop
|
|
82
|
+
* cannot block every other RPC on the object. On `enqueue` with `supersedes`
|
|
83
|
+
* the object reports the pending instances the caller must terminate. It needs
|
|
84
|
+
* a `PORULLE_WORKFLOW` binding on its environment to detect dead lock holders
|
|
85
|
+
* and to wake the next waiting instance.
|
|
75
86
|
*/
|
|
76
87
|
export declare function porulleJobCoordinator<TBase extends DurableObjectConstructor>(Base: TBase): (abstract new (...args: any[]) => {
|
|
77
88
|
readonly #logic: JobCoordinatorLogic;
|
|
78
89
|
readonly #workflow: CoordinatorWorkflowBinding;
|
|
79
90
|
readonly #state: DurableObjectStateLike;
|
|
80
|
-
|
|
91
|
+
#isStale(instanceId: string): Promise<boolean>;
|
|
92
|
+
enqueue(key: string, supersedes: boolean, instanceId: string, inputHash?: string): Promise<{
|
|
81
93
|
terminated: string[];
|
|
94
|
+
coalescedInto?: string;
|
|
82
95
|
}>;
|
|
83
96
|
acquire(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
84
97
|
/** Hands the key to the next pending instance that can still be woken; a
|
|
85
98
|
* pending instance that died or was terminated meanwhile is skipped so the
|
|
86
99
|
* key never ends up held by an instance that will never release it. */
|
|
87
100
|
release(key: string, instanceId: string): Promise<void>;
|
|
101
|
+
#releaseAndWake(key: string, holder: string): Promise<void>;
|
|
88
102
|
}) & TBase;
|
|
89
103
|
/** The RPC surface `DurableObjectConcurrencyCoordinator` calls on a
|
|
90
104
|
* `PorulleJobCoordinator` stub — the subset of `DurableObjectStub<PorulleJobCoordinator>`
|
|
91
105
|
* this package needs, so callers can inject a fake in tests without the Workers runtime. */
|
|
92
106
|
export interface CoordinatorStub {
|
|
93
|
-
enqueue(key: string, supersedes: boolean, instanceId: string): Promise<{
|
|
107
|
+
enqueue(key: string, supersedes: boolean, instanceId: string, inputHash?: string): Promise<{
|
|
94
108
|
terminated: string[];
|
|
109
|
+
coalescedInto?: string;
|
|
95
110
|
}>;
|
|
96
111
|
acquire(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
97
112
|
release(key: string, instanceId: string): Promise<void>;
|
package/dist/coordinator.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { hashJobInput } from "./index.js";
|
|
1
2
|
const STALE_INSTANCE_STATUSES = new Set(["complete", "errored", "terminated"]);
|
|
2
3
|
const TURN_EVENT_TYPE = "porulle-turn";
|
|
3
4
|
const FIRST_TURN_WAIT_MS = 60_000;
|
|
@@ -22,37 +23,82 @@ function coordinatorKey(payload) {
|
|
|
22
23
|
*/
|
|
23
24
|
export class JobCoordinatorLogic {
|
|
24
25
|
storage;
|
|
25
|
-
|
|
26
|
-
constructor(storage, isStale) {
|
|
26
|
+
constructor(storage) {
|
|
27
27
|
this.storage = storage;
|
|
28
|
-
this.isStale = isStale;
|
|
29
28
|
}
|
|
30
29
|
/** Registers `instanceId` as pending for `key` before the caller creates it, so
|
|
31
30
|
* a later supersede can see it even if it has not started running yet. When
|
|
32
31
|
* `supersedes` is set, the previously pending ids are cleared and returned for
|
|
33
32
|
* the caller to terminate. Never touches the currently running instance —
|
|
34
33
|
* matching the drizzle adapter, supersede only drops jobs that have not started. */
|
|
35
|
-
async enqueue(key, supersedes, instanceId) {
|
|
34
|
+
async enqueue(key, supersedes, instanceId, inputHash) {
|
|
36
35
|
const state = await this.getState(key);
|
|
36
|
+
if (supersedes && inputHash !== undefined) {
|
|
37
|
+
for (const pendingId of state.pending) {
|
|
38
|
+
if (state.pendingHashes[pendingId] === inputHash) {
|
|
39
|
+
return { terminated: [], coalescedInto: pendingId };
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
37
43
|
const terminated = supersedes ? state.pending.filter((id) => id !== instanceId) : [];
|
|
38
44
|
const kept = supersedes ? [] : state.pending.filter((id) => id !== instanceId);
|
|
39
|
-
|
|
45
|
+
const pendingHashes = { ...state.pendingHashes };
|
|
46
|
+
for (const id of terminated)
|
|
47
|
+
delete pendingHashes[id];
|
|
48
|
+
if (inputHash !== undefined)
|
|
49
|
+
pendingHashes[instanceId] = inputHash;
|
|
50
|
+
await this.putState(key, {
|
|
51
|
+
...state,
|
|
52
|
+
pending: [...kept, instanceId],
|
|
53
|
+
pendingHashes,
|
|
54
|
+
});
|
|
40
55
|
return { terminated };
|
|
41
56
|
}
|
|
42
|
-
|
|
43
|
-
|
|
57
|
+
/** First gate phase: storage only. Returns `needsStaleCheck` when another instance
|
|
58
|
+
* holds the key so the Durable Object can ask the Workflow binding outside the gate. */
|
|
59
|
+
async acquireRead(key, instanceId) {
|
|
60
|
+
const state = await this.getState(key);
|
|
44
61
|
if (state.running === instanceId)
|
|
45
62
|
return "granted";
|
|
46
|
-
if (state.running
|
|
47
|
-
|
|
63
|
+
if (state.running === null) {
|
|
64
|
+
await this.grantKey(key, instanceId, state);
|
|
65
|
+
return "granted";
|
|
48
66
|
}
|
|
67
|
+
return { needsStaleCheck: state.running };
|
|
68
|
+
}
|
|
69
|
+
/** Re-enter after a live holder was confirmed outside the gate. */
|
|
70
|
+
async acquireWhenHolderLive(key, instanceId) {
|
|
71
|
+
const state = await this.getState(key);
|
|
72
|
+
if (state.running === instanceId)
|
|
73
|
+
return "granted";
|
|
49
74
|
if (state.running === null) {
|
|
50
|
-
await this.
|
|
51
|
-
pending: state.pending.filter((id) => id !== instanceId),
|
|
52
|
-
running: instanceId,
|
|
53
|
-
});
|
|
75
|
+
await this.grantKey(key, instanceId, state);
|
|
54
76
|
return "granted";
|
|
55
77
|
}
|
|
78
|
+
return this.enqueuePending(key, instanceId, state);
|
|
79
|
+
}
|
|
80
|
+
/** Re-enter after the holder was stale outside the gate — only grants when the
|
|
81
|
+
* same id still holds the key, so a concurrent acquirer cannot be raced. */
|
|
82
|
+
async acquireAfterStale(key, instanceId, checkedHolderId) {
|
|
83
|
+
const state = await this.getState(key);
|
|
84
|
+
if (state.running === instanceId)
|
|
85
|
+
return "granted";
|
|
86
|
+
if (state.running === checkedHolderId) {
|
|
87
|
+
await this.grantKey(key, instanceId, state);
|
|
88
|
+
return "granted";
|
|
89
|
+
}
|
|
90
|
+
return this.acquireWhenHolderLive(key, instanceId);
|
|
91
|
+
}
|
|
92
|
+
async grantKey(key, instanceId, state) {
|
|
93
|
+
const pendingHashes = { ...state.pendingHashes };
|
|
94
|
+
delete pendingHashes[instanceId];
|
|
95
|
+
await this.putState(key, {
|
|
96
|
+
pending: state.pending.filter((id) => id !== instanceId),
|
|
97
|
+
running: instanceId,
|
|
98
|
+
pendingHashes,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
async enqueuePending(key, instanceId, state) {
|
|
56
102
|
if (!state.pending.includes(instanceId)) {
|
|
57
103
|
await this.putState(key, {
|
|
58
104
|
...state,
|
|
@@ -69,12 +115,21 @@ export class JobCoordinatorLogic {
|
|
|
69
115
|
if (state.running !== instanceId)
|
|
70
116
|
return { next: null };
|
|
71
117
|
const [next, ...rest] = state.pending;
|
|
72
|
-
|
|
118
|
+
const pendingHashes = { ...state.pendingHashes };
|
|
119
|
+
if (next)
|
|
120
|
+
delete pendingHashes[next];
|
|
121
|
+
await this.putState(key, { pending: rest, running: next ?? null, pendingHashes });
|
|
73
122
|
return { next: next ?? null };
|
|
74
123
|
}
|
|
75
124
|
async getState(key) {
|
|
76
125
|
const existing = await this.storage.get(this.storageKey(key));
|
|
77
|
-
|
|
126
|
+
if (!existing)
|
|
127
|
+
return { pending: [], running: null, pendingHashes: {} };
|
|
128
|
+
// `pendingHashes` arrived after this object was already storing state in production, so a row
|
|
129
|
+
// written by an earlier release has no such key. Backfilling on READ rather than migrating
|
|
130
|
+
// keeps a coordinator from crashing on its own history — and a crash here takes every job on
|
|
131
|
+
// that key with it.
|
|
132
|
+
return { ...existing, pendingHashes: existing.pendingHashes ?? {} };
|
|
78
133
|
}
|
|
79
134
|
async putState(key, state) {
|
|
80
135
|
await this.storage.put(this.storageKey(key), state);
|
|
@@ -93,12 +148,12 @@ export class JobCoordinatorLogic {
|
|
|
93
148
|
* export class PorulleJobCoordinator extends porulleJobCoordinator(DurableObject) {}
|
|
94
149
|
* ```
|
|
95
150
|
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
151
|
+
* State mutations run under `blockConcurrencyWhile`; Workflow binding calls
|
|
152
|
+
* (stale-holder checks and turn events) run outside it so a long release loop
|
|
153
|
+
* cannot block every other RPC on the object. On `enqueue` with `supersedes`
|
|
154
|
+
* the object reports the pending instances the caller must terminate. It needs
|
|
155
|
+
* a `PORULLE_WORKFLOW` binding on its environment to detect dead lock holders
|
|
156
|
+
* and to wake the next waiting instance.
|
|
102
157
|
*/
|
|
103
158
|
export function porulleJobCoordinator(Base) {
|
|
104
159
|
class PorulleJobCoordinator extends Base {
|
|
@@ -117,42 +172,47 @@ export function porulleJobCoordinator(Base) {
|
|
|
117
172
|
put(key, value) {
|
|
118
173
|
return ctx.storage.put(key, value);
|
|
119
174
|
},
|
|
120
|
-
}, async (instanceId) => {
|
|
121
|
-
try {
|
|
122
|
-
const handle = await env.PORULLE_WORKFLOW.get(instanceId);
|
|
123
|
-
const { status } = await handle.status();
|
|
124
|
-
return STALE_INSTANCE_STATUSES.has(status);
|
|
125
|
-
}
|
|
126
|
-
catch {
|
|
127
|
-
return true;
|
|
128
|
-
}
|
|
129
175
|
});
|
|
130
176
|
}
|
|
131
|
-
|
|
132
|
-
|
|
177
|
+
async #isStale(instanceId) {
|
|
178
|
+
try {
|
|
179
|
+
const handle = await this.#workflow.get(instanceId);
|
|
180
|
+
const { status } = await handle.status();
|
|
181
|
+
return STALE_INSTANCE_STATUSES.has(status);
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
enqueue(key, supersedes, instanceId, inputHash) {
|
|
188
|
+
return this.#state.blockConcurrencyWhile(() => this.#logic.enqueue(key, supersedes, instanceId, inputHash));
|
|
133
189
|
}
|
|
134
|
-
acquire(key, instanceId) {
|
|
135
|
-
|
|
190
|
+
async acquire(key, instanceId) {
|
|
191
|
+
const first = await this.#state.blockConcurrencyWhile(() => this.#logic.acquireRead(key, instanceId));
|
|
192
|
+
if (first === "granted")
|
|
193
|
+
return "granted";
|
|
194
|
+
const stale = await this.#isStale(first.needsStaleCheck);
|
|
195
|
+
return this.#state.blockConcurrencyWhile(() => stale
|
|
196
|
+
? this.#logic.acquireAfterStale(key, instanceId, first.needsStaleCheck)
|
|
197
|
+
: this.#logic.acquireWhenHolderLive(key, instanceId));
|
|
136
198
|
}
|
|
137
199
|
/** Hands the key to the next pending instance that can still be woken; a
|
|
138
200
|
* pending instance that died or was terminated meanwhile is skipped so the
|
|
139
201
|
* key never ends up held by an instance that will never release it. */
|
|
140
202
|
release(key, instanceId) {
|
|
141
|
-
return this.#
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
155
|
-
});
|
|
203
|
+
return this.#releaseAndWake(key, instanceId);
|
|
204
|
+
}
|
|
205
|
+
async #releaseAndWake(key, holder) {
|
|
206
|
+
const { next } = await this.#state.blockConcurrencyWhile(() => this.#logic.release(key, holder));
|
|
207
|
+
if (!next)
|
|
208
|
+
return;
|
|
209
|
+
const woken = await this.#workflow
|
|
210
|
+
.get(next)
|
|
211
|
+
.then((handle) => handle.sendEvent({ type: TURN_EVENT_TYPE }))
|
|
212
|
+
.then(() => true, () => false);
|
|
213
|
+
if (woken)
|
|
214
|
+
return;
|
|
215
|
+
return this.#releaseAndWake(key, next);
|
|
156
216
|
}
|
|
157
217
|
}
|
|
158
218
|
return PorulleJobCoordinator;
|
|
@@ -170,9 +230,12 @@ export class DurableObjectConcurrencyCoordinator {
|
|
|
170
230
|
if (!payload.concurrencyKey)
|
|
171
231
|
return create();
|
|
172
232
|
const key = coordinatorKey(payload);
|
|
173
|
-
const
|
|
233
|
+
const inputHash = hashJobInput(payload.input);
|
|
234
|
+
const { terminated, coalescedInto } = await this.options
|
|
174
235
|
.stub(key)
|
|
175
|
-
.enqueue(key, payload.supersedes, payload.jobId);
|
|
236
|
+
.enqueue(key, payload.supersedes, payload.jobId, inputHash);
|
|
237
|
+
if (coalescedInto)
|
|
238
|
+
return { id: coalescedInto };
|
|
176
239
|
await Promise.all(terminated.map((id) => this.options.workflow
|
|
177
240
|
.get(id)
|
|
178
241
|
.then((handle) => handle.terminate())
|
package/dist/index.d.ts
CHANGED
|
@@ -72,6 +72,8 @@ export interface RawWorkflowBinding {
|
|
|
72
72
|
}): Promise<void>;
|
|
73
73
|
}>;
|
|
74
74
|
}
|
|
75
|
+
/** Short, dependency-free hash of task input for coordinator coalescing. */
|
|
76
|
+
export declare function hashJobInput(input: Record<string, unknown>): string;
|
|
75
77
|
/** Wraps the real Workflows binding: Cloudflare's richer instance status folds
|
|
76
78
|
* into `JobInstanceStatus` (`paused`/`waitingForPause` → `waiting`, anything
|
|
77
79
|
* unknown → `errored`) and the error becomes its message. */
|
|
@@ -101,6 +103,12 @@ export declare class CloudflareExecutionEngine implements ExecutionEngine {
|
|
|
101
103
|
private setup;
|
|
102
104
|
constructor(options: CloudflareExecutionEngineOptions);
|
|
103
105
|
register(setup: ExecutionEngineSetup): void;
|
|
106
|
+
/** Enqueues a Workflow instance for `taskSlug`.
|
|
107
|
+
*
|
|
108
|
+
* When a superseding enqueue coalesces into an existing pending instance with identical input,
|
|
109
|
+
* instance creation is skipped. That loses nothing: a job derives its work from state read when
|
|
110
|
+
* it **starts**; the enqueue payload is identity, not data. A pending instance created before a
|
|
111
|
+
* later write will still observe that write when it runs. */
|
|
104
112
|
enqueue(taskSlug: string, input: Record<string, unknown>, options: EnqueueOptions): Promise<string>;
|
|
105
113
|
run(payload: CloudflareJobPayload, step: WorkflowStep): Promise<Record<string, unknown>>;
|
|
106
114
|
status(jobId: string): Promise<{
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,23 @@
|
|
|
1
1
|
import { TaskNonRetryableError, createPassThroughTaskStep, } from "@porulle/core/jobs";
|
|
2
|
+
/** Deterministic JSON with sorted object keys — key order must not change the hash. */
|
|
3
|
+
function stableStringify(value) {
|
|
4
|
+
if (value === null || typeof value !== "object")
|
|
5
|
+
return JSON.stringify(value);
|
|
6
|
+
if (Array.isArray(value))
|
|
7
|
+
return `[${value.map(stableStringify).join(",")}]`;
|
|
8
|
+
const obj = value;
|
|
9
|
+
const keys = Object.keys(obj).sort();
|
|
10
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${stableStringify(obj[key])}`).join(",")}}`;
|
|
11
|
+
}
|
|
12
|
+
/** Short, dependency-free hash of task input for coordinator coalescing. */
|
|
13
|
+
export function hashJobInput(input) {
|
|
14
|
+
const serialized = stableStringify(input);
|
|
15
|
+
let hash = 5381;
|
|
16
|
+
for (let index = 0; index < serialized.length; index += 1) {
|
|
17
|
+
hash = ((hash << 5) + hash) ^ serialized.charCodeAt(index);
|
|
18
|
+
}
|
|
19
|
+
return (hash >>> 0).toString(36);
|
|
20
|
+
}
|
|
2
21
|
const INSTANCE_STATUS_MAP = {
|
|
3
22
|
queued: "queued",
|
|
4
23
|
running: "running",
|
|
@@ -88,6 +107,12 @@ export class CloudflareExecutionEngine {
|
|
|
88
107
|
register(setup) {
|
|
89
108
|
this.setup = setup;
|
|
90
109
|
}
|
|
110
|
+
/** Enqueues a Workflow instance for `taskSlug`.
|
|
111
|
+
*
|
|
112
|
+
* When a superseding enqueue coalesces into an existing pending instance with identical input,
|
|
113
|
+
* instance creation is skipped. That loses nothing: a job derives its work from state read when
|
|
114
|
+
* it **starts**; the enqueue payload is identity, not data. A pending instance created before a
|
|
115
|
+
* later write will still observe that write when it runs. */
|
|
91
116
|
async enqueue(taskSlug, input, options) {
|
|
92
117
|
const task = this.requireSetup().tasks.get(taskSlug);
|
|
93
118
|
if (!task)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@porulle/jobs-cloudflare",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.31.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
}
|
|
12
12
|
},
|
|
13
13
|
"dependencies": {
|
|
14
|
-
"@porulle/core": "0.
|
|
14
|
+
"@porulle/core": "0.31.0"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
17
|
"@types/node": "^24.5.2",
|
package/src/coordinator.ts
CHANGED
|
@@ -4,6 +4,7 @@ import type {
|
|
|
4
4
|
WorkflowBinding,
|
|
5
5
|
WorkflowStep,
|
|
6
6
|
} from "./index.js";
|
|
7
|
+
import { hashJobInput } from "./index.js";
|
|
7
8
|
|
|
8
9
|
const STALE_INSTANCE_STATUSES = new Set(["complete", "errored", "terminated"]);
|
|
9
10
|
const TURN_EVENT_TYPE = "porulle-turn";
|
|
@@ -33,6 +34,8 @@ export interface CoordinatorStorage {
|
|
|
33
34
|
interface CoordinatorKeyState {
|
|
34
35
|
pending: string[];
|
|
35
36
|
running: string | null;
|
|
37
|
+
/** Hash of `payload.input` for each pending instance — pruned whenever the id leaves `pending`. */
|
|
38
|
+
pendingHashes: Record<string, string>;
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
/**
|
|
@@ -41,10 +44,7 @@ interface CoordinatorKeyState {
|
|
|
41
44
|
* with an in-memory `CoordinatorStorage` and a fake `isStale` check.
|
|
42
45
|
*/
|
|
43
46
|
export class JobCoordinatorLogic {
|
|
44
|
-
constructor(
|
|
45
|
-
private readonly storage: CoordinatorStorage,
|
|
46
|
-
private readonly isStale: (instanceId: string) => Promise<boolean>,
|
|
47
|
-
) {}
|
|
47
|
+
constructor(private readonly storage: CoordinatorStorage) {}
|
|
48
48
|
|
|
49
49
|
/** Registers `instanceId` as pending for `key` before the caller creates it, so
|
|
50
50
|
* a later supersede can see it even if it has not started running yet. When
|
|
@@ -55,27 +55,93 @@ export class JobCoordinatorLogic {
|
|
|
55
55
|
key: string,
|
|
56
56
|
supersedes: boolean,
|
|
57
57
|
instanceId: string,
|
|
58
|
-
|
|
58
|
+
inputHash?: string,
|
|
59
|
+
): Promise<{ terminated: string[]; coalescedInto?: string }> {
|
|
59
60
|
const state = await this.getState(key);
|
|
61
|
+
if (supersedes && inputHash !== undefined) {
|
|
62
|
+
for (const pendingId of state.pending) {
|
|
63
|
+
if (state.pendingHashes[pendingId] === inputHash) {
|
|
64
|
+
return { terminated: [], coalescedInto: pendingId };
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
60
68
|
const terminated = supersedes ? state.pending.filter((id) => id !== instanceId) : [];
|
|
61
69
|
const kept = supersedes ? [] : state.pending.filter((id) => id !== instanceId);
|
|
62
|
-
|
|
70
|
+
const pendingHashes = { ...state.pendingHashes };
|
|
71
|
+
for (const id of terminated) delete pendingHashes[id];
|
|
72
|
+
if (inputHash !== undefined) pendingHashes[instanceId] = inputHash;
|
|
73
|
+
await this.putState(key, {
|
|
74
|
+
...state,
|
|
75
|
+
pending: [...kept, instanceId],
|
|
76
|
+
pendingHashes,
|
|
77
|
+
});
|
|
63
78
|
return { terminated };
|
|
64
79
|
}
|
|
65
80
|
|
|
66
|
-
|
|
67
|
-
|
|
81
|
+
/** First gate phase: storage only. Returns `needsStaleCheck` when another instance
|
|
82
|
+
* holds the key so the Durable Object can ask the Workflow binding outside the gate. */
|
|
83
|
+
async acquireRead(
|
|
84
|
+
key: string,
|
|
85
|
+
instanceId: string,
|
|
86
|
+
): Promise<"granted" | { needsStaleCheck: string }> {
|
|
87
|
+
const state = await this.getState(key);
|
|
68
88
|
if (state.running === instanceId) return "granted";
|
|
69
|
-
if (state.running
|
|
70
|
-
|
|
89
|
+
if (state.running === null) {
|
|
90
|
+
await this.grantKey(key, instanceId, state);
|
|
91
|
+
return "granted";
|
|
71
92
|
}
|
|
93
|
+
return { needsStaleCheck: state.running };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Re-enter after a live holder was confirmed outside the gate. */
|
|
97
|
+
async acquireWhenHolderLive(
|
|
98
|
+
key: string,
|
|
99
|
+
instanceId: string,
|
|
100
|
+
): Promise<"granted" | "pending"> {
|
|
101
|
+
const state = await this.getState(key);
|
|
102
|
+
if (state.running === instanceId) return "granted";
|
|
72
103
|
if (state.running === null) {
|
|
73
|
-
await this.
|
|
74
|
-
pending: state.pending.filter((id) => id !== instanceId),
|
|
75
|
-
running: instanceId,
|
|
76
|
-
});
|
|
104
|
+
await this.grantKey(key, instanceId, state);
|
|
77
105
|
return "granted";
|
|
78
106
|
}
|
|
107
|
+
return this.enqueuePending(key, instanceId, state);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Re-enter after the holder was stale outside the gate — only grants when the
|
|
111
|
+
* same id still holds the key, so a concurrent acquirer cannot be raced. */
|
|
112
|
+
async acquireAfterStale(
|
|
113
|
+
key: string,
|
|
114
|
+
instanceId: string,
|
|
115
|
+
checkedHolderId: string,
|
|
116
|
+
): Promise<"granted" | "pending"> {
|
|
117
|
+
const state = await this.getState(key);
|
|
118
|
+
if (state.running === instanceId) return "granted";
|
|
119
|
+
if (state.running === checkedHolderId) {
|
|
120
|
+
await this.grantKey(key, instanceId, state);
|
|
121
|
+
return "granted";
|
|
122
|
+
}
|
|
123
|
+
return this.acquireWhenHolderLive(key, instanceId);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private async grantKey(
|
|
127
|
+
key: string,
|
|
128
|
+
instanceId: string,
|
|
129
|
+
state: CoordinatorKeyState,
|
|
130
|
+
): Promise<void> {
|
|
131
|
+
const pendingHashes = { ...state.pendingHashes };
|
|
132
|
+
delete pendingHashes[instanceId];
|
|
133
|
+
await this.putState(key, {
|
|
134
|
+
pending: state.pending.filter((id) => id !== instanceId),
|
|
135
|
+
running: instanceId,
|
|
136
|
+
pendingHashes,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private async enqueuePending(
|
|
141
|
+
key: string,
|
|
142
|
+
instanceId: string,
|
|
143
|
+
state: CoordinatorKeyState,
|
|
144
|
+
): Promise<"pending"> {
|
|
79
145
|
if (!state.pending.includes(instanceId)) {
|
|
80
146
|
await this.putState(key, {
|
|
81
147
|
...state,
|
|
@@ -92,13 +158,20 @@ export class JobCoordinatorLogic {
|
|
|
92
158
|
const state = await this.getState(key);
|
|
93
159
|
if (state.running !== instanceId) return { next: null };
|
|
94
160
|
const [next, ...rest] = state.pending;
|
|
95
|
-
|
|
161
|
+
const pendingHashes = { ...state.pendingHashes };
|
|
162
|
+
if (next) delete pendingHashes[next];
|
|
163
|
+
await this.putState(key, { pending: rest, running: next ?? null, pendingHashes });
|
|
96
164
|
return { next: next ?? null };
|
|
97
165
|
}
|
|
98
166
|
|
|
99
167
|
private async getState(key: string): Promise<CoordinatorKeyState> {
|
|
100
168
|
const existing = await this.storage.get<CoordinatorKeyState>(this.storageKey(key));
|
|
101
|
-
|
|
169
|
+
if (!existing) return { pending: [], running: null, pendingHashes: {} };
|
|
170
|
+
// `pendingHashes` arrived after this object was already storing state in production, so a row
|
|
171
|
+
// written by an earlier release has no such key. Backfilling on READ rather than migrating
|
|
172
|
+
// keeps a coordinator from crashing on its own history — and a crash here takes every job on
|
|
173
|
+
// that key with it.
|
|
174
|
+
return { ...existing, pendingHashes: existing.pendingHashes ?? {} };
|
|
102
175
|
}
|
|
103
176
|
|
|
104
177
|
private async putState(key: string, state: CoordinatorKeyState): Promise<void> {
|
|
@@ -144,12 +217,12 @@ type DurableObjectConstructor = abstract new (...args: any[]) => object;
|
|
|
144
217
|
* export class PorulleJobCoordinator extends porulleJobCoordinator(DurableObject) {}
|
|
145
218
|
* ```
|
|
146
219
|
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
220
|
+
* State mutations run under `blockConcurrencyWhile`; Workflow binding calls
|
|
221
|
+
* (stale-holder checks and turn events) run outside it so a long release loop
|
|
222
|
+
* cannot block every other RPC on the object. On `enqueue` with `supersedes`
|
|
223
|
+
* the object reports the pending instances the caller must terminate. It needs
|
|
224
|
+
* a `PORULLE_WORKFLOW` binding on its environment to detect dead lock holders
|
|
225
|
+
* and to wake the next waiting instance.
|
|
153
226
|
*/
|
|
154
227
|
export function porulleJobCoordinator<TBase extends DurableObjectConstructor>(
|
|
155
228
|
Base: TBase,
|
|
@@ -164,40 +237,47 @@ export function porulleJobCoordinator<TBase extends DurableObjectConstructor>(
|
|
|
164
237
|
const [ctx, env] = args as [DurableObjectStateLike, PorulleJobCoordinatorEnv];
|
|
165
238
|
this.#state = ctx;
|
|
166
239
|
this.#workflow = env.PORULLE_WORKFLOW;
|
|
167
|
-
this.#logic = new JobCoordinatorLogic(
|
|
168
|
-
{
|
|
169
|
-
get<T>(key
|
|
170
|
-
return ctx.storage.get<T>(key);
|
|
171
|
-
},
|
|
172
|
-
put<T>(key: string, value: T) {
|
|
173
|
-
return ctx.storage.put(key, value);
|
|
174
|
-
},
|
|
240
|
+
this.#logic = new JobCoordinatorLogic({
|
|
241
|
+
get<T>(key: string) {
|
|
242
|
+
return ctx.storage.get<T>(key);
|
|
175
243
|
},
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
const handle = await env.PORULLE_WORKFLOW.get(instanceId);
|
|
179
|
-
const { status } = await handle.status();
|
|
180
|
-
return STALE_INSTANCE_STATUSES.has(status);
|
|
181
|
-
} catch {
|
|
182
|
-
return true;
|
|
183
|
-
}
|
|
244
|
+
put<T>(key: string, value: T) {
|
|
245
|
+
return ctx.storage.put(key, value);
|
|
184
246
|
},
|
|
185
|
-
);
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async #isStale(instanceId: string): Promise<boolean> {
|
|
251
|
+
try {
|
|
252
|
+
const handle = await this.#workflow.get(instanceId);
|
|
253
|
+
const { status } = await handle.status();
|
|
254
|
+
return STALE_INSTANCE_STATUSES.has(status);
|
|
255
|
+
} catch {
|
|
256
|
+
return true;
|
|
257
|
+
}
|
|
186
258
|
}
|
|
187
259
|
|
|
188
260
|
enqueue(
|
|
189
261
|
key: string,
|
|
190
262
|
supersedes: boolean,
|
|
191
263
|
instanceId: string,
|
|
192
|
-
|
|
264
|
+
inputHash?: string,
|
|
265
|
+
): Promise<{ terminated: string[]; coalescedInto?: string }> {
|
|
193
266
|
return this.#state.blockConcurrencyWhile(() =>
|
|
194
|
-
this.#logic.enqueue(key, supersedes, instanceId),
|
|
267
|
+
this.#logic.enqueue(key, supersedes, instanceId, inputHash),
|
|
195
268
|
);
|
|
196
269
|
}
|
|
197
270
|
|
|
198
|
-
acquire(key: string, instanceId: string): Promise<"granted" | "pending"> {
|
|
271
|
+
async acquire(key: string, instanceId: string): Promise<"granted" | "pending"> {
|
|
272
|
+
const first = await this.#state.blockConcurrencyWhile(() =>
|
|
273
|
+
this.#logic.acquireRead(key, instanceId),
|
|
274
|
+
);
|
|
275
|
+
if (first === "granted") return "granted";
|
|
276
|
+
const stale = await this.#isStale(first.needsStaleCheck);
|
|
199
277
|
return this.#state.blockConcurrencyWhile(() =>
|
|
200
|
-
|
|
278
|
+
stale
|
|
279
|
+
? this.#logic.acquireAfterStale(key, instanceId, first.needsStaleCheck)
|
|
280
|
+
: this.#logic.acquireWhenHolderLive(key, instanceId),
|
|
201
281
|
);
|
|
202
282
|
}
|
|
203
283
|
|
|
@@ -205,19 +285,20 @@ export function porulleJobCoordinator<TBase extends DurableObjectConstructor>(
|
|
|
205
285
|
* pending instance that died or was terminated meanwhile is skipped so the
|
|
206
286
|
* key never ends up held by an instance that will never release it. */
|
|
207
287
|
release(key: string, instanceId: string): Promise<void> {
|
|
208
|
-
return this.#
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
288
|
+
return this.#releaseAndWake(key, instanceId);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async #releaseAndWake(key: string, holder: string): Promise<void> {
|
|
292
|
+
const { next } = await this.#state.blockConcurrencyWhile(() =>
|
|
293
|
+
this.#logic.release(key, holder),
|
|
294
|
+
);
|
|
295
|
+
if (!next) return;
|
|
296
|
+
const woken = await this.#workflow
|
|
297
|
+
.get(next)
|
|
298
|
+
.then((handle) => handle.sendEvent({ type: TURN_EVENT_TYPE }))
|
|
299
|
+
.then(() => true, () => false);
|
|
300
|
+
if (woken) return;
|
|
301
|
+
return this.#releaseAndWake(key, next);
|
|
221
302
|
}
|
|
222
303
|
}
|
|
223
304
|
return PorulleJobCoordinator;
|
|
@@ -231,7 +312,8 @@ export interface CoordinatorStub {
|
|
|
231
312
|
key: string,
|
|
232
313
|
supersedes: boolean,
|
|
233
314
|
instanceId: string,
|
|
234
|
-
|
|
315
|
+
inputHash?: string,
|
|
316
|
+
): Promise<{ terminated: string[]; coalescedInto?: string }>;
|
|
235
317
|
acquire(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
236
318
|
release(key: string, instanceId: string): Promise<void>;
|
|
237
319
|
}
|
|
@@ -259,9 +341,11 @@ export class DurableObjectConcurrencyCoordinator
|
|
|
259
341
|
): Promise<{ id: string }> {
|
|
260
342
|
if (!payload.concurrencyKey) return create();
|
|
261
343
|
const key = coordinatorKey(payload);
|
|
262
|
-
const
|
|
344
|
+
const inputHash = hashJobInput(payload.input);
|
|
345
|
+
const { terminated, coalescedInto } = await this.options
|
|
263
346
|
.stub(key)
|
|
264
|
-
.enqueue(key, payload.supersedes, payload.jobId);
|
|
347
|
+
.enqueue(key, payload.supersedes, payload.jobId, inputHash);
|
|
348
|
+
if (coalescedInto) return { id: coalescedInto };
|
|
265
349
|
await Promise.all(
|
|
266
350
|
terminated.map((id) =>
|
|
267
351
|
this.options.workflow
|
package/src/index.ts
CHANGED
|
@@ -67,6 +67,25 @@ export interface RawWorkflowBinding {
|
|
|
67
67
|
}>;
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
/** Deterministic JSON with sorted object keys — key order must not change the hash. */
|
|
71
|
+
function stableStringify(value: unknown): string {
|
|
72
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
73
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
74
|
+
const obj = value as Record<string, unknown>;
|
|
75
|
+
const keys = Object.keys(obj).sort();
|
|
76
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${stableStringify(obj[key])}`).join(",")}}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Short, dependency-free hash of task input for coordinator coalescing. */
|
|
80
|
+
export function hashJobInput(input: Record<string, unknown>): string {
|
|
81
|
+
const serialized = stableStringify(input);
|
|
82
|
+
let hash = 5381;
|
|
83
|
+
for (let index = 0; index < serialized.length; index += 1) {
|
|
84
|
+
hash = ((hash << 5) + hash) ^ serialized.charCodeAt(index);
|
|
85
|
+
}
|
|
86
|
+
return (hash >>> 0).toString(36);
|
|
87
|
+
}
|
|
88
|
+
|
|
70
89
|
const INSTANCE_STATUS_MAP: Record<string, JobInstanceStatus> = {
|
|
71
90
|
queued: "queued",
|
|
72
91
|
running: "running",
|
|
@@ -200,6 +219,12 @@ export class CloudflareExecutionEngine implements ExecutionEngine {
|
|
|
200
219
|
this.setup = setup;
|
|
201
220
|
}
|
|
202
221
|
|
|
222
|
+
/** Enqueues a Workflow instance for `taskSlug`.
|
|
223
|
+
*
|
|
224
|
+
* When a superseding enqueue coalesces into an existing pending instance with identical input,
|
|
225
|
+
* instance creation is skipped. That loses nothing: a job derives its work from state read when
|
|
226
|
+
* it **starts**; the enqueue payload is identity, not data. A pending instance created before a
|
|
227
|
+
* later write will still observe that write when it runs. */
|
|
203
228
|
async enqueue(
|
|
204
229
|
taskSlug: string,
|
|
205
230
|
input: Record<string, unknown>,
|