@porulle/jobs-cloudflare 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -8
- package/dist/coordinator.d.ts +126 -0
- package/dist/coordinator.js +213 -0
- package/dist/index.d.ts +72 -13
- package/dist/index.js +108 -17
- package/package.json +4 -4
- package/src/coordinator.ts +314 -0
- package/src/index.ts +221 -40
package/README.md
CHANGED
|
@@ -1,21 +1,116 @@
|
|
|
1
1
|
# `@porulle/jobs-cloudflare`
|
|
2
2
|
|
|
3
|
-
Cloudflare Workflows binding adapter and task runner for Porulle. Keyed exclusivity and supersession require a durable coordinator
|
|
3
|
+
Cloudflare Workflows binding adapter and task runner for Porulle. Keyed exclusivity and supersession require a durable coordinator — this package ships one, `DurableObjectConcurrencyCoordinator`, backed by a `PorulleJobCoordinator` Durable Object; the adapter fails fast when a keyed task is used without one.
|
|
4
|
+
|
|
5
|
+
## Entrypoint
|
|
4
6
|
|
|
5
7
|
```ts
|
|
6
8
|
import { WorkflowEntrypoint } from "cloudflare:workers";
|
|
7
|
-
import {
|
|
9
|
+
import { NonRetryableError } from "cloudflare:workflows";
|
|
10
|
+
import {
|
|
11
|
+
CloudflareExecutionEngine,
|
|
12
|
+
DurableObjectConcurrencyCoordinator,
|
|
13
|
+
adaptWorkflowBinding,
|
|
14
|
+
} from "@porulle/jobs-cloudflare";
|
|
8
15
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
16
|
+
// Bindings arrive with the request/event, so build the engine from `env` once you have it.
|
|
17
|
+
export function createJobsEngine(env: Env) {
|
|
18
|
+
const workflow = adaptWorkflowBinding(env.PORULLE_WORKFLOW);
|
|
19
|
+
return new CloudflareExecutionEngine({
|
|
20
|
+
workflow,
|
|
21
|
+
// Workflows stops retrying only for this exact class; the package cannot import it itself.
|
|
22
|
+
nonRetryableError: NonRetryableError,
|
|
23
|
+
coordinator: new DurableObjectConcurrencyCoordinator({
|
|
24
|
+
// One Durable Object per coordination key, so keys never queue behind each other.
|
|
25
|
+
stub: (key) => env.PORULLE_JOB_COORDINATOR.get(env.PORULLE_JOB_COORDINATOR.idFromName(key)),
|
|
26
|
+
workflow,
|
|
27
|
+
}),
|
|
28
|
+
});
|
|
29
|
+
}
|
|
13
30
|
|
|
14
|
-
export class PorulleWorkflow extends WorkflowEntrypoint {
|
|
31
|
+
export class PorulleWorkflow extends WorkflowEntrypoint<Env> {
|
|
15
32
|
async run(event, step) {
|
|
33
|
+
const jobs = createJobsEngine(this.env); // register(...) it with your tasks first
|
|
16
34
|
return jobs.run(event.payload, step);
|
|
17
35
|
}
|
|
18
36
|
}
|
|
19
37
|
```
|
|
20
38
|
|
|
21
|
-
|
|
39
|
+
`adaptWorkflowBinding` folds Cloudflare's instance status onto the six-value `JobInstanceStatus` (`paused`/`waitingForPause` → `waiting`, anything unknown → `errored`) and flattens the error to its message.
|
|
40
|
+
|
|
41
|
+
Task handlers that need durable, per-phase steps read `ctx.step` — absent on push engines that have not wired one, present here and on the drizzle engine:
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
const enrichEntityTask: TaskDefinition = {
|
|
45
|
+
slug: "loom/enrich-entity",
|
|
46
|
+
durableSteps: true,
|
|
47
|
+
handler: async ({ input, ctx }) => {
|
|
48
|
+
const loaded = await ctx.step!.do("load", () => loadEntity(input.entityId));
|
|
49
|
+
const generated = await ctx.step!.do("generate", () => callModel(loaded), {
|
|
50
|
+
timeout: 40_000,
|
|
51
|
+
});
|
|
52
|
+
if (generated.refused) {
|
|
53
|
+
throw new TaskNonRetryableError("Model refused the request");
|
|
54
|
+
}
|
|
55
|
+
await ctx.step!.do("write-ledger", () => writeLedgerRow(generated));
|
|
56
|
+
return { output: generated };
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Steps are never nested. A task that sets `durableSteps: true` runs its handler in the Workflow body: each `ctx.step.do(name, fn, options?)` call becomes its own top-level Workflow step (named `porulle:<slug>:<name>`), retried independently per `options.retries` (falling back to the task's own `retries`), and code between steps re-runs on every wake — keep side effects inside steps. A task without the flag runs as today: one step named `porulle:<slug>` retried as a unit up to `retries.attempts` times, with a pass-through `ctx.step` inside it. `TaskNonRetryableError` — thrown directly, or requested via `options.nonRetryable` — maps to Cloudflare's `NonRetryableError`, so that step (and the instance) does not retry it.
|
|
62
|
+
|
|
63
|
+
## Coordinator wiring
|
|
64
|
+
|
|
65
|
+
Add the Durable Object and Workflow bindings to `wrangler.jsonc`:
|
|
66
|
+
|
|
67
|
+
```jsonc
|
|
68
|
+
{
|
|
69
|
+
"durable_objects": {
|
|
70
|
+
"bindings": [
|
|
71
|
+
{ "name": "PORULLE_JOB_COORDINATOR", "class_name": "PorulleJobCoordinator" },
|
|
72
|
+
],
|
|
73
|
+
},
|
|
74
|
+
"migrations": [{ "tag": "v1", "new_sqlite_classes": ["PorulleJobCoordinator"] }],
|
|
75
|
+
"workflows": [
|
|
76
|
+
{ "binding": "PORULLE_WORKFLOW", "name": "porulle-jobs", "class_name": "PorulleWorkflow" },
|
|
77
|
+
],
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Build the Durable Object class on your Worker's own `DurableObject` base (this package cannot import `cloudflare:workers` itself) and export it from the Worker entrypoint alongside the Workflow:
|
|
82
|
+
|
|
83
|
+
```ts
|
|
84
|
+
import { DurableObject } from "cloudflare:workers";
|
|
85
|
+
import { porulleJobCoordinator } from "@porulle/jobs-cloudflare";
|
|
86
|
+
|
|
87
|
+
export class PorulleJobCoordinator extends porulleJobCoordinator(DurableObject) {}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The Durable Object reads `PORULLE_WORKFLOW` from its `env` — the Worker's own bindings, since the class lives in the same script — to detect dead lock holders and wake the next waiting instance.
|
|
91
|
+
|
|
92
|
+
Per key (`organizationId:taskSlug:concurrencyKey`), `enqueue` terminates every pending instance when `supersedes` is set (never the currently running one — matching the drizzle adapter, only unstarted jobs are dropped). An instance is registered with the coordinator before it is created, so a supersede also terminates instances that exist but have not started running. `run` serializes same-key instances through the DO's `acquire`/`release`; each call is its own Workflow step, so a replay of the body never re-acquires or re-releases. An instance that loses the race parks in `step.waitForEvent` for the turn event and re-acquires when the wait times out — one minute for the first round, doubling to a one-hour ceiling, so waiters recover from a holder cancelled without releasing while a long queue costs tens of steps rather than thousands. A holder the Workflow reports as `complete`, `errored`, `terminated` or no longer knows is treated as gone; a `paused` holder keeps the key.
|
|
93
|
+
|
|
94
|
+
This package imports only `@porulle/core/jobs` (types and two small helpers), so a Worker bundle does not pull the core server runtime through it. The app's own `commerce.config` still needs whatever compatibility flags it needs.
|
|
95
|
+
|
|
96
|
+
## Instance status and cancellation
|
|
97
|
+
|
|
98
|
+
```ts
|
|
99
|
+
await jobs.status(jobId); // { status: "running" | "queued" | "waiting" | "complete" | "errored" | "terminated", error? }
|
|
100
|
+
await jobs.cancel(jobId); // terminates the Workflow instance
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`adaptWorkflowBinding` (above) performs the status mapping; a hand-written `WorkflowBinding` must map Cloudflare's richer `InstanceStatus` onto this six-value set itself.
|
|
104
|
+
|
|
105
|
+
## Node fallback
|
|
106
|
+
|
|
107
|
+
In local development, or anywhere without the Workers runtime, register the drizzle engine instead — it implements the same `ExecutionEngine` contract (including `status`/`cancel`) against the app's own `commerce_jobs` table and can run tasks in-process:
|
|
108
|
+
|
|
109
|
+
```ts
|
|
110
|
+
import { DrizzleJobsAdapter } from "@porulle/core";
|
|
111
|
+
|
|
112
|
+
export const jobs = new DrizzleJobsAdapter(db);
|
|
113
|
+
// config.jobs.autorun.enabled = true polls commerce_jobs in-process.
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
A handler written against `ctx.step` runs unchanged there — the drizzle engine passes a pass-through `TaskStep` (`do` runs the callback immediately, `sleep` waits in-process).
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import type { CloudflareConcurrencyCoordinator, CloudflareJobPayload, WorkflowBinding, WorkflowStep } from "./index.js";
|
|
2
|
+
export interface CoordinatorStorage {
|
|
3
|
+
get<T>(key: string): Promise<T | undefined>;
|
|
4
|
+
put<T>(key: string, value: T): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Pure per-key lock state machine behind `PorulleJobCoordinator`, kept free of
|
|
8
|
+
* any `cloudflare:workers` dependency so it is directly unit-testable in Node
|
|
9
|
+
* with an in-memory `CoordinatorStorage` and a fake `isStale` check.
|
|
10
|
+
*/
|
|
11
|
+
export declare class JobCoordinatorLogic {
|
|
12
|
+
private readonly storage;
|
|
13
|
+
private readonly isStale;
|
|
14
|
+
constructor(storage: CoordinatorStorage, isStale: (instanceId: string) => Promise<boolean>);
|
|
15
|
+
/** Registers `instanceId` as pending for `key` before the caller creates it, so
|
|
16
|
+
* a later supersede can see it even if it has not started running yet. When
|
|
17
|
+
* `supersedes` is set, the previously pending ids are cleared and returned for
|
|
18
|
+
* the caller to terminate. Never touches the currently running instance —
|
|
19
|
+
* matching the drizzle adapter, supersede only drops jobs that have not started. */
|
|
20
|
+
enqueue(key: string, supersedes: boolean, instanceId: string): Promise<{
|
|
21
|
+
terminated: string[];
|
|
22
|
+
}>;
|
|
23
|
+
acquire(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
24
|
+
/** Releases the lock if `instanceId` holds it and hands it to the next
|
|
25
|
+
* pending instance (if any), returning that instance's id so the caller can
|
|
26
|
+
* wake it. A release from an instance that does not hold the lock is a no-op. */
|
|
27
|
+
release(key: string, instanceId: string): Promise<{
|
|
28
|
+
next: string | null;
|
|
29
|
+
}>;
|
|
30
|
+
private getState;
|
|
31
|
+
private putState;
|
|
32
|
+
private storageKey;
|
|
33
|
+
}
|
|
34
|
+
/** What the Durable Object needs from the Workflow binding — satisfied by
|
|
35
|
+
* Cloudflare's raw binding and by an adapted `WorkflowBinding` alike. */
|
|
36
|
+
export interface CoordinatorWorkflowBinding {
|
|
37
|
+
get(id: string): Promise<{
|
|
38
|
+
status(): Promise<{
|
|
39
|
+
status: string;
|
|
40
|
+
}>;
|
|
41
|
+
sendEvent(event: {
|
|
42
|
+
type: string;
|
|
43
|
+
payload?: unknown;
|
|
44
|
+
}): Promise<void>;
|
|
45
|
+
}>;
|
|
46
|
+
}
|
|
47
|
+
export interface PorulleJobCoordinatorEnv {
|
|
48
|
+
PORULLE_WORKFLOW: CoordinatorWorkflowBinding;
|
|
49
|
+
}
|
|
50
|
+
/** The subset of the real `DurableObjectState` this coordinator touches. */
|
|
51
|
+
export interface DurableObjectStateLike {
|
|
52
|
+
storage: {
|
|
53
|
+
get<T>(key: string): Promise<T | undefined>;
|
|
54
|
+
put<T>(key: string, value: T): Promise<void>;
|
|
55
|
+
};
|
|
56
|
+
blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T>;
|
|
57
|
+
}
|
|
58
|
+
type DurableObjectConstructor = abstract new (...args: any[]) => object;
|
|
59
|
+
/**
|
|
60
|
+
* Builds the coordinator Durable Object on the app's own `DurableObject` base
|
|
61
|
+
* class. `cloudflare:workers` only resolves inside the Workers runtime, so the
|
|
62
|
+
* Worker imports it and passes it in — this package stays importable under Node:
|
|
63
|
+
*
|
|
64
|
+
* ```ts
|
|
65
|
+
* import { DurableObject } from "cloudflare:workers";
|
|
66
|
+
* export class PorulleJobCoordinator extends porulleJobCoordinator(DurableObject) {}
|
|
67
|
+
* ```
|
|
68
|
+
*
|
|
69
|
+
* Every RPC runs under `blockConcurrencyWhile`: the stale-holder check is a
|
|
70
|
+
* Workflow subrequest, which would otherwise open the input gate between the
|
|
71
|
+
* read and the write and let two acquirers both be granted. On `enqueue` with
|
|
72
|
+
* `supersedes` the object reports the pending instances the caller must
|
|
73
|
+
* terminate. It needs a `PORULLE_WORKFLOW` binding on its environment to detect
|
|
74
|
+
* dead lock holders and to wake the next waiting instance.
|
|
75
|
+
*/
|
|
76
|
+
export declare function porulleJobCoordinator<TBase extends DurableObjectConstructor>(Base: TBase): (abstract new (...args: any[]) => {
|
|
77
|
+
readonly #logic: JobCoordinatorLogic;
|
|
78
|
+
readonly #workflow: CoordinatorWorkflowBinding;
|
|
79
|
+
readonly #state: DurableObjectStateLike;
|
|
80
|
+
enqueue(key: string, supersedes: boolean, instanceId: string): Promise<{
|
|
81
|
+
terminated: string[];
|
|
82
|
+
}>;
|
|
83
|
+
acquire(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
84
|
+
/** Hands the key to the next pending instance that can still be woken; a
|
|
85
|
+
* pending instance that died or was terminated meanwhile is skipped so the
|
|
86
|
+
* key never ends up held by an instance that will never release it. */
|
|
87
|
+
release(key: string, instanceId: string): Promise<void>;
|
|
88
|
+
}) & TBase;
|
|
89
|
+
/** The RPC surface `DurableObjectConcurrencyCoordinator` calls on a
|
|
90
|
+
* `PorulleJobCoordinator` stub — the subset of `DurableObjectStub<PorulleJobCoordinator>`
|
|
91
|
+
* this package needs, so callers can inject a fake in tests without the Workers runtime. */
|
|
92
|
+
export interface CoordinatorStub {
|
|
93
|
+
enqueue(key: string, supersedes: boolean, instanceId: string): Promise<{
|
|
94
|
+
terminated: string[];
|
|
95
|
+
}>;
|
|
96
|
+
acquire(key: string, instanceId: string): Promise<"granted" | "pending">;
|
|
97
|
+
release(key: string, instanceId: string): Promise<void>;
|
|
98
|
+
}
|
|
99
|
+
export interface DurableObjectConcurrencyCoordinatorOptions {
|
|
100
|
+
/** Resolves the Durable Object for a coordination key
|
|
101
|
+
* (`organizationId:taskSlug:concurrencyKey`); return one object per key so
|
|
102
|
+
* keys never queue behind each other. */
|
|
103
|
+
stub: (key: string) => CoordinatorStub;
|
|
104
|
+
workflow: WorkflowBinding;
|
|
105
|
+
}
|
|
106
|
+
/** `CloudflareConcurrencyCoordinator` backed by a `PorulleJobCoordinator` Durable
|
|
107
|
+
* Object: supersede terminates pending instances at enqueue, and `run` serialises
|
|
108
|
+
* same-key instances through the DO's `acquire`/`release`, waiting with
|
|
109
|
+
* `step.waitForEvent` when another instance already holds the key. */
|
|
110
|
+
export declare class DurableObjectConcurrencyCoordinator implements CloudflareConcurrencyCoordinator {
|
|
111
|
+
private readonly options;
|
|
112
|
+
constructor(options: DurableObjectConcurrencyCoordinatorOptions);
|
|
113
|
+
enqueue(payload: CloudflareJobPayload, create: () => Promise<{
|
|
114
|
+
id: string;
|
|
115
|
+
}>): Promise<{
|
|
116
|
+
id: string;
|
|
117
|
+
}>;
|
|
118
|
+
/** Every coordinator call is its own Workflow step, so a replay of the body
|
|
119
|
+
* neither re-acquires nor re-releases. A parked instance wakes on the turn
|
|
120
|
+
* event or, at the latest, after `turnWaitMs(round)`, and re-acquires — which
|
|
121
|
+
* is how waiters recover when the holder was terminated without releasing. A
|
|
122
|
+
* release that fails after its retries is dropped rather than masking the
|
|
123
|
+
* handler's outcome: the next acquirer sees the finished holder as stale. */
|
|
124
|
+
run<T>(payload: CloudflareJobPayload, step: WorkflowStep, handler: () => Promise<T>): Promise<T>;
|
|
125
|
+
}
|
|
126
|
+
export {};
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
const STALE_INSTANCE_STATUSES = new Set(["complete", "errored", "terminated"]);
|
|
2
|
+
const TURN_EVENT_TYPE = "porulle-turn";
|
|
3
|
+
const FIRST_TURN_WAIT_MS = 60_000;
|
|
4
|
+
const MAX_TURN_WAIT_MS = 3_600_000;
|
|
5
|
+
const COORDINATOR_STEP = {
|
|
6
|
+
retries: { limit: 3, delay: 1_000, backoff: "exponential" },
|
|
7
|
+
};
|
|
8
|
+
/** Each wait round costs two Workflow steps against the instance's step budget,
|
|
9
|
+
* so the timeout doubles per round from one minute up to one hour: a waiter
|
|
10
|
+
* still recovers from a holder that died without releasing, and a day-long
|
|
11
|
+
* queue costs tens of steps rather than thousands. */
|
|
12
|
+
function turnWaitMs(round) {
|
|
13
|
+
return Math.min(FIRST_TURN_WAIT_MS * 2 ** round, MAX_TURN_WAIT_MS);
|
|
14
|
+
}
|
|
15
|
+
function coordinatorKey(payload) {
|
|
16
|
+
return `${payload.organizationId}:${payload.taskSlug}:${payload.concurrencyKey}`;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Pure per-key lock state machine behind `PorulleJobCoordinator`, kept free of
|
|
20
|
+
* any `cloudflare:workers` dependency so it is directly unit-testable in Node
|
|
21
|
+
* with an in-memory `CoordinatorStorage` and a fake `isStale` check.
|
|
22
|
+
*/
|
|
23
|
+
export class JobCoordinatorLogic {
|
|
24
|
+
storage;
|
|
25
|
+
isStale;
|
|
26
|
+
constructor(storage, isStale) {
|
|
27
|
+
this.storage = storage;
|
|
28
|
+
this.isStale = isStale;
|
|
29
|
+
}
|
|
30
|
+
/** Registers `instanceId` as pending for `key` before the caller creates it, so
|
|
31
|
+
* a later supersede can see it even if it has not started running yet. When
|
|
32
|
+
* `supersedes` is set, the previously pending ids are cleared and returned for
|
|
33
|
+
* the caller to terminate. Never touches the currently running instance —
|
|
34
|
+
* matching the drizzle adapter, supersede only drops jobs that have not started. */
|
|
35
|
+
async enqueue(key, supersedes, instanceId) {
|
|
36
|
+
const state = await this.getState(key);
|
|
37
|
+
const terminated = supersedes ? state.pending.filter((id) => id !== instanceId) : [];
|
|
38
|
+
const kept = supersedes ? [] : state.pending.filter((id) => id !== instanceId);
|
|
39
|
+
await this.putState(key, { ...state, pending: [...kept, instanceId] });
|
|
40
|
+
return { terminated };
|
|
41
|
+
}
|
|
42
|
+
async acquire(key, instanceId) {
|
|
43
|
+
let state = await this.getState(key);
|
|
44
|
+
if (state.running === instanceId)
|
|
45
|
+
return "granted";
|
|
46
|
+
if (state.running !== null && (await this.isStale(state.running))) {
|
|
47
|
+
state = { ...state, running: null };
|
|
48
|
+
}
|
|
49
|
+
if (state.running === null) {
|
|
50
|
+
await this.putState(key, {
|
|
51
|
+
pending: state.pending.filter((id) => id !== instanceId),
|
|
52
|
+
running: instanceId,
|
|
53
|
+
});
|
|
54
|
+
return "granted";
|
|
55
|
+
}
|
|
56
|
+
if (!state.pending.includes(instanceId)) {
|
|
57
|
+
await this.putState(key, {
|
|
58
|
+
...state,
|
|
59
|
+
pending: [...state.pending, instanceId],
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
return "pending";
|
|
63
|
+
}
|
|
64
|
+
/** Releases the lock if `instanceId` holds it and hands it to the next
|
|
65
|
+
* pending instance (if any), returning that instance's id so the caller can
|
|
66
|
+
* wake it. A release from an instance that does not hold the lock is a no-op. */
|
|
67
|
+
async release(key, instanceId) {
|
|
68
|
+
const state = await this.getState(key);
|
|
69
|
+
if (state.running !== instanceId)
|
|
70
|
+
return { next: null };
|
|
71
|
+
const [next, ...rest] = state.pending;
|
|
72
|
+
await this.putState(key, { pending: rest, running: next ?? null });
|
|
73
|
+
return { next: next ?? null };
|
|
74
|
+
}
|
|
75
|
+
async getState(key) {
|
|
76
|
+
const existing = await this.storage.get(this.storageKey(key));
|
|
77
|
+
return existing ?? { pending: [], running: null };
|
|
78
|
+
}
|
|
79
|
+
async putState(key, state) {
|
|
80
|
+
await this.storage.put(this.storageKey(key), state);
|
|
81
|
+
}
|
|
82
|
+
storageKey(key) {
|
|
83
|
+
return `porulle-job-coordinator:${key}`;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Builds the coordinator Durable Object on the app's own `DurableObject` base
|
|
88
|
+
* class. `cloudflare:workers` only resolves inside the Workers runtime, so the
|
|
89
|
+
* Worker imports it and passes it in — this package stays importable under Node:
|
|
90
|
+
*
|
|
91
|
+
* ```ts
|
|
92
|
+
* import { DurableObject } from "cloudflare:workers";
|
|
93
|
+
* export class PorulleJobCoordinator extends porulleJobCoordinator(DurableObject) {}
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* Every RPC runs under `blockConcurrencyWhile`: the stale-holder check is a
|
|
97
|
+
* Workflow subrequest, which would otherwise open the input gate between the
|
|
98
|
+
* read and the write and let two acquirers both be granted. On `enqueue` with
|
|
99
|
+
* `supersedes` the object reports the pending instances the caller must
|
|
100
|
+
* terminate. It needs a `PORULLE_WORKFLOW` binding on its environment to detect
|
|
101
|
+
* dead lock holders and to wake the next waiting instance.
|
|
102
|
+
*/
|
|
103
|
+
export function porulleJobCoordinator(Base) {
|
|
104
|
+
class PorulleJobCoordinator extends Base {
|
|
105
|
+
#logic;
|
|
106
|
+
#workflow;
|
|
107
|
+
#state;
|
|
108
|
+
constructor(...args) {
|
|
109
|
+
super(...args);
|
|
110
|
+
const [ctx, env] = args;
|
|
111
|
+
this.#state = ctx;
|
|
112
|
+
this.#workflow = env.PORULLE_WORKFLOW;
|
|
113
|
+
this.#logic = new JobCoordinatorLogic({
|
|
114
|
+
get(key) {
|
|
115
|
+
return ctx.storage.get(key);
|
|
116
|
+
},
|
|
117
|
+
put(key, value) {
|
|
118
|
+
return ctx.storage.put(key, value);
|
|
119
|
+
},
|
|
120
|
+
}, async (instanceId) => {
|
|
121
|
+
try {
|
|
122
|
+
const handle = await env.PORULLE_WORKFLOW.get(instanceId);
|
|
123
|
+
const { status } = await handle.status();
|
|
124
|
+
return STALE_INSTANCE_STATUSES.has(status);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
enqueue(key, supersedes, instanceId) {
|
|
132
|
+
return this.#state.blockConcurrencyWhile(() => this.#logic.enqueue(key, supersedes, instanceId));
|
|
133
|
+
}
|
|
134
|
+
acquire(key, instanceId) {
|
|
135
|
+
return this.#state.blockConcurrencyWhile(() => this.#logic.acquire(key, instanceId));
|
|
136
|
+
}
|
|
137
|
+
/** Hands the key to the next pending instance that can still be woken; a
|
|
138
|
+
* pending instance that died or was terminated meanwhile is skipped so the
|
|
139
|
+
* key never ends up held by an instance that will never release it. */
|
|
140
|
+
release(key, instanceId) {
|
|
141
|
+
return this.#state.blockConcurrencyWhile(async () => {
|
|
142
|
+
let holder = instanceId;
|
|
143
|
+
for (;;) {
|
|
144
|
+
const { next } = await this.#logic.release(key, holder);
|
|
145
|
+
if (!next)
|
|
146
|
+
return;
|
|
147
|
+
const woken = await this.#workflow
|
|
148
|
+
.get(next)
|
|
149
|
+
.then((handle) => handle.sendEvent({ type: TURN_EVENT_TYPE }))
|
|
150
|
+
.then(() => true, () => false);
|
|
151
|
+
if (woken)
|
|
152
|
+
return;
|
|
153
|
+
holder = next;
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return PorulleJobCoordinator;
|
|
159
|
+
}
|
|
160
|
+
/** `CloudflareConcurrencyCoordinator` backed by a `PorulleJobCoordinator` Durable
|
|
161
|
+
* Object: supersede terminates pending instances at enqueue, and `run` serialises
|
|
162
|
+
* same-key instances through the DO's `acquire`/`release`, waiting with
|
|
163
|
+
* `step.waitForEvent` when another instance already holds the key. */
|
|
164
|
+
export class DurableObjectConcurrencyCoordinator {
|
|
165
|
+
options;
|
|
166
|
+
constructor(options) {
|
|
167
|
+
this.options = options;
|
|
168
|
+
}
|
|
169
|
+
async enqueue(payload, create) {
|
|
170
|
+
if (!payload.concurrencyKey)
|
|
171
|
+
return create();
|
|
172
|
+
const key = coordinatorKey(payload);
|
|
173
|
+
const { terminated } = await this.options
|
|
174
|
+
.stub(key)
|
|
175
|
+
.enqueue(key, payload.supersedes, payload.jobId);
|
|
176
|
+
await Promise.all(terminated.map((id) => this.options.workflow
|
|
177
|
+
.get(id)
|
|
178
|
+
.then((handle) => handle.terminate())
|
|
179
|
+
.catch(() => undefined)));
|
|
180
|
+
return create();
|
|
181
|
+
}
|
|
182
|
+
/** Every coordinator call is its own Workflow step, so a replay of the body
|
|
183
|
+
* neither re-acquires nor re-releases. A parked instance wakes on the turn
|
|
184
|
+
* event or, at the latest, after `turnWaitMs(round)`, and re-acquires — which
|
|
185
|
+
* is how waiters recover when the holder was terminated without releasing. A
|
|
186
|
+
* release that fails after its retries is dropped rather than masking the
|
|
187
|
+
* handler's outcome: the next acquirer sees the finished holder as stale. */
|
|
188
|
+
async run(payload, step, handler) {
|
|
189
|
+
if (!payload.concurrencyKey)
|
|
190
|
+
return handler();
|
|
191
|
+
const key = coordinatorKey(payload);
|
|
192
|
+
const stub = this.options.stub(key);
|
|
193
|
+
for (let round = 0;; round += 1) {
|
|
194
|
+
const turn = await step.do(`porulle-turn:acquire:${round}`, COORDINATOR_STEP, () => stub.acquire(key, payload.jobId));
|
|
195
|
+
if (turn === "granted")
|
|
196
|
+
break;
|
|
197
|
+
await step
|
|
198
|
+
.waitForEvent(`porulle-turn:wait:${round}`, {
|
|
199
|
+
type: TURN_EVENT_TYPE,
|
|
200
|
+
timeout: turnWaitMs(round),
|
|
201
|
+
})
|
|
202
|
+
.catch(() => undefined);
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
return await handler();
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
await step
|
|
209
|
+
.do("porulle-turn:release", COORDINATOR_STEP, () => stub.release(key, payload.jobId))
|
|
210
|
+
.catch(() => undefined);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import type { EnqueueOptions, ExecutionEngine, ExecutionEngineSetup } from "@porulle/core";
|
|
1
|
+
import type { EnqueueOptions, ExecutionEngine, ExecutionEngineSetup, JobInstanceStatus } from "@porulle/core/jobs";
|
|
2
2
|
export interface CloudflareJobPayload {
|
|
3
|
+
/** This Workflow instance's own id — the value passed as `create({ id })`. */
|
|
4
|
+
jobId: string;
|
|
3
5
|
taskSlug: string;
|
|
4
6
|
input: Record<string, unknown>;
|
|
5
7
|
organizationId: string;
|
|
@@ -9,6 +11,35 @@ export interface CloudflareJobPayload {
|
|
|
9
11
|
exclusive: boolean;
|
|
10
12
|
supersedes: boolean;
|
|
11
13
|
}
|
|
14
|
+
export interface WorkflowStepRetries {
|
|
15
|
+
limit: number;
|
|
16
|
+
delay: number;
|
|
17
|
+
backoff: "constant" | "exponential";
|
|
18
|
+
}
|
|
19
|
+
export interface WorkflowStep {
|
|
20
|
+
sleep(name: string, duration: number | string): Promise<void>;
|
|
21
|
+
do<T>(name: string, config: {
|
|
22
|
+
retries: WorkflowStepRetries;
|
|
23
|
+
timeout?: number | string;
|
|
24
|
+
}, callback: (context: {
|
|
25
|
+
attempt: number;
|
|
26
|
+
}) => Promise<T>): Promise<T>;
|
|
27
|
+
waitForEvent(name: string, options: {
|
|
28
|
+
type: string;
|
|
29
|
+
timeout?: number | string;
|
|
30
|
+
}): Promise<unknown>;
|
|
31
|
+
}
|
|
32
|
+
export interface WorkflowInstanceHandle {
|
|
33
|
+
status(): Promise<{
|
|
34
|
+
status: JobInstanceStatus;
|
|
35
|
+
error?: string;
|
|
36
|
+
}>;
|
|
37
|
+
terminate(): Promise<void>;
|
|
38
|
+
sendEvent(event: {
|
|
39
|
+
type: string;
|
|
40
|
+
payload?: unknown;
|
|
41
|
+
}): Promise<void>;
|
|
42
|
+
}
|
|
12
43
|
export interface WorkflowBinding {
|
|
13
44
|
create(options: {
|
|
14
45
|
id?: string;
|
|
@@ -16,29 +47,50 @@ export interface WorkflowBinding {
|
|
|
16
47
|
}): Promise<{
|
|
17
48
|
id: string;
|
|
18
49
|
}>;
|
|
50
|
+
get(id: string): Promise<WorkflowInstanceHandle>;
|
|
19
51
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
52
|
+
/** The shape of Cloudflare's own `Workflow` binding this package needs. */
|
|
53
|
+
export interface RawWorkflowBinding {
|
|
54
|
+
create(options: {
|
|
55
|
+
id?: string;
|
|
56
|
+
params: CloudflareJobPayload;
|
|
57
|
+
}): Promise<{
|
|
58
|
+
id: string;
|
|
59
|
+
}>;
|
|
60
|
+
get(id: string): Promise<{
|
|
61
|
+
status(): Promise<{
|
|
62
|
+
status: string;
|
|
63
|
+
error?: {
|
|
64
|
+
name: string;
|
|
65
|
+
message: string;
|
|
66
|
+
};
|
|
67
|
+
}>;
|
|
68
|
+
terminate(): Promise<void>;
|
|
69
|
+
sendEvent(event: {
|
|
70
|
+
type: string;
|
|
71
|
+
payload?: unknown;
|
|
72
|
+
}): Promise<void>;
|
|
73
|
+
}>;
|
|
31
74
|
}
|
|
75
|
+
/** Wraps the real Workflows binding: Cloudflare's richer instance status folds
|
|
76
|
+
* into `JobInstanceStatus` (`paused`/`waitingForPause` → `waiting`, anything
|
|
77
|
+
* unknown → `errored`) and the error becomes its message. */
|
|
78
|
+
export declare function adaptWorkflowBinding(binding: RawWorkflowBinding): WorkflowBinding;
|
|
32
79
|
export interface CloudflareConcurrencyCoordinator {
|
|
33
80
|
enqueue(payload: CloudflareJobPayload, create: () => Promise<{
|
|
34
81
|
id: string;
|
|
35
82
|
}>): Promise<{
|
|
36
83
|
id: string;
|
|
37
84
|
}>;
|
|
38
|
-
run<T>(
|
|
85
|
+
run<T>(payload: CloudflareJobPayload, step: WorkflowStep, handler: () => Promise<T>): Promise<T>;
|
|
39
86
|
}
|
|
87
|
+
export type NonRetryableErrorConstructor = new (message: string) => Error;
|
|
40
88
|
export interface CloudflareExecutionEngineOptions {
|
|
41
89
|
workflow: WorkflowBinding;
|
|
90
|
+
/** `NonRetryableError` from `cloudflare:workflows`. Workflows stops retrying a
|
|
91
|
+
* step only for that exact class, and this package cannot import the module
|
|
92
|
+
* outside the Workers runtime, so the Worker passes it in. */
|
|
93
|
+
nonRetryableError: NonRetryableErrorConstructor;
|
|
42
94
|
coordinator?: CloudflareConcurrencyCoordinator;
|
|
43
95
|
}
|
|
44
96
|
export declare class CloudflareExecutionEngine implements ExecutionEngine {
|
|
@@ -51,5 +103,12 @@ export declare class CloudflareExecutionEngine implements ExecutionEngine {
|
|
|
51
103
|
register(setup: ExecutionEngineSetup): void;
|
|
52
104
|
enqueue(taskSlug: string, input: Record<string, unknown>, options: EnqueueOptions): Promise<string>;
|
|
53
105
|
run(payload: CloudflareJobPayload, step: WorkflowStep): Promise<Record<string, unknown>>;
|
|
106
|
+
status(jobId: string): Promise<{
|
|
107
|
+
status: JobInstanceStatus;
|
|
108
|
+
error?: string;
|
|
109
|
+
}>;
|
|
110
|
+
cancel(jobId: string): Promise<void>;
|
|
54
111
|
private requireSetup;
|
|
55
112
|
}
|
|
113
|
+
export { DurableObjectConcurrencyCoordinator, JobCoordinatorLogic, porulleJobCoordinator, } from "./coordinator.js";
|
|
114
|
+
export type { CoordinatorStorage, CoordinatorStub, CoordinatorWorkflowBinding, DurableObjectConcurrencyCoordinatorOptions, DurableObjectStateLike, PorulleJobCoordinatorEnv, } from "./coordinator.js";
|