@porulle/jobs-cloudflare 0.45.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/coordinator.d.ts +11 -1
- package/dist/coordinator.js +20 -1
- package/dist/index.d.ts +23 -0
- package/package.json +2 -2
- package/src/coordinator.ts +20 -1
- package/src/index.ts +23 -0
- package/dist/tsconfig.tsbuildinfo +0 -1
package/dist/coordinator.d.ts
CHANGED
|
@@ -137,7 +137,17 @@ export interface DurableObjectConcurrencyCoordinatorOptions {
|
|
|
137
137
|
/** `CloudflareConcurrencyCoordinator` backed by a `PorulleJobCoordinator` Durable
|
|
138
138
|
* Object: supersede terminates pending instances at enqueue, and `run` serialises
|
|
139
139
|
* same-key instances through the DO's `acquire`/`release`, waiting with
|
|
140
|
-
* `step.waitForEvent` when another instance already holds the key.
|
|
140
|
+
* `step.waitForEvent` when another instance already holds the key.
|
|
141
|
+
*
|
|
142
|
+
* CHOOSE THE CONCURRENCY KEY FOR WHAT MAY SUPERSEDE WHAT. `supersedes` terminates
|
|
143
|
+
* EVERY other pending instance under the key (`commitEnqueue`), which is what you
|
|
144
|
+
* want when the key identifies the thing being worked — a second enqueue for one
|
|
145
|
+
* entity should replace the first. It is destructive when the key groups work that
|
|
146
|
+
* is meant to run in full: a page-per-job walk keyed on the store would have each
|
|
147
|
+
* page terminate the one before it, and the symptom is missing output, not an error.
|
|
148
|
+
* If the unit of work is a page rather than a record, either drop `supersedes` or
|
|
149
|
+
* key on the page. An uncoordinated enqueue is a bare `workflow.create()` and skips
|
|
150
|
+
* this object entirely. */
|
|
141
151
|
export declare class DurableObjectConcurrencyCoordinator implements CloudflareConcurrencyCoordinator {
|
|
142
152
|
private readonly options;
|
|
143
153
|
constructor(options: DurableObjectConcurrencyCoordinatorOptions);
|
package/dist/coordinator.js
CHANGED
|
@@ -250,7 +250,17 @@ export function porulleJobCoordinator(Base) {
|
|
|
250
250
|
/** `CloudflareConcurrencyCoordinator` backed by a `PorulleJobCoordinator` Durable
|
|
251
251
|
* Object: supersede terminates pending instances at enqueue, and `run` serialises
|
|
252
252
|
* same-key instances through the DO's `acquire`/`release`, waiting with
|
|
253
|
-
* `step.waitForEvent` when another instance already holds the key.
|
|
253
|
+
* `step.waitForEvent` when another instance already holds the key.
|
|
254
|
+
*
|
|
255
|
+
* CHOOSE THE CONCURRENCY KEY FOR WHAT MAY SUPERSEDE WHAT. `supersedes` terminates
|
|
256
|
+
* EVERY other pending instance under the key (`commitEnqueue`), which is what you
|
|
257
|
+
* want when the key identifies the thing being worked — a second enqueue for one
|
|
258
|
+
* entity should replace the first. It is destructive when the key groups work that
|
|
259
|
+
* is meant to run in full: a page-per-job walk keyed on the store would have each
|
|
260
|
+
* page terminate the one before it, and the symptom is missing output, not an error.
|
|
261
|
+
* If the unit of work is a page rather than a record, either drop `supersedes` or
|
|
262
|
+
* key on the page. An uncoordinated enqueue is a bare `workflow.create()` and skips
|
|
263
|
+
* this object entirely. */
|
|
254
264
|
export class DurableObjectConcurrencyCoordinator {
|
|
255
265
|
options;
|
|
256
266
|
constructor(options) {
|
|
@@ -266,6 +276,15 @@ export class DurableObjectConcurrencyCoordinator {
|
|
|
266
276
|
.enqueue(key, payload.supersedes, payload.jobId, inputHash);
|
|
267
277
|
if (coalescedInto)
|
|
268
278
|
return { id: coalescedInto };
|
|
279
|
+
// The swallow is deliberate, not a silenced error. `commitEnqueue` dropped these
|
|
280
|
+
// ids from the DO's pending set before we got here, and the commonest reason a
|
|
281
|
+
// terminate fails is that the instance had already finished — the same lag
|
|
282
|
+
// `#isStale` exists for. Distinguishing "already done" from "still alive and we
|
|
283
|
+
// failed to kill it" costs a status call per id on the enqueue path, to catch a
|
|
284
|
+
// case whose worst outcome is a superseded instance running once more. Every
|
|
285
|
+
// superseding task today is an idempotent sweep, so that costs duplicate work
|
|
286
|
+
// rather than wrong state. Revisit if a task ever supersedes work that is not
|
|
287
|
+
// safe to run twice.
|
|
269
288
|
await Promise.all(terminated.map((id) => this.options.workflow
|
|
270
289
|
.get(id)
|
|
271
290
|
.then((handle) => handle.terminate())
|
package/dist/index.d.ts
CHANGED
|
@@ -40,6 +40,29 @@ export interface WorkflowInstanceHandle {
|
|
|
40
40
|
payload?: unknown;
|
|
41
41
|
}): Promise<void>;
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* No `createBatch` YET, and the reason is a measurement nobody has taken rather than
|
|
45
|
+
* an absent use case. Cloudflare's binding creates up to 100 instances per call and it
|
|
46
|
+
* is what Inngest's `batchEvents` and Trigger.dev's `batchTrigger` exist for: fixing
|
|
47
|
+
* per-RECORD enqueueing.
|
|
48
|
+
*
|
|
49
|
+
* On the import path there is nothing to batch — one instance per page of work, ids in
|
|
50
|
+
* the payload, one `create`. But `enrichment-reenqueue` in the consuming app fires up
|
|
51
|
+
* to 500 enqueues by default and 5,000 capped, one per entity, and already hand-rolls
|
|
52
|
+
* bounded concurrency at six to stay under the subrequest budget. That is the case this
|
|
53
|
+
* would serve, and it survives the page-major rewrite because it is an operator route,
|
|
54
|
+
* not part of the import.
|
|
55
|
+
*
|
|
56
|
+
* What stops it being obviously worth it: those enqueues are coordinated, so each is a
|
|
57
|
+
* Durable Object round trip PLUS a `create`, and `createBatch` replaces only the second.
|
|
58
|
+
* The DO half cannot batch at all — N entity ids are N keys are N objects. So the payoff
|
|
59
|
+
* is whatever share of the measured ~0.9 s per enqueue is creation rather than
|
|
60
|
+
* coordination, and `do:enqueue` currently lumps the two. Split that class first, then
|
|
61
|
+
* decide.
|
|
62
|
+
*
|
|
63
|
+
* Keep the payload to identifiers. Params ride with the instance, so a page of ids is
|
|
64
|
+
* kilobytes and a page of product bodies is a size limit waiting to be hit.
|
|
65
|
+
*/
|
|
43
66
|
export interface WorkflowBinding {
|
|
44
67
|
create(options: {
|
|
45
68
|
id?: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@porulle/jobs-cloudflare",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.47.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
}
|
|
12
12
|
},
|
|
13
13
|
"dependencies": {
|
|
14
|
-
"@porulle/core": "0.
|
|
14
|
+
"@porulle/core": "0.47.0"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
17
|
"@types/node": "^24.5.2",
|
package/src/coordinator.ts
CHANGED
|
@@ -388,7 +388,17 @@ export interface DurableObjectConcurrencyCoordinatorOptions {
|
|
|
388
388
|
/** `CloudflareConcurrencyCoordinator` backed by a `PorulleJobCoordinator` Durable
|
|
389
389
|
* Object: supersede terminates pending instances at enqueue, and `run` serialises
|
|
390
390
|
* same-key instances through the DO's `acquire`/`release`, waiting with
|
|
391
|
-
* `step.waitForEvent` when another instance already holds the key.
|
|
391
|
+
* `step.waitForEvent` when another instance already holds the key.
|
|
392
|
+
*
|
|
393
|
+
* CHOOSE THE CONCURRENCY KEY FOR WHAT MAY SUPERSEDE WHAT. `supersedes` terminates
|
|
394
|
+
* EVERY other pending instance under the key (`commitEnqueue`), which is what you
|
|
395
|
+
* want when the key identifies the thing being worked — a second enqueue for one
|
|
396
|
+
* entity should replace the first. It is destructive when the key groups work that
|
|
397
|
+
* is meant to run in full: a page-per-job walk keyed on the store would have each
|
|
398
|
+
* page terminate the one before it, and the symptom is missing output, not an error.
|
|
399
|
+
* If the unit of work is a page rather than a record, either drop `supersedes` or
|
|
400
|
+
* key on the page. An uncoordinated enqueue is a bare `workflow.create()` and skips
|
|
401
|
+
* this object entirely. */
|
|
392
402
|
export class DurableObjectConcurrencyCoordinator
|
|
393
403
|
implements CloudflareConcurrencyCoordinator
|
|
394
404
|
{
|
|
@@ -405,6 +415,15 @@ export class DurableObjectConcurrencyCoordinator
|
|
|
405
415
|
.stub(key)
|
|
406
416
|
.enqueue(key, payload.supersedes, payload.jobId, inputHash);
|
|
407
417
|
if (coalescedInto) return { id: coalescedInto };
|
|
418
|
+
// The swallow is deliberate, not a silenced error. `commitEnqueue` dropped these
|
|
419
|
+
// ids from the DO's pending set before we got here, and the commonest reason a
|
|
420
|
+
// terminate fails is that the instance had already finished — the same lag
|
|
421
|
+
// `#isStale` exists for. Distinguishing "already done" from "still alive and we
|
|
422
|
+
// failed to kill it" costs a status call per id on the enqueue path, to catch a
|
|
423
|
+
// case whose worst outcome is a superseded instance running once more. Every
|
|
424
|
+
// superseding task today is an idempotent sweep, so that costs duplicate work
|
|
425
|
+
// rather than wrong state. Revisit if a task ever supersedes work that is not
|
|
426
|
+
// safe to run twice.
|
|
408
427
|
await Promise.all(
|
|
409
428
|
terminated.map((id) =>
|
|
410
429
|
this.options.workflow
|
package/src/index.ts
CHANGED
|
@@ -49,6 +49,29 @@ export interface WorkflowInstanceHandle {
|
|
|
49
49
|
sendEvent(event: { type: string; payload?: unknown }): Promise<void>;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
/**
|
|
53
|
+
* No `createBatch` YET, and the reason is a measurement nobody has taken rather than
|
|
54
|
+
* an absent use case. Cloudflare's binding creates up to 100 instances per call and it
|
|
55
|
+
* is what Inngest's `batchEvents` and Trigger.dev's `batchTrigger` exist for: fixing
|
|
56
|
+
* per-RECORD enqueueing.
|
|
57
|
+
*
|
|
58
|
+
* On the import path there is nothing to batch — one instance per page of work, ids in
|
|
59
|
+
* the payload, one `create`. But `enrichment-reenqueue` in the consuming app fires up
|
|
60
|
+
* to 500 enqueues by default and 5,000 capped, one per entity, and already hand-rolls
|
|
61
|
+
* bounded concurrency at six to stay under the subrequest budget. That is the case this
|
|
62
|
+
* would serve, and it survives the page-major rewrite because it is an operator route,
|
|
63
|
+
* not part of the import.
|
|
64
|
+
*
|
|
65
|
+
* What stops it being obviously worth it: those enqueues are coordinated, so each is a
|
|
66
|
+
* Durable Object round trip PLUS a `create`, and `createBatch` replaces only the second.
|
|
67
|
+
* The DO half cannot batch at all — N entity ids are N keys are N objects. So the payoff
|
|
68
|
+
* is whatever share of the measured ~0.9 s per enqueue is creation rather than
|
|
69
|
+
* coordination, and `do:enqueue` currently lumps the two. Split that class first, then
|
|
70
|
+
* decide.
|
|
71
|
+
*
|
|
72
|
+
* Keep the payload to identifiers. Params ride with the instance, so a page of ids is
|
|
73
|
+
* kilobytes and a page of product bodies is a size limit waiting to be hit.
|
|
74
|
+
*/
|
|
52
75
|
export interface WorkflowBinding {
|
|
53
76
|
create(options: {
|
|
54
77
|
id?: string;
|