bunqueue 2.8.48 → 2.8.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -9
- package/dist/application/contextFactory.d.ts +4 -2
- package/dist/application/contextFactory.js +3 -0
- package/dist/application/flowFailureRecovery.d.ts +17 -0
- package/dist/application/flowFailureRecovery.js +92 -0
- package/dist/application/operations/ack.d.ts +3 -2
- package/dist/application/operations/ack.js +34 -8
- package/dist/application/operations/flowPush.d.ts +9 -0
- package/dist/application/operations/flowPush.js +112 -0
- package/dist/application/operations/flowTopologyValidation.d.ts +3 -0
- package/dist/application/operations/flowTopologyValidation.js +101 -0
- package/dist/application/operations/flowValidation.d.ts +3 -0
- package/dist/application/operations/flowValidation.js +166 -0
- package/dist/application/operations/push.d.ts +3 -0
- package/dist/application/operations/push.js +2 -2
- package/dist/application/operations/pushInsert.d.ts +7 -1
- package/dist/application/operations/pushInsert.js +18 -8
- package/dist/application/operations/pushLocks.d.ts +1 -0
- package/dist/application/operations/pushLocks.js +5 -1
- package/dist/application/operations/queryOperations.js +7 -6
- package/dist/application/queueManager.d.ts +5 -0
- package/dist/application/queueManager.js +195 -60
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flow.d.ts +3 -6
- package/dist/client/flow.js +71 -210
- package/dist/client/flowAtomic.d.ts +9 -0
- package/dist/client/flowAtomic.js +24 -0
- package/dist/client/flowJobCoreMethods.d.ts +28 -0
- package/dist/client/flowJobCoreMethods.js +151 -0
- package/dist/client/flowJobDependencies.d.ts +7 -0
- package/dist/client/flowJobDependencies.js +64 -0
- package/dist/client/flowJobFactory.d.ts +11 -24
- package/dist/client/flowJobFactory.js +91 -360
- package/dist/client/flowJobMoveMethods.d.ts +16 -0
- package/dist/client/flowJobMoveMethods.js +112 -0
- package/dist/client/flowJobTypes.d.ts +29 -0
- package/dist/client/flowJobTypes.js +6 -0
- package/dist/client/flowLegacyPlan.d.ts +16 -0
- package/dist/client/flowLegacyPlan.js +118 -0
- package/dist/client/flowOptions.d.ts +10 -0
- package/dist/client/flowOptions.js +44 -0
- package/dist/client/flowPlan.d.ts +16 -0
- package/dist/client/flowPlan.js +93 -0
- package/dist/client/flowPush.js +4 -1
- package/dist/client/flowReader.d.ts +10 -0
- package/dist/client/flowReader.js +147 -0
- package/dist/client/jobHelpers.js +2 -0
- package/dist/client/workflow/clock.d.ts +3 -1
- package/dist/client/workflow/clock.js +18 -1
- package/dist/client/workflow/compensationChild.d.ts +30 -0
- package/dist/client/workflow/compensationChild.js +65 -0
- package/dist/client/workflow/compensationPass.d.ts +16 -0
- package/dist/client/workflow/compensationPass.js +96 -0
- package/dist/client/workflow/compensationSupport.d.ts +24 -0
- package/dist/client/workflow/compensationSupport.js +57 -0
- package/dist/client/workflow/compensator.d.ts +11 -50
- package/dist/client/workflow/compensator.js +22 -374
- package/dist/client/workflow/definitionGuard.d.ts +11 -0
- package/dist/client/workflow/definitionGuard.js +22 -0
- package/dist/client/workflow/engine.d.ts +2 -2
- package/dist/client/workflow/engine.js +8 -8
- package/dist/client/workflow/eventTypes.d.ts +44 -0
- package/dist/client/workflow/eventTypes.js +2 -0
- package/dist/client/workflow/executionTypes.d.ts +133 -0
- package/dist/client/workflow/executionTypes.js +2 -0
- package/dist/client/workflow/executor.d.ts +4 -17
- package/dist/client/workflow/executor.js +50 -208
- package/dist/client/workflow/executorLifecycle.d.ts +18 -0
- package/dist/client/workflow/executorLifecycle.js +69 -0
- package/dist/client/workflow/executorNodes.d.ts +17 -0
- package/dist/client/workflow/executorNodes.js +138 -0
- package/dist/client/workflow/identity.d.ts +2 -0
- package/dist/client/workflow/identity.js +8 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.d.ts +3 -5
- package/dist/client/workflow/loops.js +25 -23
- package/dist/client/workflow/mapRunner.d.ts +4 -0
- package/dist/client/workflow/mapRunner.js +45 -0
- package/dist/client/workflow/recovery.js +10 -3
- package/dist/client/workflow/runner.d.ts +2 -21
- package/dist/client/workflow/runner.js +39 -104
- package/dist/client/workflow/runnerTiming.d.ts +14 -0
- package/dist/client/workflow/runnerTiming.js +66 -0
- package/dist/client/workflow/stepTypes.d.ts +171 -0
- package/dist/client/workflow/stepTypes.js +3 -0
- package/dist/client/workflow/store.d.ts +17 -17
- package/dist/client/workflow/store.js +65 -100
- package/dist/client/workflow/storeExecutionCodec.d.ts +11 -0
- package/dist/client/workflow/storeExecutionCodec.js +34 -0
- package/dist/client/workflow/storeListing.d.ts +11 -0
- package/dist/client/workflow/storeListing.js +45 -0
- package/dist/client/workflow/storeMaintenance.d.ts +4 -0
- package/dist/client/workflow/storeMaintenance.js +40 -0
- package/dist/client/workflow/storeSignals.d.ts +9 -0
- package/dist/client/workflow/storeSignals.js +38 -2
- package/dist/client/workflow/subWorkflowRunner.d.ts +13 -0
- package/dist/client/workflow/subWorkflowRunner.js +40 -0
- package/dist/client/workflow/types.d.ts +4 -356
- package/dist/client/workflow/types.js +1 -3
- package/dist/client/workflow/waitFor.js +39 -26
- package/dist/client/workflow/workflow.d.ts +16 -59
- package/dist/client/workflow/workflow.js +53 -179
- package/dist/client/workflow/workflowDecisions.d.ts +11 -0
- package/dist/client/workflow/workflowDecisions.js +27 -0
- package/dist/client/workflow/workflowDefinition.d.ts +16 -0
- package/dist/client/workflow/workflowDefinition.js +123 -0
- package/dist/client/workflow/workflowIntrospection.d.ts +5 -0
- package/dist/client/workflow/workflowIntrospection.js +46 -0
- package/dist/client/workflow/workflowValidation.d.ts +44 -0
- package/dist/client/workflow/workflowValidation.js +143 -0
- package/dist/domain/types/command.d.ts +7 -1
- package/dist/domain/types/flow.d.ts +25 -0
- package/dist/domain/types/flow.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +52 -3
- package/dist/infrastructure/persistence/sqlite.d.ts +16 -0
- package/dist/infrastructure/persistence/sqlite.js +129 -7
- package/dist/infrastructure/persistence/sqliteBatch.js +10 -4
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +2 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +18 -5
- package/dist/infrastructure/persistence/statements.d.ts +4 -0
- package/dist/infrastructure/persistence/statements.js +8 -2
- package/dist/infrastructure/server/handlerRoutes.js +3 -0
- package/dist/infrastructure/server/handlers/advanced.js +2 -2
- package/dist/infrastructure/server/handlers/flow.d.ts +7 -0
- package/dist/infrastructure/server/handlers/flow.js +11 -0
- package/dist/infrastructure/server/handlers/index.d.ts +1 -0
- package/dist/infrastructure/server/handlers/index.js +1 -0
- package/package.json +4 -2
|
@@ -7,7 +7,7 @@ import { Database } from 'bun:sqlite';
|
|
|
7
7
|
import { createDlqEntry, } from '../../domain/types/dlq';
|
|
8
8
|
import { PRAGMA_SETTINGS, SCHEMA, MIGRATION_TABLE, SCHEMA_VERSION, MIGRATIONS } from './schema';
|
|
9
9
|
import { prepareStatements, } from './statements';
|
|
10
|
-
import { pack, persistedStallCount, reconstructDlqEntry, rowToJob, unpack, } from './sqliteSerializer';
|
|
10
|
+
import { pack, persistedInitialState, persistedStallCount, reconstructDlqEntry, rowToJob, unpack, } from './sqliteSerializer';
|
|
11
11
|
import { BatchInsertManager, WriteBuffer } from './sqliteBatch';
|
|
12
12
|
import { storageLog } from '../../shared/logger';
|
|
13
13
|
/** Check if an error is a SQLITE_FULL (disk full) error */
|
|
@@ -256,7 +256,7 @@ export class SqliteStorage {
|
|
|
256
256
|
runInsertJobStmt(job) {
|
|
257
257
|
this.statements
|
|
258
258
|
.get('insertJob')
|
|
259
|
-
.run(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, job.
|
|
259
|
+
.run(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, persistedInitialState(job), job.lifo ? 1 : 0, job.groupId, job.removeOnComplete ? 1 : 0, job.removeOnFail ? 1 : 0, job.failParentOnFailure ? 1 : 0, job.removeDependencyOnFailure ? 1 : 0, job.continueParentOnFailure ? 1 : 0, job.ignoreDependencyOnFailure ? 1 : 0, job.stallTimeout, persistedStallCount(job), job.timeline.length > 0 ? pack(job.timeline) : null);
|
|
260
260
|
}
|
|
261
261
|
/**
|
|
262
262
|
* Ensure a job's buffered INSERT has been written to disk before issuing a
|
|
@@ -308,6 +308,35 @@ export class SqliteStorage {
|
|
|
308
308
|
.run(entry.job.id, entry.job.queue, pack(entry), entry.enteredAt);
|
|
309
309
|
});
|
|
310
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Commit a terminal child outcome, its optional DLQ row, and its flow outbox
|
|
313
|
+
* record together. Recovery therefore never observes only half the transition.
|
|
314
|
+
*/
|
|
315
|
+
commitFailedJob(jobId, entry, flowFailure) {
|
|
316
|
+
this.writeBuffer.removePending(jobId);
|
|
317
|
+
this.safeWrite(() => {
|
|
318
|
+
const tx = this.db.transaction(() => {
|
|
319
|
+
if (entry) {
|
|
320
|
+
this.statements
|
|
321
|
+
.get('insertDlq')
|
|
322
|
+
.run(entry.job.id, entry.job.queue, pack(entry), entry.enteredAt);
|
|
323
|
+
}
|
|
324
|
+
if (flowFailure) {
|
|
325
|
+
this.db
|
|
326
|
+
.prepare(`INSERT INTO flow_failures
|
|
327
|
+
(parent_id, child_id, child_queue, mode, error, created_at)
|
|
328
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
329
|
+
ON CONFLICT(parent_id, child_id) DO UPDATE SET
|
|
330
|
+
child_queue=excluded.child_queue, mode=excluded.mode,
|
|
331
|
+
error=excluded.error, created_at=excluded.created_at`)
|
|
332
|
+
.run(flowFailure.parentId, flowFailure.childId, flowFailure.childQueue, flowFailure.mode, flowFailure.error, flowFailure.createdAt);
|
|
333
|
+
}
|
|
334
|
+
this.statements.get('deleteJob').run(jobId);
|
|
335
|
+
this.statements.get('deleteJobResult').run(jobId);
|
|
336
|
+
});
|
|
337
|
+
tx();
|
|
338
|
+
});
|
|
339
|
+
}
|
|
311
340
|
/** Delete DLQ entry by job ID */
|
|
312
341
|
deleteDlqEntry(jobId) {
|
|
313
342
|
this.safeWrite(() => {
|
|
@@ -388,6 +417,7 @@ export class SqliteStorage {
|
|
|
388
417
|
const tx = this.db.transaction((id) => {
|
|
389
418
|
deleteStmt.run(id);
|
|
390
419
|
deleteResultStmt.run(id);
|
|
420
|
+
this.db.prepare('DELETE FROM flow_failures WHERE parent_id = ?').run(id);
|
|
391
421
|
});
|
|
392
422
|
tx(jobId);
|
|
393
423
|
});
|
|
@@ -438,12 +468,104 @@ export class SqliteStorage {
|
|
|
438
468
|
}
|
|
439
469
|
/** Update a job's children_ids blob and parent_id */
|
|
440
470
|
updateJobChildrenIds(jobId, childrenIds) {
|
|
471
|
+
this.flushIfBuffered(jobId);
|
|
441
472
|
this.safeWrite(() => {
|
|
442
473
|
this.db
|
|
443
474
|
.prepare('UPDATE jobs SET children_ids = ? WHERE id = ?')
|
|
444
475
|
.run(childrenIds.length > 0 ? pack(childrenIds) : null, jobId);
|
|
445
476
|
});
|
|
446
477
|
}
|
|
478
|
+
/** Persist both sides of a legacy parent link in one SQLite transaction. */
|
|
479
|
+
updateFlowLink(child, parent, parentState) {
|
|
480
|
+
this.flushIfBuffered(child.id);
|
|
481
|
+
this.flushIfBuffered(parent.id);
|
|
482
|
+
this.safeWrite(() => {
|
|
483
|
+
const tx = this.db.transaction(() => {
|
|
484
|
+
this.db
|
|
485
|
+
.prepare('UPDATE jobs SET parent_id = ?, data = ? WHERE id = ?')
|
|
486
|
+
.run(child.parentId, pack(child.data), child.id);
|
|
487
|
+
this.db
|
|
488
|
+
.prepare('UPDATE jobs SET children_ids = ?, depends_on = ?, data = ?, state = ? WHERE id = ?')
|
|
489
|
+
.run(parent.childrenIds.length > 0 ? pack(parent.childrenIds) : null, parent.dependsOn.length > 0 ? pack(parent.dependsOn) : null, pack(parent.data), parentState, parent.id);
|
|
490
|
+
});
|
|
491
|
+
tx();
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
/** Persist both sides of a removed parent/child relationship atomically. */
|
|
495
|
+
removeFlowLink(child, parent, parentState) {
|
|
496
|
+
this.flushIfBuffered(child.id);
|
|
497
|
+
this.flushIfBuffered(parent.id);
|
|
498
|
+
this.safeWrite(() => {
|
|
499
|
+
const tx = this.db.transaction(() => {
|
|
500
|
+
this.db
|
|
501
|
+
.prepare('UPDATE jobs SET parent_id = ?, data = ? WHERE id = ?')
|
|
502
|
+
.run(child.parentId, pack(child.data), child.id);
|
|
503
|
+
this.db
|
|
504
|
+
.prepare(`UPDATE jobs
|
|
505
|
+
SET children_ids = ?, depends_on = ?, data = ?, run_at = ?, state = ?
|
|
506
|
+
WHERE id = ?`)
|
|
507
|
+
.run(parent.childrenIds.length > 0 ? pack(parent.childrenIds) : null, parent.dependsOn.length > 0 ? pack(parent.dependsOn) : null, pack(parent.data), parent.runAt, parentState, parent.id);
|
|
508
|
+
this.db
|
|
509
|
+
.prepare('DELETE FROM flow_failures WHERE parent_id = ? AND child_id = ?')
|
|
510
|
+
.run(parent.id, child.id);
|
|
511
|
+
});
|
|
512
|
+
tx();
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
/** Record terminal child propagation before applying its parent-side effect. */
|
|
516
|
+
saveFlowFailure(record) {
|
|
517
|
+
this.safeWrite(() => {
|
|
518
|
+
this.db
|
|
519
|
+
.prepare(`INSERT INTO flow_failures
|
|
520
|
+
(parent_id, child_id, child_queue, mode, error, created_at)
|
|
521
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
522
|
+
ON CONFLICT(parent_id, child_id) DO UPDATE SET
|
|
523
|
+
child_queue=excluded.child_queue, mode=excluded.mode,
|
|
524
|
+
error=excluded.error, created_at=excluded.created_at`)
|
|
525
|
+
.run(record.parentId, record.childId, record.childQueue, record.mode, record.error, record.createdAt);
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
loadFlowFailures() {
|
|
529
|
+
const rows = this.db
|
|
530
|
+
.query('SELECT * FROM flow_failures ORDER BY created_at, parent_id, child_id')
|
|
531
|
+
.all();
|
|
532
|
+
return rows.map((row) => ({
|
|
533
|
+
parentId: row.parent_id,
|
|
534
|
+
childId: row.child_id,
|
|
535
|
+
childQueue: row.child_queue,
|
|
536
|
+
mode: row.mode,
|
|
537
|
+
error: row.error,
|
|
538
|
+
createdAt: row.created_at,
|
|
539
|
+
}));
|
|
540
|
+
}
|
|
541
|
+
deleteFlowFailure(parentId, childId) {
|
|
542
|
+
this.safeWrite(() => {
|
|
543
|
+
if (childId) {
|
|
544
|
+
this.db
|
|
545
|
+
.prepare('DELETE FROM flow_failures WHERE parent_id = ? AND child_id = ?')
|
|
546
|
+
.run(parentId, childId);
|
|
547
|
+
}
|
|
548
|
+
else {
|
|
549
|
+
this.db.prepare('DELETE FROM flow_failures WHERE parent_id = ?').run(parentId);
|
|
550
|
+
}
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
/** Persist dependency removal/promotion as one parent-side recovery checkpoint. */
|
|
554
|
+
updateFlowParentResolution(job) {
|
|
555
|
+
this.flushIfBuffered(job.id);
|
|
556
|
+
const state = job.dependsOn.length > 0
|
|
557
|
+
? 'waiting-children'
|
|
558
|
+
: job.runAt > Date.now()
|
|
559
|
+
? 'delayed'
|
|
560
|
+
: job.priority > 0
|
|
561
|
+
? 'prioritized'
|
|
562
|
+
: 'waiting';
|
|
563
|
+
this.safeWrite(() => {
|
|
564
|
+
this.db
|
|
565
|
+
.prepare('UPDATE jobs SET depends_on = ?, run_at = ?, state = ? WHERE id = ?')
|
|
566
|
+
.run(job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.runAt, state, job.id);
|
|
567
|
+
});
|
|
568
|
+
}
|
|
447
569
|
getJob(id) {
|
|
448
570
|
const row = this.statements.get('getJob').get(id);
|
|
449
571
|
return row ? rowToJob(row) : null;
|
|
@@ -573,15 +695,15 @@ export class SqliteStorage {
|
|
|
573
695
|
const predicateParams = [];
|
|
574
696
|
const requested = new Set(uniqueStates);
|
|
575
697
|
if (requested.has('waiting')) {
|
|
576
|
-
predicates.push("(state IN ('waiting', 'delayed') AND run_at <= ? AND priority <= 0)");
|
|
698
|
+
predicates.push("(state IN ('waiting', 'prioritized', 'delayed') AND run_at <= ? AND priority <= 0)");
|
|
577
699
|
predicateParams.push(options.now);
|
|
578
700
|
}
|
|
579
701
|
if (requested.has('prioritized')) {
|
|
580
|
-
predicates.push("(state IN ('waiting', 'delayed') AND run_at <= ? AND priority > 0)");
|
|
702
|
+
predicates.push("(state IN ('waiting', 'prioritized', 'delayed') AND run_at <= ? AND priority > 0)");
|
|
581
703
|
predicateParams.push(options.now);
|
|
582
704
|
}
|
|
583
705
|
if (requested.has('delayed')) {
|
|
584
|
-
predicates.push("(state IN ('waiting', 'delayed') AND run_at > ?)");
|
|
706
|
+
predicates.push("(state IN ('waiting', 'prioritized', 'delayed') AND run_at > ?)");
|
|
585
707
|
predicateParams.push(options.now);
|
|
586
708
|
}
|
|
587
709
|
const persistedStates = uniqueStates.filter((state) => state !== 'waiting' && state !== 'prioritized' && state !== 'delayed');
|
|
@@ -607,7 +729,7 @@ export class SqliteStorage {
|
|
|
607
729
|
*/
|
|
608
730
|
loadPendingJobs(limit = 10000, offset = 0) {
|
|
609
731
|
const rows = this.db
|
|
610
|
-
.query("SELECT * FROM jobs WHERE state IN ('waiting', 'delayed') ORDER BY priority DESC, run_at ASC, id ASC LIMIT ? OFFSET ?")
|
|
732
|
+
.query("SELECT * FROM jobs WHERE state IN ('waiting', 'prioritized', 'waiting-children', 'delayed') ORDER BY priority DESC, run_at ASC, id ASC LIMIT ? OFFSET ?")
|
|
611
733
|
.all(limit, offset);
|
|
612
734
|
return rows.map((row) => rowToJob(row));
|
|
613
735
|
}
|
|
@@ -639,7 +761,7 @@ export class SqliteStorage {
|
|
|
639
761
|
*/
|
|
640
762
|
countPendingJobs() {
|
|
641
763
|
const result = this.db
|
|
642
|
-
.query("SELECT COUNT(*) as count FROM jobs WHERE state IN ('waiting', 'delayed')")
|
|
764
|
+
.query("SELECT COUNT(*) as count FROM jobs WHERE state IN ('waiting', 'prioritized', 'waiting-children', 'delayed')")
|
|
643
765
|
.get();
|
|
644
766
|
return result?.count ?? 0;
|
|
645
767
|
}
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* SQLite Batch Operations
|
|
3
3
|
* High-performance batch insert with prepared statement caching
|
|
4
4
|
*/
|
|
5
|
-
import { pack, persistedStallCount } from './sqliteSerializer';
|
|
6
|
-
const COLS_PER_ROW =
|
|
5
|
+
import { pack, persistedInitialState, persistedStallCount } from './sqliteSerializer';
|
|
6
|
+
const COLS_PER_ROW = 29;
|
|
7
7
|
// SQLite has a limit of ~999 variables, so batch in chunks
|
|
8
8
|
const MAX_ROWS_PER_INSERT = Math.floor(999 / COLS_PER_ROW);
|
|
9
9
|
/**
|
|
@@ -75,7 +75,7 @@ export class BatchInsertManager {
|
|
|
75
75
|
getBatchInsertStmt(size) {
|
|
76
76
|
let stmt = this.cache.get(size);
|
|
77
77
|
if (!stmt) {
|
|
78
|
-
const rowPlaceholder = '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
|
|
78
|
+
const rowPlaceholder = '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
|
|
79
79
|
const placeholders = Array(size).fill(rowPlaceholder).join(', ');
|
|
80
80
|
// ON CONFLICT(id) DO UPDATE (upsert): a colliding id overwrites in place instead
|
|
81
81
|
// of failing the whole flush. Without it, one stale ORPHAN row (left when
|
|
@@ -87,6 +87,8 @@ export class BatchInsertManager {
|
|
|
87
87
|
id, queue, data, priority, created_at, run_at, attempts, max_attempts,
|
|
88
88
|
backoff, ttl, timeout, unique_key, custom_id, depends_on, parent_id,
|
|
89
89
|
children_ids, tags, state, lifo, group_id, remove_on_complete, remove_on_fail,
|
|
90
|
+
fail_parent_on_failure, remove_dependency_on_failure,
|
|
91
|
+
continue_parent_on_failure, ignore_dependency_on_failure,
|
|
90
92
|
stall_timeout, stall_count, timeline
|
|
91
93
|
) VALUES ${placeholders}
|
|
92
94
|
ON CONFLICT(id) DO UPDATE SET
|
|
@@ -97,6 +99,10 @@ export class BatchInsertManager {
|
|
|
97
99
|
depends_on=excluded.depends_on, parent_id=excluded.parent_id, children_ids=excluded.children_ids,
|
|
98
100
|
tags=excluded.tags, state=excluded.state, lifo=excluded.lifo, group_id=excluded.group_id,
|
|
99
101
|
remove_on_complete=excluded.remove_on_complete, remove_on_fail=excluded.remove_on_fail,
|
|
102
|
+
fail_parent_on_failure=excluded.fail_parent_on_failure,
|
|
103
|
+
remove_dependency_on_failure=excluded.remove_dependency_on_failure,
|
|
104
|
+
continue_parent_on_failure=excluded.continue_parent_on_failure,
|
|
105
|
+
ignore_dependency_on_failure=excluded.ignore_dependency_on_failure,
|
|
100
106
|
stall_timeout=excluded.stall_timeout, stall_count=excluded.stall_count,
|
|
101
107
|
timeline=excluded.timeline,
|
|
102
108
|
started_at=excluded.started_at, completed_at=excluded.completed_at,
|
|
@@ -116,7 +122,7 @@ export class BatchInsertManager {
|
|
|
116
122
|
// Flatten all values
|
|
117
123
|
const values = [];
|
|
118
124
|
for (const job of jobs) {
|
|
119
|
-
values.push(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, job.
|
|
125
|
+
values.push(job.id, job.queue, pack(job.data), job.priority, job.createdAt, job.runAt, job.attempts, job.maxAttempts, job.backoff, job.ttl, job.timeout, job.uniqueKey, job.customId, job.dependsOn.length > 0 ? pack(job.dependsOn) : null, job.parentId, job.childrenIds.length > 0 ? pack(job.childrenIds) : null, job.tags.length > 0 ? pack(job.tags) : null, persistedInitialState(job, now), job.lifo ? 1 : 0, job.groupId, job.removeOnComplete ? 1 : 0, job.removeOnFail ? 1 : 0, job.failParentOnFailure ? 1 : 0, job.removeDependencyOnFailure ? 1 : 0, job.continueParentOnFailure ? 1 : 0, job.ignoreDependencyOnFailure ? 1 : 0, job.stallTimeout, persistedStallCount(job), job.timeline.length > 0 ? pack(job.timeline) : null);
|
|
120
126
|
}
|
|
121
127
|
stmt.run(...values);
|
|
122
128
|
}
|
|
@@ -16,6 +16,8 @@ export declare function unpack<T>(buffer: Uint8Array | null, fallback: T, contex
|
|
|
16
16
|
* field receives the same zero default as a freshly created Job.
|
|
17
17
|
*/
|
|
18
18
|
export declare function persistedStallCount(job: Pick<Job, 'stallCount'>): number;
|
|
19
|
+
/** Derive the durable initial state from the authoritative timeline. */
|
|
20
|
+
export declare function persistedInitialState(job: Job, now?: number): string;
|
|
19
21
|
/**
|
|
20
22
|
* Symbol marker stamped on a Job whose `depends_on` blob failed to decode.
|
|
21
23
|
*
|
|
@@ -30,6 +30,19 @@ export function unpack(buffer, fallback, context) {
|
|
|
30
30
|
export function persistedStallCount(job) {
|
|
31
31
|
return job.stallCount ?? 0;
|
|
32
32
|
}
|
|
33
|
+
/** Derive the durable initial state from the authoritative timeline. */
|
|
34
|
+
export function persistedInitialState(job, now = Date.now()) {
|
|
35
|
+
const latest = job.timeline[job.timeline.length - 1]?.state;
|
|
36
|
+
if (latest === 'waiting-children' ||
|
|
37
|
+
latest === 'waiting' ||
|
|
38
|
+
latest === 'prioritized' ||
|
|
39
|
+
latest === 'delayed') {
|
|
40
|
+
return latest;
|
|
41
|
+
}
|
|
42
|
+
if (job.runAt > now)
|
|
43
|
+
return 'delayed';
|
|
44
|
+
return job.priority > 0 ? 'prioritized' : 'waiting';
|
|
45
|
+
}
|
|
33
46
|
/**
|
|
34
47
|
* Symbol marker stamped on a Job whose `depends_on` blob failed to decode.
|
|
35
48
|
*
|
|
@@ -113,14 +126,14 @@ export function rowToJob(row) {
|
|
|
113
126
|
lastHeartbeat: row.last_heartbeat ?? row.created_at,
|
|
114
127
|
stallTimeout: row.stall_timeout,
|
|
115
128
|
stallCount: row.stall_count ?? 0,
|
|
116
|
-
// BullMQ v5 additional fields
|
|
129
|
+
// BullMQ v5 additional fields
|
|
117
130
|
stackTraceLimit: 10,
|
|
118
131
|
keepLogs: null,
|
|
119
132
|
sizeLimit: null,
|
|
120
|
-
failParentOnFailure:
|
|
121
|
-
removeDependencyOnFailure:
|
|
122
|
-
continueParentOnFailure:
|
|
123
|
-
ignoreDependencyOnFailure:
|
|
133
|
+
failParentOnFailure: row.fail_parent_on_failure === 1,
|
|
134
|
+
removeDependencyOnFailure: row.remove_dependency_on_failure === 1,
|
|
135
|
+
continueParentOnFailure: row.continue_parent_on_failure === 1,
|
|
136
|
+
ignoreDependencyOnFailure: row.ignore_dependency_on_failure === 1,
|
|
124
137
|
deduplicationTtl: null,
|
|
125
138
|
deduplicationExtend: false,
|
|
126
139
|
deduplicationReplace: false,
|
|
@@ -37,6 +37,10 @@ export interface DbJob {
|
|
|
37
37
|
progress_msg: string | null;
|
|
38
38
|
remove_on_complete: number;
|
|
39
39
|
remove_on_fail: number;
|
|
40
|
+
fail_parent_on_failure: number;
|
|
41
|
+
remove_dependency_on_failure: number;
|
|
42
|
+
continue_parent_on_failure: number;
|
|
43
|
+
ignore_dependency_on_failure: number;
|
|
40
44
|
stall_timeout: number | null;
|
|
41
45
|
last_heartbeat: number | null;
|
|
42
46
|
stall_count: number;
|
|
@@ -15,12 +15,14 @@ export const SQL_STATEMENTS = {
|
|
|
15
15
|
id, queue, data, priority, created_at, run_at, attempts,
|
|
16
16
|
max_attempts, backoff, ttl, timeout, unique_key, custom_id,
|
|
17
17
|
depends_on, parent_id, children_ids, tags, state, lifo, group_id,
|
|
18
|
-
remove_on_complete, remove_on_fail,
|
|
18
|
+
remove_on_complete, remove_on_fail, fail_parent_on_failure,
|
|
19
|
+
remove_dependency_on_failure, continue_parent_on_failure,
|
|
20
|
+
ignore_dependency_on_failure, stall_timeout, stall_count, timeline
|
|
19
21
|
) VALUES (
|
|
20
22
|
?, ?, ?, ?, ?, ?, ?,
|
|
21
23
|
?, ?, ?, ?, ?, ?,
|
|
22
24
|
?, ?, ?, ?, ?, ?, ?,
|
|
23
|
-
?, ?, ?, ?, ?
|
|
25
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
24
26
|
)
|
|
25
27
|
ON CONFLICT(id) DO UPDATE SET
|
|
26
28
|
queue=excluded.queue, data=excluded.data, priority=excluded.priority,
|
|
@@ -30,6 +32,10 @@ export const SQL_STATEMENTS = {
|
|
|
30
32
|
depends_on=excluded.depends_on, parent_id=excluded.parent_id, children_ids=excluded.children_ids,
|
|
31
33
|
tags=excluded.tags, state=excluded.state, lifo=excluded.lifo, group_id=excluded.group_id,
|
|
32
34
|
remove_on_complete=excluded.remove_on_complete, remove_on_fail=excluded.remove_on_fail,
|
|
35
|
+
fail_parent_on_failure=excluded.fail_parent_on_failure,
|
|
36
|
+
remove_dependency_on_failure=excluded.remove_dependency_on_failure,
|
|
37
|
+
continue_parent_on_failure=excluded.continue_parent_on_failure,
|
|
38
|
+
ignore_dependency_on_failure=excluded.ignore_dependency_on_failure,
|
|
33
39
|
stall_timeout=excluded.stall_timeout, stall_count=excluded.stall_count,
|
|
34
40
|
timeline=excluded.timeline,
|
|
35
41
|
-- Per-execution columns are NOT in the INSERT column list, so excluded.<col>
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
// Import handlers
|
|
5
5
|
import { handlePush, handlePushBatch, handlePull, handlePullBatch, handleAck, handleAckBatch, handleFail, } from './handlers/core';
|
|
6
|
+
import { handlePushFlow } from './handlers/flow';
|
|
6
7
|
import { handleGetJob, handleGetState, handleGetResult, handleGetJobCounts, handleGetCountsPerPriority, handleGetJobByCustomId, handleGetJobs, handleGetChildrenValues, } from './handlers/query';
|
|
7
8
|
import { handleCancel, handleProgress, handleGetProgress, handlePause, handleResume, handleDrain, handleStats, handleMetrics, handleStorageStatus, } from './handlers/management';
|
|
8
9
|
import { handleDlq, handleRetryDlq, handlePurgeDlq, handleRetryCompleted } from './handlers/dlq';
|
|
@@ -18,6 +19,8 @@ export async function routeCoreCommand(cmd, ctx, reqId) {
|
|
|
18
19
|
return handlePush(cmd, ctx, reqId);
|
|
19
20
|
case 'PUSHB':
|
|
20
21
|
return handlePushBatch(cmd, ctx, reqId);
|
|
22
|
+
case 'PUSHF':
|
|
23
|
+
return handlePushFlow(cmd, ctx, reqId);
|
|
21
24
|
case 'PULL':
|
|
22
25
|
return handlePull(cmd, ctx, reqId);
|
|
23
26
|
case 'PULLB':
|
|
@@ -78,8 +78,8 @@ export async function handleUpdateParent(cmd, ctx, reqId) {
|
|
|
78
78
|
await ctx.queueManager.updateJobParent(jobId(cmd.childId), jobId(cmd.parentId));
|
|
79
79
|
return resp.ok(undefined, reqId);
|
|
80
80
|
}
|
|
81
|
-
catch {
|
|
82
|
-
return resp.error('Failed to update parent', reqId);
|
|
81
|
+
catch (error) {
|
|
82
|
+
return resp.error(error instanceof Error ? error.message : 'Failed to update parent', reqId);
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
/** Handle MoveToDelayed command */
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Command } from '../../../domain/types/command';
|
|
2
|
+
import type { Response } from '../../../domain/types/response';
|
|
3
|
+
import type { HandlerContext } from '../types';
|
|
4
|
+
/** Commit the whole graph in one broker operation; validation happens pre-mutation. */
|
|
5
|
+
export declare function handlePushFlow(cmd: Extract<Command, {
|
|
6
|
+
cmd: 'PUSHF';
|
|
7
|
+
}>, ctx: HandlerContext, reqId?: string): Promise<Response>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import * as resp from '../../../domain/types/response';
|
|
2
|
+
/** Commit the whole graph in one broker operation; validation happens pre-mutation. */
|
|
3
|
+
export async function handlePushFlow(cmd, ctx, reqId) {
|
|
4
|
+
try {
|
|
5
|
+
const result = await ctx.queueManager.pushFlow({ jobs: cmd.jobs });
|
|
6
|
+
return resp.data(result, reqId);
|
|
7
|
+
}
|
|
8
|
+
catch (error) {
|
|
9
|
+
return resp.error(error instanceof Error ? error.message : String(error), reqId);
|
|
10
|
+
}
|
|
11
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bunqueue",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.50",
|
|
4
4
|
"description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external infrastructure.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/main.js",
|
|
@@ -59,13 +59,15 @@
|
|
|
59
59
|
"docs:api": "bun run scripts/build-api-reference.ts",
|
|
60
60
|
"check:docs-data": "bun run scripts/check-docs-data.ts",
|
|
61
61
|
"test": "BUNQUEUE_EMBEDDED=1 bun test",
|
|
62
|
-
"test:model": "BUNQUEUE_EMBEDDED=1 bun test test/model-based/queue-model.test.ts test/model-based/workflow-model.test.ts test/model-based/backup-model.test.ts test/model-based/monitoring-model.test.ts test/model-based/enterprise-telemetry-model.test.ts",
|
|
62
|
+
"test:model": "BUNQUEUE_EMBEDDED=1 bun test test/model-based/queue-model.test.ts test/model-based/workflow-model.test.ts test/model-based/flow-producer-model.test.ts test/model-based/backup-model.test.ts test/model-based/monitoring-model.test.ts test/model-based/enterprise-telemetry-model.test.ts",
|
|
63
63
|
"test:sandbox": "bun run scripts/test-sandbox.ts",
|
|
64
64
|
"test:sandbox:sdk": "bun run scripts/test-sdk-sandbox.ts",
|
|
65
65
|
"bench": "bun run bench/comprehensive.ts",
|
|
66
66
|
"bench:tcp": "bun run bench/tcp-bench.ts",
|
|
67
67
|
"bench:autobatch": "bun run bench/local-autobatch.ts",
|
|
68
68
|
"bench:pushbulk": "bun run bench/pushbulk-delta.ts",
|
|
69
|
+
"bench:workflow": "bun run bench/workflow-engine.ts",
|
|
70
|
+
"bench:workflow:scale": "bun run bench/workflow-engine/scale.ts",
|
|
69
71
|
"bench:joblist": "bun run bench/job-list-perf.ts",
|
|
70
72
|
"bench:compare": "bun run bench/comparison/run.ts",
|
|
71
73
|
"lint": "biome lint src",
|