effect-mq 0.3.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -14
- package/dist/Job.d.ts +131 -9
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +81 -5
- package/dist/Job.js.map +1 -1
- package/dist/JobStore.d.ts +63 -2
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +159 -119
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Worker.d.ts +18 -0
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +37 -9
- package/dist/Worker.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +239 -49
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/drizzle-postgres/schema.d.ts +35 -27
- package/dist/drizzle-postgres/schema.d.ts.map +1 -1
- package/dist/drizzle-postgres/schema.js +8 -2
- package/dist/drizzle-postgres/schema.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +173 -36
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +24 -1
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +126 -21
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +266 -0
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/Job.ts +268 -13
- package/src/JobStore.ts +77 -2
- package/src/MemoryJobStore.ts +102 -53
- package/src/Worker.ts +61 -9
- package/src/drizzle-postgres/DrizzleJobStore.ts +273 -66
- package/src/drizzle-postgres/schema.ts +33 -25
- package/src/redis/RedisJobStore.ts +236 -47
- package/src/redis/scripts.ts +153 -21
- package/src/testing/conformance.ts +350 -0
|
@@ -113,6 +113,7 @@ type JobRow = {
|
|
|
113
113
|
readonly timeoutMs: number | string | null
|
|
114
114
|
readonly cancelRequested: boolean
|
|
115
115
|
readonly dedupeKey: string | null
|
|
116
|
+
readonly trace: JobStore.TraceContext | null
|
|
116
117
|
readonly runAt: Date
|
|
117
118
|
readonly enqueuedAt: Date
|
|
118
119
|
readonly processedAt: Date | null
|
|
@@ -171,6 +172,7 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
|
|
|
171
172
|
timeoutMs: row.timeoutMs === null || row.timeoutMs === undefined ? undefined : Number(row.timeoutMs),
|
|
172
173
|
cancelRequested: row.cancelRequested,
|
|
173
174
|
dedupeKey: row.dedupeKey ?? undefined,
|
|
175
|
+
trace: row.trace ?? undefined,
|
|
174
176
|
runAt: row.runAt.getTime(),
|
|
175
177
|
enqueuedAt: row.enqueuedAt.getTime(),
|
|
176
178
|
processedAt: row.processedAt?.getTime(),
|
|
@@ -220,6 +222,7 @@ export const make = (
|
|
|
220
222
|
"timeoutMs",
|
|
221
223
|
"cancelRequested",
|
|
222
224
|
"dedupeKey",
|
|
225
|
+
"trace",
|
|
223
226
|
"runAt",
|
|
224
227
|
"enqueuedAt",
|
|
225
228
|
"processedAt",
|
|
@@ -490,13 +493,15 @@ export const make = (
|
|
|
490
493
|
: sql`'j-' || ${seqExpr}::text`
|
|
491
494
|
const rows = rowsOf(yield* exec.execute<{ id: string }>(sql`
|
|
492
495
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
493
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, run_at, enqueued_at${extraColumnNames})
|
|
496
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
494
497
|
VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
|
|
495
498
|
${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
|
|
496
499
|
${request.attemptsMax},
|
|
497
500
|
${request.backoff === undefined ? null : JSON.stringify(request.backoff)}::jsonb,
|
|
498
501
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
499
|
-
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
502
|
+
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
503
|
+
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
504
|
+
${runAt}, ${now}${extraColumnValues(request)})
|
|
500
505
|
ON CONFLICT (id) DO NOTHING
|
|
501
506
|
RETURNING ${jobs.id} AS id
|
|
502
507
|
`).pipe(Effect.mapError(storeError("enqueue failed"))))
|
|
@@ -572,6 +577,7 @@ export const make = (
|
|
|
572
577
|
backoff = ${request.backoff === undefined ? null : JSON.stringify(request.backoff)}::jsonb,
|
|
573
578
|
keep = ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
574
579
|
timeout_ms = ${request.timeoutMs ?? null},
|
|
580
|
+
trace = ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
575
581
|
run_at = ${new Date(now.getTime() + Math.max(0, request.delayMs))}${extraColumnAssignments(request)}
|
|
576
582
|
WHERE ${jobs.id} = ${entry.jobId} AND ${jobs.state} = 'delayed'
|
|
577
583
|
RETURNING ${jobs.id} AS id, ${jobs.queue} AS "queue"
|
|
@@ -650,22 +656,224 @@ export const make = (
|
|
|
650
656
|
AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
|
|
651
657
|
`).pipe(Effect.asVoid)
|
|
652
658
|
|
|
659
|
+
// Shared by cancel and cancelByDedupe.
|
|
660
|
+
const cancelJob = (id: JobStore.JobId) =>
|
|
661
|
+
db.transaction((tx) =>
|
|
662
|
+
Effect.gen(function*() {
|
|
663
|
+
const now = yield* nowDate
|
|
664
|
+
// One guarded statement: waiting/delayed become terminal, active
|
|
665
|
+
// gets the cancel-request flag; anything else is reported by state.
|
|
666
|
+
const rows = rowsOf(yield* tx.execute<
|
|
667
|
+
{
|
|
668
|
+
id: string
|
|
669
|
+
state: string
|
|
670
|
+
processedAt: Date | null
|
|
671
|
+
name: string
|
|
672
|
+
keep: JobStore.KeepPolicy | null
|
|
673
|
+
dedupeKey: string | null
|
|
674
|
+
}
|
|
675
|
+
>(sql`
|
|
676
|
+
UPDATE ${jobs} SET
|
|
677
|
+
state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN 'cancelled' ELSE ${jobs.state} END,
|
|
678
|
+
finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
|
|
679
|
+
cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
|
|
680
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active')
|
|
681
|
+
RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
|
|
682
|
+
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
683
|
+
${jobs.dedupeKey} AS "dedupeKey"
|
|
684
|
+
`))
|
|
685
|
+
const row = rows[0]
|
|
686
|
+
if (row === undefined) {
|
|
687
|
+
const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
|
|
688
|
+
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
689
|
+
`))
|
|
690
|
+
const found = existing[0]
|
|
691
|
+
if (found === undefined) {
|
|
692
|
+
return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
693
|
+
}
|
|
694
|
+
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
|
|
695
|
+
}
|
|
696
|
+
if (row.state === "cancelled") {
|
|
697
|
+
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
698
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
699
|
+
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
700
|
+
}
|
|
701
|
+
})
|
|
702
|
+
).pipe(
|
|
703
|
+
Effect.mapError((error) =>
|
|
704
|
+
error instanceof JobStore.JobNotFoundError ||
|
|
705
|
+
error instanceof JobStore.JobNotCancellableError ||
|
|
706
|
+
error instanceof JobStore.JobStoreError
|
|
707
|
+
? error
|
|
708
|
+
: storeError("cancel failed")(error)
|
|
709
|
+
),
|
|
710
|
+
Effect.asVoid
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
const enqueueOne = (request: JobStore.EnqueueRequest) =>
|
|
714
|
+
Effect.gen(function*() {
|
|
715
|
+
if (request.dedupe !== undefined) {
|
|
716
|
+
const result = yield* enqueueDeduped(request, request.dedupe)
|
|
717
|
+
if (result.wake) {
|
|
718
|
+
yield* wakeUp("wakeQueue" in result && result.wakeQueue !== undefined ? result.wakeQueue : request.queue)
|
|
719
|
+
}
|
|
720
|
+
return { id: result.id, duplicate: result.duplicate }
|
|
721
|
+
}
|
|
722
|
+
const now = yield* nowDate
|
|
723
|
+
const result = yield* insertJob(db, request, now)
|
|
724
|
+
if (!result.duplicate) {
|
|
725
|
+
yield* wakeUp(request.queue)
|
|
726
|
+
}
|
|
727
|
+
return result
|
|
728
|
+
})
|
|
729
|
+
|
|
730
|
+
// Multi-row insert with every id resolved client-side, so the ON
|
|
731
|
+
// CONFLICT outcome maps back to items unambiguously (RETURNING only
|
|
732
|
+
// yields rows that actually inserted, in no guaranteed order). Auto ids
|
|
733
|
+
// draw from the seq sequence one statement per round; a conflicted auto
|
|
734
|
+
// id (user squatting on "j-<n>", or a colliding generator) re-draws.
|
|
735
|
+
const insertBatch = (requests: ReadonlyArray<JobStore.EnqueueRequest>) =>
|
|
736
|
+
Effect.gen(function*() {
|
|
737
|
+
const now = yield* nowDate
|
|
738
|
+
const generate = options.idGenerator
|
|
739
|
+
const results: Array<JobStore.EnqueueResult | undefined> = requests.map(() => undefined)
|
|
740
|
+
interface PendingItem {
|
|
741
|
+
readonly request: JobStore.EnqueueRequest
|
|
742
|
+
readonly index: number
|
|
743
|
+
id: JobStore.JobId | undefined
|
|
744
|
+
}
|
|
745
|
+
let pending: Array<PendingItem> = requests.map((request, index) => ({ request, index, id: request.id }))
|
|
746
|
+
for (let round = 0; round < 5 && pending.length > 0; round++) {
|
|
747
|
+
const needIds = pending.filter((item) => item.id === undefined)
|
|
748
|
+
if (needIds.length > 0) {
|
|
749
|
+
if (generate !== undefined) {
|
|
750
|
+
for (const item of needIds) {
|
|
751
|
+
const raw = generate(item.request)
|
|
752
|
+
item.id = JobId(Effect.isEffect(raw) ? yield* raw : raw)
|
|
753
|
+
}
|
|
754
|
+
} else {
|
|
755
|
+
const rows = rowsOf(yield* db.execute<{ id: string }>(sql`
|
|
756
|
+
SELECT 'j-' || ${seqExpr}::text AS id FROM generate_series(1, ${needIds.length})
|
|
757
|
+
`).pipe(Effect.mapError(storeError("enqueueMany failed"))))
|
|
758
|
+
for (let i = 0; i < needIds.length; i++) {
|
|
759
|
+
const row = rows[i]
|
|
760
|
+
const item = needIds[i]
|
|
761
|
+
if (row !== undefined && item !== undefined) {
|
|
762
|
+
item.id = JobId(row.id)
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
// Intra-batch repeats: only an id's first occurrence inserts. A
|
|
768
|
+
// later explicit repeat is a duplicate; an auto collision re-draws.
|
|
769
|
+
const seen = new Set<string>()
|
|
770
|
+
const toInsert: Array<{ request: JobStore.EnqueueRequest; index: number; id: JobStore.JobId }> = []
|
|
771
|
+
const stillPending: Array<PendingItem> = []
|
|
772
|
+
for (const item of pending) {
|
|
773
|
+
const id = item.id
|
|
774
|
+
if (id === undefined) continue
|
|
775
|
+
if (seen.has(id)) {
|
|
776
|
+
if (item.request.id !== undefined) {
|
|
777
|
+
results[item.index] = { id, duplicate: true }
|
|
778
|
+
} else {
|
|
779
|
+
item.id = undefined
|
|
780
|
+
stillPending.push(item)
|
|
781
|
+
}
|
|
782
|
+
continue
|
|
783
|
+
}
|
|
784
|
+
seen.add(id)
|
|
785
|
+
toInsert.push({ request: item.request, index: item.index, id })
|
|
786
|
+
}
|
|
787
|
+
pending = stillPending
|
|
788
|
+
// One INSERT per chunk keeps the bind-parameter count bounded.
|
|
789
|
+
for (let start = 0; start < toInsert.length; start += 500) {
|
|
790
|
+
const chunk = toInsert.slice(start, start + 500)
|
|
791
|
+
const values = chunk.map(({ id, request }) =>
|
|
792
|
+
sql`(${id}, ${request.name}, ${request.queue}, ${
|
|
793
|
+
request.delayMs > 0 ? "delayed" : "waiting"
|
|
794
|
+
}, ${request.priority},
|
|
795
|
+
${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
|
|
796
|
+
${request.attemptsMax},
|
|
797
|
+
${request.backoff === undefined ? null : JSON.stringify(request.backoff)}::jsonb,
|
|
798
|
+
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
799
|
+
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
800
|
+
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
801
|
+
${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`
|
|
802
|
+
)
|
|
803
|
+
const rows = rowsOf(yield* db.execute<{ id: string }>(sql`
|
|
804
|
+
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
805
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
806
|
+
VALUES ${sql.join(values, sql`, `)}
|
|
807
|
+
ON CONFLICT (id) DO NOTHING
|
|
808
|
+
RETURNING ${jobs.id} AS id
|
|
809
|
+
`).pipe(Effect.mapError(storeError("enqueueMany failed"))))
|
|
810
|
+
const inserted = new Set(rows.map((row) => row.id))
|
|
811
|
+
// Wake per committed chunk, immediately: a later chunk's failure
|
|
812
|
+
// (or id exhaustion below) must not strand these durable rows
|
|
813
|
+
// unwoken until the poll interval.
|
|
814
|
+
const freshQueues = new Set<JobStore.QueueName>()
|
|
815
|
+
for (const item of chunk) {
|
|
816
|
+
if (inserted.has(item.id)) {
|
|
817
|
+
results[item.index] = { id: item.id, duplicate: false }
|
|
818
|
+
freshQueues.add(item.request.queue)
|
|
819
|
+
} else if (item.request.id !== undefined) {
|
|
820
|
+
results[item.index] = { id: item.id, duplicate: true }
|
|
821
|
+
} else {
|
|
822
|
+
// Auto/generated id squatted by an existing row: re-draw and
|
|
823
|
+
// re-insert next round. NOTE this lands the item after its
|
|
824
|
+
// batch-mates in seq (FIFO) order — acceptable for a
|
|
825
|
+
// pathological collision, and documented on the contract.
|
|
826
|
+
pending.push({ request: item.request, index: item.index, id: undefined })
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
for (const queue of freshQueues) {
|
|
830
|
+
yield* wakeUp(queue)
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
const resolved: Array<JobStore.EnqueueResult> = []
|
|
835
|
+
for (const result of results) {
|
|
836
|
+
if (result === undefined) {
|
|
837
|
+
return yield* new JobStore.JobStoreError({
|
|
838
|
+
message: "enqueueMany failed: could not generate unique job ids"
|
|
839
|
+
})
|
|
840
|
+
}
|
|
841
|
+
resolved.push(result)
|
|
842
|
+
}
|
|
843
|
+
return resolved
|
|
844
|
+
})
|
|
845
|
+
|
|
653
846
|
const store: JobStore.Service = {
|
|
654
|
-
enqueue:
|
|
847
|
+
enqueue: enqueueOne,
|
|
848
|
+
|
|
849
|
+
enqueueMany: (requests) =>
|
|
655
850
|
Effect.gen(function*() {
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
851
|
+
const results: Array<JobStore.EnqueueResult> = []
|
|
852
|
+
let batch: Array<JobStore.EnqueueRequest> = []
|
|
853
|
+
const flush = () =>
|
|
854
|
+
Effect.gen(function*() {
|
|
855
|
+
if (batch.length === 0) return
|
|
856
|
+
const items = batch
|
|
857
|
+
batch = []
|
|
858
|
+
// No spread: a six-figure batch would blow the engine's
|
|
859
|
+
// argument-count limit after the rows already committed.
|
|
860
|
+
for (const result of yield* insertBatch(items)) {
|
|
861
|
+
results.push(result)
|
|
862
|
+
}
|
|
863
|
+
})
|
|
864
|
+
for (const request of requests) {
|
|
865
|
+
// Dedup items run through the transactional single-enqueue path
|
|
866
|
+
// in order; runs of plain items between them batch into one
|
|
867
|
+
// INSERT with multi-row VALUES.
|
|
868
|
+
if (request.dedupe !== undefined) {
|
|
869
|
+
yield* flush()
|
|
870
|
+
results.push(yield* enqueueOne(request))
|
|
871
|
+
} else {
|
|
872
|
+
batch.push(request)
|
|
660
873
|
}
|
|
661
|
-
return { id: result.id, duplicate: result.duplicate }
|
|
662
|
-
}
|
|
663
|
-
const now = yield* nowDate
|
|
664
|
-
const result = yield* insertJob(db, request, now)
|
|
665
|
-
if (!result.duplicate) {
|
|
666
|
-
yield* wakeUp(request.queue)
|
|
667
874
|
}
|
|
668
|
-
|
|
875
|
+
yield* flush()
|
|
876
|
+
return results
|
|
669
877
|
}),
|
|
670
878
|
|
|
671
879
|
claim: (claimOptions) =>
|
|
@@ -709,6 +917,7 @@ export const make = (
|
|
|
709
917
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
710
918
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
711
919
|
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
920
|
+
${jobs.trace} AS "trace",
|
|
712
921
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
713
922
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
714
923
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -1064,59 +1273,24 @@ export const make = (
|
|
|
1064
1273
|
yield* wakeUp(JobStore.QueueName(rows[0]?.queue ?? ""))
|
|
1065
1274
|
}),
|
|
1066
1275
|
|
|
1067
|
-
cancel: (id) =>
|
|
1068
|
-
db.transaction((tx) =>
|
|
1069
|
-
Effect.gen(function*() {
|
|
1070
|
-
const now = yield* nowDate
|
|
1071
|
-
// One guarded statement: waiting/delayed become terminal, active
|
|
1072
|
-
// gets the cancel-request flag; anything else is reported by state.
|
|
1073
|
-
const rows = rowsOf(yield* tx.execute<
|
|
1074
|
-
{
|
|
1075
|
-
id: string
|
|
1076
|
-
state: string
|
|
1077
|
-
processedAt: Date | null
|
|
1078
|
-
name: string
|
|
1079
|
-
keep: JobStore.KeepPolicy | null
|
|
1080
|
-
dedupeKey: string | null
|
|
1081
|
-
}
|
|
1082
|
-
>(sql`
|
|
1083
|
-
UPDATE ${jobs} SET
|
|
1084
|
-
state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN 'cancelled' ELSE ${jobs.state} END,
|
|
1085
|
-
finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
|
|
1086
|
-
cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
|
|
1087
|
-
WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active')
|
|
1088
|
-
RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
|
|
1089
|
-
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
1090
|
-
${jobs.dedupeKey} AS "dedupeKey"
|
|
1091
|
-
`))
|
|
1092
|
-
const row = rows[0]
|
|
1093
|
-
if (row === undefined) {
|
|
1094
|
-
const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
|
|
1095
|
-
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
1096
|
-
`))
|
|
1097
|
-
const found = existing[0]
|
|
1098
|
-
if (found === undefined) {
|
|
1099
|
-
return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
1100
|
-
}
|
|
1101
|
-
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
|
|
1102
|
-
}
|
|
1103
|
-
if (row.state === "cancelled") {
|
|
1104
|
-
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
1105
|
-
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1106
|
-
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
1107
|
-
}
|
|
1108
|
-
})
|
|
1109
|
-
).pipe(
|
|
1110
|
-
Effect.mapError((error) =>
|
|
1111
|
-
error instanceof JobStore.JobNotFoundError ||
|
|
1112
|
-
error instanceof JobStore.JobNotCancellableError ||
|
|
1113
|
-
error instanceof JobStore.JobStoreError
|
|
1114
|
-
? error
|
|
1115
|
-
: storeError("cancel failed")(error)
|
|
1116
|
-
),
|
|
1117
|
-
Effect.asVoid
|
|
1118
|
-
),
|
|
1276
|
+
cancel: (id) => cancelJob(id),
|
|
1119
1277
|
|
|
1278
|
+
cancelByDedupe: (name, key) =>
|
|
1279
|
+
Effect.gen(function*() {
|
|
1280
|
+
const rows = rowsOf(yield* db.execute<{ jobId: string }>(sql`
|
|
1281
|
+
SELECT ${dedupe.jobId} AS "jobId" FROM ${dedupe}
|
|
1282
|
+
WHERE ${dedupe.name} = ${name} AND ${dedupe.key} = ${key}
|
|
1283
|
+
`).pipe(Effect.mapError(storeError("cancelByDedupe failed"))))
|
|
1284
|
+
const jobId = rows[0]?.jobId
|
|
1285
|
+
// "" is the in-transaction placeholder; treat like no entry.
|
|
1286
|
+
if (jobId === undefined || jobId === "") return false
|
|
1287
|
+
return yield* cancelJob(JobId(jobId)).pipe(
|
|
1288
|
+
Effect.as(true),
|
|
1289
|
+
// Idempotent: a vanished or already-terminal keyed job is
|
|
1290
|
+
// "nothing pending", not an error.
|
|
1291
|
+
Effect.catchTag(["JobNotFoundError", "JobNotCancellableError"], () => Effect.succeed(false))
|
|
1292
|
+
)
|
|
1293
|
+
}),
|
|
1120
1294
|
promote: (id) =>
|
|
1121
1295
|
Effect.gen(function*() {
|
|
1122
1296
|
const now = yield* nowDate
|
|
@@ -1251,6 +1425,39 @@ export const make = (
|
|
|
1251
1425
|
Effect.asVoid
|
|
1252
1426
|
),
|
|
1253
1427
|
|
|
1428
|
+
tickSchedule: (key, expectedRunAt, nextRunAt, request) =>
|
|
1429
|
+
Effect.gen(function*() {
|
|
1430
|
+
if (request.id === undefined) {
|
|
1431
|
+
return yield* new JobStore.JobStoreError({
|
|
1432
|
+
message: "tickSchedule requires an explicit request.id"
|
|
1433
|
+
})
|
|
1434
|
+
}
|
|
1435
|
+
// One transaction: the nextRunAt CAS claims the slot and the job
|
|
1436
|
+
// INSERT commits with it, so a stale sweeper can never re-fire an
|
|
1437
|
+
// occurrence — even after retention pruned the previous slot's job.
|
|
1438
|
+
const fired = yield* db.transaction((tx) =>
|
|
1439
|
+
Effect.gen(function*() {
|
|
1440
|
+
const claimed = rowsOf(yield* tx.execute<{ key: string }>(sql`
|
|
1441
|
+
UPDATE ${schedules} SET next_run_at = ${new Date(nextRunAt)}
|
|
1442
|
+
WHERE ${schedules.key} = ${key} AND ${schedules.nextRunAt} = ${new Date(expectedRunAt)}
|
|
1443
|
+
RETURNING ${schedules.key} AS key
|
|
1444
|
+
`))
|
|
1445
|
+
if (claimed.length === 0) return false
|
|
1446
|
+
const now = yield* nowDate
|
|
1447
|
+
const result = yield* insertJob(tx, request, now)
|
|
1448
|
+
// A pre-existing slot row (pre-0.4 crash between enqueue and
|
|
1449
|
+
// advance) still advances the schedule but fires nothing new.
|
|
1450
|
+
return !result.duplicate
|
|
1451
|
+
})
|
|
1452
|
+
).pipe(Effect.mapError((error) =>
|
|
1453
|
+
error instanceof JobStore.JobStoreError ? error : storeError("tickSchedule failed")(error)
|
|
1454
|
+
))
|
|
1455
|
+
if (fired) {
|
|
1456
|
+
yield* wakeUp(request.queue)
|
|
1457
|
+
}
|
|
1458
|
+
return fired
|
|
1459
|
+
}),
|
|
1460
|
+
|
|
1254
1461
|
counts: (queue) =>
|
|
1255
1462
|
db.execute<{ state: JobStore.JobState; count: number }>(sql`
|
|
1256
1463
|
SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
|
|
@@ -55,6 +55,7 @@ type JobState = JobStore.JobState
|
|
|
55
55
|
type BackoffPolicy = JobStore.BackoffPolicy
|
|
56
56
|
type KeepPolicy = JobStore.KeepPolicy
|
|
57
57
|
type AttemptOutcome = JobStore.AttemptRecord["outcome"]
|
|
58
|
+
type TraceContext = JobStore.TraceContext
|
|
58
59
|
|
|
59
60
|
/**
|
|
60
61
|
* Table-factory options: `extraConfig` receives the table's columns (exactly
|
|
@@ -69,10 +70,10 @@ export interface MqTableOptions<Columns extends Record<string, AnyPgColumnBuilde
|
|
|
69
70
|
| undefined
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
const jobsColumns = <JobName extends string>() => ({
|
|
73
|
+
const jobsColumns = <JobName extends string, Queue extends string>() => ({
|
|
73
74
|
id: text("id").primaryKey().$type<JobId>(),
|
|
74
75
|
name: text("name").notNull().$type<JobName>(),
|
|
75
|
-
queue: text("queue").notNull().$type<
|
|
76
|
+
queue: text("queue").notNull().$type<Queue>(),
|
|
76
77
|
state: text("state").notNull().$type<JobState>(),
|
|
77
78
|
priority: integer("priority").notNull().default(0),
|
|
78
79
|
/** FIFO order within a priority; bumped on retry so retries go to the tail. */
|
|
@@ -87,6 +88,7 @@ const jobsColumns = <JobName extends string>() => ({
|
|
|
87
88
|
timeoutMs: bigint("timeout_ms", { mode: "number" }),
|
|
88
89
|
cancelRequested: boolean("cancel_requested").notNull().default(false),
|
|
89
90
|
dedupeKey: text("dedupe_key"),
|
|
91
|
+
trace: jsonb("trace").$type<TraceContext>(),
|
|
90
92
|
runAt: timestamp("run_at", { withTimezone: true, mode: "date" }).notNull(),
|
|
91
93
|
enqueuedAt: timestamp("enqueued_at", { withTimezone: true, mode: "date" }).notNull(),
|
|
92
94
|
processedAt: timestamp("processed_at", { withTimezone: true, mode: "date" }),
|
|
@@ -100,6 +102,8 @@ const jobsColumns = <JobName extends string>() => ({
|
|
|
100
102
|
/**
|
|
101
103
|
* The jobs table factory. `JobName` types the `name` column — derive it from
|
|
102
104
|
* your job definitions: `mqJobs<typeof GenerateInvoice._tag | typeof Report._tag>()`.
|
|
105
|
+
* `Queue` types the `queue` column the same way — pass your own branded type
|
|
106
|
+
* or literal union (defaults to effect-mq's `QueueName` brand).
|
|
103
107
|
*
|
|
104
108
|
* `extend` adds your own columns (tenant ids, object ids, ...) to the table.
|
|
105
109
|
* At enqueue the Postgres store fills each extended column from the job's
|
|
@@ -122,16 +126,17 @@ const jobsColumns = <JobName extends string>() => ({
|
|
|
122
126
|
*/
|
|
123
127
|
export const mqJobs = <
|
|
124
128
|
JobName extends string = string,
|
|
129
|
+
Queue extends string = QueueName,
|
|
125
130
|
Extend extends Record<string, AnyPgColumnBuilder> = Record<never, never>
|
|
126
131
|
>(
|
|
127
132
|
tableName = "effect_mq_jobs",
|
|
128
|
-
options?: MqTableOptions<ReturnType<typeof jobsColumns<JobName>> & Extend> & {
|
|
133
|
+
options?: MqTableOptions<ReturnType<typeof jobsColumns<JobName, Queue>> & Extend> & {
|
|
129
134
|
/** Extra columns appended to the factory's own (see the JSDoc example). */
|
|
130
135
|
readonly extend?: Extend | undefined
|
|
131
136
|
}
|
|
132
137
|
) =>
|
|
133
138
|
pgTable(tableName, {
|
|
134
|
-
...jobsColumns<JobName>(),
|
|
139
|
+
...jobsColumns<JobName, Queue>(),
|
|
135
140
|
// SAFETY: when `extend` is absent, `Extend` was never inferred from a
|
|
136
141
|
// value and stays at its empty-record default, which {} satisfies.
|
|
137
142
|
...options?.extend ?? ({} as Extend)
|
|
@@ -173,7 +178,7 @@ const attemptsColumns = (jobs: MqJobsTable) => ({
|
|
|
173
178
|
* @since 0.1.0
|
|
174
179
|
*/
|
|
175
180
|
export const mqJobAttempts = (
|
|
176
|
-
jobs:
|
|
181
|
+
jobs: MqJobsTable,
|
|
177
182
|
tableName = "effect_mq_job_attempts",
|
|
178
183
|
options?: MqTableOptions<ReturnType<typeof attemptsColumns>>
|
|
179
184
|
) =>
|
|
@@ -182,10 +187,10 @@ export const mqJobAttempts = (
|
|
|
182
187
|
...options?.extraConfig?.(table) ?? []
|
|
183
188
|
])
|
|
184
189
|
|
|
185
|
-
const schedulesColumns = () => ({
|
|
190
|
+
const schedulesColumns = <JobName extends string, Queue extends string>() => ({
|
|
186
191
|
key: text("key").primaryKey().$type<ScheduleKey>(),
|
|
187
|
-
jobName: text("job_name").notNull(),
|
|
188
|
-
queue: text("queue").notNull().$type<
|
|
192
|
+
jobName: text("job_name").notNull().$type<JobName>(),
|
|
193
|
+
queue: text("queue").notNull().$type<Queue>(),
|
|
189
194
|
cron: text("cron"),
|
|
190
195
|
tz: text("tz"),
|
|
191
196
|
everyMs: bigint("every_ms", { mode: "number" }),
|
|
@@ -202,19 +207,22 @@ const schedulesColumns = () => ({
|
|
|
202
207
|
/**
|
|
203
208
|
* Repeatable-job schedules (one row per `Job.schedule` key).
|
|
204
209
|
*
|
|
210
|
+
* `JobName` and `Queue` type the `jobName`/`queue` columns, exactly like
|
|
211
|
+
* `mqJobs` (both default to plain string / effect-mq's `QueueName` brand).
|
|
212
|
+
*
|
|
205
213
|
* @since 0.2.0
|
|
206
214
|
*/
|
|
207
|
-
export const mqSchedules = (
|
|
215
|
+
export const mqSchedules = <JobName extends string = string, Queue extends string = QueueName>(
|
|
208
216
|
tableName = "effect_mq_schedules",
|
|
209
|
-
options?: MqTableOptions<ReturnType<typeof schedulesColumns
|
|
217
|
+
options?: MqTableOptions<ReturnType<typeof schedulesColumns<JobName, Queue>>>
|
|
210
218
|
) =>
|
|
211
|
-
pgTable(tableName, schedulesColumns(), (table) => [
|
|
219
|
+
pgTable(tableName, schedulesColumns<JobName, Queue>(), (table) => [
|
|
212
220
|
index(`${tableName}_due_idx`).on(table.nextRunAt),
|
|
213
221
|
...options?.extraConfig?.(table) ?? []
|
|
214
222
|
])
|
|
215
223
|
|
|
216
|
-
const dedupeColumns = () => ({
|
|
217
|
-
name: text("name").notNull(),
|
|
224
|
+
const dedupeColumns = <JobName extends string>() => ({
|
|
225
|
+
name: text("name").notNull().$type<JobName>(),
|
|
218
226
|
key: text("key").notNull(),
|
|
219
227
|
jobId: text("job_id").notNull().$type<JobId>(),
|
|
220
228
|
/** Set for ttl/throttle windows; NULL rows live as long as their job is pending. */
|
|
@@ -226,17 +234,17 @@ const dedupeColumns = () => ({
|
|
|
226
234
|
*
|
|
227
235
|
* @since 0.3.0
|
|
228
236
|
*/
|
|
229
|
-
export const mqDedupe = (
|
|
237
|
+
export const mqDedupe = <JobName extends string = string>(
|
|
230
238
|
tableName = "effect_mq_dedupe",
|
|
231
|
-
options?: MqTableOptions<ReturnType<typeof dedupeColumns
|
|
239
|
+
options?: MqTableOptions<ReturnType<typeof dedupeColumns<JobName>>>
|
|
232
240
|
) =>
|
|
233
|
-
pgTable(tableName, dedupeColumns(), (table) => [
|
|
241
|
+
pgTable(tableName, dedupeColumns<JobName>(), (table) => [
|
|
234
242
|
primaryKey({ columns: [table.name, table.key] }),
|
|
235
243
|
...options?.extraConfig?.(table) ?? []
|
|
236
244
|
])
|
|
237
245
|
|
|
238
|
-
const queueControlColumns = () => ({
|
|
239
|
-
queue: text("queue").primaryKey().$type<
|
|
246
|
+
const queueControlColumns = <Queue extends string>() => ({
|
|
247
|
+
queue: text("queue").primaryKey().$type<Queue>(),
|
|
240
248
|
paused: boolean("paused").notNull().default(false)
|
|
241
249
|
})
|
|
242
250
|
|
|
@@ -245,18 +253,18 @@ const queueControlColumns = () => ({
|
|
|
245
253
|
*
|
|
246
254
|
* @since 0.2.0
|
|
247
255
|
*/
|
|
248
|
-
export const mqQueueControl = (
|
|
256
|
+
export const mqQueueControl = <Queue extends string = QueueName>(
|
|
249
257
|
tableName = "effect_mq_queue_control",
|
|
250
|
-
options?: MqTableOptions<ReturnType<typeof queueControlColumns
|
|
258
|
+
options?: MqTableOptions<ReturnType<typeof queueControlColumns<Queue>>>
|
|
251
259
|
) =>
|
|
252
|
-
pgTable(tableName, queueControlColumns(), (table) => [
|
|
260
|
+
pgTable(tableName, queueControlColumns<Queue>(), (table) => [
|
|
253
261
|
...options?.extraConfig?.(table) ?? []
|
|
254
262
|
])
|
|
255
263
|
|
|
256
264
|
/**
|
|
257
265
|
* @since 0.1.0
|
|
258
266
|
*/
|
|
259
|
-
export type MqJobsTable = ReturnType<typeof mqJobs<any>>
|
|
267
|
+
export type MqJobsTable = ReturnType<typeof mqJobs<any, any>>
|
|
260
268
|
|
|
261
269
|
/**
|
|
262
270
|
* @since 0.1.0
|
|
@@ -266,14 +274,14 @@ export type MqJobAttemptsTable = ReturnType<typeof mqJobAttempts>
|
|
|
266
274
|
/**
|
|
267
275
|
* @since 0.2.0
|
|
268
276
|
*/
|
|
269
|
-
export type MqSchedulesTable = ReturnType<typeof mqSchedules
|
|
277
|
+
export type MqSchedulesTable = ReturnType<typeof mqSchedules<any, any>>
|
|
270
278
|
|
|
271
279
|
/**
|
|
272
280
|
* @since 0.2.0
|
|
273
281
|
*/
|
|
274
|
-
export type MqQueueControlTable = ReturnType<typeof mqQueueControl
|
|
282
|
+
export type MqQueueControlTable = ReturnType<typeof mqQueueControl<any>>
|
|
275
283
|
|
|
276
284
|
/**
|
|
277
285
|
* @since 0.3.0
|
|
278
286
|
*/
|
|
279
|
-
export type MqDedupeTable = ReturnType<typeof mqDedupe
|
|
287
|
+
export type MqDedupeTable = ReturnType<typeof mqDedupe<any>>
|