@kronos-ts/kysely 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/kysely-dead-letter-queue.d.ts +53 -0
- package/dist/kysely-dead-letter-queue.d.ts.map +1 -0
- package/dist/kysely-dead-letter-queue.js +219 -0
- package/dist/kysely-dead-letter-queue.js.map +1 -0
- package/package.json +3 -3
- package/src/index.ts +5 -0
- package/src/kysely-dead-letter-queue.ts +301 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { kyselyTransactionManager, type KyselyDatabaseLike, type KyselyTransaction, } from "./kysely-transaction-manager.js";
|
|
2
2
|
export { kyselyTokenStore, type KyselyDbLike, } from "./kysely-token-store.js";
|
|
3
|
+
export { kyselyDeadLetterQueue, type KyselyDeadLetterQueueConfig, } from "./kysely-dead-letter-queue.js";
|
|
3
4
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,GACvB,MAAM,iCAAiC,CAAA;AAExC,OAAO,EACL,gBAAgB,EAChB,KAAK,YAAY,GAClB,MAAM,yBAAyB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,GACvB,MAAM,iCAAiC,CAAA;AAExC,OAAO,EACL,gBAAgB,EAChB,KAAK,YAAY,GAClB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EACL,qBAAqB,EACrB,KAAK,2BAA2B,GACjC,MAAM,+BAA+B,CAAA"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,GAGzB,MAAM,iCAAiC,CAAA;AAExC,OAAO,EACL,gBAAgB,GAEjB,MAAM,yBAAyB,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,GAGzB,MAAM,iCAAiC,CAAA;AAExC,OAAO,EACL,gBAAgB,GAEjB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EACL,qBAAqB,GAEtB,MAAM,+BAA+B,CAAA"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { SequencedDeadLetterQueue } from "@kronos-ts/messaging";
|
|
2
|
+
import type { KyselyDbLike } from "./kysely-token-store.js";
|
|
3
|
+
/**
|
|
4
|
+
* Dead letter table interface for Kysely. Users must define this table in
|
|
5
|
+
* their Kysely database interface:
|
|
6
|
+
*
|
|
7
|
+
* ```typescript
|
|
8
|
+
* interface Database {
|
|
9
|
+
* kronos_dead_letters: {
|
|
10
|
+
* dead_letter_id: string
|
|
11
|
+
* processing_group: string
|
|
12
|
+
* sequence_identifier: string
|
|
13
|
+
* sequence_index: number
|
|
14
|
+
* message: string
|
|
15
|
+
* cause_type: string | null
|
|
16
|
+
* cause_message: string | null
|
|
17
|
+
* diagnostics: string
|
|
18
|
+
* enqueued_at: string
|
|
19
|
+
* last_touched: string
|
|
20
|
+
* processing_started: string | null
|
|
21
|
+
* }
|
|
22
|
+
* }
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* Persistent {@link SequencedDeadLetterQueue} backed by Kysely — mirrors the
|
|
26
|
+
* Drizzle reference implementation in query semantics.
|
|
27
|
+
*
|
|
28
|
+
* Like {@link kyselyTokenStore} it reads the active transaction via
|
|
29
|
+
* `getActiveTransaction()`, so enqueue/evict/requeue commit in the **same
|
|
30
|
+
* transaction as the token update** — a crash cannot advance the processor's
|
|
31
|
+
* token while losing the parked letter.
|
|
32
|
+
*
|
|
33
|
+
* The table is shared across processors and partitioned by `processingGroup`.
|
|
34
|
+
* Per-sequence FIFO order is held by a monotonic `sequence_index`; a
|
|
35
|
+
* `processing_started` lease column makes `process()` safe across multiple
|
|
36
|
+
* nodes (Axon parity).
|
|
37
|
+
*/
|
|
38
|
+
export interface KyselyDeadLetterQueueConfig {
|
|
39
|
+
/** The Kysely database instance (or transaction). */
|
|
40
|
+
db: KyselyDbLike;
|
|
41
|
+
/** Processing group (the processor name) this queue serves. */
|
|
42
|
+
processingGroup: string;
|
|
43
|
+
/** Table name. Default: `kronos_dead_letters`. */
|
|
44
|
+
tableName?: string;
|
|
45
|
+
/** Maximum number of sequences. Default: 1024 (Axon parity). */
|
|
46
|
+
maxSequences?: number;
|
|
47
|
+
/** Maximum letters per sequence. Default: 1024 (Axon parity). */
|
|
48
|
+
maxSequenceSize?: number;
|
|
49
|
+
/** Lease duration for in-flight processing, ms. Default: 30000 (Axon parity). */
|
|
50
|
+
claimDurationMs?: number;
|
|
51
|
+
}
|
|
52
|
+
export declare function kyselyDeadLetterQueue(config: KyselyDeadLetterQueueConfig): SequencedDeadLetterQueue;
|
|
53
|
+
//# sourceMappingURL=kysely-dead-letter-queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kysely-dead-letter-queue.d.ts","sourceRoot":"","sources":["../src/kysely-dead-letter-queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA+B,wBAAwB,EAAE,MAAM,sBAAsB,CAAA;AAGjG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,WAAW,2BAA2B;IAC1C,qDAAqD;IACrD,EAAE,EAAE,YAAY,CAAA;IAChB,+DAA+D;IAC/D,eAAe,EAAE,MAAM,CAAA;IACvB,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,iEAAiE;IACjE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,iFAAiF;IACjF,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AA0BD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,wBAAwB,CA6NnG"}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import { getActiveTransaction, DeadLetterQueueOverflowError } from "@kronos-ts/messaging";
|
|
2
|
+
/** Reserved diagnostics key carrying the persistent row id across read → evict/requeue. */
|
|
3
|
+
const DL_ID = "__dlqId";
|
|
4
|
+
let idCounter = 0;
|
|
5
|
+
function newId(group) {
|
|
6
|
+
// Unique within the table: time + per-process counter + group.
|
|
7
|
+
idCounter += 1;
|
|
8
|
+
return `${group}:${Date.now()}:${idCounter}`;
|
|
9
|
+
}
|
|
10
|
+
export function kyselyDeadLetterQueue(config) {
|
|
11
|
+
const { processingGroup } = config;
|
|
12
|
+
const table = config.tableName ?? "kronos_dead_letters";
|
|
13
|
+
const maxSequences = config.maxSequences ?? 1024;
|
|
14
|
+
const maxSequenceSize = config.maxSequenceSize ?? 1024;
|
|
15
|
+
const claimDurationMs = config.claimDurationMs ?? 30000;
|
|
16
|
+
function getDb() {
|
|
17
|
+
return getActiveTransaction() ?? config.db;
|
|
18
|
+
}
|
|
19
|
+
function rowToLetter(row) {
|
|
20
|
+
const cause = new Error(row.cause_message ?? "");
|
|
21
|
+
if (row.cause_type)
|
|
22
|
+
cause.name = row.cause_type;
|
|
23
|
+
return {
|
|
24
|
+
message: JSON.parse(row.message),
|
|
25
|
+
cause,
|
|
26
|
+
enqueuedAt: Number(row.enqueued_at),
|
|
27
|
+
lastTouched: Number(row.last_touched),
|
|
28
|
+
diagnostics: { ...JSON.parse(row.diagnostics), [DL_ID]: row.dead_letter_id },
|
|
29
|
+
sequenceIdentifier: row.sequence_identifier,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function letterToRow(letter, sequenceIndex, deadLetterId) {
|
|
33
|
+
const { [DL_ID]: _omit, ...diagnostics } = letter.diagnostics;
|
|
34
|
+
return {
|
|
35
|
+
dead_letter_id: deadLetterId,
|
|
36
|
+
processing_group: processingGroup,
|
|
37
|
+
sequence_identifier: letter.sequenceIdentifier,
|
|
38
|
+
sequence_index: sequenceIndex,
|
|
39
|
+
message: JSON.stringify(letter.message),
|
|
40
|
+
cause_type: letter.cause.name,
|
|
41
|
+
cause_message: letter.cause.message,
|
|
42
|
+
diagnostics: JSON.stringify(diagnostics),
|
|
43
|
+
enqueued_at: String(letter.enqueuedAt),
|
|
44
|
+
last_touched: String(letter.lastTouched),
|
|
45
|
+
processing_started: null,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
async function sequenceRows(db, seqId) {
|
|
49
|
+
return db.selectFrom(table)
|
|
50
|
+
.selectAll()
|
|
51
|
+
.where("processing_group", "=", processingGroup)
|
|
52
|
+
.where("sequence_identifier", "=", seqId)
|
|
53
|
+
.orderBy("sequence_index", "asc")
|
|
54
|
+
.execute();
|
|
55
|
+
}
|
|
56
|
+
async function distinctSequences(db) {
|
|
57
|
+
const rows = await db.selectFrom(table)
|
|
58
|
+
.select("sequence_identifier")
|
|
59
|
+
.distinct()
|
|
60
|
+
.where("processing_group", "=", processingGroup)
|
|
61
|
+
.execute();
|
|
62
|
+
return rows.map((r) => r.sequence_identifier);
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
async enqueue(letter) {
|
|
66
|
+
const db = getDb();
|
|
67
|
+
const existing = await sequenceRows(db, letter.sequenceIdentifier);
|
|
68
|
+
if (existing.length === 0) {
|
|
69
|
+
if ((await distinctSequences(db)).length >= maxSequences) {
|
|
70
|
+
throw new DeadLetterQueueOverflowError(`max sequences ${maxSequences} reached`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
else if (existing.length >= maxSequenceSize) {
|
|
74
|
+
throw new DeadLetterQueueOverflowError(`sequence "${letter.sequenceIdentifier}" has reached max size ${maxSequenceSize}`);
|
|
75
|
+
}
|
|
76
|
+
const idx = existing.length === 0 ? 0 : existing[existing.length - 1].sequence_index + 1;
|
|
77
|
+
await db.insertInto(table).values(letterToRow(letter, idx, newId(processingGroup))).execute();
|
|
78
|
+
},
|
|
79
|
+
async enqueueIfPresent(sequenceIdentifier, letterSupplier) {
|
|
80
|
+
const db = getDb();
|
|
81
|
+
const existing = await sequenceRows(db, sequenceIdentifier);
|
|
82
|
+
if (existing.length === 0)
|
|
83
|
+
return false;
|
|
84
|
+
if (existing.length >= maxSequenceSize) {
|
|
85
|
+
throw new DeadLetterQueueOverflowError(`sequence "${sequenceIdentifier}" has reached max size ${maxSequenceSize}`);
|
|
86
|
+
}
|
|
87
|
+
const idx = existing[existing.length - 1].sequence_index + 1;
|
|
88
|
+
await db.insertInto(table).values(letterToRow(letterSupplier(), idx, newId(processingGroup))).execute();
|
|
89
|
+
return true;
|
|
90
|
+
},
|
|
91
|
+
async evict(_sequenceIdentifier, letter) {
|
|
92
|
+
const db = getDb();
|
|
93
|
+
const id = letter.diagnostics[DL_ID];
|
|
94
|
+
if (typeof id !== "string")
|
|
95
|
+
return;
|
|
96
|
+
await db.deleteFrom(table)
|
|
97
|
+
.where("processing_group", "=", processingGroup)
|
|
98
|
+
.where("dead_letter_id", "=", id)
|
|
99
|
+
.execute();
|
|
100
|
+
},
|
|
101
|
+
async requeue(letter, update) {
|
|
102
|
+
const db = getDb();
|
|
103
|
+
const id = letter.diagnostics[DL_ID];
|
|
104
|
+
if (typeof id !== "string")
|
|
105
|
+
return;
|
|
106
|
+
const { [DL_ID]: _omit, ...baseDiag } = letter.diagnostics;
|
|
107
|
+
const cause = update?.cause ?? letter.cause;
|
|
108
|
+
const diagnostics = update?.diagnostics ? { ...baseDiag, ...update.diagnostics } : baseDiag;
|
|
109
|
+
await db.updateTable(table)
|
|
110
|
+
.set({
|
|
111
|
+
cause_type: cause.name,
|
|
112
|
+
cause_message: cause.message,
|
|
113
|
+
diagnostics: JSON.stringify(diagnostics),
|
|
114
|
+
last_touched: String(Date.now()),
|
|
115
|
+
})
|
|
116
|
+
.where("processing_group", "=", processingGroup)
|
|
117
|
+
.where("dead_letter_id", "=", id)
|
|
118
|
+
.execute();
|
|
119
|
+
},
|
|
120
|
+
async contains(sequenceIdentifier) {
|
|
121
|
+
const db = getDb();
|
|
122
|
+
const row = await db.selectFrom(table)
|
|
123
|
+
.select("dead_letter_id")
|
|
124
|
+
.where("processing_group", "=", processingGroup)
|
|
125
|
+
.where("sequence_identifier", "=", sequenceIdentifier)
|
|
126
|
+
.limit(1)
|
|
127
|
+
.executeTakeFirst();
|
|
128
|
+
return row != null;
|
|
129
|
+
},
|
|
130
|
+
async deadLetterSequence(sequenceIdentifier) {
|
|
131
|
+
const db = getDb();
|
|
132
|
+
return (await sequenceRows(db, sequenceIdentifier)).map(rowToLetter);
|
|
133
|
+
},
|
|
134
|
+
async sequenceIdentifiers() {
|
|
135
|
+
return distinctSequences(getDb());
|
|
136
|
+
},
|
|
137
|
+
async process(sequenceFilter, processingTask) {
|
|
138
|
+
const db = getDb();
|
|
139
|
+
const candidates = (await distinctSequences(db)).filter(sequenceFilter);
|
|
140
|
+
if (candidates.length === 0)
|
|
141
|
+
return false;
|
|
142
|
+
// Pick the oldest sequence by its head letter's lastTouched, skipping
|
|
143
|
+
// sequences under an unexpired processing lease (multi-node safety).
|
|
144
|
+
const cutoff = Date.now() - claimDurationMs;
|
|
145
|
+
let chosen;
|
|
146
|
+
let oldest = Infinity;
|
|
147
|
+
for (const seqId of candidates) {
|
|
148
|
+
const rows = await sequenceRows(db, seqId);
|
|
149
|
+
if (rows.length === 0)
|
|
150
|
+
continue;
|
|
151
|
+
const head = rows[0];
|
|
152
|
+
const leased = head.processing_started != null && Number(head.processing_started) > cutoff;
|
|
153
|
+
if (leased)
|
|
154
|
+
continue;
|
|
155
|
+
if (Number(head.last_touched) < oldest) {
|
|
156
|
+
oldest = Number(head.last_touched);
|
|
157
|
+
chosen = seqId;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (!chosen)
|
|
161
|
+
return false;
|
|
162
|
+
// Claim the sequence head's lease for the duration of this pass.
|
|
163
|
+
const headRows = await sequenceRows(db, chosen);
|
|
164
|
+
await db.updateTable(table)
|
|
165
|
+
.set({ processing_started: String(Date.now()) })
|
|
166
|
+
.where("processing_group", "=", processingGroup)
|
|
167
|
+
.where("dead_letter_id", "=", headRows[0].dead_letter_id)
|
|
168
|
+
.execute();
|
|
169
|
+
try {
|
|
170
|
+
for (const row of headRows) {
|
|
171
|
+
const letter = rowToLetter(row);
|
|
172
|
+
const decision = await processingTask(letter);
|
|
173
|
+
if (decision.shouldEnqueue) {
|
|
174
|
+
await this.requeue(letter, { cause: decision.cause, diagnostics: decision.diagnostics });
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
await this.evict(chosen, letter);
|
|
178
|
+
}
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
// Release any lease still set on a surviving head.
|
|
183
|
+
const remaining = await sequenceRows(db, chosen);
|
|
184
|
+
if (remaining.length > 0 && remaining[0].processing_started != null) {
|
|
185
|
+
await db.updateTable(table)
|
|
186
|
+
.set({ processing_started: null })
|
|
187
|
+
.where("processing_group", "=", processingGroup)
|
|
188
|
+
.where("dead_letter_id", "=", remaining[0].dead_letter_id)
|
|
189
|
+
.execute();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
async size() {
|
|
194
|
+
const db = getDb();
|
|
195
|
+
const rows = await db.selectFrom(table)
|
|
196
|
+
.select("dead_letter_id")
|
|
197
|
+
.where("processing_group", "=", processingGroup)
|
|
198
|
+
.execute();
|
|
199
|
+
return rows.length;
|
|
200
|
+
},
|
|
201
|
+
async amountOfSequences() {
|
|
202
|
+
return (await distinctSequences(getDb())).length;
|
|
203
|
+
},
|
|
204
|
+
async clear() {
|
|
205
|
+
const db = getDb();
|
|
206
|
+
await db.deleteFrom(table)
|
|
207
|
+
.where("processing_group", "=", processingGroup)
|
|
208
|
+
.execute();
|
|
209
|
+
},
|
|
210
|
+
async isFull(sequenceIdentifier) {
|
|
211
|
+
const db = getDb();
|
|
212
|
+
const rows = await sequenceRows(db, sequenceIdentifier);
|
|
213
|
+
if (rows.length > 0)
|
|
214
|
+
return rows.length >= maxSequenceSize;
|
|
215
|
+
return (await distinctSequences(db)).length >= maxSequences;
|
|
216
|
+
},
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
//# sourceMappingURL=kysely-dead-letter-queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kysely-dead-letter-queue.js","sourceRoot":"","sources":["../src/kysely-dead-letter-queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,MAAM,sBAAsB,CAAA;AAsDzF,2FAA2F;AAC3F,MAAM,KAAK,GAAG,SAAS,CAAA;AAEvB,IAAI,SAAS,GAAG,CAAC,CAAA;AACjB,SAAS,KAAK,CAAC,KAAa;IAC1B,+DAA+D;IAC/D,SAAS,IAAI,CAAC,CAAA;IACd,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,SAAS,EAAE,CAAA;AAC9C,CAAC;AAgBD,MAAM,UAAU,qBAAqB,CAAC,MAAmC;IACvE,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,CAAA;IAClC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,qBAAqB,CAAA;IACvD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAA;IAChD,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI,CAAA;IACtD,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,KAAK,CAAA;IAEvD,SAAS,KAAK;QACZ,OAAO,oBAAoB,EAAqB,IAAI,MAAM,CAAC,EAAE,CAAA;IAC/D,CAAC;IAED,SAAS,WAAW,CAAC,GAAkB;QACrC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAAA;QAChD,IAAI,GAAG,CAAC,UAAU;YAAE,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,UAAU,CAAA;QAC/C,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;YAChC,KAAK;YACL,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC;YACrC,WAAW,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,cAAc,EAAE;YAC5E,kBAAkB,EAAE,GAAG,CAAC,mBAAmB;SAC5C,CAAA;IACH,CAAC;IAED,SAAS,WAAW,CAAC,MAAkB,EAAE,aAAqB,EAAE,YAAoB;QAClF,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,WAAW,EAAE,GAAG,MAAM,CAAC,WAAsC,CAAA;QACxF,OAAO;YACL,cAAc,EAAE,YAAY;YAC5B,gBAAgB,EAAE,eAAe;YACjC,mBAAmB,EAAE,MAAM,CAAC,kBAAkB;YAC9C,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;YACvC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI;YAC7B,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;YACnC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;YACxC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;YACtC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;YACxC,kBAAkB,EAAE,IAAI;SACzB,CAAA;IACH,CAAC;IAED,KAAK,UAAU,YAAY,CAAC,EAAgB,EAAE,KAAa;QACzD,OAAO,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;aACxB,SAAS,EAAE;aACX,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;aAC/C,KAAK,CAAC,qBAAqB,EAAE,GAAG,EAAE,KAAK,CAAC;aACxC,OAAO,CAAC,gBAAgB,EAAE,KAAK,CAAC;aAChC,OAAO,EAAE,CAAA;IACd,CAAC;IAED,KAAK,UAAU,iBAAiB,CAAC,EAAgB;QAC/C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;aACpC,MAAM,CAAC,qBAAqB,CAAC;aAC7B,QAAQ,EAAE;aACV,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;aAC/C,OAAO,EAAE,CAAA;QACZ,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAkC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAA;IAChF,CAAC;IAED,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAA;YAClE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,MAAM,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;oBACzD,MAAM,IAAI,4BAA4B,CAAC,iBAAiB,YAAY,UAAU,CAAC,CAAA;gBACjF,CAAC;YACH,CAAC;iBAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;gBAC9C,MAAM,IAAI,4BAA4B,CACpC,aAAa,MAAM,CAAC,kBAAkB,0BAA0B,eAAe,EAAE,CAClF,CAAA;YACH,CAAC;YACD,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAA;YACxF,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;QAC/F,CAAC;QAED,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,cAAc;YACvD,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;YAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACvC,IAAI,QAAQ,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;gBACvC,MAAM,IAAI,4BAA4B,CACpC,aAAa,kBAAkB,0BAA0B,eAAe,EAAE,CAC3E,CAAA;YACH,CAAC;YACD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAA;YAC5D,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;YACvG,OAAO,IAAI,CAAA;QACb,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,mBAAmB,EAAE,MAAM;YACrC,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,EAAE,GAAI,MAAM,CAAC,WAAuC,CAAC,KAAK,CAAC,CAAA;YACjE,IAAI,OAAO,EAAE,KAAK,QAAQ;gBAAE,OAAM;YAClC,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;iBACvB,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;iBAC/C,KAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,EAAE,CAAC;iBAChC,OAAO,EAAE,CAAA;QACd,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM;YAC1B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,EAAE,GAAI,MAAM,CAAC,WAAuC,CAAC,KAAK,CAAC,CAAA;YACjE,IAAI,OAAO,EAAE,KAAK,QAAQ;gBAAE,OAAM;YAClC,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,WAAsC,CAAA;YACrF,MAAM,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAA;YAC3C,MAAM,WAAW,GAAG,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAA;YAC3F,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC;iBACxB,GAAG,CAAC;gBACH,UAAU,EAAE,KAAK,CAAC,IAAI;gBACtB,aAAa,EAAE,KAAK,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;gBACxC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;aACjC,CAAC;iBACD,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;iBAC/C,KAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,EAAE,CAAC;iBAChC,OAAO,EAAE,CAAA;QACd,CAAC;QAED,KAAK,CAAC,QAAQ,CAAC,kBAAkB;YAC/B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;iBACnC,MAAM,CAAC,gBAAgB,CAAC;iBACxB,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;iBAC/C,KAAK,CAAC,qBAAqB,EAAE,GAAG,EAAE,kBAAkB,CAAC;iBACrD,KAAK,CAAC,CAAC,CAAC;iBACR,gBAAgB,EAAE,CAAA;YACrB,OAAO,GAAG,IAAI,IAAI,CAAA;QACpB,CAAC;QAED,KAAK,CAAC,kBAAkB,CAAC,kBAAkB;YACzC,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,OAAO,CAAC,MAAM,YAAY,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QACtE,CAAC;QAED,KAAK,CAAC,mBAAmB;YACvB,OAAO,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAA;QACnC,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,cAAc;YAC1C,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,UAAU,GAAG,CAAC,MAAM,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;YACvE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YAEzC,sEAAsE;YACtE,qEAAqE;YACrE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAA;YAC3C,IAAI,MAA0B,CAAA;YAC9B,IAAI,MAAM,GAAG,QAAQ,CAAA;YACrB,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;gBAC1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAQ;gBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;gBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,MAAM,CAAA;gBAC1F,IAAI,MAAM;oBAAE,SAAQ;gBACpB,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,MAAM,EAAE,CAAC;oBACvC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;oBAClC,MAAM,GAAG,KAAK,CAAA;gBAChB,CAAC;YACH,CAAC;YACD,IAAI,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAA;YAEzB,iEAAiE;YACjE,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;YAC/C,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC;iBACxB,GAAG,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;iBAC/C,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;iBAC/C,KAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;iBACxD,OAAO,EAAE,CAAA;YAEZ,IAAI,CAAC;gBACH,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAA;oBAC/B,MAAM,QAAQ,GAAoB,MAAM,cAAc,CAAC,MAAM,CAAC,CAAA;oBAC9D,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;wBAC3B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAA;wBACxF,OAAO,IAAI,CAAA;oBACb,CAAC;oBACD,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;gBAClC,CAAC;gBACD,OAAO,IAAI,CAAA;YACb,CAAC;oBAAS,CAAC;gBACT,mDAAmD;gBACnD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;gBAChD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,IAAI,IAAI,EAAE,CAAC;oBACpE,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC;yBACxB,GAAG,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;yBACjC,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;yBAC/C,KAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;yBACzD,OAAO,EAAE,CAAA;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI;YACR,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;iBACpC,MAAM,CAAC,gBAAgB,CAAC;iBACxB,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;iBAC/C,OAAO,EAAE,CAAA;YACZ,OAAO,IAAI,CAAC,MAAM,CAAA;QACpB,CAAC;QAED,KAAK,CAAC,iBAAiB;YACrB,OAAO,CAAC,MAAM,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAA;QAClD,CAAC;QAED,KAAK,CAAC,KAAK;YACT,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;iBACvB,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,eAAe,CAAC;iBAC/C,OAAO,EAAE,CAAA;QACd,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,kBAAkB;YAC7B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;YAClB,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;YACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC,MAAM,IAAI,eAAe,CAAA;YAC1D,OAAO,CAAC,MAAM,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,YAAY,CAAA;QAC7D,CAAC;KACF,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kronos-ts/kysely",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Kysely extension for Kronos.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -48,8 +48,8 @@
|
|
|
48
48
|
}
|
|
49
49
|
},
|
|
50
50
|
"dependencies": {
|
|
51
|
-
"@kronos-ts/common": "0.1.
|
|
52
|
-
"@kronos-ts/messaging": "0.
|
|
51
|
+
"@kronos-ts/common": "0.1.1",
|
|
52
|
+
"@kronos-ts/messaging": "0.4.0"
|
|
53
53
|
},
|
|
54
54
|
"peerDependencies": {
|
|
55
55
|
"kysely": ">=0.27.0"
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
import type { DeadLetter, EnqueueDecision, SequencedDeadLetterQueue } from "@kronos-ts/messaging"
|
|
2
|
+
import { getActiveTransaction, DeadLetterQueueOverflowError } from "@kronos-ts/messaging"
|
|
3
|
+
import type { KyselyTransaction } from "./kysely-transaction-manager.js"
|
|
4
|
+
import type { KyselyDbLike } from "./kysely-token-store.js"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Dead letter table interface for Kysely. Users must define this table in
|
|
8
|
+
* their Kysely database interface:
|
|
9
|
+
*
|
|
10
|
+
* ```typescript
|
|
11
|
+
* interface Database {
|
|
12
|
+
* kronos_dead_letters: {
|
|
13
|
+
* dead_letter_id: string
|
|
14
|
+
* processing_group: string
|
|
15
|
+
* sequence_identifier: string
|
|
16
|
+
* sequence_index: number
|
|
17
|
+
* message: string
|
|
18
|
+
* cause_type: string | null
|
|
19
|
+
* cause_message: string | null
|
|
20
|
+
* diagnostics: string
|
|
21
|
+
* enqueued_at: string
|
|
22
|
+
* last_touched: string
|
|
23
|
+
* processing_started: string | null
|
|
24
|
+
* }
|
|
25
|
+
* }
|
|
26
|
+
* ```
|
|
27
|
+
*
|
|
28
|
+
* Persistent {@link SequencedDeadLetterQueue} backed by Kysely — mirrors the
|
|
29
|
+
* Drizzle reference implementation in query semantics.
|
|
30
|
+
*
|
|
31
|
+
* Like {@link kyselyTokenStore} it reads the active transaction via
|
|
32
|
+
* `getActiveTransaction()`, so enqueue/evict/requeue commit in the **same
|
|
33
|
+
* transaction as the token update** — a crash cannot advance the processor's
|
|
34
|
+
* token while losing the parked letter.
|
|
35
|
+
*
|
|
36
|
+
* The table is shared across processors and partitioned by `processingGroup`.
|
|
37
|
+
* Per-sequence FIFO order is held by a monotonic `sequence_index`; a
|
|
38
|
+
* `processing_started` lease column makes `process()` safe across multiple
|
|
39
|
+
* nodes (Axon parity).
|
|
40
|
+
*/
|
|
41
|
+
export interface KyselyDeadLetterQueueConfig {
|
|
42
|
+
/** The Kysely database instance (or transaction). */
|
|
43
|
+
db: KyselyDbLike
|
|
44
|
+
/** Processing group (the processor name) this queue serves. */
|
|
45
|
+
processingGroup: string
|
|
46
|
+
/** Table name. Default: `kronos_dead_letters`. */
|
|
47
|
+
tableName?: string
|
|
48
|
+
/** Maximum number of sequences. Default: 1024 (Axon parity). */
|
|
49
|
+
maxSequences?: number
|
|
50
|
+
/** Maximum letters per sequence. Default: 1024 (Axon parity). */
|
|
51
|
+
maxSequenceSize?: number
|
|
52
|
+
/** Lease duration for in-flight processing, ms. Default: 30000 (Axon parity). */
|
|
53
|
+
claimDurationMs?: number
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Reserved diagnostics key carrying the persistent row id across read → evict/requeue. */
|
|
57
|
+
const DL_ID = "__dlqId"
|
|
58
|
+
|
|
59
|
+
let idCounter = 0
|
|
60
|
+
function newId(group: string): string {
|
|
61
|
+
// Unique within the table: time + per-process counter + group.
|
|
62
|
+
idCounter += 1
|
|
63
|
+
return `${group}:${Date.now()}:${idCounter}`
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
interface DeadLetterRow {
|
|
67
|
+
dead_letter_id: string
|
|
68
|
+
processing_group: string
|
|
69
|
+
sequence_identifier: string
|
|
70
|
+
sequence_index: number
|
|
71
|
+
message: string
|
|
72
|
+
cause_type: string | null
|
|
73
|
+
cause_message: string | null
|
|
74
|
+
diagnostics: string
|
|
75
|
+
enqueued_at: string
|
|
76
|
+
last_touched: string
|
|
77
|
+
processing_started: string | null
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function kyselyDeadLetterQueue(config: KyselyDeadLetterQueueConfig): SequencedDeadLetterQueue {
|
|
81
|
+
const { processingGroup } = config
|
|
82
|
+
const table = config.tableName ?? "kronos_dead_letters"
|
|
83
|
+
const maxSequences = config.maxSequences ?? 1024
|
|
84
|
+
const maxSequenceSize = config.maxSequenceSize ?? 1024
|
|
85
|
+
const claimDurationMs = config.claimDurationMs ?? 30000
|
|
86
|
+
|
|
87
|
+
function getDb(): KyselyDbLike {
|
|
88
|
+
return getActiveTransaction<KyselyTransaction>() ?? config.db
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function rowToLetter(row: DeadLetterRow): DeadLetter {
|
|
92
|
+
const cause = new Error(row.cause_message ?? "")
|
|
93
|
+
if (row.cause_type) cause.name = row.cause_type
|
|
94
|
+
return {
|
|
95
|
+
message: JSON.parse(row.message),
|
|
96
|
+
cause,
|
|
97
|
+
enqueuedAt: Number(row.enqueued_at),
|
|
98
|
+
lastTouched: Number(row.last_touched),
|
|
99
|
+
diagnostics: { ...JSON.parse(row.diagnostics), [DL_ID]: row.dead_letter_id },
|
|
100
|
+
sequenceIdentifier: row.sequence_identifier,
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function letterToRow(letter: DeadLetter, sequenceIndex: number, deadLetterId: string): DeadLetterRow {
|
|
105
|
+
const { [DL_ID]: _omit, ...diagnostics } = letter.diagnostics as Record<string, unknown>
|
|
106
|
+
return {
|
|
107
|
+
dead_letter_id: deadLetterId,
|
|
108
|
+
processing_group: processingGroup,
|
|
109
|
+
sequence_identifier: letter.sequenceIdentifier,
|
|
110
|
+
sequence_index: sequenceIndex,
|
|
111
|
+
message: JSON.stringify(letter.message),
|
|
112
|
+
cause_type: letter.cause.name,
|
|
113
|
+
cause_message: letter.cause.message,
|
|
114
|
+
diagnostics: JSON.stringify(diagnostics),
|
|
115
|
+
enqueued_at: String(letter.enqueuedAt),
|
|
116
|
+
last_touched: String(letter.lastTouched),
|
|
117
|
+
processing_started: null,
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function sequenceRows(db: KyselyDbLike, seqId: string): Promise<DeadLetterRow[]> {
|
|
122
|
+
return db.selectFrom(table)
|
|
123
|
+
.selectAll()
|
|
124
|
+
.where("processing_group", "=", processingGroup)
|
|
125
|
+
.where("sequence_identifier", "=", seqId)
|
|
126
|
+
.orderBy("sequence_index", "asc")
|
|
127
|
+
.execute()
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
async function distinctSequences(db: KyselyDbLike): Promise<string[]> {
|
|
131
|
+
const rows = await db.selectFrom(table)
|
|
132
|
+
.select("sequence_identifier")
|
|
133
|
+
.distinct()
|
|
134
|
+
.where("processing_group", "=", processingGroup)
|
|
135
|
+
.execute()
|
|
136
|
+
return rows.map((r: { sequence_identifier: string }) => r.sequence_identifier)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return {
|
|
140
|
+
async enqueue(letter) {
|
|
141
|
+
const db = getDb()
|
|
142
|
+
const existing = await sequenceRows(db, letter.sequenceIdentifier)
|
|
143
|
+
if (existing.length === 0) {
|
|
144
|
+
if ((await distinctSequences(db)).length >= maxSequences) {
|
|
145
|
+
throw new DeadLetterQueueOverflowError(`max sequences ${maxSequences} reached`)
|
|
146
|
+
}
|
|
147
|
+
} else if (existing.length >= maxSequenceSize) {
|
|
148
|
+
throw new DeadLetterQueueOverflowError(
|
|
149
|
+
`sequence "${letter.sequenceIdentifier}" has reached max size ${maxSequenceSize}`,
|
|
150
|
+
)
|
|
151
|
+
}
|
|
152
|
+
const idx = existing.length === 0 ? 0 : existing[existing.length - 1].sequence_index + 1
|
|
153
|
+
await db.insertInto(table).values(letterToRow(letter, idx, newId(processingGroup))).execute()
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
async enqueueIfPresent(sequenceIdentifier, letterSupplier) {
|
|
157
|
+
const db = getDb()
|
|
158
|
+
const existing = await sequenceRows(db, sequenceIdentifier)
|
|
159
|
+
if (existing.length === 0) return false
|
|
160
|
+
if (existing.length >= maxSequenceSize) {
|
|
161
|
+
throw new DeadLetterQueueOverflowError(
|
|
162
|
+
`sequence "${sequenceIdentifier}" has reached max size ${maxSequenceSize}`,
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
const idx = existing[existing.length - 1].sequence_index + 1
|
|
166
|
+
await db.insertInto(table).values(letterToRow(letterSupplier(), idx, newId(processingGroup))).execute()
|
|
167
|
+
return true
|
|
168
|
+
},
|
|
169
|
+
|
|
170
|
+
async evict(_sequenceIdentifier, letter) {
|
|
171
|
+
const db = getDb()
|
|
172
|
+
const id = (letter.diagnostics as Record<string, unknown>)[DL_ID]
|
|
173
|
+
if (typeof id !== "string") return
|
|
174
|
+
await db.deleteFrom(table)
|
|
175
|
+
.where("processing_group", "=", processingGroup)
|
|
176
|
+
.where("dead_letter_id", "=", id)
|
|
177
|
+
.execute()
|
|
178
|
+
},
|
|
179
|
+
|
|
180
|
+
async requeue(letter, update) {
|
|
181
|
+
const db = getDb()
|
|
182
|
+
const id = (letter.diagnostics as Record<string, unknown>)[DL_ID]
|
|
183
|
+
if (typeof id !== "string") return
|
|
184
|
+
const { [DL_ID]: _omit, ...baseDiag } = letter.diagnostics as Record<string, unknown>
|
|
185
|
+
const cause = update?.cause ?? letter.cause
|
|
186
|
+
const diagnostics = update?.diagnostics ? { ...baseDiag, ...update.diagnostics } : baseDiag
|
|
187
|
+
await db.updateTable(table)
|
|
188
|
+
.set({
|
|
189
|
+
cause_type: cause.name,
|
|
190
|
+
cause_message: cause.message,
|
|
191
|
+
diagnostics: JSON.stringify(diagnostics),
|
|
192
|
+
last_touched: String(Date.now()),
|
|
193
|
+
})
|
|
194
|
+
.where("processing_group", "=", processingGroup)
|
|
195
|
+
.where("dead_letter_id", "=", id)
|
|
196
|
+
.execute()
|
|
197
|
+
},
|
|
198
|
+
|
|
199
|
+
async contains(sequenceIdentifier) {
|
|
200
|
+
const db = getDb()
|
|
201
|
+
const row = await db.selectFrom(table)
|
|
202
|
+
.select("dead_letter_id")
|
|
203
|
+
.where("processing_group", "=", processingGroup)
|
|
204
|
+
.where("sequence_identifier", "=", sequenceIdentifier)
|
|
205
|
+
.limit(1)
|
|
206
|
+
.executeTakeFirst()
|
|
207
|
+
return row != null
|
|
208
|
+
},
|
|
209
|
+
|
|
210
|
+
async deadLetterSequence(sequenceIdentifier) {
|
|
211
|
+
const db = getDb()
|
|
212
|
+
return (await sequenceRows(db, sequenceIdentifier)).map(rowToLetter)
|
|
213
|
+
},
|
|
214
|
+
|
|
215
|
+
async sequenceIdentifiers() {
|
|
216
|
+
return distinctSequences(getDb())
|
|
217
|
+
},
|
|
218
|
+
|
|
219
|
+
async process(sequenceFilter, processingTask) {
|
|
220
|
+
const db = getDb()
|
|
221
|
+
const candidates = (await distinctSequences(db)).filter(sequenceFilter)
|
|
222
|
+
if (candidates.length === 0) return false
|
|
223
|
+
|
|
224
|
+
// Pick the oldest sequence by its head letter's lastTouched, skipping
|
|
225
|
+
// sequences under an unexpired processing lease (multi-node safety).
|
|
226
|
+
const cutoff = Date.now() - claimDurationMs
|
|
227
|
+
let chosen: string | undefined
|
|
228
|
+
let oldest = Infinity
|
|
229
|
+
for (const seqId of candidates) {
|
|
230
|
+
const rows = await sequenceRows(db, seqId)
|
|
231
|
+
if (rows.length === 0) continue
|
|
232
|
+
const head = rows[0]
|
|
233
|
+
const leased = head.processing_started != null && Number(head.processing_started) > cutoff
|
|
234
|
+
if (leased) continue
|
|
235
|
+
if (Number(head.last_touched) < oldest) {
|
|
236
|
+
oldest = Number(head.last_touched)
|
|
237
|
+
chosen = seqId
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (!chosen) return false
|
|
241
|
+
|
|
242
|
+
// Claim the sequence head's lease for the duration of this pass.
|
|
243
|
+
const headRows = await sequenceRows(db, chosen)
|
|
244
|
+
await db.updateTable(table)
|
|
245
|
+
.set({ processing_started: String(Date.now()) })
|
|
246
|
+
.where("processing_group", "=", processingGroup)
|
|
247
|
+
.where("dead_letter_id", "=", headRows[0].dead_letter_id)
|
|
248
|
+
.execute()
|
|
249
|
+
|
|
250
|
+
try {
|
|
251
|
+
for (const row of headRows) {
|
|
252
|
+
const letter = rowToLetter(row)
|
|
253
|
+
const decision: EnqueueDecision = await processingTask(letter)
|
|
254
|
+
if (decision.shouldEnqueue) {
|
|
255
|
+
await this.requeue(letter, { cause: decision.cause, diagnostics: decision.diagnostics })
|
|
256
|
+
return true
|
|
257
|
+
}
|
|
258
|
+
await this.evict(chosen, letter)
|
|
259
|
+
}
|
|
260
|
+
return true
|
|
261
|
+
} finally {
|
|
262
|
+
// Release any lease still set on a surviving head.
|
|
263
|
+
const remaining = await sequenceRows(db, chosen)
|
|
264
|
+
if (remaining.length > 0 && remaining[0].processing_started != null) {
|
|
265
|
+
await db.updateTable(table)
|
|
266
|
+
.set({ processing_started: null })
|
|
267
|
+
.where("processing_group", "=", processingGroup)
|
|
268
|
+
.where("dead_letter_id", "=", remaining[0].dead_letter_id)
|
|
269
|
+
.execute()
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
},
|
|
273
|
+
|
|
274
|
+
async size() {
|
|
275
|
+
const db = getDb()
|
|
276
|
+
const rows = await db.selectFrom(table)
|
|
277
|
+
.select("dead_letter_id")
|
|
278
|
+
.where("processing_group", "=", processingGroup)
|
|
279
|
+
.execute()
|
|
280
|
+
return rows.length
|
|
281
|
+
},
|
|
282
|
+
|
|
283
|
+
async amountOfSequences() {
|
|
284
|
+
return (await distinctSequences(getDb())).length
|
|
285
|
+
},
|
|
286
|
+
|
|
287
|
+
async clear() {
|
|
288
|
+
const db = getDb()
|
|
289
|
+
await db.deleteFrom(table)
|
|
290
|
+
.where("processing_group", "=", processingGroup)
|
|
291
|
+
.execute()
|
|
292
|
+
},
|
|
293
|
+
|
|
294
|
+
async isFull(sequenceIdentifier) {
|
|
295
|
+
const db = getDb()
|
|
296
|
+
const rows = await sequenceRows(db, sequenceIdentifier)
|
|
297
|
+
if (rows.length > 0) return rows.length >= maxSequenceSize
|
|
298
|
+
return (await distinctSequences(db)).length >= maxSequences
|
|
299
|
+
},
|
|
300
|
+
}
|
|
301
|
+
}
|