@iterativeflow/mysql 2.7.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +11 -4
- package/dist/index.mjs +208 -149
- package/package.json +2 -2
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Backend, IdGen } from "@iterativeflow/core/backend";
|
|
1
|
+
import { Backend, IdGen, Json } from "@iterativeflow/core/backend";
|
|
2
2
|
import { Pool } from "mysql2/promise";
|
|
3
3
|
//#region src/sql.d.ts
|
|
4
4
|
/** What a write reports — MySQL has no `RETURNING`, so first-writer-wins reads `affectedRows`. */
|
|
@@ -6,6 +6,13 @@ interface WriteResult {
|
|
|
6
6
|
affectedRows: number;
|
|
7
7
|
insertId: number;
|
|
8
8
|
}
|
|
9
|
+
type SqlScalar = string | number | boolean | Date | null;
|
|
10
|
+
/** A value bound to a positional `?`; an array expands to a list (`IN (?)`). */
|
|
11
|
+
type SqlParam = SqlScalar | readonly SqlScalar[];
|
|
12
|
+
/** A column value as `mysql2` decodes it. */
|
|
13
|
+
type SqlValue = Json | bigint | Date | Buffer;
|
|
14
|
+
/** One result row, keyed by column name or alias. */
|
|
15
|
+
type SqlRow = Readonly<Record<string, SqlValue>>;
|
|
9
16
|
/**
|
|
10
17
|
* The minimal SQL surface the backend needs: positional-`?` `query` for reads, `exec` for writes
|
|
11
18
|
* (returning `affectedRows`, since MySQL has no `RETURNING`), and a `tx` that runs a unit of work on
|
|
@@ -14,8 +21,8 @@ interface WriteResult {
|
|
|
14
21
|
* gives contention-free batch claims.
|
|
15
22
|
*/
|
|
16
23
|
interface Sql {
|
|
17
|
-
query
|
|
18
|
-
exec(text: string, params?: readonly
|
|
24
|
+
query(text: string, params?: readonly SqlParam[]): Promise<SqlRow[]>;
|
|
25
|
+
exec(text: string, params?: readonly SqlParam[]): Promise<WriteResult>;
|
|
19
26
|
tx<T>(fn: (t: Sql) => Promise<T>): Promise<T>;
|
|
20
27
|
}
|
|
21
28
|
/** Options for {@link mysqlPool}. */
|
|
@@ -80,4 +87,4 @@ declare const applySchema: (sql: Sql, prefix?: string) => Promise<void>;
|
|
|
80
87
|
*/
|
|
81
88
|
declare const inTx: <T>(pool: Pool, fn: (backend: Backend, tx: Sql) => Promise<T>, opts?: MysqlBackendOpts) => Promise<T>;
|
|
82
89
|
//#endregion
|
|
83
|
-
export { type MysqlBackendOpts, type Sql, applySchema, createMysqlBackend, ddl, inTx, mysqlPool };
|
|
90
|
+
export { type MysqlBackendOpts, type Sql, type SqlParam, type SqlRow, type SqlValue, applySchema, createMysqlBackend, ddl, inTx, mysqlPool };
|
package/dist/index.mjs
CHANGED
|
@@ -1,16 +1,147 @@
|
|
|
1
|
-
import { ACTIVE_STATUSES, NON_SUCCESS_TERMINAL_STATUSES, RECONCILABLE_STATUSES, TERMINAL_STATUSES, assertSqlIdentifier, createLocalWakeup, isTerminal, newId, orphanedRunsSql, purgeWhereSql, queueDepthOf, runSetWhereSql, statusList, zeroRunStats } from "@iterativeflow/core/backend";
|
|
1
|
+
import { ACTIVE_STATUSES, CRON_OVERLAPS, NON_SUCCESS_TERMINAL_STATUSES, RECONCILABLE_STATUSES, RUN_STATUSES, STEP_STATUSES, TERMINAL_STATUSES, assertSqlIdentifier, createLocalWakeup, decodeFlowError, decodeOneOf, decodeTags, distinctEnqueues, isTerminal, newId, orphanedRunsSql, purgeWhereSql, queueDepthOf, runSetWhereSql, statusList, zeroRunStats } from "@iterativeflow/core/backend";
|
|
2
|
+
//#region src/codec.ts
|
|
3
|
+
/** @internal */
|
|
4
|
+
const j = (value) => value === void 0 ? null : JSON.stringify(value);
|
|
5
|
+
const cell = (row, column) => {
|
|
6
|
+
const value = row[column];
|
|
7
|
+
if (value === void 0) throw new Error(`mysql: result has no column "${column}"`);
|
|
8
|
+
return value;
|
|
9
|
+
};
|
|
10
|
+
/** @internal */
|
|
11
|
+
const textOrNull = (row, column) => {
|
|
12
|
+
const value = cell(row, column);
|
|
13
|
+
return value === null ? null : String(value);
|
|
14
|
+
};
|
|
15
|
+
/** @internal */
|
|
16
|
+
const text = (row, column) => {
|
|
17
|
+
const value = textOrNull(row, column);
|
|
18
|
+
if (value === null) throw new Error(`mysql: column "${column}" is NULL`);
|
|
19
|
+
return value;
|
|
20
|
+
};
|
|
21
|
+
/** @internal */
|
|
22
|
+
const intOrNull = (row, column) => {
|
|
23
|
+
const value = cell(row, column);
|
|
24
|
+
return value === null ? null : Number(value);
|
|
25
|
+
};
|
|
26
|
+
/** @internal */
|
|
27
|
+
const int = (row, column) => Number(text(row, column));
|
|
28
|
+
const oneOf = (allowed, row, column) => decodeOneOf(allowed, text(row, column), `mysql: column "${column}"`);
|
|
29
|
+
/** @internal */
|
|
30
|
+
const runStatus = (row) => oneOf(RUN_STATUSES, row, "status");
|
|
31
|
+
const optionalText = (row, column) => textOrNull(row, column) ?? void 0;
|
|
32
|
+
const json = (row, column) => {
|
|
33
|
+
const value = textOrNull(row, column);
|
|
34
|
+
return value === null ? void 0 : JSON.parse(value);
|
|
35
|
+
};
|
|
36
|
+
const at = (row, column) => new Date(int(row, column));
|
|
37
|
+
const mapRun = (r) => ({
|
|
38
|
+
id: text(r, "id"),
|
|
39
|
+
name: text(r, "name"),
|
|
40
|
+
version: int(r, "version"),
|
|
41
|
+
status: runStatus(r),
|
|
42
|
+
input: json(r, "input"),
|
|
43
|
+
attempts: int(r, "attempts"),
|
|
44
|
+
output: json(r, "output"),
|
|
45
|
+
error: decodeFlowError(json(r, "error"), `mysql: column "error"`),
|
|
46
|
+
idempotencyKey: optionalText(r, "idempotency_key"),
|
|
47
|
+
tags: decodeTags(json(r, "tags"), `mysql: column "tags"`),
|
|
48
|
+
parentRunId: optionalText(r, "parent_run_id"),
|
|
49
|
+
parentCursorKey: optionalText(r, "parent_cursor_key"),
|
|
50
|
+
depth: int(r, "depth"),
|
|
51
|
+
createdAt: at(r, "created_at")
|
|
52
|
+
});
|
|
53
|
+
/** @internal */
|
|
54
|
+
const STEP_COLUMNS = "status, result, error, attempts, shape AS memo_call";
|
|
55
|
+
const mapStep = (r) => ({
|
|
56
|
+
status: oneOf(STEP_STATUSES, r, "status"),
|
|
57
|
+
result: json(r, "result"),
|
|
58
|
+
error: decodeFlowError(json(r, "error"), `mysql: column "error"`),
|
|
59
|
+
attempts: int(r, "attempts"),
|
|
60
|
+
call: optionalText(r, "memo_call")
|
|
61
|
+
});
|
|
62
|
+
const mapSignal = (r) => ({
|
|
63
|
+
id: text(r, "id"),
|
|
64
|
+
name: text(r, "name"),
|
|
65
|
+
payload: json(r, "payload")
|
|
66
|
+
});
|
|
67
|
+
const mapCron = (r) => {
|
|
68
|
+
const lastRunAt = intOrNull(r, "last_run_at");
|
|
69
|
+
return {
|
|
70
|
+
name: text(r, "name"),
|
|
71
|
+
schedule: text(r, "schedule"),
|
|
72
|
+
flowName: text(r, "flow_name"),
|
|
73
|
+
flowVersion: int(r, "flow_version"),
|
|
74
|
+
input: json(r, "input"),
|
|
75
|
+
overlap: oneOf(CRON_OVERLAPS, r, "overlap"),
|
|
76
|
+
nextRunAt: at(r, "next_run_at"),
|
|
77
|
+
lastRunAt: lastRunAt === null ? void 0 : new Date(lastRunAt)
|
|
78
|
+
};
|
|
79
|
+
};
|
|
80
|
+
//#endregion
|
|
81
|
+
//#region src/statements.ts
|
|
82
|
+
const ENQUEUE_ROWS_PER_STATEMENT = 1e3;
|
|
83
|
+
const inList$1 = (n) => `(${Array.from({ length: n }, () => "?").join(",")})`;
|
|
84
|
+
/** @internal */
|
|
85
|
+
const enqueueManyStmt = async (sql, t, requests) => {
|
|
86
|
+
const rows = distinctEnqueues(requests);
|
|
87
|
+
for (let i = 0; i < rows.length; i += ENQUEUE_ROWS_PER_STATEMENT) {
|
|
88
|
+
const chunk = rows.slice(i, i + ENQUEUE_ROWS_PER_STATEMENT);
|
|
89
|
+
await sql.exec(`INSERT INTO ${t.job} (run_id, run_at, priority, version)
|
|
90
|
+
VALUES ${chunk.map(() => `(?, ?, COALESCE(?, (SELECT priority FROM ${t.run} WHERE id = ?), 0), 1)`).join(", ")}
|
|
91
|
+
ON DUPLICATE KEY UPDATE run_at = VALUES(run_at), priority = VALUES(priority), version = version + 1`, chunk.flatMap(([runId, opts]) => [
|
|
92
|
+
runId,
|
|
93
|
+
opts?.runAt ? opts.runAt.getTime() : 0,
|
|
94
|
+
opts?.priority ?? null,
|
|
95
|
+
runId
|
|
96
|
+
]));
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
/** @internal */
|
|
100
|
+
const enqueueStmt = (sql, t, runId, opts) => enqueueManyStmt(sql, t, [{
|
|
101
|
+
runId,
|
|
102
|
+
opts
|
|
103
|
+
}]);
|
|
104
|
+
/** @internal */
|
|
105
|
+
const scheduleStmt = (sql, t, runId, fireAt) => sql.exec(`INSERT INTO ${t.timer} (run_id, fire_at) VALUES (?, ?)
|
|
106
|
+
ON DUPLICATE KEY UPDATE fire_at = VALUES(fire_at)`, [runId, fireAt.getTime()]);
|
|
107
|
+
/** @internal */
|
|
108
|
+
const applyOutbox = async (sql, t, fx) => {
|
|
109
|
+
for (const s of fx.spawn ?? []) await sql.exec(`INSERT IGNORE INTO ${t.run}
|
|
110
|
+
(id, name, version, status, input, idempotency_key, tags, parent_run_id, parent_cursor_key, depth, priority, created_at)
|
|
111
|
+
VALUES (?, ?, ?, 'pending', ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
112
|
+
s.runId,
|
|
113
|
+
s.spec.name,
|
|
114
|
+
s.spec.version,
|
|
115
|
+
j(s.spec.input),
|
|
116
|
+
s.spec.idempotencyKey ?? null,
|
|
117
|
+
j(s.spec.tags),
|
|
118
|
+
s.spec.parentRunId ?? null,
|
|
119
|
+
s.spec.parentCursorKey ?? null,
|
|
120
|
+
s.spec.depth ?? 0,
|
|
121
|
+
s.spec.priority ?? 0,
|
|
122
|
+
(s.spec.createdAt ?? /* @__PURE__ */ new Date()).getTime()
|
|
123
|
+
]);
|
|
124
|
+
const enqueues = [...(fx.spawn ?? []).map((s) => ({
|
|
125
|
+
runId: s.runId,
|
|
126
|
+
opts: s.enqueue
|
|
127
|
+
})), ...fx.enqueue ?? []];
|
|
128
|
+
if (enqueues.length) await enqueueManyStmt(sql, t, enqueues);
|
|
129
|
+
for (const tm of fx.timers ?? []) await scheduleStmt(sql, t, tm.runId, tm.fireAt);
|
|
130
|
+
if (fx.cancelTimers?.length) await sql.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inList$1(fx.cancelTimers.length)}`, fx.cancelTimers);
|
|
131
|
+
if (fx.consumeSignals?.length) await sql.exec(`UPDATE ${t.signal} SET consumed = 1 WHERE id IN ${inList$1(fx.consumeSignals.length)}`, fx.consumeSignals);
|
|
132
|
+
if (fx.joinTarget) await sql.exec(`UPDATE ${t.run} SET join_remaining = ? WHERE id = ?`, [fx.joinTarget.count, fx.joinTarget.runId]);
|
|
133
|
+
};
|
|
134
|
+
//#endregion
|
|
2
135
|
//#region src/queue.ts
|
|
3
136
|
/** @internal */
|
|
4
137
|
const createMysqlQueue = (sql, t, id) => {
|
|
5
138
|
const ms = (d) => (d ?? /* @__PURE__ */ new Date()).getTime();
|
|
6
139
|
return {
|
|
7
140
|
async enqueue(runId, opts) {
|
|
8
|
-
await sql
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
opts?.priority ?? 0
|
|
13
|
-
]);
|
|
141
|
+
await enqueueStmt(sql, t, runId, opts);
|
|
142
|
+
},
|
|
143
|
+
async enqueueMany(requests) {
|
|
144
|
+
await enqueueManyStmt(sql, t, requests);
|
|
14
145
|
},
|
|
15
146
|
async claim({ limit, leaseMs, now, names }) {
|
|
16
147
|
if (names && names.length === 0) return [];
|
|
@@ -34,17 +165,18 @@ const createMysqlQueue = (sql, t, id) => {
|
|
|
34
165
|
ORDER BY j.priority, j.run_at LIMIT ? FOR UPDATE OF j SKIP LOCKED`, params);
|
|
35
166
|
const leases = [];
|
|
36
167
|
for (const row of due) {
|
|
37
|
-
const
|
|
168
|
+
const runId = text(row, "run_id");
|
|
169
|
+
const token = `${id()}:${runId}`;
|
|
38
170
|
await tx.exec(`UPDATE ${t.job} SET lease_token = ?, lease_expires = ? WHERE run_id = ?`, [
|
|
39
171
|
token,
|
|
40
172
|
at + leaseMs,
|
|
41
|
-
|
|
173
|
+
runId
|
|
42
174
|
]);
|
|
43
175
|
leases.push({
|
|
44
|
-
runId
|
|
176
|
+
runId,
|
|
45
177
|
token,
|
|
46
178
|
expiresAt,
|
|
47
|
-
version:
|
|
179
|
+
version: int(row, "version")
|
|
48
180
|
});
|
|
49
181
|
}
|
|
50
182
|
return leases;
|
|
@@ -81,7 +213,7 @@ const createMysqlQueue = (sql, t, id) => {
|
|
|
81
213
|
if (names && names.length === 0) return queueDepthOf([], now.getTime());
|
|
82
214
|
const nowMs = now.getTime();
|
|
83
215
|
const namePredicate = names ? ` WHERE r.name IS NULL OR r.name IN (${names.map(() => "?").join(",")})` : "";
|
|
84
|
-
const
|
|
216
|
+
const [row] = await sql.query(`SELECT SUM(claimable) AS claimable, SUM(leased) AS leased,
|
|
85
217
|
MIN(CASE WHEN claimable = 1 THEN run_at END) AS oldest
|
|
86
218
|
FROM (SELECT j.run_at AS run_at,
|
|
87
219
|
(j.run_at <= ? AND (j.lease_expires IS NULL OR j.lease_expires <= ?)) AS claimable,
|
|
@@ -91,11 +223,12 @@ const createMysqlQueue = (sql, t, id) => {
|
|
|
91
223
|
nowMs,
|
|
92
224
|
nowMs,
|
|
93
225
|
...names ?? []
|
|
94
|
-
])
|
|
226
|
+
]);
|
|
227
|
+
const oldest = intOrNull(row, "oldest");
|
|
95
228
|
return {
|
|
96
|
-
claimable:
|
|
97
|
-
leased:
|
|
98
|
-
oldestClaimableAgeMs:
|
|
229
|
+
claimable: intOrNull(row, "claimable") ?? 0,
|
|
230
|
+
leased: intOrNull(row, "leased") ?? 0,
|
|
231
|
+
oldestClaimableAgeMs: oldest === null ? null : nowMs - oldest
|
|
99
232
|
};
|
|
100
233
|
}
|
|
101
234
|
};
|
|
@@ -146,6 +279,7 @@ const ddl = (prefix = "") => {
|
|
|
146
279
|
parent_cursor_key VARCHAR(191),
|
|
147
280
|
depth INT NOT NULL DEFAULT 0,
|
|
148
281
|
join_remaining INT NOT NULL DEFAULT 0,
|
|
282
|
+
priority INT NOT NULL DEFAULT 0,
|
|
149
283
|
created_at BIGINT NOT NULL,
|
|
150
284
|
UNIQUE KEY run_idem (name, version, idempotency_key),
|
|
151
285
|
KEY run_created (created_at),
|
|
@@ -217,99 +351,18 @@ const ddl = (prefix = "") => {
|
|
|
217
351
|
)`
|
|
218
352
|
];
|
|
219
353
|
};
|
|
220
|
-
const
|
|
221
|
-
const cols = await sql.query(`SELECT COUNT(*) AS n FROM information_schema.columns
|
|
222
|
-
WHERE table_schema = DATABASE() AND table_name = ? AND column_name =
|
|
223
|
-
if (
|
|
224
|
-
await sql.query(`ALTER TABLE ${
|
|
354
|
+
const addColumn = async (sql, table, column, def) => {
|
|
355
|
+
const [cols] = await sql.query(`SELECT COUNT(*) AS n FROM information_schema.columns
|
|
356
|
+
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`, [table.replace(/`/g, ""), column]);
|
|
357
|
+
if (int(cols, "n") > 0) return;
|
|
358
|
+
await sql.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${def}`);
|
|
225
359
|
};
|
|
226
360
|
/** Apply the schema DDL (idempotent). Run once before use. */
|
|
227
361
|
const applySchema = async (sql, prefix = "") => {
|
|
228
362
|
for (const stmt of ddl(prefix)) await sql.query(stmt);
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
//#region src/codec.ts
|
|
233
|
-
/** @internal */
|
|
234
|
-
const j = (value) => value === void 0 ? null : JSON.stringify(value);
|
|
235
|
-
const p = (text) => text === null ? void 0 : JSON.parse(text);
|
|
236
|
-
const orUndef = (v) => v === null ? void 0 : v;
|
|
237
|
-
const at = (v) => new Date(Number(v));
|
|
238
|
-
const mapRun = (r) => ({
|
|
239
|
-
id: r.id,
|
|
240
|
-
name: r.name,
|
|
241
|
-
version: r.version,
|
|
242
|
-
status: r.status,
|
|
243
|
-
input: p(r.input),
|
|
244
|
-
attempts: r.attempts,
|
|
245
|
-
output: p(r.output),
|
|
246
|
-
error: p(r.error),
|
|
247
|
-
idempotencyKey: orUndef(r.idempotency_key),
|
|
248
|
-
tags: p(r.tags),
|
|
249
|
-
parentRunId: orUndef(r.parent_run_id),
|
|
250
|
-
parentCursorKey: orUndef(r.parent_cursor_key),
|
|
251
|
-
depth: r.depth,
|
|
252
|
-
createdAt: at(r.created_at)
|
|
253
|
-
});
|
|
254
|
-
const mapStep = (r) => ({
|
|
255
|
-
status: r.status,
|
|
256
|
-
result: p(r.result),
|
|
257
|
-
error: p(r.error),
|
|
258
|
-
attempts: r.attempts,
|
|
259
|
-
shape: orUndef(r.shape ?? null)
|
|
260
|
-
});
|
|
261
|
-
const mapSignal = (r) => ({
|
|
262
|
-
id: r.id,
|
|
263
|
-
name: r.name,
|
|
264
|
-
payload: p(r.payload)
|
|
265
|
-
});
|
|
266
|
-
const mapCron = (r) => ({
|
|
267
|
-
name: r.name,
|
|
268
|
-
schedule: r.schedule,
|
|
269
|
-
flowName: r.flow_name,
|
|
270
|
-
flowVersion: r.flow_version,
|
|
271
|
-
input: p(r.input),
|
|
272
|
-
overlap: r.overlap,
|
|
273
|
-
nextRunAt: at(r.next_run_at),
|
|
274
|
-
lastRunAt: r.last_run_at === null ? void 0 : at(r.last_run_at)
|
|
275
|
-
});
|
|
276
|
-
//#endregion
|
|
277
|
-
//#region src/statements.ts
|
|
278
|
-
const inList$1 = (n) => `(${Array.from({ length: n }, () => "?").join(",")})`;
|
|
279
|
-
/** @internal */
|
|
280
|
-
const enqueueStmt = (sql, t, runId, opts) => sql.exec(`INSERT INTO ${t.job} (run_id, run_at, priority, version) VALUES (?, ?, ?, 1)
|
|
281
|
-
ON DUPLICATE KEY UPDATE run_at = VALUES(run_at), priority = VALUES(priority), version = version + 1`, [
|
|
282
|
-
runId,
|
|
283
|
-
opts?.runAt ? opts.runAt.getTime() : 0,
|
|
284
|
-
opts?.priority ?? 0
|
|
285
|
-
]);
|
|
286
|
-
/** @internal */
|
|
287
|
-
const scheduleStmt = (sql, t, runId, fireAt) => sql.exec(`INSERT INTO ${t.timer} (run_id, fire_at) VALUES (?, ?)
|
|
288
|
-
ON DUPLICATE KEY UPDATE fire_at = VALUES(fire_at)`, [runId, fireAt.getTime()]);
|
|
289
|
-
/** @internal */
|
|
290
|
-
const applyOutbox = async (sql, t, fx) => {
|
|
291
|
-
for (const s of fx.spawn ?? []) {
|
|
292
|
-
await sql.exec(`INSERT IGNORE INTO ${t.run}
|
|
293
|
-
(id, name, version, status, input, idempotency_key, tags, parent_run_id, parent_cursor_key, depth, created_at)
|
|
294
|
-
VALUES (?, ?, ?, 'pending', ?, ?, ?, ?, ?, ?, ?)`, [
|
|
295
|
-
s.runId,
|
|
296
|
-
s.spec.name,
|
|
297
|
-
s.spec.version,
|
|
298
|
-
j(s.spec.input),
|
|
299
|
-
s.spec.idempotencyKey ?? null,
|
|
300
|
-
j(s.spec.tags),
|
|
301
|
-
s.spec.parentRunId ?? null,
|
|
302
|
-
s.spec.parentCursorKey ?? null,
|
|
303
|
-
s.spec.depth ?? 0,
|
|
304
|
-
(s.spec.createdAt ?? /* @__PURE__ */ new Date()).getTime()
|
|
305
|
-
]);
|
|
306
|
-
await enqueueStmt(sql, t, s.runId, s.enqueue);
|
|
307
|
-
}
|
|
308
|
-
for (const e of fx.enqueue ?? []) await enqueueStmt(sql, t, e.runId, e.opts);
|
|
309
|
-
for (const tm of fx.timers ?? []) await scheduleStmt(sql, t, tm.runId, tm.fireAt);
|
|
310
|
-
if (fx.cancelTimers?.length) await sql.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inList$1(fx.cancelTimers.length)}`, fx.cancelTimers);
|
|
311
|
-
if (fx.consumeSignals?.length) await sql.exec(`UPDATE ${t.signal} SET consumed = 1 WHERE id IN ${inList$1(fx.consumeSignals.length)}`, fx.consumeSignals);
|
|
312
|
-
if (fx.joinTarget) await sql.exec(`UPDATE ${t.run} SET join_remaining = ? WHERE id = ?`, [fx.joinTarget.count, fx.joinTarget.runId]);
|
|
363
|
+
const t = tables(prefix);
|
|
364
|
+
await addColumn(sql, t.signal, "consumed", "TINYINT(1) NOT NULL DEFAULT 0");
|
|
365
|
+
await addColumn(sql, t.run, "priority", "INT NOT NULL DEFAULT 0");
|
|
313
366
|
};
|
|
314
367
|
//#endregion
|
|
315
368
|
//#region src/store.ts
|
|
@@ -320,13 +373,14 @@ const NON_SUCCESS_TERMINAL = sqlTuple(NON_SUCCESS_TERMINAL_STATUSES);
|
|
|
320
373
|
const MS_PURGE = {
|
|
321
374
|
placeholder: () => "?",
|
|
322
375
|
time: (at) => at.getTime(),
|
|
323
|
-
statusTuple: sqlTuple
|
|
376
|
+
statusTuple: sqlTuple,
|
|
377
|
+
tag: (p) => `JSON_CONTAINS(tags, JSON_QUOTE(${p}))`
|
|
324
378
|
};
|
|
325
379
|
const inList = (n) => `(${Array.from({ length: n }, () => "?").join(",")})`;
|
|
326
380
|
/** @internal */
|
|
327
381
|
const createMysqlStore = (sql, t, id) => {
|
|
328
382
|
const loadStep = async (exec, runId, cursorKey) => {
|
|
329
|
-
const row = (await exec.query(`SELECT
|
|
383
|
+
const row = (await exec.query(`SELECT ${STEP_COLUMNS} FROM ${t.step} WHERE run_id = ? AND cursor_key = ?`, [runId, cursorKey]))[0];
|
|
330
384
|
if (!row) throw new Error(`checkpointStep: step ${runId}/${cursorKey} vanished after write`);
|
|
331
385
|
return mapStep(row);
|
|
332
386
|
};
|
|
@@ -334,8 +388,8 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
334
388
|
const runId = id();
|
|
335
389
|
if (spec.idempotencyKey) {
|
|
336
390
|
if ((await exec.exec(`INSERT IGNORE INTO ${t.run}
|
|
337
|
-
(id, name, version, status, input, idempotency_key, tags, parent_run_id, parent_cursor_key, depth, created_at)
|
|
338
|
-
VALUES (?, ?, ?, 'pending', ?, ?, ?, ?, ?, ?, ?)`, [
|
|
391
|
+
(id, name, version, status, input, idempotency_key, tags, parent_run_id, parent_cursor_key, depth, priority, created_at)
|
|
392
|
+
VALUES (?, ?, ?, 'pending', ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
339
393
|
runId,
|
|
340
394
|
spec.name,
|
|
341
395
|
spec.version,
|
|
@@ -345,6 +399,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
345
399
|
spec.parentRunId ?? null,
|
|
346
400
|
spec.parentCursorKey ?? null,
|
|
347
401
|
spec.depth ?? 0,
|
|
402
|
+
spec.priority ?? 0,
|
|
348
403
|
(spec.createdAt ?? /* @__PURE__ */ new Date()).getTime()
|
|
349
404
|
])).affectedRows === 1) return {
|
|
350
405
|
runId,
|
|
@@ -357,14 +412,14 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
357
412
|
spec.idempotencyKey
|
|
358
413
|
]);
|
|
359
414
|
return {
|
|
360
|
-
runId: hit[0]
|
|
415
|
+
runId: text(hit[0], "id"),
|
|
361
416
|
created: false,
|
|
362
|
-
status: hit[0]
|
|
417
|
+
status: runStatus(hit[0])
|
|
363
418
|
};
|
|
364
419
|
}
|
|
365
420
|
await exec.exec(`INSERT INTO ${t.run}
|
|
366
|
-
(id, name, version, status, input, tags, parent_run_id, parent_cursor_key, depth, created_at)
|
|
367
|
-
VALUES (?, ?, ?, 'pending', ?, ?, ?, ?, ?, ?)`, [
|
|
421
|
+
(id, name, version, status, input, tags, parent_run_id, parent_cursor_key, depth, priority, created_at)
|
|
422
|
+
VALUES (?, ?, ?, 'pending', ?, ?, ?, ?, ?, ?, ?)`, [
|
|
368
423
|
runId,
|
|
369
424
|
spec.name,
|
|
370
425
|
spec.version,
|
|
@@ -373,6 +428,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
373
428
|
spec.parentRunId ?? null,
|
|
374
429
|
spec.parentCursorKey ?? null,
|
|
375
430
|
spec.depth ?? 0,
|
|
431
|
+
spec.priority ?? 0,
|
|
376
432
|
(spec.createdAt ?? /* @__PURE__ */ new Date()).getTime()
|
|
377
433
|
]);
|
|
378
434
|
return {
|
|
@@ -389,7 +445,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
389
445
|
return sql.tx(async (tx) => {
|
|
390
446
|
const ids = (await tx.query(`SELECT id FROM ${t.run}
|
|
391
447
|
WHERE ${q.where}
|
|
392
|
-
${order} LIMIT ?`, params)).map((r) => r
|
|
448
|
+
${order} LIMIT ?`, params)).map((r) => text(r, "id"));
|
|
393
449
|
if (ids.length === 0) return 0;
|
|
394
450
|
const inIds = inList(ids.length);
|
|
395
451
|
await tx.exec(`DELETE FROM ${t.step} WHERE run_id IN ${inIds}`, ids);
|
|
@@ -414,14 +470,14 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
414
470
|
async loadRun(runId) {
|
|
415
471
|
const [runRows, stepRows, sigRows] = await Promise.all([
|
|
416
472
|
sql.query(`SELECT * FROM ${t.run} WHERE id = ?`, [runId]),
|
|
417
|
-
sql.query(`SELECT cursor_key,
|
|
473
|
+
sql.query(`SELECT cursor_key, ${STEP_COLUMNS} FROM ${t.step} WHERE run_id = ?`, [runId]),
|
|
418
474
|
sql.query(`SELECT id, name, payload FROM ${t.signal} WHERE run_id = ? AND consumed = 0 ORDER BY seq`, [runId])
|
|
419
475
|
]);
|
|
420
476
|
const runRow = runRows[0];
|
|
421
477
|
if (!runRow) return void 0;
|
|
422
478
|
return {
|
|
423
479
|
run: mapRun(runRow),
|
|
424
|
-
steps: new Map(stepRows.map((r) => [r
|
|
480
|
+
steps: new Map(stepRows.map((r) => [text(r, "cursor_key"), mapStep(r)])),
|
|
425
481
|
signals: sigRows.map(mapSignal)
|
|
426
482
|
};
|
|
427
483
|
},
|
|
@@ -432,14 +488,14 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
432
488
|
async loadRunRows(runIds) {
|
|
433
489
|
if (runIds.length === 0) return [];
|
|
434
490
|
const rows = await sql.query(`SELECT * FROM ${t.run} WHERE id IN ${inList(runIds.length)}`, runIds);
|
|
435
|
-
const byId = new Map(rows.map((
|
|
491
|
+
const byId = new Map(rows.map(mapRun).map((run) => [run.id, run]));
|
|
436
492
|
return runIds.map((runId) => byId.get(runId));
|
|
437
493
|
},
|
|
438
494
|
arriveAtJoin(parentRunId) {
|
|
439
495
|
return sql.tx(async (tx) => {
|
|
440
496
|
if ((await tx.exec(`UPDATE ${t.run} SET join_remaining = join_remaining - 1 WHERE id = ?`, [parentRunId])).affectedRows === 0) return void 0;
|
|
441
497
|
const rows = await tx.query(`SELECT join_remaining FROM ${t.run} WHERE id = ?`, [parentRunId]);
|
|
442
|
-
return rows[0] ?
|
|
498
|
+
return rows[0] ? int(rows[0], "join_remaining") : void 0;
|
|
443
499
|
});
|
|
444
500
|
},
|
|
445
501
|
async postSignal(runId, name, payload, opts) {
|
|
@@ -457,10 +513,10 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
457
513
|
},
|
|
458
514
|
markRunning(runId) {
|
|
459
515
|
return sql.tx(async (tx) => {
|
|
460
|
-
const cur = await tx.query(`SELECT status, attempts FROM ${t.run} WHERE id = ? FOR UPDATE`, [runId]);
|
|
461
|
-
if (!cur
|
|
462
|
-
const attempts =
|
|
463
|
-
if (isTerminal(cur
|
|
516
|
+
const [cur] = await tx.query(`SELECT status, attempts FROM ${t.run} WHERE id = ? FOR UPDATE`, [runId]);
|
|
517
|
+
if (!cur) throw new Error(`markRunning: run ${runId} not found`);
|
|
518
|
+
const attempts = int(cur, "attempts");
|
|
519
|
+
if (isTerminal(runStatus(cur))) return attempts;
|
|
464
520
|
await tx.exec(`UPDATE ${t.run} SET status = 'running', attempts = attempts + 1 WHERE id = ?`, [runId]);
|
|
465
521
|
return attempts + 1;
|
|
466
522
|
});
|
|
@@ -471,7 +527,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
471
527
|
if (fx?.requireVersion !== void 0) {
|
|
472
528
|
if ((await tx.query(`SELECT 1 AS ok FROM ${t.step} WHERE run_id = ? AND cursor_key = ?`, [c.runId, c.cursorKey])).length === 0) {
|
|
473
529
|
const job = await tx.query(`SELECT version FROM ${t.job} WHERE run_id = ? FOR UPDATE`, [c.runId]);
|
|
474
|
-
if (!job[0] ||
|
|
530
|
+
if (!job[0] || int(job[0], "version") !== fx.requireVersion) return {
|
|
475
531
|
status: c.status,
|
|
476
532
|
attempts: c.attempts,
|
|
477
533
|
committed: false
|
|
@@ -486,7 +542,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
486
542
|
j(c.result),
|
|
487
543
|
j(c.error),
|
|
488
544
|
c.attempts,
|
|
489
|
-
c.
|
|
545
|
+
c.call ?? null
|
|
490
546
|
])).affectedRows === 1 && fx) await applyOutbox(tx, t, fx);
|
|
491
547
|
return loadStep(tx, c.runId, c.cursorKey);
|
|
492
548
|
});
|
|
@@ -502,7 +558,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
502
558
|
const output = outcome.status === "done" ? outcome.output : void 0;
|
|
503
559
|
const error = outcome.status === "done" ? void 0 : outcome.error;
|
|
504
560
|
if ((await tx.exec(`UPDATE ${t.run} SET status = ?, output = ?, error = ?
|
|
505
|
-
WHERE id = ? AND status
|
|
561
|
+
WHERE id = ? AND status NOT IN ${TERMINAL}`, [
|
|
506
562
|
outcome.status,
|
|
507
563
|
j(output),
|
|
508
564
|
j(error),
|
|
@@ -538,9 +594,9 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
538
594
|
const clause = where.length ? `WHERE ${where.join(" AND ")}` : "";
|
|
539
595
|
const rows = await sql.query(`SELECT * FROM ${t.run} ${clause} ORDER BY seq DESC LIMIT ?`, params);
|
|
540
596
|
const last = rows[rows.length - 1];
|
|
541
|
-
const cursor = rows.length === page.limit && last ?
|
|
597
|
+
const cursor = rows.length === page.limit && last ? text(last, "seq") : void 0;
|
|
542
598
|
return {
|
|
543
|
-
runs: rows.map(
|
|
599
|
+
runs: rows.map(mapRun),
|
|
544
600
|
cursor
|
|
545
601
|
};
|
|
546
602
|
},
|
|
@@ -549,7 +605,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
549
605
|
if (!q) return 0;
|
|
550
606
|
const params = [...q.params, limit];
|
|
551
607
|
return sql.tx(async (tx) => {
|
|
552
|
-
const ids = (await tx.query(`SELECT id FROM ${t.run} WHERE ${q.where} LIMIT ? FOR UPDATE`, params)).map((r) => r
|
|
608
|
+
const ids = (await tx.query(`SELECT id FROM ${t.run} WHERE ${q.where} LIMIT ? FOR UPDATE`, params)).map((r) => text(r, "id"));
|
|
553
609
|
if (ids.length === 0) return 0;
|
|
554
610
|
const inIds = inList(ids.length);
|
|
555
611
|
await tx.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inIds}`, ids);
|
|
@@ -562,21 +618,21 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
562
618
|
if (!q) return 0;
|
|
563
619
|
const params = [...q.params, limit];
|
|
564
620
|
return sql.tx(async (tx) => {
|
|
565
|
-
const ids = (await tx.query(`SELECT id FROM ${t.run} WHERE ${q.where} LIMIT ? FOR UPDATE`, params)).map((r) => r
|
|
621
|
+
const ids = (await tx.query(`SELECT id FROM ${t.run} WHERE ${q.where} LIMIT ? FOR UPDATE`, params)).map((r) => text(r, "id"));
|
|
566
622
|
if (ids.length === 0) return 0;
|
|
567
623
|
const inIds = inList(ids.length);
|
|
568
624
|
await tx.exec(`UPDATE ${t.run} SET status = 'pending', error = NULL, attempts = 0 WHERE id IN ${inIds}`, ids);
|
|
569
|
-
|
|
625
|
+
await enqueueManyStmt(tx, t, ids.map((runId) => ({ runId })));
|
|
570
626
|
return ids.length;
|
|
571
627
|
});
|
|
572
628
|
},
|
|
573
629
|
async childrenOf(runId) {
|
|
574
|
-
return (await sql.query(`SELECT * FROM ${t.run} WHERE parent_run_id = ?`, [runId])).map(
|
|
630
|
+
return (await sql.query(`SELECT * FROM ${t.run} WHERE parent_run_id = ?`, [runId])).map(mapRun);
|
|
575
631
|
},
|
|
576
632
|
async runStats() {
|
|
577
633
|
const rows = await sql.query(`SELECT status, count(*) AS n FROM ${t.run} GROUP BY status`);
|
|
578
634
|
const stats = zeroRunStats();
|
|
579
|
-
for (const r of rows) stats[r
|
|
635
|
+
for (const r of rows) stats[runStatus(r)] = int(r, "n");
|
|
580
636
|
return stats;
|
|
581
637
|
},
|
|
582
638
|
async orphanedRuns(limit) {
|
|
@@ -589,7 +645,7 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
589
645
|
nonSuccessTerminal: NON_SUCCESS_TERMINAL,
|
|
590
646
|
order: "r.seq",
|
|
591
647
|
limit: "?"
|
|
592
|
-
}), [limit])).map((r) => r
|
|
648
|
+
}), [limit])).map((r) => text(r, "id"));
|
|
593
649
|
},
|
|
594
650
|
deleteRuns,
|
|
595
651
|
deleteRunsOlderThan: (before, limit) => deleteRuns({ before }, limit),
|
|
@@ -635,8 +691,8 @@ const createMysqlStore = (sql, t, id) => {
|
|
|
635
691
|
if (names && names.length === 0) return 0;
|
|
636
692
|
const namePredicate = names ? ` AND flow_name IN ${inList(names.length)}` : "";
|
|
637
693
|
const params = names ? [now.getTime(), ...names] : [now.getTime()];
|
|
638
|
-
const
|
|
639
|
-
return
|
|
694
|
+
const [row] = await sql.query(`SELECT count(*) AS n FROM ${t.cron} WHERE next_run_at <= ?${namePredicate}`, params);
|
|
695
|
+
return int(row, "n");
|
|
640
696
|
},
|
|
641
697
|
async advanceCron(name, expectedNextRunAt, nextRunAt, lastRunAt) {
|
|
642
698
|
return (await sql.exec(`UPDATE ${t.cron} SET next_run_at = ?, last_run_at = ?
|
|
@@ -663,7 +719,7 @@ const createMysqlTimer = (sql, t) => {
|
|
|
663
719
|
return sql.tx(async (tx) => {
|
|
664
720
|
const due = await tx.query(`SELECT run_id FROM ${t.timer} WHERE fire_at <= ? ORDER BY fire_at LIMIT ? FOR UPDATE SKIP LOCKED`, [at, limit]);
|
|
665
721
|
if (!due.length) return [];
|
|
666
|
-
const ids = due.map((r) => r
|
|
722
|
+
const ids = due.map((r) => text(r, "run_id"));
|
|
667
723
|
const holes = ids.map(() => "?").join(", ");
|
|
668
724
|
await tx.exec(`DELETE FROM ${t.timer} WHERE run_id IN (${holes})`, ids);
|
|
669
725
|
return ids;
|
|
@@ -673,16 +729,17 @@ const createMysqlTimer = (sql, t) => {
|
|
|
673
729
|
if (names && names.length === 0) return 0;
|
|
674
730
|
const namePredicate = names ? ` AND (r.name IS NULL OR r.name IN (${names.map(() => "?").join(",")}))` : "";
|
|
675
731
|
const params = names ? [now.getTime(), ...names] : [now.getTime()];
|
|
676
|
-
const
|
|
732
|
+
const [row] = await sql.query(`SELECT count(*) AS n FROM ${t.timer} tm LEFT JOIN ${t.run} r ON r.id = tm.run_id
|
|
677
733
|
WHERE tm.fire_at <= ?${namePredicate}`, params);
|
|
678
|
-
return
|
|
734
|
+
return int(row, "n");
|
|
679
735
|
},
|
|
680
736
|
async cancel(runId) {
|
|
681
737
|
await sql.exec(`DELETE FROM ${t.timer} WHERE run_id = ?`, [runId]);
|
|
682
738
|
},
|
|
683
739
|
async nextDueAt(now) {
|
|
684
|
-
const
|
|
685
|
-
|
|
740
|
+
const [row] = await sql.query(`SELECT min(fire_at) AS fire_at FROM ${t.timer} WHERE fire_at > ?`, [now.getTime()]);
|
|
741
|
+
const at = intOrNull(row, "fire_at");
|
|
742
|
+
return at === null ? null : new Date(at);
|
|
686
743
|
}
|
|
687
744
|
};
|
|
688
745
|
};
|
|
@@ -705,16 +762,18 @@ const createMysqlBackend = (sql, opts = {}) => {
|
|
|
705
762
|
};
|
|
706
763
|
//#endregion
|
|
707
764
|
//#region src/sql.ts
|
|
708
|
-
const write = (r) => {
|
|
709
|
-
const h = r[0];
|
|
710
|
-
return {
|
|
711
|
-
affectedRows: h.affectedRows,
|
|
712
|
-
insertId: h.insertId
|
|
713
|
-
};
|
|
714
|
-
};
|
|
715
765
|
const bind = (q) => ({
|
|
716
|
-
query:
|
|
717
|
-
|
|
766
|
+
query: async (text, params) => {
|
|
767
|
+
const [result] = await q.query(text, params && [...params]);
|
|
768
|
+
return Array.isArray(result) ? result : [];
|
|
769
|
+
},
|
|
770
|
+
exec: async (text, params) => {
|
|
771
|
+
const [{ affectedRows, insertId }] = await q.query(text, params && [...params]);
|
|
772
|
+
return {
|
|
773
|
+
affectedRows,
|
|
774
|
+
insertId
|
|
775
|
+
};
|
|
776
|
+
}
|
|
718
777
|
});
|
|
719
778
|
const onConn = (c) => ({
|
|
720
779
|
...bind(c),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@iterativeflow/mysql",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"access": "public"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@iterativeflow/core": "
|
|
28
|
+
"@iterativeflow/core": "3.0.0"
|
|
29
29
|
},
|
|
30
30
|
"peerDependencies": {
|
|
31
31
|
"mysql2": ">=3.0.0"
|