@iterativeflow/mysql 2.2.0 → 2.4.0

package/README.md

@@ -13,9 +13,7 @@ import { createEngine } from "@iterativeflow/core";
 
 const sql = mysqlPool(createPool(process.env.MYSQL_URL));
 await applySchema(sql);
-const engine = createEngine(createMysqlBackend(sql), [
-  /* your flows */
-]);
+const engine = createEngine(createMysqlBackend(sql), [/* your flows */]);
 ```
 
 ## Notes

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { NON_SUCCESS_TERMINAL_STATUSES, RECONCILABLE_STATUSES, TERMINAL_STATUSES, assertSqlIdentifier, createLocalWakeup, isTerminal, newId, orphanedRunsSql, purgeWhereSql, queueDepthOf, statusList, zeroRunStats } from "@iterativeflow/core/backend";
+import { ACTIVE_STATUSES, NON_SUCCESS_TERMINAL_STATUSES, RECONCILABLE_STATUSES, TERMINAL_STATUSES, assertSqlIdentifier, createLocalWakeup, isTerminal, newId, orphanedRunsSql, purgeWhereSql, queueDepthOf, runSetWhereSql, statusList, zeroRunStats } from "@iterativeflow/core/backend";
 //#region src/queue.ts
 /** @internal */
 const createMysqlQueue = (sql, t, id) => {
@@ -183,6 +183,7 @@ const ddl = (prefix = "") => {
       name     VARCHAR(191) NOT NULL,
       payload  LONGTEXT,
       idem_key VARCHAR(191),
+      consumed TINYINT(1) NOT NULL DEFAULT 0,
       KEY signal_inbox (run_id, seq),
       UNIQUE KEY signal_idem (run_id, idem_key)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`,
@@ -216,9 +217,16 @@ const ddl = (prefix = "") => {
      )`
 	];
 };
+const addSignalConsumed = async (sql, t) => {
+	const cols = await sql.query(`SELECT COUNT(*) AS n FROM information_schema.columns
+      WHERE table_schema = DATABASE() AND table_name = ? AND column_name = 'consumed'`, [t.signal.replace(/`/g, "")]);
+	if (Number(cols[0]?.n ?? 0) > 0) return;
+	await sql.query(`ALTER TABLE ${t.signal} ADD COLUMN consumed TINYINT(1) NOT NULL DEFAULT 0`);
+};
 /** Apply the schema DDL (idempotent). Run once before use. */
 const applySchema = async (sql, prefix = "") => {
 	for (const stmt of ddl(prefix)) await sql.query(stmt);
+	await addSignalConsumed(sql, tables(prefix));
 };
 //#endregion
 //#region src/codec.ts
@@ -300,7 +308,7 @@ const applyOutbox = async (sql, t, fx) => {
 	for (const e of fx.enqueue ?? []) await enqueueStmt(sql, t, e.runId, e.opts);
 	for (const tm of fx.timers ?? []) await scheduleStmt(sql, t, tm.runId, tm.fireAt);
 	if (fx.cancelTimers?.length) await sql.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inList$1(fx.cancelTimers.length)}`, fx.cancelTimers);
-	if (fx.consumeSignals?.length) await sql.exec(`DELETE FROM ${t.signal} WHERE id IN ${inList$1(fx.consumeSignals.length)}`, fx.consumeSignals);
+	if (fx.consumeSignals?.length) await sql.exec(`UPDATE ${t.signal} SET consumed = 1 WHERE id IN ${inList$1(fx.consumeSignals.length)}`, fx.consumeSignals);
 	if (fx.joinTarget) await sql.exec(`UPDATE ${t.run} SET join_remaining = ? WHERE id = ?`, [fx.joinTarget.count, fx.joinTarget.runId]);
 };
 //#endregion
@@ -311,7 +319,8 @@ const RECONCILABLE = sqlTuple(RECONCILABLE_STATUSES);
 const NON_SUCCESS_TERMINAL = sqlTuple(NON_SUCCESS_TERMINAL_STATUSES);
 const MS_PURGE = {
 	placeholder: () => "?",
-	time: (at) => at.getTime()
+	time: (at) => at.getTime(),
+	statusTuple: sqlTuple
 };
 const inList = (n) => `(${Array.from({ length: n }, () => "?").join(",")})`;
 /** @internal */
@@ -375,11 +384,12 @@ const createMysqlStore = (sql, t, id) => {
 	const deleteRuns = async (filter, limit) => {
 		const q = purgeWhereSql(filter, MS_PURGE);
 		if (!q) return 0;
+		const order = filter.before === void 0 ? "" : " ORDER BY seq";
 		const params = [...q.params, limit];
 		return sql.tx(async (tx) => {
 			const ids = (await tx.query(`SELECT id FROM ${t.run}
              WHERE ${q.where}
-             ORDER BY seq LIMIT ?`, params)).map((r) => r.id);
+             ${order} LIMIT ?`, params)).map((r) => r.id);
 			if (ids.length === 0) return 0;
 			const inIds = inList(ids.length);
 			await tx.exec(`DELETE FROM ${t.step} WHERE run_id IN ${inIds}`, ids);
@@ -405,7 +415,7 @@ const createMysqlStore = (sql, t, id) => {
 			const [runRows, stepRows, sigRows] = await Promise.all([
 				sql.query(`SELECT * FROM ${t.run} WHERE id = ?`, [runId]),
 				sql.query(`SELECT cursor_key, status, result, error, attempts, shape FROM ${t.step} WHERE run_id = ?`, [runId]),
-				sql.query(`SELECT id, name, payload FROM ${t.signal} WHERE run_id = ? ORDER BY seq`, [runId])
+				sql.query(`SELECT id, name, payload FROM ${t.signal} WHERE run_id = ? AND consumed = 0 ORDER BY seq`, [runId])
 			]);
 			const runRow = runRows[0];
 			if (!runRow) return void 0;
@@ -512,6 +522,10 @@ const createMysqlStore = (sql, t, id) => {
 				where.push(`name = ?`);
 				params.push(filter.name);
 			}
+			if (filter.version !== void 0) {
+				where.push(`version = ?`);
+				params.push(filter.version);
+			}
 			if (filter.tag) {
 				where.push(`JSON_CONTAINS(tags, JSON_QUOTE(?))`);
 				params.push(filter.tag);
@@ -530,6 +544,32 @@ const createMysqlStore = (sql, t, id) => {
 				cursor
 			};
 		},
+		async cancelRuns(filter, limit) {
+			const q = runSetWhereSql(filter, ACTIVE_STATUSES, "cancelRuns", MS_PURGE);
+			if (!q) return 0;
+			const params = [...q.params, limit];
+			return sql.tx(async (tx) => {
+				const ids = (await tx.query(`SELECT id FROM ${t.run} WHERE ${q.where} LIMIT ? FOR UPDATE`, params)).map((r) => r.id);
+				if (ids.length === 0) return 0;
+				const inIds = inList(ids.length);
+				await tx.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inIds}`, ids);
+				await tx.exec(`UPDATE ${t.run} SET status = 'canceled' WHERE id IN ${inIds}`, ids);
+				return ids.length;
+			});
+		},
+		async retryRuns(filter, limit) {
+			const q = runSetWhereSql(filter, ["failed"], "retryRuns", MS_PURGE);
+			if (!q) return 0;
+			const params = [...q.params, limit];
+			return sql.tx(async (tx) => {
+				const ids = (await tx.query(`SELECT id FROM ${t.run} WHERE ${q.where} LIMIT ? FOR UPDATE`, params)).map((r) => r.id);
+				if (ids.length === 0) return 0;
+				const inIds = inList(ids.length);
+				await tx.exec(`UPDATE ${t.run} SET status = 'pending', error = NULL, attempts = 0 WHERE id IN ${inIds}`, ids);
+				for (const runId of ids) await enqueueStmt(tx, t, runId);
+				return ids.length;
+			});
+		},
 		async childrenOf(runId) {
 			return (await sql.query(`SELECT * FROM ${t.run} WHERE parent_run_id = ?`, [runId])).map((r) => mapRun(r));
 		},
@@ -555,7 +595,8 @@ const createMysqlStore = (sql, t, id) => {
 		deleteRunsOlderThan: (before, limit) => deleteRuns({ before }, limit),
 		retryRun(runId) {
 			return sql.tx(async (tx) => {
-				if ((await tx.exec(`UPDATE ${t.run} SET status = 'pending', error = NULL WHERE id = ? AND status = 'failed'`, [runId])).affectedRows !== 1) return { retried: false };
+				if ((await tx.exec(`UPDATE ${t.run} SET status = 'pending', error = NULL, attempts = 0
+           WHERE id = ? AND status = 'failed'`, [runId])).affectedRows !== 1) return { retried: false };
 				await enqueueStmt(tx, t, runId);
 				return { retried: true };
 			});
@@ -565,6 +606,8 @@ const createMysqlStore = (sql, t, id) => {
            (name, schedule, flow_name, flow_version, input, overlap, next_run_at)
          VALUES (?, ?, ?, ?, ?, ?, ?)
          ON DUPLICATE KEY UPDATE
+           -- MySQL applies these left to right, so compare the schedule BEFORE overwriting it
+           next_run_at = IF(schedule <> VALUES(schedule), VALUES(next_run_at), next_run_at),
            schedule = VALUES(schedule),
            flow_name = VALUES(flow_name),
            flow_version = VALUES(flow_version),
@@ -582,6 +625,12 @@ const createMysqlStore = (sql, t, id) => {
 		async dueCrons(now, limit) {
 			return (await sql.query(`SELECT * FROM ${t.cron} WHERE next_run_at <= ? ORDER BY next_run_at LIMIT ?`, [now.getTime(), limit])).map(mapCron);
 		},
+		async listCrons() {
+			return (await sql.query(`SELECT * FROM ${t.cron} ORDER BY name`)).map(mapCron);
+		},
+		async removeCron(name) {
+			return (await sql.exec(`DELETE FROM ${t.cron} WHERE name = ?`, [name])).affectedRows > 0;
+		},
 		async dueCronCount(now, names) {
 			if (names && names.length === 0) return 0;
 			const namePredicate = names ? ` AND flow_name IN ${inList(names.length)}` : "";

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@iterativeflow/mysql",
-  "version": "2.2.0",
+  "version": "2.4.0",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ahmedrowaihi/iterativeflow.git",
-    "directory": "v2/packages/mysql"
+    "directory": "packages/mysql"
   },
   "files": [
     "dist",
@@ -25,11 +25,14 @@
     "access": "public"
   },
   "dependencies": {
-    "@iterativeflow/core": "2.2.0"
+    "@iterativeflow/core": "2.4.0"
   },
   "peerDependencies": {
     "mysql2": ">=3.0.0"
   },
+  "engines": {
+    "node": ">=20"
+  },
   "scripts": {
     "build": "tsdown"
   }