@iterativeflow/mysql 2.1.0 → 2.2.0

package/dist/index.d.mts

@@ -53,6 +53,11 @@ declare const createMysqlBackend: (sql: Sql, opts?: MysqlBackendOpts) => Backend
  * MySQL has no implicit rowid). Indexed string columns are `VARCHAR(191)` to stay under the utf8mb4
  * index-key limit. The idempotency/signal-dedup UNIQUE KEYs need no partial `WHERE`: MySQL treats
  * NULLs as distinct, so unkeyed rows never collide.
+ *
+ * Also creates the `pending_work` function, so applying this needs `CREATE ROUTINE` as well as
+ * `CREATE TABLE`. MySQL has no `CREATE OR REPLACE FUNCTION`, so that one statement is a `DROP` +
+ * `CREATE` rather than `IF NOT EXISTS` — a scaler polling the function across a boot can see one
+ * failed read.
  */
 declare const ddl: (prefix?: string) => string[];
 /** Apply the schema DDL (idempotent). Run once before use. */

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { NON_SUCCESS_TERMINAL_STATUSES, RECONCILABLE_STATUSES, TERMINAL_STATUSES, assertSqlIdentifier, createLocalWakeup, newId, orphanedRunsSql, queueDepthOf, statusList, zeroRunStats } from "@iterativeflow/core/backend";
+import { NON_SUCCESS_TERMINAL_STATUSES, RECONCILABLE_STATUSES, TERMINAL_STATUSES, assertSqlIdentifier, createLocalWakeup, isTerminal, newId, orphanedRunsSql, purgeWhereSql, queueDepthOf, statusList, zeroRunStats } from "@iterativeflow/core/backend";
 //#region src/queue.ts
 /** @internal */
 const createMysqlQueue = (sql, t, id) => {
@@ -17,7 +17,7 @@ const createMysqlQueue = (sql, t, id) => {
 			const at = ms(now);
 			const expiresAt = new Date(at + leaseMs);
 			return sql.tx(async (tx) => {
-				const namePredicate = names ? ` AND r.name IN (${names.map(() => "?").join(",")})` : "";
+				const namePredicate = names ? ` AND (r.name IS NULL OR r.name IN (${names.map(() => "?").join(",")}))` : "";
 				const params = names ? [
 					at,
 					at,
@@ -80,7 +80,7 @@ const createMysqlQueue = (sql, t, id) => {
 		async depth(now, names) {
 			if (names && names.length === 0) return queueDepthOf([], now.getTime());
 			const nowMs = now.getTime();
-			const namePredicate = names ? ` WHERE r.name IN (${names.map(() => "?").join(",")})` : "";
+			const namePredicate = names ? ` WHERE r.name IS NULL OR r.name IN (${names.map(() => "?").join(",")})` : "";
 			const r = (await sql.query(`SELECT SUM(claimable) AS claimable, SUM(leased) AS leased,
                 MIN(CASE WHEN claimable = 1 THEN run_at END) AS oldest
            FROM (SELECT j.run_at AS run_at,
@@ -121,6 +121,11 @@ const tables = (prefix) => {
 * MySQL has no implicit rowid). Indexed string columns are `VARCHAR(191)` to stay under the utf8mb4
 * index-key limit. The idempotency/signal-dedup UNIQUE KEYs need no partial `WHERE`: MySQL treats
 * NULLs as distinct, so unkeyed rows never collide.
+*
+* Also creates the `pending_work` function, so applying this needs `CREATE ROUTINE` as well as
+* `CREATE TABLE`. MySQL has no `CREATE OR REPLACE FUNCTION`, so that one statement is a `DROP` +
+* `CREATE` rather than `IF NOT EXISTS` — a scaler polling the function across a boot can see one
+* failed read.
 */
 const ddl = (prefix = "") => {
 	const t = tables(prefix);
@@ -196,11 +201,16 @@ const ddl = (prefix = "") => {
 		`CREATE FUNCTION \`${prefix}pending_work\`(flow_names JSON, as_of BIGINT)
      RETURNS BIGINT READS SQL DATA
      RETURN (
-       (SELECT COUNT(*) FROM ${t.job} j LEFT JOIN ${t.run} r ON r.id = j.run_id
+       (SELECT COUNT(*) FROM ${t.job} j
           WHERE j.run_at <= as_of AND (j.lease_expires IS NULL OR j.lease_expires <= as_of)
-            AND (flow_names IS NULL OR JSON_CONTAINS(flow_names, JSON_QUOTE(r.name))))
-     + (SELECT COUNT(*) FROM ${t.timer} tm LEFT JOIN ${t.run} r ON r.id = tm.run_id
-          WHERE tm.fire_at <= as_of AND (flow_names IS NULL OR JSON_CONTAINS(flow_names, JSON_QUOTE(r.name))))
+            AND (flow_names IS NULL OR (JSON_LENGTH(flow_names) > 0 AND NOT EXISTS (
+                  SELECT 1 FROM ${t.run} r
+                   WHERE r.id = j.run_id AND NOT JSON_CONTAINS(flow_names, JSON_QUOTE(r.name))))))
+     + (SELECT COUNT(*) FROM ${t.timer} tm
+          WHERE tm.fire_at <= as_of
+            AND (flow_names IS NULL OR (JSON_LENGTH(flow_names) > 0 AND NOT EXISTS (
+                  SELECT 1 FROM ${t.run} r
+                   WHERE r.id = tm.run_id AND NOT JSON_CONTAINS(flow_names, JSON_QUOTE(r.name))))))
      + (SELECT COUNT(*) FROM ${t.cron} c
           WHERE c.next_run_at <= as_of AND (flow_names IS NULL OR JSON_CONTAINS(flow_names, JSON_QUOTE(c.flow_name))))
      )`
@@ -299,6 +309,10 @@ const sqlTuple = (statuses) => `(${statuses.map((s) => `'${s}'`).join(",")})`;
 const TERMINAL = sqlTuple(TERMINAL_STATUSES);
 const RECONCILABLE = sqlTuple(RECONCILABLE_STATUSES);
 const NON_SUCCESS_TERMINAL = sqlTuple(NON_SUCCESS_TERMINAL_STATUSES);
+const MS_PURGE = {
+	placeholder: () => "?",
+	time: (at) => at.getTime()
+};
 const inList = (n) => `(${Array.from({ length: n }, () => "?").join(",")})`;
 /** @internal */
 const createMysqlStore = (sql, t, id) => {
@@ -358,6 +372,24 @@ const createMysqlStore = (sql, t, id) => {
 			status: "pending"
 		};
 	};
+	const deleteRuns = async (filter, limit) => {
+		const q = purgeWhereSql(filter, MS_PURGE);
+		if (!q) return 0;
+		const params = [...q.params, limit];
+		return sql.tx(async (tx) => {
+			const ids = (await tx.query(`SELECT id FROM ${t.run}
+             WHERE ${q.where}
+             ORDER BY seq LIMIT ?`, params)).map((r) => r.id);
+			if (ids.length === 0) return 0;
+			const inIds = inList(ids.length);
+			await tx.exec(`DELETE FROM ${t.step} WHERE run_id IN ${inIds}`, ids);
+			await tx.exec(`DELETE FROM ${t.signal} WHERE run_id IN ${inIds}`, ids);
+			await tx.exec(`DELETE FROM ${t.job} WHERE run_id IN ${inIds}`, ids);
+			await tx.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inIds}`, ids);
+			await tx.exec(`DELETE FROM ${t.run} WHERE id IN ${inIds}`, ids);
+			return ids.length;
+		});
+	};
 	return {
 		startRun(spec) {
 			return startOne(sql, spec);
@@ -418,7 +450,7 @@ const createMysqlStore = (sql, t, id) => {
 				const cur = await tx.query(`SELECT status, attempts FROM ${t.run} WHERE id = ? FOR UPDATE`, [runId]);
 				if (!cur[0]) throw new Error(`markRunning: run ${runId} not found`);
 				const attempts = Number(cur[0].attempts);
-				if (TERMINAL_STATUSES.includes(cur[0].status)) return attempts;
+				if (isTerminal(cur[0].status)) return attempts;
 				await tx.exec(`UPDATE ${t.run} SET status = 'running', attempts = attempts + 1 WHERE id = ?`, [runId]);
 				return attempts + 1;
 			});
@@ -519,21 +551,8 @@ const createMysqlStore = (sql, t, id) => {
 				limit: "?"
 			}), [limit])).map((r) => r.id);
 		},
-		deleteRunsOlderThan(before, limit) {
-			return sql.tx(async (tx) => {
-				const ids = (await tx.query(`SELECT id FROM ${t.run}
-             WHERE status IN ${TERMINAL} AND created_at < ?
-             ORDER BY seq LIMIT ?`, [before.getTime(), limit])).map((r) => r.id);
-				if (ids.length === 0) return 0;
-				const inIds = inList(ids.length);
-				await tx.exec(`DELETE FROM ${t.step} WHERE run_id IN ${inIds}`, ids);
-				await tx.exec(`DELETE FROM ${t.signal} WHERE run_id IN ${inIds}`, ids);
-				await tx.exec(`DELETE FROM ${t.job} WHERE run_id IN ${inIds}`, ids);
-				await tx.exec(`DELETE FROM ${t.timer} WHERE run_id IN ${inIds}`, ids);
-				await tx.exec(`DELETE FROM ${t.run} WHERE id IN ${inIds}`, ids);
-				return ids.length;
-			});
-		},
+		deleteRuns,
+		deleteRunsOlderThan: (before, limit) => deleteRuns({ before }, limit),
 		retryRun(runId) {
 			return sql.tx(async (tx) => {
 				if ((await tx.exec(`UPDATE ${t.run} SET status = 'pending', error = NULL WHERE id = ? AND status = 'failed'`, [runId])).affectedRows !== 1) return { retried: false };
@@ -603,7 +622,7 @@ const createMysqlTimer = (sql, t) => {
 		},
 		async dueCount(now, names) {
 			if (names && names.length === 0) return 0;
-			const namePredicate = names ? ` AND r.name IN (${names.map(() => "?").join(",")})` : "";
+			const namePredicate = names ? ` AND (r.name IS NULL OR r.name IN (${names.map(() => "?").join(",")}))` : "";
 			const params = names ? [now.getTime(), ...names] : [now.getTime()];
 			const rows = await sql.query(`SELECT count(*) AS n FROM ${t.timer} tm LEFT JOIN ${t.run} r ON r.id = tm.run_id
          WHERE tm.fire_at <= ?${namePredicate}`, params);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iterativeflow/mysql",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -25,7 +25,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@iterativeflow/core": "2.1.0"
+    "@iterativeflow/core": "2.2.0"
   },
   "peerDependencies": {
     "mysql2": ">=3.0.0"