@eventferry/mysql 2.0.0

package/README.md

@@ -0,0 +1,183 @@
+# @eventferry/mysql
+
+[![npm](https://img.shields.io/npm/v/@eventferry/mysql.svg)](https://www.npmjs.com/package/@eventferry/mysql)
+
+The **MySQL / MariaDB store** for [eventferry](https://github.com/SametGoktepe/eventferry) —
+a transactional outbox toolkit for relational databases + Kafka/Redpanda.
+
+Provides:
+
+- `MysqlStore` — transaction-joining `enqueue` and a lock-free `claimBatch`
+  using `SELECT ... FOR UPDATE SKIP LOCKED`, with **strict per-aggregate
+  ordering** and a **crash-recovery reaper** (visibility timeout).
+- `createMigrationSql` — idempotent DDL generator for the outbox table
+  (InnoDB + utf8mb4 + DATETIME(3) + JSON).
+- `purgeDone` — batched retention of published rows.
+
+## Requirements
+
+- **MySQL 8.0.1+** or **MariaDB 10.6+** (needs `FOR UPDATE SKIP LOCKED`)
+- **InnoDB** storage engine (default)
+- Node.js **18+**
+
+> Older MySQL versions don't support `SKIP LOCKED` — concurrent relays would
+> serialize on the same rows. There is no fallback in this MVP.
+
+## Install
+
+```bash
+npm i @eventferry/mysql @eventferry/core mysql2
+```
+
+`mysql2` is an optional peer (the driver).
+
+## Usage
+
+```ts
+import mysql from "mysql2/promise";
+import { MysqlStore, createMigrationSql } from "@eventferry/mysql";
+
+const pool = mysql.createPool({
+  host: "localhost",
+  user: "app",
+  password: "...",
+  database: "shop",
+  // Recommended for the outbox: stable date handling and bigint-safe ids.
+  dateStrings: false,
+  supportBigNumbers: true,
+});
+
+await pool.query(createMigrationSql("outbox")); // idempotent DDL
+
+const store = new MysqlStore({ pool });
+
+// Inside your business transaction:
+const conn = await pool.getConnection();
+try {
+  await conn.beginTransaction();
+  await conn.query("INSERT INTO orders ...");
+  await store.enqueue(conn, {
+    topic: "orders.created",
+    aggregateType: "order",
+    aggregateId: order.id,
+    payload: { orderId: order.id, total: order.total },
+  });
+  await conn.commit();
+} catch (err) {
+  await conn.rollback();
+  throw err;
+} finally {
+  conn.release();
+}
+```
+
+Hand the store to a `Relay` from `@eventferry/core` together with a publisher
+from `@eventferry/kafka`:
+
+```ts
+import { Relay } from "@eventferry/core";
+import { KafkaPublisher } from "@eventferry/kafka";
+
+const publisher = new KafkaPublisher({
+  driver: "kafkajs",
+  brokers: ["localhost:19092"],
+  idempotent: true,
+});
+
+const relay = new Relay({ store, publisher, dlq: { topic: "orders.dlq" } });
+await relay.start();
+process.on("SIGTERM", () => relay.stop());
+```
+
+## Binlog streaming (CDC) mode
+
+For high-throughput workloads, use the `MysqlBinlogRelay` to tail the **MySQL
+binary log** directly — the same mechanism Debezium uses — and bypass the
+polling claim loop entirely. Latency drops from "one poll interval" to a few
+milliseconds.
+
+**MySQL server requirements:**
+
+```ini
+# my.cnf
+[mysqld]
+binlog_format    = ROW
+binlog_row_image = FULL
+server_id        = 1            # any value, must be unique in the cluster
+log_bin          = mysql-bin
+gtid_mode        = ON           # recommended for safer resumption
+enforce_gtid_consistency = ON
+```
+
+**Grants the reader user needs:**
+
+```sql
+CREATE USER 'outbox_reader'@'%' IDENTIFIED BY 'strong-password';
+GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'outbox_reader'@'%';
+GRANT SELECT ON shop.outbox TO 'outbox_reader'@'%';
+```
+
+**Install the optional peer dep:**
+
+```bash
+npm i @vlasky/zongji
+```
+
+**Usage:**
+
+```ts
+import { MysqlStore, MysqlBinlogRelay } from "@eventferry/mysql";
+import { KafkaPublisher } from "@eventferry/kafka";
+
+// IMPORTANT: claimFailedOnly=true so the internal retry loop only drains
+// failures — pending rows are owned by the binlog stream.
+const store = new MysqlStore({ pool, claimFailedOnly: true });
+const publisher = new KafkaPublisher({ driver: "kafkajs", brokers, idempotent: true });
+
+const relay = new MysqlBinlogRelay({
+  store,
+  publisher,
+  binlog: {
+    host: "localhost",
+    user: "outbox_reader",
+    password: "strong-password",
+    database: "shop",
+    table: "outbox",
+    // serverId: 1_000_001,        // override if you have multiple readers
+    // startPosition: { filename: "mysql-bin.000042", position: 12345 },
+  },
+});
+await relay.start();
+process.on("SIGTERM", () => relay.stop());
+```
+
+**Position persistence (at-least-once across restarts):**
+
+MySQL has no server-side ack like Postgres logical replication, so binlog
+position tracking lives outside the server. Hook `onCommit` to persist the
+position to your own KV store, then pass it back via `startPosition` on the
+next start — without it the relay starts at the **end** of the binlog (tail
+mode) and won't replay rows written before it connected.
+
+## What's still not in this package
+
+- **No native low-latency waker** for the polling relay. MySQL has no
+  `LISTEN/NOTIFY`; if you don't want binlog mode either, tune the relay's
+  polling interval down (e.g. 100ms).
+- **Built-in position persistence** for the binlog relay — for now, you persist
+  the position yourself via the `onCommit` hook.
+
+## Retention
+
+The outbox table grows; `purgeDone` batch-deletes old `done` rows (run from
+your own cron):
+
+```ts
+await store.purgeDone({ olderThanMs: 7 * 24 * 60 * 60 * 1000 }); // 7 days
+```
+
+📖 **Full documentation:** [github.com/SametGoktepe/eventferry](https://github.com/SametGoktepe/eventferry#readme)
+
+## License
+
+MIT

package/dist/chunk-W2AQTNYH.js

@@ -0,0 +1,50 @@
+// src/ident.ts
+function assertIdent(name) {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid identifier "${name}": must match /^[a-zA-Z_][a-zA-Z0-9_]*$/`
+    );
+  }
+  return name;
+}
+
+// src/migrations.ts
+function createMigrationSql(tableName = "outbox") {
+  const t = assertIdent(tableName);
+  return `
+CREATE TABLE IF NOT EXISTS \`${t}\` (
+  id              BIGINT NOT NULL AUTO_INCREMENT,
+  message_id      CHAR(36) NOT NULL,
+  aggregate_type  VARCHAR(255) NOT NULL,
+  aggregate_id    VARCHAR(255) NOT NULL,
+  topic           VARCHAR(255) NOT NULL,
+  \`key\`           TEXT,
+  payload         JSON NOT NULL,
+  headers         JSON NOT NULL,
+  trace_id        VARCHAR(64),
+  status          TINYINT NOT NULL DEFAULT 0,
+  attempts        INT NOT NULL DEFAULT 0,
+  next_retry_at   DATETIME(3),
+  claimed_at      DATETIME(3),
+  created_at      DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+  processed_at    DATETIME(3),
+  PRIMARY KEY (id),
+  UNIQUE KEY uq_${t}_message_id (message_id),
+  KEY idx_${t}_ready (status, id),
+  KEY idx_${t}_agg_order (aggregate_id, id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+`.trim();
+}
+function createRetentionIndexSql(tableName = "outbox") {
+  const t = assertIdent(tableName);
+  return `
+CREATE INDEX idx_${t}_done_processed ON \`${t}\` (processed_at);
+`.trim();
+}
+
+export {
+  assertIdent,
+  createMigrationSql,
+  createRetentionIndexSql
+};
+//# sourceMappingURL=chunk-W2AQTNYH.js.map

package/dist/chunk-W2AQTNYH.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/ident.ts","../src/migrations.ts"],"sourcesContent":["/**\n * Allow only safe SQL identifier characters. Used wherever a user-supplied name\n * (table) is interpolated into SQL, to prevent injection.\n */\nexport function assertIdent(name: string): string {\n  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {\n    throw new Error(\n      `Invalid identifier \"${name}\": must match /^[a-zA-Z_][a-zA-Z0-9_]*$/`,\n    );\n  }\n  return name;\n}\n","import { assertIdent } from \"./ident.js\";\n\n/**\n * Generate the DDL for the MySQL outbox table, parameterized by table name.\n * Kept as a string template (not a file read) so it works regardless of how\n * the package is bundled or where it's installed.\n *\n * Requirements:\n *  - **MySQL 8.0.1+** or **MariaDB 10.6+** (needs `SELECT ... FOR UPDATE SKIP LOCKED`).\n *  - **InnoDB** engine — required for transactions and row-level locking.\n *  - `DATETIME(3)` is used (not `TIMESTAMP`) so values are tz-stable and free of\n *    the 2038 problem; millisecond precision matches the reaper's claim window.\n *\n * MySQL has no partial indexes, so `idx_${t}_ready` covers all statuses (it\n * still helps because the planner picks the index for `WHERE status IN (...)`).\n * Pair with `createRetentionIndexSql` only if `purgeDone` scans become hot.\n */\nexport function createMigrationSql(tableName = \"outbox\"): string {\n  const t = assertIdent(tableName);\n  return `\nCREATE TABLE IF NOT EXISTS \\`${t}\\` (\n  id              BIGINT NOT NULL AUTO_INCREMENT,\n  message_id      CHAR(36) NOT NULL,\n  aggregate_type  VARCHAR(255) NOT NULL,\n  aggregate_id    VARCHAR(255) NOT NULL,\n  topic           VARCHAR(255) NOT NULL,\n  \\`key\\`           TEXT,\n  payload         JSON NOT NULL,\n  headers         JSON NOT NULL,\n  trace_id        VARCHAR(64),\n  status          TINYINT NOT NULL DEFAULT 0,\n  attempts        INT NOT NULL DEFAULT 0,\n  next_retry_at   DATETIME(3),\n  claimed_at      DATETIME(3),\n  created_at      DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),\n  processed_at    DATETIME(3),\n  PRIMARY KEY (id),\n  UNIQUE KEY uq_${t}_message_id (message_id),\n  KEY idx_${t}_ready (status, id),\n  KEY idx_${t}_agg_order (aggregate_id, id)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;\n`.trim();\n}\n\n/**\n * Optional index that speeds up `purgeDone` on high-volume tables. The default\n * indexes don't cover `processed_at`, so the retention scan otherwise filesorts\n * across all done rows; this index makes it index-driven. Skip unless retention\n * scans are slow — it adds write/space overhead on the bulk (done) segment.\n */\nexport function createRetentionIndexSql(tableName = \"outbox\"): string {\n  const t = assertIdent(tableName);\n  return `\nCREATE INDEX idx_${t}_done_processed ON \\`${t}\\` (processed_at);\n`.trim();\n}\n"],"mappings":";AAIO,SAAS,YAAY,MAAsB;AAChD,MAAI,CAAC,2BAA2B,KAAK,IAAI,GAAG;AAC1C,UAAM,IAAI;AAAA,MACR,uBAAuB,IAAI;AAAA,IAC7B;AAAA,EACF;AACA,SAAO;AACT;;;ACMO,SAAS,mBAAmB,YAAY,UAAkB;AAC/D,QAAM,IAAI,YAAY,SAAS;AAC/B,SAAO;AAAA,+BACsB,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,kBAiBd,CAAC;AAAA,YACP,CAAC;AAAA,YACD,CAAC;AAAA;AAAA,EAEX,KAAK;AACP;AAQO,SAAS,wBAAwB,YAAY,UAAkB;AACpE,QAAM,IAAI,YAAY,SAAS;AAC/B,SAAO;AAAA,mBACU,CAAC,wBAAwB,CAAC;AAAA,EAC3C,KAAK;AACP;","names":[]}