@usehenri/jobs 0.0.0 → 1.2.0

package/src/store/schema.js

@@ -0,0 +1,499 @@
+/**
+ * The tables `@usehenri/jobs` owns, and the DDL of every SQL dialect henri
+ * can talk to.
+ *
+ * The queue never goes through a henri model: it owns four tables of its own
+ * so it cannot collide with the application's schema, and so a store that
+ * has no models at all (a fresh application) still has a queue.
+ *
+ * ## The upgrade block
+ *
+ * These tables are `CREATE TABLE IF NOT EXISTS` and there is no migration
+ * chain behind them, so a table an older henri created is the table an
+ * installation still has. A **new table** is therefore free -- the guarded
+ * create makes it appear -- and a **new column** is not.
+ *
+ * `upgrade()` is the answer: the statements that bring an existing table up
+ * to what this version writes, every one of them idempotent, and every one
+ * of them **tolerated** by the store (see `SqlStore#install`). A database
+ * user who may not `ALTER` never fails a boot over a feature the
+ * application does not use; the feature itself asks whether its column is
+ * there (`SqlStore#concurrent`) and refuses with the install line when it is
+ * not. The same statements run on a fresh database, where they find their
+ * work already done.
+ *
+ * Every moment is stored as a BIGINT of milliseconds since the epoch rather
+ * than a timestamp column: sqlite has no date type, MySQL, PostgreSQL and
+ * MSSQL disagree on the precision and on the time zone of a bare
+ * `TIMESTAMP`, and the claim compares `run_at` to the runner's clock -- a
+ * comparison that has to mean the same thing on every dialect. Numbers read
+ * back as strings on some drivers (BIGINT over the pg protocol), so every
+ * read coerces.
+ */
+
+/** Names an application may give the tables */
+const { coded } = require('../errors');
+
+const SAFE_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+const DIALECTS = {
+  mssql: {
+    /**
+     * Adds a column, only when the table has none by that name
+     *
+     * @param {string} table The table name
+     * @param {string} column The column name
+     * @param {string} type The column type
+     * @returns {string} The statement
+     */
+    addColumn: (table, column, type) =>
+      `IF COL_LENGTH('${table}', '${column}') IS NULL ALTER TABLE [${table}] ADD ${column} ${type} NULL`,
+
+    /**
+     * Wraps a statement so it only runs when the index is missing
+     *
+     * @param {string} table The table name
+     * @param {string} index The index name
+     * @param {string} statement The CREATE INDEX statement
+     * @returns {string} The guarded statement
+     */
+    guardIndex: (table, index, statement) =>
+      `IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = '${index}' AND object_id = OBJECT_ID('${table}')) ${statement}`,
+
+    /**
+     * Wraps a CREATE TABLE so it only runs when the table is missing
+     *
+     * @param {string} table The table name
+     * @param {string} statement The CREATE TABLE statement
+     * @returns {string} The guarded statement
+     */
+    guardTable: (table, statement) =>
+      `IF OBJECT_ID('${table}', 'U') IS NULL ${statement}`,
+
+    ifNotExists: '',
+    indexIfNotExists: '',
+    inlineIndexes: false,
+    int: 'INT',
+    // MSSQL is the one dialect that treats NULLs as equal in a unique index
+    partialUnique: true,
+    quote: (identifier) => `[${identifier}]`,
+    text: 'NVARCHAR(MAX)',
+  },
+  mysql: {
+    // MySQL has no ADD COLUMN IF NOT EXISTS: a second run answers 1060,
+    // which the store tolerates like every other upgrade statement
+    addColumn: (table, column, type) =>
+      `ALTER TABLE \`${table}\` ADD COLUMN ${column} ${type} NULL`,
+    ifNotExists: 'IF NOT EXISTS',
+    // MySQL has no CREATE INDEX IF NOT EXISTS: the indexes are declared in
+    // the CREATE TABLE, which is guarded. An index the create cannot carry
+    // (a `late` one, on a table that is already there) is written bare and
+    // tolerated when it answers 1061
+    indexIfNotExists: '',
+    inlineIndexes: true,
+    int: 'INT',
+    partialUnique: false,
+    quote: (identifier) => `\`${identifier}\``,
+    // TEXT stops at 65535 bytes and the arguments alone may reach 65536
+    text: 'MEDIUMTEXT',
+  },
+  postgres: {
+    addColumn: (table, column, type) =>
+      `ALTER TABLE "${table}" ADD COLUMN IF NOT EXISTS ${column} ${type} NULL`,
+    ifNotExists: 'IF NOT EXISTS',
+    indexIfNotExists: 'IF NOT EXISTS',
+    inlineIndexes: false,
+    int: 'INTEGER',
+    partialUnique: false,
+    quote: (identifier) => `"${identifier}"`,
+    text: 'TEXT',
+  },
+  sqlite: {
+    // SQLite has no ADD COLUMN IF NOT EXISTS either: a second run answers
+    // "duplicate column name", which the store tolerates
+    addColumn: (table, column, type) =>
+      `ALTER TABLE "${table}" ADD COLUMN ${column} ${type} NULL`,
+    ifNotExists: 'IF NOT EXISTS',
+    indexIfNotExists: 'IF NOT EXISTS',
+    inlineIndexes: false,
+    int: 'INTEGER',
+    partialUnique: false,
+    quote: (identifier) => `"${identifier}"`,
+    text: 'TEXT',
+  },
+};
+
+/**
+ * The columns an older henri did not write, and the type they take.
+ *
+ * One entry per column added after a version that shipped: the create
+ * statement declares them and `upgrade()` adds them to a table that has
+ * them not.
+ */
+const ADDED = [
+  { column: 'concurrency_key', type: 'VARCHAR(190)' },
+  { column: 'batch_id', type: 'VARCHAR(36)' },
+  // 190 is the width core gives a tenant (`base/tenancy.js`, MAX_TENANT)
+  // and the width `@usehenri/webhooks` gives an endpoint's owner: what
+  // MySQL indexes in a utf8mb4 key. A tenant is never truncated to fit
+  { column: 'tenant', type: 'VARCHAR(190)' },
+];
+
+/**
+ * The columns of the jobs table, in order
+ *
+ * @param {object} dialect A dialect description
+ * @returns {Array<string>} The column definitions
+ */
+const jobColumns = (dialect) => [
+  'id VARCHAR(36) NOT NULL',
+  'queue VARCHAR(120) NOT NULL',
+  'name VARCHAR(120) NOT NULL',
+  `args ${dialect.text} NOT NULL`,
+  'state VARCHAR(16) NOT NULL',
+  `priority ${dialect.int} NOT NULL`,
+  `attempts ${dialect.int} NOT NULL`,
+  `max_attempts ${dialect.int} NOT NULL`,
+  `timeout_ms ${dialect.int} NULL`,
+  'run_at BIGINT NOT NULL',
+  'created_at BIGINT NOT NULL',
+  'updated_at BIGINT NOT NULL',
+  'started_at BIGINT NULL',
+  'finished_at BIGINT NULL',
+  `duration_ms ${dialect.int} NULL`,
+  'claimed_by VARCHAR(120) NULL',
+  'claimed_at BIGINT NULL',
+  'heartbeat_at BIGINT NULL',
+  'claim_token VARCHAR(36) NULL',
+  `error_message ${dialect.text} NULL`,
+  `error_stack ${dialect.text} NULL`,
+  `history ${dialect.text} NULL`,
+  'unique_key VARCHAR(190) NULL',
+  'concurrency_key VARCHAR(190) NULL',
+  'batch_id VARCHAR(36) NULL',
+  'tenant VARCHAR(190) NULL',
+  'PRIMARY KEY (id)',
+];
+
+/**
+ * The columns of the concurrency table, in order
+ *
+ * One row is one slot of one key: `(limit_key, slot)` is the primary key, so
+ * `slot` counts from zero to the job's limit and an INSERT is what takes it.
+ * That unique index is the whole bound -- see `SqlStore#takeSlot`.
+ *
+ * @param {object} dialect A dialect description
+ * @returns {Array<string>} The column definitions
+ */
+const limitColumns = (dialect) => [
+  'limit_key VARCHAR(190) NOT NULL',
+  `slot ${dialect.int} NOT NULL`,
+  'job_id VARCHAR(36) NULL',
+  'runner VARCHAR(120) NOT NULL',
+  'taken_at BIGINT NOT NULL',
+  'heartbeat_at BIGINT NOT NULL',
+  'PRIMARY KEY (limit_key, slot)',
+];
+
+/**
+ * The indexes of the concurrency table
+ *
+ * @param {string} table The table name
+ * @returns {Array<object>} `{ name, columns, unique }` entries
+ */
+const limitIndexes = (table) => [
+  { columns: ['heartbeat_at'], name: `${table}_stale`, unique: false },
+];
+
+/**
+ * The columns of the batches table, in order
+ *
+ * A batch counts: `total` is what was enqueued under it and is written once,
+ * when the batch is sealed; `done` and `failed` are advanced by the same
+ * token-guarded write that records an attempt's outcome, one statement per
+ * job. `finished_at` is stamped after the callback has been enqueued, so a
+ * crash in between leaves the batch unfinished and the sweep settles it
+ * again -- the enqueue is idempotent (see `../keys.js`).
+ *
+ * @param {object} dialect A dialect description
+ * @returns {Array<string>} The column definitions
+ */
+const batchColumns = (dialect) => [
+  'id VARCHAR(36) NOT NULL',
+  'name VARCHAR(190) NULL',
+  'callback VARCHAR(120) NULL',
+  `callback_args ${dialect.text} NULL`,
+  `callback_options ${dialect.text} NULL`,
+  'callback_id VARCHAR(36) NULL',
+  `total ${dialect.int} NOT NULL`,
+  `done ${dialect.int} NOT NULL`,
+  `failed ${dialect.int} NOT NULL`,
+  'created_at BIGINT NOT NULL',
+  'updated_at BIGINT NOT NULL',
+  'sealed_at BIGINT NULL',
+  'finished_at BIGINT NULL',
+  'PRIMARY KEY (id)',
+];
+
+/**
+ * The indexes of the batches table
+ *
+ * @param {string} table The table name
+ * @returns {Array<object>} `{ name, columns, unique }` entries
+ */
+const batchIndexes = (table) => [
+  {
+    columns: ['finished_at', 'updated_at'],
+    name: `${table}_open`,
+    unique: false,
+  },
+];
+
+/**
+ * The columns of the schedules table, in order
+ *
+ * @param {object} dialect A dialect description
+ * @returns {Array<string>} The column definitions
+ */
+const scheduleColumns = (dialect) => [
+  'name VARCHAR(190) NOT NULL',
+  'job VARCHAR(120) NOT NULL',
+  'spec VARCHAR(190) NOT NULL',
+  'next_run_at BIGINT NOT NULL',
+  'last_run_at BIGINT NULL',
+  'token VARCHAR(36) NULL',
+  'created_at BIGINT NOT NULL',
+  'updated_at BIGINT NOT NULL',
+  'PRIMARY KEY (name)',
+];
+
+/**
+ * The indexes of the jobs table
+ *
+ * @param {string} table The table name
+ * @returns {Array<object>} `{ name, columns, unique }` entries
+ */
+const jobIndexes = (table) => [
+  {
+    columns: ['state', 'queue', 'priority', 'run_at'],
+    name: `${table}_claim`,
+    unique: false,
+  },
+  { columns: ['claim_token'], name: `${table}_token`, unique: false },
+  {
+    columns: ['state', 'finished_at'],
+    name: `${table}_finished`,
+    unique: false,
+  },
+  { columns: ['unique_key'], name: `${table}_unique`, unique: true },
+  // `late`: it arrived with a column an older table has not, so it belongs
+  // to the upgrade block on every dialect rather than to the create
+  {
+    columns: ['state', 'concurrency_key', 'run_at'],
+    late: true,
+    name: `${table}_limited`,
+    unique: false,
+  },
+  {
+    columns: ['batch_id'],
+    late: true,
+    name: `${table}_batch`,
+    unique: false,
+  },
+  // The listing `henri jobs:list --tenant` makes, and nothing else: the
+  // claim is deliberately **not** narrowed by tenant, so this index is
+  // never on the hot path (see `SqlStore#claimStatement`)
+  {
+    columns: ['tenant', 'state', 'run_at'],
+    late: true,
+    name: `${table}_tenant`,
+    unique: false,
+  },
+];
+
+/**
+ * The statement that creates one index
+ *
+ * @param {object} dialect A dialect description
+ * @param {string} table The table name
+ * @param {object} index An index description
+ * @returns {string} The statement
+ */
+const indexStatement = (dialect, table, index) => {
+  const filter =
+    index.unique && dialect.partialUnique
+      ? ` WHERE ${index.columns[0]} IS NOT NULL`
+      : '';
+  const statement = [
+    `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX`,
+    dialect.guardIndex ? '' : dialect.indexIfNotExists,
+    `${dialect.quote(index.name)} ON ${dialect.quote(table)} (${index.columns.join(', ')})${filter}`,
+  ]
+    .filter(Boolean)
+    .join(' ');
+
+  return dialect.guardIndex
+    ? dialect.guardIndex(table, index.name, statement)
+    : statement;
+};
+
+/**
+ * The statements that create a table and its indexes, in order
+ *
+ * An index marked `late` is left out: it names a column an older table does
+ * not have, so it is the upgrade block's, on every dialect at once.
+ *
+ * @param {object} dialect A dialect description
+ * @param {string} table The table name
+ * @param {Array<string>} columns The column definitions
+ * @param {Array<object>} all The indexes
+ * @returns {Array<string>} The statements to run, in order
+ */
+const statementsFor = (dialect, table, columns, all) => {
+  const quoted = dialect.quote(table);
+  const definitions = [...columns];
+  const statements = [];
+  const indexes = all.filter((index) => !index.late);
+
+  if (dialect.inlineIndexes) {
+    for (const index of indexes) {
+      definitions.push(
+        `${index.unique ? 'UNIQUE KEY' : 'KEY'} ${dialect.quote(index.name)} (${index.columns.join(', ')})`
+      );
+    }
+  }
+
+  const create = [
+    'CREATE TABLE',
+    dialect.ifNotExists,
+    `${quoted} (\n  ${definitions.join(',\n  ')}\n)`,
+  ]
+    .filter(Boolean)
+    .join(' ');
+
+  statements.push(
+    dialect.guardTable ? dialect.guardTable(table, create) : create
+  );
+
+  if (dialect.inlineIndexes) {
+    return statements;
+  }
+
+  for (const index of indexes) {
+    statements.push(indexStatement(dialect, table, index));
+  }
+
+  return statements;
+};
+
+/**
+ * The statements that bring a table an older henri created up to date
+ *
+ * Every one of them is idempotent and every one of them is tolerated by the
+ * store: a database user who may not `ALTER` never fails a boot over a
+ * feature the application does not use. What decides whether the feature
+ * works is asking the table, not whether these ran.
+ *
+ * @param {string} name The dialect (sqlite, postgres, mysql, mssql)
+ * @param {object} tables `{ jobs, schedules, limits, batches }` table names
+ * @returns {Array<string>} The statements
+ * @throws {Error} When the dialect is unknown
+ */
+const upgrade = (name, tables) => {
+  const dialect = DIALECTS[name];
+
+  if (!dialect) {
+    throw coded(
+      'HENRI_JOB_UNSUPPORTED_STORE',
+      `@usehenri/jobs: unsupported SQL dialect "${name}"`
+    );
+  }
+
+  return [
+    ...ADDED.map((added) =>
+      dialect.addColumn(tables.jobs, added.column, added.type)
+    ),
+    ...jobIndexes(tables.jobs)
+      .filter((index) => index.late)
+      .map((index) => indexStatement(dialect, tables.jobs, index)),
+  ];
+};
+
+/**
+ * Every statement `henri jobs:install` runs, in order
+ *
+ * All of them are idempotent: running the install twice, or against a
+ * database another runner already prepared, changes nothing.
+ *
+ * @param {string} name The dialect (sqlite, postgres, mysql, mssql)
+ * @param {object} tables `{ jobs, schedules, limits, batches }` table names
+ * @returns {Array<string>} The statements
+ * @throws {Error} When the dialect or a table name is unknown
+ */
+const install = (name, tables) => {
+  const dialect = DIALECTS[name];
+
+  if (!dialect) {
+    throw coded(
+      'HENRI_JOB_UNSUPPORTED_STORE',
+      `@usehenri/jobs: unsupported SQL dialect "${name}"`
+    );
+  }
+
+  for (const table of Object.values(tables)) {
+    if (!SAFE_NAME.test(table)) {
+      throw coded(
+        'HENRI_CONFIG_INVALID',
+        `@usehenri/jobs: invalid table name "${table}": letters, digits and underscores only`
+      );
+    }
+  }
+
+  return [
+    ...statementsFor(
+      dialect,
+      tables.jobs,
+      jobColumns(dialect),
+      jobIndexes(tables.jobs)
+    ),
+    ...statementsFor(dialect, tables.schedules, scheduleColumns(dialect), []),
+    ...statementsFor(
+      dialect,
+      tables.limits,
+      limitColumns(dialect),
+      limitIndexes(tables.limits)
+    ),
+    ...statementsFor(
+      dialect,
+      tables.batches,
+      batchColumns(dialect),
+      batchIndexes(tables.batches)
+    ),
+    ...upgrade(name, tables),
+  ];
+};
+
+/**
+ * The statements that drop the tables, newest first
+ *
+ * @param {string} name The dialect
+ * @param {object} tables `{ jobs, schedules, limits, batches }` table names
+ * @returns {Array<string>} The statements
+ * @throws {Error} When the dialect is unknown
+ */
+const uninstall = (name, tables) => {
+  const dialect = DIALECTS[name];
+
+  if (!dialect) {
+    throw coded(
+      'HENRI_JOB_UNSUPPORTED_STORE',
+      `@usehenri/jobs: unsupported SQL dialect "${name}"`
+    );
+  }
+
+  return [tables.batches, tables.limits, tables.schedules, tables.jobs].map(
+    (table) => `DROP TABLE IF EXISTS ${dialect.quote(table)}`
+  );
+};
+
+module.exports = { ADDED, DIALECTS, install, uninstall, upgrade };