@usehenri/webhooks 0.0.0 → 1.2.0

package/src/store/schema.js

@@ -0,0 +1,230 @@
+const { coded } = require('../errors');
+
+/**
+ * The table `@usehenri/webhooks` owns, and the DDL of every SQL dialect
+ * henri can talk to.
+ *
+ * Endpoints are henri's data, not the application's: a url, a set of
+ * events, signing secrets and whether the thing is still enabled. They are
+ * never a henri model, for the reason the queue is not one either -- an
+ * application would then have to carry a migration for a table it does not
+ * own, and a store with no models at all would have no endpoints. So this
+ * is one table reached through the adapter's `query()`, or one MongoDB
+ * collection, exactly like `henri_jobs`.
+ *
+ * There is no deliveries table, and that is deliberate: a delivery is a job
+ * in `henri_jobs`, so what succeeded, what is waiting, what is dead and why
+ * is already answered by `henri jobs:list`, `henri jobs:dead` and
+ * `henri jobs:show`. A second table would be a second, worse copy of it.
+ *
+ * Every moment is a BIGINT of milliseconds since the epoch, as in the
+ * queue: sqlite has no date type and the other three disagree on the
+ * precision and the time zone of a bare `TIMESTAMP`.
+ */
+
+/** Table names an application may give */
+const SAFE_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/u;
+
+const DIALECTS = {
+  mssql: {
+    /**
+     * Wraps a statement so it only runs when the index is missing
+     *
+     * @param {string} table The table name
+     * @param {string} index The index name
+     * @param {string} statement The CREATE INDEX statement
+     * @returns {string} The guarded statement
+     */
+    guardIndex: (table, index, statement) =>
+      `IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = '${index}' AND object_id = OBJECT_ID('${table}')) ${statement}`,
+
+    /**
+     * Wraps a CREATE TABLE so it only runs when the table is missing
+     *
+     * @param {string} table The table name
+     * @param {string} statement The CREATE TABLE statement
+     * @returns {string} The guarded statement
+     */
+    guardTable: (table, statement) =>
+      `IF OBJECT_ID('${table}', 'U') IS NULL ${statement}`,
+
+    ifNotExists: '',
+    inlineIndexes: false,
+    quote: (identifier) => `[${identifier}]`,
+    text: 'NVARCHAR(MAX)',
+    url: 'NVARCHAR(2048)',
+  },
+  mysql: {
+    ifNotExists: 'IF NOT EXISTS',
+    // MySQL has no CREATE INDEX IF NOT EXISTS: the indexes are declared in
+    // the CREATE TABLE, which is guarded
+    inlineIndexes: true,
+    quote: (identifier) => `\`${identifier}\``,
+    text: 'MEDIUMTEXT',
+    url: 'VARCHAR(2048)',
+  },
+  postgres: {
+    ifNotExists: 'IF NOT EXISTS',
+    inlineIndexes: false,
+    quote: (identifier) => `"${identifier}"`,
+    text: 'TEXT',
+    url: 'VARCHAR(2048)',
+  },
+  sqlite: {
+    ifNotExists: 'IF NOT EXISTS',
+    inlineIndexes: false,
+    quote: (identifier) => `"${identifier}"`,
+    text: 'TEXT',
+    url: 'VARCHAR(2048)',
+  },
+};
+
+/** The columns of the endpoints table, in order */
+const COLUMNS = [
+  'id',
+  'owner',
+  'url',
+  'events',
+  'secrets',
+  'headers',
+  'description',
+  'disabled_at',
+  'disabled_reason',
+  'created_at',
+  'updated_at',
+];
+
+/**
+ * The column definitions of the endpoints table
+ *
+ * @param {object} dialect A dialect description
+ * @returns {Array<string>} The definitions
+ */
+const columns = (dialect) => [
+  'id VARCHAR(36) NOT NULL',
+  'owner VARCHAR(190) NULL',
+  `url ${dialect.url} NOT NULL`,
+  `events ${dialect.text} NOT NULL`,
+  `secrets ${dialect.text} NOT NULL`,
+  `headers ${dialect.text} NULL`,
+  'description VARCHAR(190) NULL',
+  'disabled_at BIGINT NULL',
+  'disabled_reason VARCHAR(190) NULL',
+  'created_at BIGINT NOT NULL',
+  'updated_at BIGINT NOT NULL',
+  'PRIMARY KEY (id)',
+];
+
+/**
+ * The indexes of the endpoints table
+ *
+ * One index, on what a lookup filters by: the tenant an event belongs to
+ * and whether the endpoint still takes deliveries. Which events an endpoint
+ * subscribes to is a JSON list, matched in this process, because no two of
+ * these four dialects agree on how to ask that question.
+ *
+ * @param {string} table The table name
+ * @returns {Array<object>} `{ name, columns }` entries
+ */
+const indexes = (table) => [
+  { columns: ['owner', 'disabled_at'], name: `${table}_owner` },
+];
+
+/**
+ * Every statement `henri webhooks:install` runs, in order
+ *
+ * All of them are idempotent: running the install twice, or against a
+ * database another process already prepared, changes nothing.
+ *
+ * @param {string} name The dialect (sqlite, postgres, mysql, mssql)
+ * @param {object} tables `{ endpoints }` table names
+ * @returns {Array<string>} The statements
+ * @throws {Error} When the dialect or the table name is unknown
+ */
+const install = (name, tables) => {
+  const dialect = DIALECTS[name];
+
+  if (!dialect) {
+    throw coded(
+      'HENRI_WEBHOOK_UNSUPPORTED_STORE',
+      `@usehenri/webhooks: unsupported SQL dialect "${name}"`
+    );
+  }
+
+  if (!SAFE_NAME.test(tables.endpoints)) {
+    throw coded(
+      'HENRI_CONFIG_INVALID',
+      `@usehenri/webhooks: invalid table name "${tables.endpoints}": letters, digits and underscores only`
+    );
+  }
+
+  const table = tables.endpoints;
+  const quoted = dialect.quote(table);
+  const definitions = [...columns(dialect)];
+  const statements = [];
+
+  if (dialect.inlineIndexes) {
+    for (const index of indexes(table)) {
+      definitions.push(
+        `KEY ${dialect.quote(index.name)} (${index.columns.join(', ')})`
+      );
+    }
+  }
+
+  const create = [
+    'CREATE TABLE',
+    dialect.ifNotExists,
+    `${quoted} (\n  ${definitions.join(',\n  ')}\n)`,
+  ]
+    .filter(Boolean)
+    .join(' ');
+
+  statements.push(
+    dialect.guardTable ? dialect.guardTable(table, create) : create
+  );
+
+  if (dialect.inlineIndexes) {
+    return statements;
+  }
+
+  for (const index of indexes(table)) {
+    const statement = [
+      'CREATE INDEX',
+      dialect.guardIndex ? '' : dialect.ifNotExists,
+      `${dialect.quote(index.name)} ON ${quoted} (${index.columns.join(', ')})`,
+    ]
+      .filter(Boolean)
+      .join(' ');
+
+    statements.push(
+      dialect.guardIndex
+        ? dialect.guardIndex(table, index.name, statement)
+        : statement
+    );
+  }
+
+  return statements;
+};
+
+/**
+ * The statement that drops the table
+ *
+ * @param {string} name The dialect
+ * @param {object} tables `{ endpoints }` table names
+ * @returns {Array<string>} The statements
+ * @throws {Error} When the dialect is unknown
+ */
+const uninstall = (name, tables) => {
+  const dialect = DIALECTS[name];
+
+  if (!dialect) {
+    throw coded(
+      'HENRI_WEBHOOK_UNSUPPORTED_STORE',
+      `@usehenri/webhooks: unsupported SQL dialect "${name}"`
+    );
+  }
+
+  return [`DROP TABLE IF EXISTS ${dialect.quote(tables.endpoints)}`];
+};
+
+module.exports = { COLUMNS, DIALECTS, columns, indexes, install, uninstall };

package/src/store/sql.js

@@ -0,0 +1,450 @@
+const debug = require('debug')('henri:webhooks:sql');
+
+const { COLUMNS, install, uninstall } = require('./schema');
+const { WebhookError } = require('../errors');
+
+/**
+ * The SQL backend of the endpoints table.
+ *
+ * Everything goes through the store adapter's own `query()`, so no henri
+ * model is involved and an application whose store has no models at all
+ * still has endpoints. There is no claiming and no contention here -- an
+ * endpoint is written by an operator and read by the runners -- so this is
+ * six plain statements, and the concurrency of the queue is not repeated.
+ */
+
+/**
+ * Errors that mean the object was created by someone else in between
+ *
+ * `CREATE TABLE IF NOT EXISTS` is not atomic against a concurrent creation
+ * on PostgreSQL: two processes booting together can both find the table
+ * missing, and one of them then fails on the catalogue's own unique index.
+ * The install is idempotent by intent, so that failure means it is done.
+ */
+const ALREADY_THERE =
+  /already exists|duplicate key|duplicate table|there is already an object named/iu;
+
+/** Errors that mean "another writer got there first, try again" */
+const RETRYABLE =
+  /deadlock|lock wait timeout|database is locked|database table is locked|SQLITE_BUSY/iu;
+
+/**
+ * Everything an error says about itself, wrappers included
+ *
+ * Sequelize keeps the driver error on `parent`, drizzle on `cause`.
+ *
+ * @param {*} error An error
+ * @param {number} [depth=4] How far to unwrap
+ * @returns {string} The messages, joined
+ */
+const reasons = (error, depth = 4) => {
+  const said = [];
+  let current = error;
+
+  for (let step = 0; step < depth && current; step += 1) {
+    said.push(String(current.message || ''), String(current.code || ''));
+    current = current.parent || current.original || current.cause;
+  }
+
+  return said.join(' ');
+};
+
+/**
+ * A number read back from any driver (pg hands BIGINT over as a string)
+ *
+ * @param {*} value The stored value
+ * @returns {?number} The number, or null
+ */
+const toNumber = (value) => {
+  if (value === null || typeof value === 'undefined' || value === '') {
+    return null;
+  }
+
+  const number = Number(value);
+
+  return Number.isNaN(number) ? null : number;
+};
+
+/**
+ * The `?` placeholders of a list
+ *
+ * @param {Array} values The values
+ * @returns {string} `?, ?, ?`
+ */
+const marks = (values) => values.map(() => '?').join(', ');
+
+/**
+ * The SQL store
+ *
+ * @class SqlStore
+ */
+class SqlStore {
+  /**
+   * Creates an instance of SqlStore.
+   *
+   * @param {object} adapter A henri store adapter with `query()`
+   * @param {object} options Options
+   * @param {string} options.dialect sqlite, postgres, mysql or mssql
+   * @param {boolean} [options.dollars=false] The driver numbers its
+   *   placeholders (`$1`), as node-postgres does
+   * @param {object} options.tables `{ endpoints }` table names
+   * @memberof SqlStore
+   */
+  constructor(adapter, { dialect, dollars = false, tables }) {
+    this.adapter = adapter;
+    this.dialect = dialect;
+    this.dollars = dollars;
+    this.tables = tables;
+    this.kind = 'sql';
+  }
+
+  /**
+   * The statement with the placeholders the driver expects
+   *
+   * @param {string} sql A statement written with `?` placeholders
+   * @returns {string} The statement
+   * @memberof SqlStore
+   */
+  prepare(sql) {
+    if (!this.dollars) {
+      return sql;
+    }
+
+    let index = 0;
+
+    return sql.replace(/\?/gu, () => {
+      index += 1;
+
+      return `$${index}`;
+    });
+  }
+
+  /**
+   * Runs an operation again when the database says another writer won
+   *
+   * @param {Function} fn The operation
+   * @param {number} [attempts=8] How many times to try
+   * @returns {Promise<*>} What fn returns
+   * @throws {Error} The last error when every attempt failed
+   * @memberof SqlStore
+   */
+  async retrying(fn, attempts = 8) {
+    let last = null;
+
+    for (let attempt = 0; attempt < attempts; attempt += 1) {
+      try {
+        return await fn();
+      } catch (error) {
+        if (!RETRYABLE.test(reasons(error))) {
+          throw error;
+        }
+
+        last = error;
+        await new Promise((resolve) => setTimeout(resolve, 15 * (attempt + 1)));
+      }
+    }
+
+    throw last;
+  }
+
+  /**
+   * Runs a statement that returns no rows
+   *
+   * @param {string} sql The statement, with `?` placeholders
+   * @param {Array} [params=[]] The parameters
+   * @returns {Promise<void>} Resolves when done
+   * @memberof SqlStore
+   */
+  async run(sql, params = []) {
+    debug('run %s', sql);
+
+    await this.retrying(() => this.adapter.query(this.prepare(sql), params));
+  }
+
+  /**
+   * Runs a query and returns its rows
+   *
+   * @param {string} sql The query, with `?` placeholders
+   * @param {Array} [params=[]] The parameters
+   * @returns {Promise<Array<object>>} The rows
+   * @memberof SqlStore
+   */
+  async select(sql, params = []) {
+    debug('select %s', sql);
+
+    const result = await this.retrying(() =>
+      this.adapter.query(this.prepare(sql), params, { type: 'SELECT' })
+    );
+
+    return Array.isArray(result) ? result : [];
+  }
+
+  /**
+   * Creates the table and its index; idempotent
+   *
+   * @returns {Promise<Array<string>>} The statements that ran
+   * @memberof SqlStore
+   */
+  async install() {
+    const statements = install(this.dialect, this.tables);
+
+    for (const statement of statements) {
+      try {
+        await this.run(statement);
+      } catch (error) {
+        if (!ALREADY_THERE.test(reasons(error))) {
+          throw error;
+        }
+
+        debug('another process created it first: %s', error.message);
+      }
+    }
+
+    return statements;
+  }
+
+  /**
+   * Drops the table
+   *
+   * @returns {Promise<Array<string>>} The statements that ran
+   * @memberof SqlStore
+   */
+  async uninstall() {
+    const statements = uninstall(this.dialect, this.tables);
+
+    for (const statement of statements) {
+      await this.run(statement);
+    }
+
+    return statements;
+  }
+
+  /**
+   * Whether the table is there
+   *
+   * @returns {Promise<boolean>} true when it answers
+   * @memberof SqlStore
+   */
+  async installed() {
+    try {
+      await this.select(
+        `SELECT COUNT(*) AS total FROM ${this.tables.endpoints}`
+      );
+
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  /**
+   * Inserts an endpoint
+   *
+   * @param {object} row A row, in database shape
+   * @returns {Promise<object>} The endpoint, read back
+   * @memberof SqlStore
+   */
+  async insert(row) {
+    const values = COLUMNS.map((column) =>
+      typeof row[column] === 'undefined' ? null : row[column]
+    );
+
+    await this.run(
+      `INSERT INTO ${this.tables.endpoints} (${COLUMNS.join(', ')}) VALUES (${marks(COLUMNS)})`,
+      values
+    );
+
+    return this.find(row.id);
+  }
+
+  /**
+   * One endpoint by id
+   *
+   * @param {string} id The endpoint id
+   * @returns {Promise<?object>} The row, or null
+   * @memberof SqlStore
+   */
+  async find(id) {
+    const [row] = await this.select(
+      `SELECT * FROM ${this.tables.endpoints} WHERE id = ?`,
+      [id]
+    );
+
+    return row || null;
+  }
+
+  /**
+   * Writes a few columns of an endpoint
+   *
+   * @param {string} id The endpoint id
+   * @param {object} changes The columns to write
+   * @returns {Promise<?object>} The row, read back
+   * @memberof SqlStore
+   */
+  async update(id, changes) {
+    const keys = Object.keys(changes).filter((key) => COLUMNS.includes(key));
+
+    if (keys.length === 0) {
+      return this.find(id);
+    }
+
+    await this.run(
+      `UPDATE ${this.tables.endpoints} SET ${keys
+        .map((key) => `${key} = ?`)
+        .join(', ')} WHERE id = ?`,
+      [...keys.map((key) => changes[key]), id]
+    );
+
+    return this.find(id);
+  }
+
+  /**
+   * Deletes an endpoint
+   *
+   * @param {string} id The endpoint id
+   * @returns {Promise<boolean>} Whether there was one to delete
+   * @memberof SqlStore
+   */
+  async remove(id) {
+    const before = await this.find(id);
+
+    await this.run(`DELETE FROM ${this.tables.endpoints} WHERE id = ?`, [id]);
+
+    return Boolean(before);
+  }
+
+  /**
+   * The endpoints of an application, or of one owner
+   *
+   * @param {object} [filter={}] `owner`, `disabled`, `limit`, `offset`
+   * @returns {Promise<Array<object>>} The rows
+   * @memberof SqlStore
+   */
+  async list(filter = {}) {
+    const where = [];
+    const params = [];
+
+    if (typeof filter.owner === 'string') {
+      where.push('owner = ?');
+      params.push(filter.owner);
+    }
+
+    // `null` is a filter, not the absence of one: an event emitted without
+    // an owner reaches the endpoints that have none, and never a tenant's
+    if (filter.owner === null) {
+      where.push('owner IS NULL');
+    }
+
+    if (filter.disabled === false) {
+      where.push('disabled_at IS NULL');
+    }
+
+    if (filter.disabled === true) {
+      where.push('disabled_at IS NOT NULL');
+    }
+
+    const limit = Math.max(1, Math.min(Number(filter.limit) || 1000, 10000));
+    const offset = Math.max(0, Number(filter.offset) || 0);
+    const clause = where.length > 0 ? ` WHERE ${where.join(' AND ')}` : '';
+    const paging =
+      this.dialect === 'mssql'
+        ? ` OFFSET ${offset} ROWS FETCH NEXT ${limit} ROWS ONLY`
+        : ` LIMIT ${limit} OFFSET ${offset}`;
+
+    return this.select(
+      `SELECT * FROM ${this.tables.endpoints}${clause} ORDER BY created_at ASC, id ASC${paging}`,
+      params
+    );
+  }
+
+  /**
+   * How many endpoints there are
+   *
+   * @param {object} [filter={}] `owner`, `disabled`
+   * @returns {Promise<number>} The count
+   * @memberof SqlStore
+   */
+  async count(filter = {}) {
+    const where = [];
+    const params = [];
+
+    if (typeof filter.owner === 'string') {
+      where.push('owner = ?');
+      params.push(filter.owner);
+    }
+
+    if (filter.owner === null) {
+      where.push('owner IS NULL');
+    }
+
+    if (filter.disabled === false) {
+      where.push('disabled_at IS NULL');
+    }
+
+    if (filter.disabled === true) {
+      where.push('disabled_at IS NOT NULL');
+    }
+
+    const clause = where.length > 0 ? ` WHERE ${where.join(' AND ')}` : '';
+    const [row] = await this.select(
+      `SELECT COUNT(*) AS total FROM ${this.tables.endpoints}${clause}`,
+      params
+    );
+
+    return toNumber(row && (row.total || row.TOTAL)) || 0;
+  }
+}
+
+/**
+ * The dialect of a store adapter, or nothing when it is not SQL
+ *
+ * @param {object} adapter A henri store adapter
+ * @returns {?object} `{ dialect, dollars }`
+ */
+const describe = (adapter) => {
+  // The drizzle adapter names its dialect and its placeholder style
+  if (adapter.dialect && typeof adapter.dialect === 'object') {
+    return {
+      dialect: adapter.dialect.name,
+      dollars: adapter.dialect.placeholder(1) === '$1',
+    };
+  }
+
+  // The sequelize adapters: the dialect comes from the connector, and
+  // sequelize renders `?` replacements itself on every dialect
+  if (typeof adapter.ensureConnector === 'function') {
+    return { dialect: adapter.ensureConnector().getDialect(), dollars: false };
+  }
+
+  return null;
+};
+
+/**
+ * Builds the SQL store of an adapter
+ *
+ * @param {object} adapter A henri store adapter
+ * @param {object} tables `{ endpoints }` table names
+ * @returns {SqlStore} The store
+ * @throws {WebhookError} When the dialect cannot hold the endpoints
+ */
+const create = (adapter, tables) => {
+  const described = describe(adapter);
+
+  if (!described) {
+    throw new WebhookError(
+      'HENRI_WEBHOOK_UNSUPPORTED_STORE',
+      `@usehenri/webhooks: the ${adapter.adapterName} adapter has no SQL surface`
+    );
+  }
+
+  if (!['mssql', 'mysql', 'postgres', 'sqlite'].includes(described.dialect)) {
+    throw new WebhookError(
+      'HENRI_WEBHOOK_UNSUPPORTED_STORE',
+      `@usehenri/webhooks: the ${described.dialect} dialect is not supported`
+    );
+  }
+
+  return new SqlStore(adapter, { ...described, tables });
+};
+
+module.exports = { SqlStore, create, describe, reasons, toNumber };