@usehenri/webhooks 0.0.0 → 1.2.0

package/src/config.js

@@ -0,0 +1,85 @@
+const { duration } = require('@usehenri/jobs');
+
+const { coded } = require('./errors');
+
+/**
+ * The `webhooks` block of `config/<env>.json`, with every default filled in.
+ *
+ * The durations are read by the queue's own parser, so `'10s'` means the
+ * same thing in `webhooks.timeout` as it does in `jobs.timeout`.
+ *
+ * The retry policy is the queue's, and only its numbers are different:
+ * eight attempts with a base of ten seconds tripling up to six hours is
+ * roughly three days of trying, which is what a receiver that is down for a
+ * weekend needs and what Stripe settled on. A delivery that runs out of
+ * attempts is in the dead letter queue, not gone.
+ */
+
+const DEFAULTS = {
+  allowHttp: false,
+  allowPrivate: false,
+  backoff: { base: '10s', factor: 3, jitter: 0.2, max: '6h' },
+  install: true,
+  maxAttempts: 8,
+  maxFanout: 1000,
+  queue: 'webhooks',
+  store: 'default',
+  table: 'henri_webhooks',
+  timeout: '10s',
+};
+
+/**
+ * Checks a table name before it is written into every statement
+ *
+ * The table name is configuration and not request input, but an application
+ * may set any key from the environment, and a name that is not a plain
+ * identifier gives a syntax error deep in a query instead of a sentence
+ * here.
+ *
+ * @param {string} value The name
+ * @returns {string} The name
+ * @throws {Error} HENRI_CONFIG_INVALID when it is not a plain identifier
+ */
+const table = (value) => {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(value)) {
+    throw coded(
+      'HENRI_CONFIG_INVALID',
+      `@usehenri/webhooks: invalid table name "${value}": letters, digits and underscores only`
+    );
+  }
+
+  return value;
+};
+
+/**
+ * The webhooks configuration of an application
+ *
+ * @param {object} [config={}] The `webhooks` block
+ * @returns {object} The configuration, with the defaults filled in
+ * @throws {Error} When a duration or the table name is invalid
+ */
+const normalize = (config = {}) => {
+  const value = config || {};
+  const backoff = { ...DEFAULTS.backoff, ...(value.backoff || {}) };
+  const name = table(value.table || DEFAULTS.table);
+
+  return {
+    allowHttp: value.allowHttp === true,
+    allowPrivate: value.allowPrivate === true,
+    backoff: {
+      base: duration(backoff.base, 10000),
+      factor: Number(backoff.factor) || DEFAULTS.backoff.factor,
+      jitter: Math.min(Math.max(Number(backoff.jitter) || 0, 0), 1),
+      max: duration(backoff.max, 21600000),
+    },
+    install: value.install !== false,
+    maxAttempts: Math.max(1, Number(value.maxAttempts) || DEFAULTS.maxAttempts),
+    maxFanout: Math.max(1, Number(value.maxFanout) || DEFAULTS.maxFanout),
+    queue: value.queue || DEFAULTS.queue,
+    store: value.store || DEFAULTS.store,
+    tables: { endpoints: name },
+    timeout: duration(value.timeout, 10000),
+  };
+};
+
+module.exports = { DEFAULTS, normalize, table };

package/src/deliver.js

@@ -0,0 +1,271 @@
+const http = require('http');
+const https = require('https');
+const debug = require('debug')('henri:webhooks');
+
+const { WebhookDeliveryError, WebhookError } = require('./errors');
+const { check } = require('./address');
+
+/**
+ * One HTTP request to a receiver, and what to make of the answer.
+ *
+ * Everything here assumes the receiver is hostile until it has proven
+ * otherwise, which it never quite does:
+ *
+ * - the socket connects to the address `address.js` checked, and to no
+ *   other: the agent is handed a `lookup` that answers that one address, so
+ *   the name cannot resolve to something else between the check and the
+ *   connection. TLS still validates against the *name*, not the address, so
+ *   pinning costs no certificate warning;
+ * - a redirect is not followed. A `3xx` is a receiver choosing the next
+ *   url, after henri checked the one it was given, and following it would
+ *   hand back the hole the address check just closed. Stripe treats a
+ *   redirect as a failure too; henri goes one step further and does not
+ *   retry it, because the fix is a registration change, not time;
+ * - the answer is read up to `maxBytes` and then the socket is destroyed,
+ *   so a receiver that streams forever holds a runner for the length of one
+ *   timeout and not a byte more. Nothing in the answer is parsed: it is
+ *   kept as a short excerpt, for the operator to read;
+ * - one deadline covers the whole exchange -- resolution, connection,
+ *   headers and body -- because three separate timeouts add up to a wait
+ *   nobody configured.
+ */
+
+/** How much of an answer is read, and kept for the operator */
+const MAX_BYTES = 65536;
+
+/** How much of the answer is shown in the queue's error message */
+const EXCERPT = 500;
+
+/**
+ * The statuses a delivery may not be retried on, with what they mean
+ *
+ * Everything else is retried, `4xx` included: a `401` is a token that was
+ * rotated a minute ago, a `404` is a deploy that is half done, and the
+ * queue's backoff is patient enough to outlive both. The two exceptions are
+ * the answers where waiting cannot help:
+ *
+ * - `3xx`: the url is wrong. It has to be re-registered.
+ * - `410 Gone`: the receiver is saying "stop". henri stops, and disables
+ *   the endpoint so nothing else is queued for it.
+ */
+const FINAL = {
+  gone: 410,
+  redirect: [300, 399],
+};
+
+/**
+ * Whether an answer means the delivery arrived
+ *
+ * @param {number} status The HTTP status
+ * @returns {boolean} Whether it is a 2xx
+ */
+const accepted = (status) => status >= 200 && status < 300;
+
+/**
+ * A short, single-line excerpt of what a receiver answered
+ *
+ * @param {string} body The answer body
+ * @returns {string} The excerpt
+ */
+const excerpt = (body) =>
+  String(body || '')
+    .replace(/\s+/gu, ' ')
+    .trim()
+    .slice(0, EXCERPT);
+
+/**
+ * Sends one request, to one address, with one deadline
+ *
+ * @param {object} options Options
+ * @param {URL} options.url The url
+ * @param {string} options.address The address the socket connects to
+ * @param {number} options.family 4 or 6
+ * @param {string} options.body The body, as it was signed
+ * @param {object} options.headers The headers to send
+ * @param {number} options.timeout The deadline, in milliseconds
+ * @param {number} [options.maxBytes=MAX_BYTES] How much answer to read
+ * @returns {Promise<object>} `{ status, headers, body, duration }`
+ * @throws {WebhookError} HENRI_WEBHOOK_TIMEOUT, or the socket's own error
+ */
+const send = ({
+  address,
+  body,
+  family,
+  headers,
+  maxBytes = MAX_BYTES,
+  timeout,
+  url,
+}) =>
+  new Promise((resolve, reject) => {
+    const started = Date.now();
+    const client = url.protocol === 'https:' ? https : http;
+    const payload = Buffer.from(body, 'utf8');
+    let settled = false;
+    let timer = null;
+
+    /**
+     * Answers once, whichever of the deadline and the socket got there first
+     *
+     * @param {?Error} error What went wrong, or nothing
+     * @param {object} [answer] The answer
+     * @returns {void}
+     */
+    const done = (error, answer) => {
+      if (settled) {
+        return;
+      }
+
+      settled = true;
+      clearTimeout(timer);
+
+      if (error) {
+        reject(error);
+
+        return;
+      }
+
+      resolve({ ...answer, duration: Date.now() - started });
+    };
+
+    const request = client.request(
+      url,
+      {
+        agent: false,
+        headers: {
+          ...headers,
+          'content-length': String(payload.length),
+        },
+        // The name resolved once, was checked, and is not asked again: this
+        // is what closes the window between the check and the connection.
+        // `autoSelectFamily` (on by default since Node 20) asks with
+        // `all: true` and wants a list back, so both shapes are answered
+        lookup: (hostname, settings, callback) =>
+          settings && settings.all
+            ? callback(null, [{ address, family }])
+            : callback(null, address, family),
+        method: 'POST',
+      },
+      (response) => {
+        const chunks = [];
+        let size = 0;
+
+        response.on('data', (chunk) => {
+          size += chunk.length;
+          chunks.push(size > maxBytes ? chunk.subarray(0, maxBytes) : chunk);
+
+          if (size >= maxBytes) {
+            response.destroy();
+          }
+        });
+
+        /**
+         * Hands back what was read, however the answer ended
+         *
+         * @returns {void}
+         */
+        const finish = () =>
+          done(null, {
+            body: Buffer.concat(chunks).toString('utf8').slice(0, maxBytes),
+            headers: response.headers,
+            status: response.statusCode,
+          });
+
+        response.on('end', finish);
+        // A body cut short by maxBytes emits `close`, never `end`
+        response.on('close', finish);
+        response.on('error', (error) => done(error));
+      }
+    );
+
+    timer = setTimeout(() => {
+      request.destroy();
+      done(
+        new WebhookError(
+          'HENRI_WEBHOOK_TIMEOUT',
+          `the receiver did not answer within ${timeout}ms`,
+          { timeout }
+        )
+      );
+    }, timeout);
+
+    request.on('error', (error) => done(error));
+    request.end(payload);
+  });
+
+/**
+ * Delivers a signed body to a url, and says what the answer means
+ *
+ * @param {object} options Options
+ * @param {string} options.url Where to deliver
+ * @param {string} options.body The body, as it was signed
+ * @param {object} options.headers The headers, signature included
+ * @param {number} options.timeout The deadline, in milliseconds
+ * @param {boolean} [options.allowHttp] Allow a plaintext url
+ * @param {boolean} [options.allowPrivate] Allow a private address
+ * @param {Function} [options.lookup] A `dns.promises.lookup` stand-in
+ * @param {number} [options.maxBytes] How much answer to read
+ * @returns {Promise<object>} `{ status, address, duration, body }`
+ * @throws {WebhookAddressError} When the address may not be reached
+ * @throws {WebhookDeliveryError} When the receiver refused the delivery
+ */
+const deliver = async (options) => {
+  const { address, family, url } = await check(options.url, options);
+
+  debug('POST %s (%s)', url.href, address);
+
+  const answer = await send({
+    address,
+    body: options.body,
+    family,
+    headers: options.headers,
+    maxBytes: options.maxBytes,
+    timeout: options.timeout,
+    url,
+  });
+  const result = {
+    address,
+    body: excerpt(answer.body),
+    duration: answer.duration,
+    status: answer.status,
+  };
+
+  if (accepted(answer.status)) {
+    return result;
+  }
+
+  const [from, to] = FINAL.redirect;
+  const location = answer.headers.location;
+
+  if (answer.status >= from && answer.status <= to) {
+    throw new WebhookDeliveryError(
+      `the receiver answered ${answer.status} and pointed at ${location || 'nowhere'}; a delivery does not follow a redirect`,
+      {
+        ...result,
+        hint: 'Register the url the redirect names: henri webhooks:update <id> --url <url>',
+        retryable: false,
+      }
+    );
+  }
+
+  if (answer.status === FINAL.gone) {
+    throw new WebhookDeliveryError(
+      'the receiver answered 410 Gone: it is asking not to be sent to again',
+      { ...result, gone: true, retryable: false }
+    );
+  }
+
+  throw new WebhookDeliveryError(
+    `the receiver answered ${answer.status}${result.body ? `: ${result.body}` : ''}`,
+    result
+  );
+};
+
+module.exports = {
+  EXCERPT,
+  FINAL,
+  MAX_BYTES,
+  accepted,
+  deliver,
+  excerpt,
+  send,
+};

package/src/errors.js

@@ -0,0 +1,107 @@
+/**
+ * The errors `@usehenri/webhooks` throws.
+ *
+ * Every one of them carries a `code` from henri's own catalogue
+ * (`@usehenri/core/error-codes.json`), so an application branches on the
+ * reason instead of matching a message. A code is a string and nothing
+ * more: raising one imports nothing.
+ *
+ * `retryable` is the other half. A delivery is a job, so a failure that the
+ * queue should try again says so by staying silent (the default), and one
+ * that a retry cannot fix -- a redirect, an address that must not be
+ * reached, a receiver that answered `410 Gone` -- carries
+ * `retryable: false`, which buries the job on the spot instead of spending
+ * eight attempts to learn the same thing.
+ */
+
+/**
+ * Base error of the package
+ *
+ * @class WebhookError
+ * @extends {Error}
+ */
+class WebhookError extends Error {
+  /**
+   * Creates an instance of WebhookError.
+   *
+   * @param {string} code A henri error code (ex: HENRI_WEBHOOK_UNKNOWN)
+   * @param {string} message What went wrong
+   * @param {object} [options={}] `cause`, `retryable` and anything to carry
+   * @memberof WebhookError
+   */
+  constructor(code, message, options = {}) {
+    const { cause, ...rest } = options;
+
+    super(message, cause ? { cause } : undefined);
+
+    this.name = 'WebhookError';
+    this.code = code;
+    Object.assign(this, rest);
+  }
+}
+
+/**
+ * A url a delivery must not open
+ *
+ * The address is checked when the request is made and not when the endpoint
+ * was registered, because DNS answers differently later: the name that
+ * resolved to a public address yesterday resolves to 169.254.169.254 today,
+ * and only the resolution that the request itself does is evidence.
+ *
+ * @class WebhookAddressError
+ * @extends {WebhookError}
+ */
+class WebhookAddressError extends WebhookError {
+  /**
+   * Creates an instance of WebhookAddressError.
+   *
+   * @param {string} message What is wrong with the address
+   * @param {object} [options={}] `url`, `address` and the usual options
+   * @memberof WebhookAddressError
+   */
+  constructor(message, options = {}) {
+    super('HENRI_WEBHOOK_ADDRESS_REFUSED', message, {
+      retryable: false,
+      ...options,
+    });
+    this.name = 'WebhookAddressError';
+  }
+}
+
+/**
+ * A receiver that answered something other than a 2xx
+ *
+ * @class WebhookDeliveryError
+ * @extends {WebhookError}
+ */
+class WebhookDeliveryError extends WebhookError {
+  /**
+   * Creates an instance of WebhookDeliveryError.
+   *
+   * @param {string} message What the receiver said
+   * @param {object} [options={}] `status`, `retryable` and the usual options
+   * @memberof WebhookDeliveryError
+   */
+  constructor(message, options = {}) {
+    super('HENRI_WEBHOOK_DELIVERY_FAILED', message, options);
+    this.name = 'WebhookDeliveryError';
+  }
+}
+
+/**
+ * An Error carrying one of henri's error codes
+ *
+ * @param {string} code The henri error code
+ * @param {string} message What went wrong
+ * @param {object} [rest={}] Anything else to carry (`hint`, `retryable`)
+ * @returns {Error} The error to throw
+ */
+const coded = (code, message, rest = {}) =>
+  Object.assign(new Error(message), { code, ...rest });
+
+module.exports = {
+  WebhookAddressError,
+  WebhookDeliveryError,
+  WebhookError,
+  coded,
+};

package/src/job.js

@@ -0,0 +1,49 @@
+const { DELIVERY_JOB } = require('./webhooks');
+
+/**
+ * The job that performs one delivery.
+ *
+ * A delivery is a job and nothing else, which is the whole design: the
+ * retries, the exponential backoff, the dead letter queue, the visibility
+ * (`henri jobs:list --queue webhooks`, `henri jobs:dead`,
+ * `henri jobs:show <id>`) and the recovery of a runner that died mid-flight
+ * are the queue's, already written, already covered on four databases. A
+ * second delivery mechanism would be a second, worse copy of it.
+ *
+ * The job's own timeout is the HTTP deadline plus a margin, on purpose: the
+ * request's deadline should be what fires, because it says
+ * "the receiver did not answer within 10000ms" and the job's timeout says
+ * only that the attempt ran long. The margin is the backstop for the case
+ * where something outside the request hangs.
+ */
+
+/** How much longer than the request the job itself may take */
+const MARGIN = 5000;
+
+/**
+ * The definition of the delivery job, for `henri.jobs.define()`
+ *
+ * @param {object} webhooks A started Webhooks
+ * @returns {object} `{ name, definition }`
+ */
+const definition = (webhooks) => ({
+  definition: {
+    backoff: webhooks.config.backoff,
+    maxAttempts: webhooks.config.maxAttempts,
+
+    /**
+     * Delivers one webhook
+     *
+     * @param {object} args `{ body, endpoint, event, id }`
+     * @param {object} context The job context
+     * @returns {Promise<object>} What happened
+     */
+    perform: (args, context) => webhooks.perform(args, context),
+
+    queue: webhooks.config.queue,
+    timeout: webhooks.config.timeout + MARGIN,
+  },
+  name: DELIVERY_JOB,
+});
+
+module.exports = { MARGIN, definition };