@usehenri/jobs 0.0.0 → 1.2.0

package/src/cron.js

@@ -0,0 +1,237 @@
+/**
+ * A five field cron parser (minute hour day-of-month month day-of-week).
+ *
+ * Everything is computed in UTC: a recurring job runs at the same absolute
+ * moment wherever the runner is deployed, and no daylight saving change can
+ * make an hourly schedule fire twice or not at all. The documentation says
+ * so; if a schedule has to follow a wall clock, `every` is not the answer
+ * either -- enqueue from a job that knows the zone.
+ */
+
+const { coded } = require('./errors');
+
+const MONTHS = {
+  apr: 4,
+  aug: 8,
+  dec: 12,
+  feb: 2,
+  jan: 1,
+  jul: 7,
+  jun: 6,
+  mar: 3,
+  may: 5,
+  nov: 11,
+  oct: 10,
+  sep: 9,
+};
+
+const DAYS = { fri: 5, mon: 1, sat: 6, sun: 0, thu: 4, tue: 2, wed: 3 };
+
+const ALIASES = {
+  '@annually': '0 0 1 1 *',
+  '@daily': '0 0 * * *',
+  '@hourly': '0 * * * *',
+  '@midnight': '0 0 * * *',
+  '@monthly': '0 0 1 * *',
+  '@weekly': '0 0 * * 0',
+  '@yearly': '0 0 1 1 *',
+};
+
+const MINUTE = 60000;
+
+/** Four years of minutes is enough for `0 0 29 2 *` (a leap day) */
+const HORIZON = 4 * 366 * 24 * 60;
+
+/**
+ * The numeric value of one entry of a field
+ *
+ * @param {string} token The entry (a number or a name)
+ * @param {object} names The names this field accepts
+ * @param {string} field The field name, for the error
+ * @returns {number} The value
+ * @throws {Error} When the entry is not a number or a known name
+ */
+const value = (token, names, field) => {
+  const lower = token.toLowerCase();
+
+  if (Object.prototype.hasOwnProperty.call(names, lower)) {
+    return names[lower];
+  }
+
+  if (!/^\d+$/.test(token)) {
+    throw coded(
+      'HENRI_JOB_INVALID_CRON',
+      `Invalid ${field} "${token}" in cron expression`
+    );
+  }
+
+  return Number(token);
+};
+
+/**
+ * The allowed values of one cron field
+ *
+ * @param {string} spec The field (`*`, `1-5`, `0/15`, `mon,wed`)
+ * @param {number} min The lowest value of the field
+ * @param {number} max The highest value of the field
+ * @param {string} name The field name, for the errors
+ * @param {object} [names={}] The names the field accepts
+ * @returns {Set<number>} The values that match
+ * @throws {Error} When the field cannot be parsed
+ */
+const field = (spec, min, max, name, names = {}) => {
+  const values = new Set();
+
+  for (const part of String(spec).split(',')) {
+    const [range, step = '1'] = part.split('/');
+
+    if (!/^\d+$/.test(step) || Number(step) < 1) {
+      throw coded(
+        'HENRI_JOB_INVALID_CRON',
+        `Invalid step "${step}" in ${name} of a cron expression`
+      );
+    }
+
+    const by = Number(step);
+    let from = min;
+    let to = max;
+
+    if (range !== '*' && range !== '') {
+      const [start, end] = range.split('-');
+
+      from = value(start, names, name);
+      to = typeof end === 'undefined' ? from : value(end, names, name);
+
+      // `5/10` means "from 5 to the end of the field, every 10"
+      if (typeof end === 'undefined' && part.includes('/')) {
+        to = max;
+      }
+    }
+
+    if (from < min || to > max || from > to) {
+      throw coded(
+        'HENRI_JOB_INVALID_CRON',
+        `Out of range "${part}" in ${name} of a cron expression (${min}-${max})`
+      );
+    }
+
+    for (let entry = from; entry <= to; entry += by) {
+      values.add(entry);
+    }
+  }
+
+  return values;
+};
+
+/**
+ * Parses a cron expression
+ *
+ * @param {string} expression Five fields, or `@daily`, `@hourly`, ...
+ * @returns {object} The parsed schedule
+ * @throws {Error} When the expression is invalid
+ */
+const parse = (expression) => {
+  const text = String(expression || '').trim();
+  const normalized = ALIASES[text.toLowerCase()] || text;
+  const parts = normalized.split(/\s+/);
+
+  if (parts.length !== 5) {
+    throw coded(
+      'HENRI_JOB_INVALID_CRON',
+      `Invalid cron expression "${expression}": expected 5 fields (minute hour day month weekday)`
+    );
+  }
+
+  const [minute, hour, day, month, weekday] = parts;
+  const weekdays = field(weekday, 0, 7, 'weekday', DAYS);
+
+  // Both 0 and 7 are Sunday
+  if (weekdays.has(7)) {
+    weekdays.add(0);
+    weekdays.delete(7);
+  }
+
+  return {
+    days: field(day, 1, 31, 'day of month'),
+    everyDay: day === '*',
+    everyWeekday: weekday === '*',
+    hours: field(hour, 0, 23, 'hour'),
+    minutes: field(minute, 0, 59, 'minute'),
+    months: field(month, 1, 12, 'month', MONTHS),
+    weekdays,
+  };
+};
+
+/**
+ * Does this date fall on a day the schedule wants?
+ *
+ * Cron's oddity: when both the day of the month and the weekday are
+ * restricted, either one matching is enough.
+ *
+ * @param {object} schedule A parsed schedule
+ * @param {Date} date The date to test (read in UTC)
+ * @returns {boolean} Whether the day matches
+ */
+const matchesDay = (schedule, date) => {
+  const day = schedule.days.has(date.getUTCDate());
+  const weekday = schedule.weekdays.has(date.getUTCDay());
+
+  if (schedule.everyDay && schedule.everyWeekday) {
+    return true;
+  }
+
+  if (schedule.everyDay) {
+    return weekday;
+  }
+
+  if (schedule.everyWeekday) {
+    return day;
+  }
+
+  return day || weekday;
+};
+
+/**
+ * The first moment after `from` that the expression matches
+ *
+ * @param {(string|object)} expression A cron expression or a parsed schedule
+ * @param {number} [from=Date.now()] The moment to start from (exclusive)
+ * @returns {?number} A timestamp in milliseconds, or null when the
+ *   expression can never match again (`0 0 30 2 *`)
+ * @throws {Error} When the expression is invalid
+ */
+const next = (expression, from = Date.now()) => {
+  const schedule =
+    typeof expression === 'string' ? parse(expression) : expression;
+  const date = new Date(Math.floor(from / MINUTE) * MINUTE + MINUTE);
+
+  for (let guard = 0; guard < HORIZON; guard += 1) {
+    if (!schedule.months.has(date.getUTCMonth() + 1)) {
+      date.setUTCMonth(date.getUTCMonth() + 1, 1);
+      date.setUTCHours(0, 0, 0, 0);
+      continue;
+    }
+
+    if (!matchesDay(schedule, date)) {
+      date.setUTCDate(date.getUTCDate() + 1);
+      date.setUTCHours(0, 0, 0, 0);
+      continue;
+    }
+
+    if (!schedule.hours.has(date.getUTCHours())) {
+      date.setUTCHours(date.getUTCHours() + 1, 0, 0, 0);
+      continue;
+    }
+
+    if (!schedule.minutes.has(date.getUTCMinutes())) {
+      date.setUTCMinutes(date.getUTCMinutes() + 1, 0, 0);
+      continue;
+    }
+
+    return date.getTime();
+  }
+
+  return null;
+};
+
+module.exports = { next, parse };

package/src/definitions.js

@@ -0,0 +1,236 @@
+const path = require('path');
+const { globSync } = require('glob');
+
+const { duration } = require('./duration');
+const { JobError } = require('./errors');
+
+/**
+ * Job definitions live in `app/jobs`, in the shape henri already uses for
+ * models and controllers: a file exports an object. The name of a job is its
+ * path under `app/jobs` without the extension, so `app/jobs/mail/welcome.js`
+ * is the job `mail/welcome`.
+ */
+
+/** The widest a concurrency key may be: the column that holds it */
+const KEY_LENGTH = 190;
+
+/**
+ * Reads a job's `concurrency` declaration
+ *
+ * `1` is `{ limit: 1 }`; a `key` is a field of the arguments or a function
+ * of them, and a `group` is the name several jobs share a bound under. The
+ * default group is the job's own name, so a limit is that job's alone
+ * unless it says otherwise.
+ *
+ * The limit belongs to the **job**, never to the call: an option of
+ * `perform()` would let one caller step outside a bound the job declared,
+ * which is the one thing a bound is for.
+ *
+ * @param {string} name The job name
+ * @param {(number|object|null)} value What the file declared
+ * @returns {?object} `{ group, key, limit }`, or null
+ * @throws {JobError} HENRI_JOB_INVALID_CONCURRENCY on anything else
+ */
+const concurrency = (name, value) => {
+  if (value === null || typeof value === 'undefined' || value === false) {
+    return null;
+  }
+
+  const declared = typeof value === 'number' ? { limit: value } : value;
+  const refuse = (why) => {
+    throw new JobError(
+      'HENRI_JOB_INVALID_CONCURRENCY',
+      `The job "${name}" declares a concurrency limit that cannot be read: ${why}`,
+      {
+        hint: '`concurrency: 3`, or `concurrency: { limit: 3, key: "tenantId" }` to bound each key of its own',
+        job: name,
+      }
+    );
+  };
+
+  if (typeof declared !== 'object') {
+    refuse('it is neither a number nor an object');
+  }
+
+  const limit = Number(declared.limit);
+
+  if (!Number.isInteger(limit) || limit < 1) {
+    refuse(
+      `its limit is ${JSON.stringify(declared.limit)}, not a whole number above zero`
+    );
+  }
+
+  const group =
+    typeof declared.group === 'undefined' || declared.group === null
+      ? name
+      : declared.group;
+
+  if (typeof group !== 'string' || group === '') {
+    refuse('its group is not a name');
+  }
+
+  if (group.length > KEY_LENGTH) {
+    refuse(`its group is longer than ${KEY_LENGTH} characters`);
+  }
+
+  const { key } = declared;
+
+  if (
+    typeof key !== 'undefined' &&
+    key !== null &&
+    typeof key !== 'string' &&
+    typeof key !== 'function'
+  ) {
+    refuse('its key is neither the name of an argument nor a function of them');
+  }
+
+  return {
+    group,
+    key:
+      typeof key === 'string'
+        ? (args) => (args ? args[key] : null)
+        : key || null,
+    limit,
+  };
+};
+
+/**
+ * The concurrency key of one call, or null when the job is unbounded
+ *
+ * A key that resolves to nothing is the group's own bucket, which is also
+ * where a job enqueued before the limit was declared sits: the two mean the
+ * same thing, so they share a bound rather than each getting one.
+ *
+ * @param {object} definition A validated definition
+ * @param {*} args What perform() will receive
+ * @returns {?string} The key to store
+ * @throws {JobError} HENRI_JOB_INVALID_CONCURRENCY when the key cannot be read
+ */
+const keyOf = (definition, args) => {
+  const bound = definition && definition.concurrency;
+
+  if (!bound) {
+    return null;
+  }
+
+  if (!bound.key) {
+    return bound.group;
+  }
+
+  let value;
+
+  try {
+    value = bound.key(args);
+  } catch (error) {
+    throw new JobError(
+      'HENRI_JOB_INVALID_CONCURRENCY',
+      `The concurrency key of "${definition.name}" could not be read: ${error.message}`,
+      { cause: error, job: definition.name }
+    );
+  }
+
+  if (value === null || typeof value === 'undefined' || value === '') {
+    return bound.group;
+  }
+
+  if (typeof value === 'object') {
+    throw new JobError(
+      'HENRI_JOB_INVALID_CONCURRENCY',
+      `The concurrency key of "${definition.name}" is an object; it has to be a value that names one bound`,
+      { job: definition.name }
+    );
+  }
+
+  const key = `${bound.group}:${String(value)}`;
+
+  if (key.length > KEY_LENGTH) {
+    throw new JobError(
+      'HENRI_JOB_INVALID_CONCURRENCY',
+      `The concurrency key of "${definition.name}" is ${key.length} characters, over the ${KEY_LENGTH} that are stored`,
+      {
+        hint: 'A key names a bound, so it is an id or a tenant name; hash it yourself if it has to be longer',
+        job: definition.name,
+      }
+    );
+  }
+
+  return key;
+};
+
+/**
+ * Reads and checks one definition
+ *
+ * @param {string} name The job name
+ * @param {object} definition What the file exports
+ * @param {object} defaults The queue defaults (`queue`, `maxAttempts`, ...)
+ * @returns {object} The definition, with the defaults filled in
+ * @throws {JobError} HENRI_JOB_INVALID_DEFINITION without a `perform`
+ */
+const validate = (name, definition, defaults) => {
+  if (!definition || typeof definition.perform !== 'function') {
+    throw new JobError(
+      'HENRI_JOB_INVALID_DEFINITION',
+      `app/jobs/${name}.js does not export a perform(args, context) function`,
+      { job: name }
+    );
+  }
+
+  const backoff = definition.backoff || {};
+
+  return {
+    backoff: {
+      base: duration(backoff.base, defaults.backoff.base),
+      factor: Number(backoff.factor) || defaults.backoff.factor,
+      jitter:
+        typeof backoff.jitter === 'number'
+          ? backoff.jitter
+          : defaults.backoff.jitter,
+      max: duration(backoff.max, defaults.backoff.max),
+    },
+    concurrency: concurrency(name, definition.concurrency),
+    maxAttempts: Math.max(
+      1,
+      Number(definition.maxAttempts) || defaults.maxAttempts
+    ),
+    name,
+    perform: definition.perform,
+    priority:
+      typeof definition.priority === 'number'
+        ? definition.priority
+        : defaults.priority,
+    queue: definition.queue || defaults.queue,
+    timeout: duration(definition.timeout, defaults.timeout),
+  };
+};
+
+/**
+ * Loads every job of an application
+ *
+ * @param {string} location The `app/jobs` directory
+ * @param {object} defaults The queue defaults
+ * @returns {object} The definitions, by name
+ * @throws {JobError} BAD_JOB when a file is not a job
+ */
+const load = (location, defaults) => {
+  const dirname = path.resolve(location);
+  const definitions = {};
+  const files = globSync('**/*.js', {
+    cwd: dirname,
+    ignore: ['**/node_modules/**'],
+    nodir: true,
+    posix: true,
+  }).sort();
+
+  for (const file of files) {
+    const full = path.join(dirname, file);
+    const name = file.replace(/\.js$/, '');
+
+    delete require.cache[require.resolve(full)];
+
+    definitions[name] = validate(name, require(full), defaults);
+  }
+
+  return definitions;
+};
+
+module.exports = { KEY_LENGTH, concurrency, keyOf, load, validate };

package/src/duration.js

@@ -0,0 +1,112 @@
+/**
+ * Durations are written the way a human says them (`'30s'`, `'5m'`, `'2h'`,
+ * `'1d'`) or in milliseconds. Everything the queue stores is milliseconds.
+ */
+
+const { coded } = require('./errors');
+
+// A Map, not an object: the keys are one letter long on purpose
+const UNITS = new Map([
+  ['ms', 1],
+  ['s', 1000],
+  ['m', 60000],
+  ['h', 3600000],
+  ['d', 86400000],
+  ['w', 604800000],
+]);
+
+/**
+ * The written form: an amount, then a unit.
+ *
+ * The surrounding whitespace is trimmed before this runs rather than
+ * matched by it. `^\s*…\s*$` around an optional group is quadratic: on
+ * `'1' + ' '.repeat(60000) + '!'` the two star quantifiers split the run
+ * between them one position at a time, which measured 2 seconds before this
+ * was one `\s*` with nothing to share with. A duration reaches here from
+ * `--in=` and from whatever an application passes to `enqueue()`, so it is
+ * not always a value the author typed.
+ */
+const PATTERN = /^(\d+(?:\.\d+)?)\s*(ms|[smhdw])?$/i;
+
+/**
+ * Milliseconds of a duration
+ *
+ * @param {(number|string|null)} value `'5m'`, `300000`, or nothing
+ * @param {?number} [fallback=null] What to answer for an empty value
+ * @returns {?number} The duration in milliseconds
+ * @throws {Error} When the value is not a duration
+ */
+const duration = (value, fallback = null) => {
+  if (value === null || typeof value === 'undefined' || value === '') {
+    return fallback;
+  }
+
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value) || value < 0) {
+      throw coded('HENRI_JOB_INVALID_DURATION', `Invalid duration: ${value}`);
+    }
+
+    return Math.round(value);
+  }
+
+  const match = PATTERN.exec(String(value).trim());
+
+  if (!match) {
+    throw coded(
+      'HENRI_JOB_INVALID_DURATION',
+      `Invalid duration "${value}": use a number of milliseconds or 30s, 5m, 2h, 1d`
+    );
+  }
+
+  const [, amount, unit] = match;
+
+  return Math.round(Number(amount) * UNITS.get((unit || 'ms').toLowerCase()));
+};
+
+/**
+ * The moment a job should run, from `at` or `wait`
+ *
+ * @param {object} [options={}] `at` (a Date, an ISO string or a timestamp)
+ *   and `wait` (a duration from now)
+ * @param {number} [now=Date.now()] The current time
+ * @returns {number} A timestamp in milliseconds
+ * @throws {Error} When `at` is not a date
+ */
+const runAt = (options = {}, now = Date.now()) => {
+  const { at, wait } = options;
+
+  if (typeof at !== 'undefined' && at !== null) {
+    const date = at instanceof Date ? at : new Date(at);
+    const time = date.getTime();
+
+    if (Number.isNaN(time)) {
+      throw coded('HENRI_JOB_INVALID_DURATION', `Invalid date: ${String(at)}`);
+    }
+
+    return time;
+  }
+
+  return now + (duration(wait, 0) || 0);
+};
+
+/**
+ * A stored moment, as the API hands it out
+ *
+ * Every moment the queue stores is a BIGINT of milliseconds, and every
+ * moment it answers with is an ISO string. Some drivers read a BIGINT back
+ * as a string, so the number is taken first.
+ *
+ * @param {*} value A timestamp in milliseconds
+ * @returns {?string} An ISO string, or null
+ */
+const iso = (value) => {
+  if (value === null || typeof value === 'undefined' || value === '') {
+    return null;
+  }
+
+  const number = Number(value);
+
+  return Number.isNaN(number) ? null : new Date(number).toISOString();
+};
+
+module.exports = { duration, iso, runAt };

package/src/errors.js

@@ -0,0 +1,115 @@
+/**
+ * The errors `@usehenri/jobs` throws.
+ *
+ * Every one of them carries a `code`, so an application (or a test) can
+ * branch on the reason instead of matching the message. The codes are
+ * henri's own, from the catalogue of `@usehenri/core/error-codes.json`: a
+ * code is a string, so nothing is imported to raise one.
+ */
+
+/**
+ * Base error of the package
+ *
+ * @class JobError
+ * @extends {Error}
+ */
+class JobError extends Error {
+  /**
+   * Creates an instance of JobError.
+   *
+   * @param {string} code A henri error code (ex: HENRI_JOB_UNKNOWN)
+   * @param {string} message What went wrong
+   * @param {object} [options={}] `cause` and any extra property to carry
+   * @memberof JobError
+   */
+  constructor(code, message, options = {}) {
+    const { cause, ...rest } = options;
+
+    super(message, cause ? { cause } : undefined);
+
+    this.name = 'JobError';
+    this.code = code;
+    Object.assign(this, rest);
+  }
+}
+
+/**
+ * An argument that cannot be stored (a function, a bigint, a cycle, ...)
+ *
+ * @class JobArgumentError
+ * @extends {JobError}
+ */
+class JobArgumentError extends JobError {
+  /**
+   * Creates an instance of JobArgumentError.
+   *
+   * @param {string} message What went wrong
+   * @param {object} [options={}] `path` and the usual options
+   * @memberof JobArgumentError
+   */
+  constructor(message, options = {}) {
+    super('HENRI_JOB_INVALID_ARGUMENTS', message, options);
+    this.name = 'JobArgumentError';
+  }
+}
+
+/**
+ * An attempt that ran past the job's timeout
+ *
+ * @class JobTimeoutError
+ * @extends {JobError}
+ */
+class JobTimeoutError extends JobError {
+  /**
+   * Creates an instance of JobTimeoutError.
+   *
+   * @param {string} name The job name
+   * @param {number} timeout The timeout in milliseconds
+   * @memberof JobTimeoutError
+   */
+  constructor(name, timeout) {
+    super('HENRI_JOB_TIMEOUT', `${name} timed out after ${timeout}ms`);
+    this.name = 'JobTimeoutError';
+    this.timeout = timeout;
+  }
+}
+
+/**
+ * A store that cannot back the queue
+ *
+ * @class JobStoreError
+ * @extends {JobError}
+ */
+class JobStoreError extends JobError {
+  /**
+   * Creates an instance of JobStoreError.
+   *
+   * @param {string} message What went wrong
+   * @param {object} [options={}] The usual options
+   * @memberof JobStoreError
+   */
+  constructor(message, options = {}) {
+    super('HENRI_JOB_UNSUPPORTED_STORE', message, options);
+    this.name = 'JobStoreError';
+  }
+}
+
+/**
+ * An Error carrying one of henri's error codes
+ *
+ * A code is a string and nothing more (`@usehenri/core/error-codes.json` is
+ * the catalogue), so a failure names itself without importing anything.
+ *
+ * @param {string} code The henri error code (HENRI_JOB_UNKNOWN, ...)
+ * @param {string} message What went wrong
+ * @returns {Error} The error to throw
+ */
+const coded = (code, message) => Object.assign(new Error(message), { code });
+
+module.exports = {
+  JobArgumentError,
+  JobError,
+  JobStoreError,
+  JobTimeoutError,
+  coded,
+};