@toa.io/storages.mongodb 1.0.0-alpha.31 → 1.0.0-alpha.310

package/src/indexes.js

@@ -0,0 +1,84 @@
+import { console } from 'openspan'
+
+/**
+ * The indexes the runtime keeps on collections of its own — the outbox's and the inbox's. They
+ * are declared in code rather than in a migration a component would have to write, so making
+ * and pruning them is the same job in both places and is written once.
+ */
+
+/**
+ * The name of one of these is fixed, so changing what it is made of — a retention that changes
+ * `expireAfterSeconds`, say — leaves that name held by an index of the old shape, which MongoDB
+ * refuses to overwrite. The old one is dropped and the declared one made.
+ *
+ * @param {import('mongodb').Collection} collection
+ * @param {object} fields
+ * @param {object} options
+ */
+export async function index(collection, fields, options) {
+  try {
+    await collection.createIndex(fields, options)
+  } catch (e) {
+    if (!CONFLICTS.includes(e.code))
+      return console.warn('MongoDB index creation failed', {
+        collection: collection.collectionName,
+        name: options.name,
+        error: e
+      })
+
+    console.info('Recreating an index whose declaration changed', {
+      collection: collection.collectionName,
+      name: options.name
+    })
+
+    await drop(collection, options.name)
+    await collection.createIndex(fields, options)
+  }
+}
+
+/**
+ * Removes what this collection carries that is no longer declared, so that a change to what the
+ * runtime wants does not leave the shape it wanted before in place forever.
+ *
+ * @param {import('mongodb').Collection} collection
+ * @param {string[]} desired the names that are declared
+ */
+export async function prune(collection, desired) {
+  let current
+
+  try {
+    current = await collection.listIndexes().toArray()
+  } catch {
+    return
+  }
+
+  const obsolete = current
+    .map(({ name }) => name)
+    .filter((name) => name !== '_id_' && !desired.includes(name))
+
+  if (obsolete.length === 0) return
+
+  console.info('Removing obsolete indexes', {
+    collection: collection.collectionName,
+    indexes: obsolete.join(', ')
+  })
+
+  await Promise.all(obsolete.map((name) => drop(collection, name)))
+}
+
+/**
+ * Concurrent replicas prune the same index, and losing that race is not an error.
+ *
+ * @param {import('mongodb').Collection} collection
+ * @param {string} name
+ */
+async function drop(collection, name) {
+  try {
+    await collection.dropIndex(name)
+  } catch (e) {
+    if (e.code !== ERR_INDEX_NOT_FOUND) throw e
+  }
+}
+
+const ERR_INDEX_NOT_FOUND = 27
+const CONFLICTS = [85, 86] // IndexOptionsConflict, IndexKeySpecsConflict

package/src/measurements.js

@@ -0,0 +1,39 @@
+import { registry } from 'openspan'
+
+/**
+ * What this storage measures. `provider` is which database it is, so that a deployment running more
+ * than one can be read a kind at a time. No database label: the database is the context, which
+ * every series already carries as its resource.
+ */
+const meters = registry()
+
+/** Prometheus' own ladder, in seconds: from five milliseconds to ten seconds. */
+const DURATIONS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
+
+const PROVIDER = 'mongodb'
+
+const duration = meters.histogram(
+  'toa.storage.query.duration',
+  { buckets: DURATIONS, unit: 's' },
+  { provider: null, collection: null, operation: null }
+)
+
+/**
+ * A compare-and-swap that lost. Whether it is retried or raised is the operation's to declare, and
+ * the retried one succeeds — so without this it is a duration that grew and nothing else.
+ */
+const conflicts = meters.counter('toa.storage.conflicts', {
+  provider: null,
+  collection: null
+})
+
+export function query(collection, operation) {
+  return {
+    histogram: duration,
+    labels: { provider: PROVIDER, collection, operation }
+  }
+}
+
+export function conflicted(collection) {
+  conflicts.add(1, { provider: PROVIDER, collection })
+}

package/src/migrations.js

@@ -0,0 +1,343 @@
+import { randomUUID } from 'node:crypto'
+import { setTimeout as sleep } from 'node:timers/promises'
+
+import { console } from 'openspan'
+
+/**
+ * Applies a component's migrations to its collection, in the order they were declared, and
+ * records each one so that it is applied once for the database rather than once per replica.
+ *
+ * The record is also the lock: the row that says a migration ran is inserted before it runs,
+ * and MongoDB refuses the second insert of the same `_id`. Nothing else is needed to make the
+ * group agree, which is why this works where there is no Redis.
+ */
+export class Migrations {
+  /** @type {import('mongodb').Collection} */
+  #collection
+
+  /** @type {import('mongodb').Collection} */
+  #state
+
+  /** @type {Array<{ id: string, steps: object[] }>} */
+  #list
+
+  /**
+   * @param {import('mongodb').Db} db
+   * @param {import('mongodb').Collection} collection the entity's own
+   * @param {Array<{ id: string, steps: object[] }>} list
+   */
+  constructor(db, collection, list) {
+    this.#collection = collection
+    this.#state = db.collection(STATE)
+    this.#list = list
+  }
+
+  /**
+   * A migration is applied only after the one before it, because a later one is written
+   * against what an earlier one leaves behind.
+   */
+  async run() {
+    let applied = 0
+
+    for (const migration of this.#list) if (await this.#apply(migration)) applied++
+
+    // debug, because the ordinary start has nothing to report: every migration was applied
+    // long ago by whoever started first. But a run that says nothing at all cannot be told
+    // from one that never looked
+    console.debug('Migrations checked', {
+      collection: this.#collection.collectionName,
+      declared: this.#list.length,
+      applied
+    })
+  }
+
+  /**
+   * Answers whether this replica was the one that applied it.
+   *
+   * @private
+   */
+  async #apply(migration) {
+    const id = `${this.#collection.collectionName}:${migration.id}`
+
+    /** when the wait for another replica was last reported */
+    let announced
+
+    while (true) {
+      if (await this.#claim(id)) {
+        await this.#run(id, migration)
+
+        return true
+      }
+
+      const row = await this.#state.findOne({ _id: id })
+
+      // removed between the claim and the read; whoever did that wants it applied again
+      if (row === null) continue
+
+      if (row.state === DONE) {
+        console.debug('Migration was applied already', { migration: id })
+
+        return false
+      }
+
+      const stale = Date.now() - row.heartbeat.getTime() > LEASE
+
+      if (!stale) {
+        /*
+         * The component does not serve until this returns, so a replica waiting here is a pod
+         * that is simply not up, with nothing anywhere saying why. On its own cadence rather
+         * than the poll's, which is a second.
+         */
+        if (announced === undefined || Date.now() - announced > PROGRESS) {
+          announced = Date.now()
+
+          console.info('Waiting for another replica to apply a migration', {
+            migration: id,
+            owner: row.owner
+          })
+        }
+
+        await sleep(POLL)
+
+        continue
+      }
+
+      console.warn('Taking over an abandoned migration', {
+        migration: id,
+        owner: row.owner,
+        heartbeat: row.heartbeat
+      })
+
+      if (await this.#steal(id, row.heartbeat)) {
+        await this.#run(id, migration)
+
+        return true
+      }
+    }
+  }
+
+  /**
+   * Writes the row that says this replica is applying the migration. Answers whether it won.
+   *
+   * @private
+   */
+  async #claim(id) {
+    try {
+      await this.#state.insertOne({
+        _id: id,
+        state: RUNNING,
+        owner: OWNER,
+        started: new Date(),
+        heartbeat: new Date()
+      })
+
+      return true
+    } catch (error) {
+      if (error.code === ERR_DUPLICATE_KEY) return false
+
+      throw error
+    }
+  }
+
+  /**
+   * Takes a claim whose owner stopped saying it was alive. The heartbeat it read is part of
+   * the criteria, so only one of several waiting replicas takes it.
+   *
+   * @private
+   */
+  async #steal(id, heartbeat) {
+    const result = await this.#state.updateOne(
+      { _id: id, state: RUNNING, heartbeat },
+      { $set: { owner: OWNER, heartbeat: new Date() } }
+    )
+
+    return result.modifiedCount === 1
+  }
+
+  /**
+   * Applies the steps and marks the migration done. A step that throws leaves the row as it
+   * is: the component does not start, and the next replica to reach a stale claim runs the
+   * migration again from its first step.
+   *
+   * @private
+   */
+  async #run(id, migration) {
+    const total = migration.steps.length
+    const started = Date.now()
+
+    console.info('Applying migration', { migration: id, steps: total })
+
+    const beat = setInterval(() => {
+      this.#state
+        .updateOne({ _id: id }, { $set: { heartbeat: new Date() } })
+        .catch((error) =>
+          console.warn('Migration heartbeat failed', { migration: id, error })
+        )
+    }, HEARTBEAT)
+
+    beat.unref?.()
+
+    let applying = 0
+
+    /*
+     * A backfill takes as long as the collection is large, and every replica of the group waits
+     * out the whole of it. On its own cadence rather than the heartbeat's, which is every five
+     * seconds and would say this a dozen times a minute.
+     */
+    const progress = setInterval(() => {
+      console.info('Migration is still being applied', {
+        migration: id,
+        step: applying,
+        steps: total,
+        elapsed: Date.now() - started
+      })
+    }, PROGRESS)
+
+    progress.unref?.()
+
+    try {
+      for (const step of migration.steps) {
+        applying++
+
+        await this.#step(id, step)
+      }
+
+      await this.#state.updateOne(
+        { _id: id },
+        { $set: { state: DONE, completed: new Date() } }
+      )
+    } finally {
+      clearInterval(beat)
+      clearInterval(progress)
+    }
+
+    console.info('Migration applied', { migration: id, elapsed: Date.now() - started })
+  }
+
+  /** @private */
+  async #step(id, step) {
+    const verbs = Object.keys(step ?? {})
+
+    if (verbs.length !== 1 || !(verbs[0] in STEPS))
+      throw new Error(
+        `Migration '${id}' has a step that is not one of ` +
+          `${Object.keys(STEPS).join(', ')}: ${JSON.stringify(step)}`
+      )
+
+    await STEPS[verbs[0]](this.#collection, step[verbs[0]], id)
+  }
+}
+
+/**
+ * Creates the index the step declares. Where the name is held by an index of a different shape,
+ * that one is dropped: the declaration is what the component is to run against, and refusing
+ * to reconcile is what left a database diverged from its manifest before migrations existed.
+ */
+async function index(collection, { name, keys, ...rest }, id) {
+  if (name === undefined || keys === undefined)
+    throw new Error(`Migration '${id}' declares an index without a name or keys`)
+
+  const spec = Object.fromEntries(
+    Object.entries(keys).map(([field, direction]) => [
+      field,
+      DIRECTIONS[direction] ?? direction
+    ])
+  )
+
+  const options = { name }
+
+  for (const [key, value] of Object.entries(rest)) {
+    if (!(key in OPTIONS))
+      throw new Error(`Migration '${id}' declares an unknown index option '${key}'`)
+
+    options[OPTIONS[key]] = value
+  }
+
+  try {
+    await collection.createIndex(spec, options)
+  } catch (error) {
+    if (!CONFLICTS.includes(error.code)) throw error
+
+    console.info('Recreating an index whose declaration changed', { index: name })
+
+    await collection.dropIndex(name)
+    await collection.createIndex(spec, options)
+  }
+}
+
+async function dropIndex(collection, { name }, id) {
+  if (name === undefined)
+    throw new Error(`Migration '${id}' drops an index without a name`)
+
+  try {
+    await collection.dropIndex(name)
+  } catch (error) {
+    if (error.code !== ERR_INDEX_NOT_FOUND) throw error
+  }
+}
+
+async function update(collection, { filter, update: changeset }, id) {
+  if (changeset === undefined) throw new Error(`Migration '${id}' updates with nothing`)
+
+  const result = await collection.updateMany(filter ?? {}, changeset)
+
+  console.info('Migration updated records', {
+    migration: id,
+    records: result.modifiedCount
+  })
+}
+
+async function remove(collection, { filter }, id) {
+  if (filter === undefined)
+    throw new Error(
+      `Migration '${id}' deletes without a filter; pass {} to mean every record`
+    )
+
+  const result = await collection.deleteMany(filter)
+
+  console.info('Migration deleted records', {
+    migration: id,
+    records: result.deletedCount
+  })
+}
+
+const STEPS = { index, dropIndex, update, delete: remove }
+
+const DIRECTIONS = { asc: 1, desc: -1, hash: 'hashed' }
+
+/** what an index step may say, and what the driver calls it */
+const OPTIONS = {
+  unique: 'unique',
+  sparse: 'sparse',
+  partial: 'partialFilterExpression',
+  ttl: 'expireAfterSeconds'
+}
+
+/**
+ * A component's namespace may not be `system`, so this cannot be a component's own collection;
+ * and MongoDB reserves the `system.` prefix, with a dot, which this is not.
+ */
+export const STATE = 'system_migrations'
+
+const RUNNING = 'running'
+const DONE = 'done'
+
+/** how long a claim outlives its last heartbeat before another replica may take it */
+const LEASE = 30_000
+const HEARTBEAT = 5_000
+const POLL = 1_000
+
+/** how often a run that is taking its time says it is still going, and a wait that it is waiting */
+const PROGRESS = 30_000
+
+/**
+ * Which process holds a claim, and nothing more: it is read by whoever takes an abandoned
+ * one over, and told to whoever reads the collection. Random, because there is no identity
+ * a process is guaranteed to have — a host name says nothing about two replicas on one
+ * machine, and a pid is reused.
+ */
+const OWNER = randomUUID()
+
+const ERR_DUPLICATE_KEY = 11000
+const ERR_INDEX_NOT_FOUND = 27
+const CONFLICTS = [85, 86] // IndexOptionsConflict, IndexKeySpecsConflict

package/src/outbox.js

@@ -0,0 +1,131 @@
+import { environment } from '@toa.io/generic'
+import { index, prune } from './indexes.js'
+
+/**
+ * The outbox rows of one component. Its lifecycle is the Client's, so it is not a Connector.
+ *
+ * Core owns what a row means — its id, its lane and when it becomes due. This only writes
+ * them, reads back what is due, and marks what has been published.
+ */
+export class Outbox {
+  /** @type {import('mongodb').Collection} */
+  #collection
+
+  #retention
+
+  constructor(collection) {
+    this.#collection = collection
+    this.#retention = retention()
+  }
+
+  /** @param {import('mongodb').ClientSession} session */
+  async insert(row, session) {
+    await this.#collection.insertOne(to(row), { session })
+  }
+
+  /** @param {import('mongodb').ClientSession} session */
+  async insertMany(rows, session) {
+    if (rows.length === 0) return
+
+    await this.#collection.insertMany(rows.map(to), { session })
+  }
+
+  /**
+   * One page of what this replica should publish: due, still unpublished, and in a lane it
+   * owns. In steady state the first page is empty.
+   *
+   * `after` continues from the last id of the page before. Ids are uuid v7, so their order is
+   * the order rows were written and a page is never read twice within a cycle — which matters
+   * because a row stays unpublished in the database until the cycle that sent it marks it.
+   */
+  async pending(lanes, now, limit, after = undefined) {
+    const criteria = { lane: { $in: lanes }, published: false, pending: { $lte: now } }
+
+    if (after !== undefined) criteria._id = { $gt: after }
+
+    const rows = await this.#collection
+      .find(criteria)
+      .sort({ _id: 1 })
+      .limit(limit)
+      .toArray()
+
+    return rows.map(from)
+  }
+
+  /**
+   * Takes those destinations out of what those rows are outstanding for, and marks published
+   * the ones left outstanding for nothing — which is what `pending` selects on and what the
+   * TTL reaps by, so neither changes.
+   *
+   * One batched write for many rows, which is why the ids are held in memory until the tick
+   * rather than updated one by one. A row written before this property existed reads as
+   * outstanding for `events` alone, which is all there was.
+   */
+  async settle(ids, destinations) {
+    if (ids.length === 0) return
+
+    await this.#collection.updateMany({ _id: { $in: ids } }, [
+      {
+        $set: {
+          outstanding: {
+            $setDifference: [{ $ifNull: ['$outstanding', [EVENTS]] }, destinations]
+          }
+        }
+      },
+      {
+        $set: {
+          published: { $eq: [{ $size: '$outstanding' }, 0] },
+          // written once, when the last destination lands: an unpublished row has none, and
+          // the TTL monitor skips a document that lacks the field it expires by
+          publishedAt: {
+            $cond: [{ $eq: [{ $size: '$outstanding' }, 0] }, '$$NOW', '$publishedAt']
+          }
+        }
+      }
+    ])
+  }
+
+  /**
+   * These are the runtime's indexes, not the component's, so they are declared here rather
+   * than in a migration a component would have to write — including pruning, or a later
+   * change leaves the old index behind forever.
+   */
+  async index() {
+    const desired = {
+      // holds only what is not published yet, so it stays at in-flight size
+      outbox_pending: {
+        fields: { lane: 1, pending: 1 },
+        options: { name: 'outbox_pending', partialFilterExpression: { published: false } }
+      },
+      // an unpublished row has no `publishedAt`, and the TTL monitor skips those — so a row
+      // that never made it out is never reaped
+      outbox_published_at: {
+        fields: { publishedAt: 1 },
+        options: { name: 'outbox_published_at', expireAfterSeconds: this.#retention }
+      }
+    }
+
+    for (const { fields, options } of Object.values(desired))
+      await index(this.#collection, fields, options)
+
+    await prune(this.#collection, Object.keys(desired))
+  }
+}
+
+const to = ({ id, ...rest }) => ({ _id: id, ...rest })
+
+// a row written before destinations existed is outstanding for the events, which is all a row
+// was ever published to then
+const from = ({ _id, ...rest }) => ({ id: _id, outstanding: [EVENTS], ...rest })
+
+function retention() {
+  const value = Number(environment.get('TOA_OUTBOX_RETENTION'))
+
+  return Number.isNaN(value) || value < 0 ? RETENTION : value
+}
+
+/** what a component's own events are outstanding for; `Emission.name` */
+const EVENTS = 'events'
+
+/** seconds a published row is kept as a change log before the TTL monitor reaps it */
+const RETENTION = 86400