@toa.io/storages.mongodb 1.0.0-alpha.284 → 1.0.0-alpha.287

package/CHANGELOG.md

@@ -3,145 +3,8 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
-# [1.0.0-alpha.284](https://github.com/toa-io/toa/compare/v1.0.0-alpha.283...v1.0.0-alpha.284) (2026-09-05)
+# [1.0.0-alpha.287](https://github.com/toa-io/toa/compare/v1.0.0-alpha.286...v1.0.0-alpha.287) (2026-09-06)
 
-**Note:** Version bump only for package @toa.io/storages.mongodb
+### Features
 
-
-
-
-
-# [1.0.0-alpha.283](https://github.com/toa-io/toa/compare/v1.0.0-alpha.282...v1.0.0-alpha.283) (2026-09-04)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.282](https://github.com/toa-io/toa/compare/v1.0.0-alpha.281...v1.0.0-alpha.282) (2026-09-03)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.278](https://github.com/toa-io/toa/compare/v1.0.0-alpha.277...v1.0.0-alpha.278) (2026-09-03)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.277](https://github.com/toa-io/toa/compare/v1.0.0-alpha.276...v1.0.0-alpha.277) (2026-09-03)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.274](https://github.com/toa-io/toa/compare/v1.0.0-alpha.273...v1.0.0-alpha.274) (2026-09-02)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.273](https://github.com/toa-io/toa/compare/v1.0.0-alpha.272...v1.0.0-alpha.273) (2026-09-02)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.272](https://github.com/toa-io/toa/compare/v1.0.0-alpha.271...v1.0.0-alpha.272) (2026-09-01)
-
-
-* refactor(core)!: pump the outbox in one cycle ([bf590fe](https://github.com/toa-io/toa/commit/bf590fe8ed59c701b5fd1a91ba4874685b79242f))
-* feat(core)!: commit events with the state that produced them ([1eb68cc](https://github.com/toa-io/toa/commit/1eb68cc435dbfa03faa16009fceb866693d22e1a)), closes [#20](https://github.com/toa-io/toa/issues/20)
-
-
-### BREAKING CHANGES
-
-* `Storage.outbox.pending` takes a fourth argument, the id to
-continue from.
-
-Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
-* `event.changeset` is removed; use `origin` and `state`. `State`
-takes an `Outbox` in place of an `Emission`. `difference` is dropped from
-`@toa.io/generic` along with its last caller.
-
-Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
-
-
-
-
-
-# [1.0.0-alpha.270](https://github.com/toa-io/toa/compare/v1.0.0-alpha.269...v1.0.0-alpha.270) (2026-08-31)
-
-
-### Performance Improvements
-
-* **boot:** stop paying for what a composition does not need to start ([1619c27](https://github.com/toa-io/toa/commit/1619c2743072f4706939ce72f3074e615d26a91e))
-* **storages.mongodb:** time the call instead of monitoring the command ([135f1a9](https://github.com/toa-io/toa/commit/135f1a97c753b1caed9291f85cea0521ef68b0e3))
-
-
-
-
-
-# [1.0.0-alpha.266](https://github.com/toa-io/toa/compare/v1.0.0-alpha.265...v1.0.0-alpha.266) (2026-08-29)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.265](https://github.com/toa-io/toa/compare/v1.0.0-alpha.264...v1.0.0-alpha.265) (2026-08-29)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.264](https://github.com/toa-io/toa/compare/v1.0.0-alpha.263...v1.0.0-alpha.264) (2026-08-29)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.263](https://github.com/toa-io/toa/compare/v1.0.0-alpha.262...v1.0.0-alpha.263) (2026-08-29)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.262](https://github.com/toa-io/toa/compare/v1.0.0-alpha.261...v1.0.0-alpha.262) (2026-08-28)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.259](https://github.com/toa-io/toa/compare/v1.0.0-alpha.258...v1.0.0-alpha.259) (2026-08-24)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
-
-
-
-
-
-# [1.0.0-alpha.257](https://github.com/toa-io/toa/compare/v1.0.0-alpha.256...v1.0.0-alpha.257) (2026-08-24)
-
-**Note:** Version bump only for package @toa.io/storages.mongodb
+* **mongodb:** a migration says what it is doing ([a917a81](https://github.com/toa-io/toa/commit/a917a81fdc94eb23fd73182c0edf891e6b2df843))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/storages.mongodb",
-  "version": "1.0.0-alpha.284",
+  "version": "1.0.0-alpha.287",
   "type": "module",
   "description": "Toa MongoDB Storage Connector",
   "author": "temich <tema.gurtovoy@gmail.com>",
@@ -20,13 +20,13 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/conveyor": "1.0.0-alpha.283",
-    "@toa.io/core": "1.0.0-alpha.284",
-    "@toa.io/generic": "1.0.0-alpha.283",
-    "@toa.io/pointer": "1.0.0-alpha.283",
+    "@toa.io/conveyor": "1.0.0-alpha.286",
+    "@toa.io/core": "1.0.0-alpha.287",
+    "@toa.io/generic": "1.0.0-alpha.286",
+    "@toa.io/pointer": "1.0.0-alpha.287",
     "mongodb": "7.6.0",
-    "openspan": "1.0.0-alpha.277",
+    "openspan": "1.0.0-alpha.286",
     "saslprep": "1.0.3"
   },
-  "gitHead": "006755556879e91e8d68fa86753aa3b475bdf4bf"
+  "gitHead": "5c7944a70c16065ab6372072385cc1f02f5db77c"
 }

package/src/client.js

@@ -43,6 +43,15 @@ export class Client extends Connector {
    */
   transactional = false
 
+  /**
+   * The database this component's collections live in, which is where the migration state
+   * is kept as well.
+   *
+   * @public
+   * @type {import('mongodb').Db}
+   */
+  db
+
   /**
    * @private
    * @type {Locator}
@@ -71,7 +80,7 @@ export class Client extends Connector {
    * @param {Locator} locator
    * @param {boolean} [publishes] whether this component publishes anything
    */
-  constructor (locator, publishes = false) {
+  constructor(locator, publishes = false) {
     super()
 
     this.locator = locator
@@ -84,7 +93,7 @@ export class Client extends Connector {
    * @override
    * @return {Promise<void>}
    */
-  async open () {
+  async open() {
     const urls = await this.resolveURLs()
     const dbname = this.resolveDB()
 
@@ -101,6 +110,7 @@ export class Client extends Connector {
 
     const db = this.instance.client.db(dbname)
 
+    this.db = db
     this.collection = await collection(db, this.name)
     this.transactional = await transactional(db)
 
@@ -108,8 +118,10 @@ export class Client extends Connector {
 
     if (this.transactional) this.outbox = await collection(db, this.name + OUTBOX)
     else
-      console.warn('MongoDB is not a replica set; events are emitted inline, without an outbox',
-        { collection: this.name })
+      console.warn(
+        'MongoDB is not a replica set; events are emitted inline, without an outbox',
+        { collection: this.name }
+      )
   }
 
   /**
@@ -122,9 +134,10 @@ export class Client extends Connector {
    * @param {(session: import('mongodb').ClientSession) => Promise<T>} fn
    * @return {Promise<T>}
    */
-  async transaction (fn) {
+  async transaction(fn) {
     return this.instance.client.withSession(async (session) =>
-      session.withTransaction(async () => fn(session)))
+      session.withTransaction(async () => fn(session))
+    )
   }
 
   /**
@@ -132,7 +145,7 @@ export class Client extends Connector {
    * @override
    * @return {Promise<void>}
    */
-  async close () {
+  async close() {
     const instance = await INSTANCES[this.key]
 
     instance.count--
@@ -148,7 +161,7 @@ export class Client extends Connector {
    * @param {string[]} urls
    * @return {Promise<Instance>}
    */
-  async createInstance (urls) {
+  async createInstance(urls) {
     const client = new MongoClient(urls.join(','), OPTIONS)
     const hosts = urls.map((str) => new URL(str).host)
 
@@ -166,9 +179,11 @@ export class Client extends Connector {
    * @private
    * @return {Promise<string[]>}
    */
-  async resolveURLs () {
+  async resolveURLs() {
+    // Toa's own development stack is not on the conventional ports: the applications built on
+    // Toa are, and they share the machine. See CONTRIBUTING.md.
     if (process.env.TOA_DEV === '1') {
-      return ['mongodb://developer:secret@localhost']
+      return ['mongodb://developer:secret@localhost:31020']
     } else {
       return await resolve(ID, this.locator.id)
     }
@@ -178,7 +193,7 @@ export class Client extends Connector {
    * @private
    * @return {string}
    */
-  resolveDB () {
+  resolveDB() {
     if (process.env.TOA_CONTEXT !== undefined) {
       return process.env.TOA_CONTEXT
     }
@@ -191,14 +206,14 @@ export class Client extends Connector {
   }
 }
 
-function getKey (db, urls) {
+function getKey(db, urls) {
   return db + ':' + urls.sort().join(' ')
 }
 
 /**
  * Concurrent pods race to create the same collection, and losing that race is not an error.
  */
-async function collection (db, name) {
+async function collection(db, name) {
   try {
     return await db.createCollection(name)
   } catch (e) {
@@ -208,7 +223,7 @@ async function collection (db, name) {
   }
 }
 
-async function transactional (db) {
+async function transactional(db) {
   try {
     const hello = await db.admin().command({ hello: 1 })
 

package/src/deployment.js

@@ -7,7 +7,7 @@ export const deployment = (instances, annotation) => {
   return { variables }
 }
 
-function createRequest (instance) {
+function createRequest(instance) {
   return {
     group: instance.locator.label,
     selectors: [instance.locator.id]

package/src/factory.js

@@ -2,7 +2,7 @@ import { Client } from './client.js'
 import { Storage } from './storage.js'
 
 export class Factory {
-  storage (locator, entity, options = {}) {
+  storage(locator, entity, options = {}) {
     const client = new Client(locator, options.outbox === true)
 
     return new Storage(client, entity)

package/src/migrations.js

@@ -0,0 +1,343 @@
+import { randomUUID } from 'node:crypto'
+import { setTimeout as sleep } from 'node:timers/promises'
+
+import { console } from 'openspan'
+
+/**
+ * Applies a component's migrations to its collection, in the order they were declared, and
+ * records each one so that it is applied once for the database rather than once per replica.
+ *
+ * The record is also the lock: the row that says a migration ran is inserted before it runs,
+ * and MongoDB refuses the second insert of the same `_id`. Nothing else is needed to make the
+ * group agree, which is why this works where there is no Redis.
+ */
+export class Migrations {
+  /** @type {import('mongodb').Collection} */
+  #collection
+
+  /** @type {import('mongodb').Collection} */
+  #state
+
+  /** @type {Array<{ id: string, steps: object[] }>} */
+  #list
+
+  /**
+   * @param {import('mongodb').Db} db
+   * @param {import('mongodb').Collection} collection the entity's own
+   * @param {Array<{ id: string, steps: object[] }>} list
+   */
+  constructor(db, collection, list) {
+    this.#collection = collection
+    this.#state = db.collection(STATE)
+    this.#list = list
+  }
+
+  /**
+   * A migration is applied only after the one before it, because a later one is written
+   * against what an earlier one leaves behind.
+   */
+  async run() {
+    let applied = 0
+
+    for (const migration of this.#list) if (await this.#apply(migration)) applied++
+
+    // debug, because the ordinary start has nothing to report: every migration was applied
+    // long ago by whoever started first. But a run that says nothing at all cannot be told
+    // from one that never looked
+    console.debug('Migrations checked', {
+      collection: this.#collection.collectionName,
+      declared: this.#list.length,
+      applied
+    })
+  }
+
+  /**
+   * Answers whether this replica was the one that applied it.
+   *
+   * @private
+   */
+  async #apply(migration) {
+    const id = `${this.#collection.collectionName}:${migration.id}`
+
+    /** when the wait for another replica was last reported */
+    let announced
+
+    while (true) {
+      if (await this.#claim(id)) {
+        await this.#run(id, migration)
+
+        return true
+      }
+
+      const row = await this.#state.findOne({ _id: id })
+
+      // removed between the claim and the read; whoever did that wants it applied again
+      if (row === null) continue
+
+      if (row.state === DONE) {
+        console.debug('Migration was applied already', { migration: id })
+
+        return false
+      }
+
+      const stale = Date.now() - row.heartbeat.getTime() > LEASE
+
+      if (!stale) {
+        /*
+         * The component does not serve until this returns, so a replica waiting here is a pod
+         * that is simply not up, with nothing anywhere saying why. On its own cadence rather
+         * than the poll's, which is a second.
+         */
+        if (announced === undefined || Date.now() - announced > PROGRESS) {
+          announced = Date.now()
+
+          console.info('Waiting for another replica to apply a migration', {
+            migration: id,
+            owner: row.owner
+          })
+        }
+
+        await sleep(POLL)
+
+        continue
+      }
+
+      console.warn('Taking over an abandoned migration', {
+        migration: id,
+        owner: row.owner,
+        heartbeat: row.heartbeat
+      })
+
+      if (await this.#steal(id, row.heartbeat)) {
+        await this.#run(id, migration)
+
+        return true
+      }
+    }
+  }
+
+  /**
+   * Writes the row that says this replica is applying the migration. Answers whether it won.
+   *
+   * @private
+   */
+  async #claim(id) {
+    try {
+      await this.#state.insertOne({
+        _id: id,
+        state: RUNNING,
+        owner: OWNER,
+        started: new Date(),
+        heartbeat: new Date()
+      })
+
+      return true
+    } catch (error) {
+      if (error.code === ERR_DUPLICATE_KEY) return false
+
+      throw error
+    }
+  }
+
+  /**
+   * Takes a claim whose owner stopped saying it was alive. The heartbeat it read is part of
+   * the criteria, so only one of several waiting replicas takes it.
+   *
+   * @private
+   */
+  async #steal(id, heartbeat) {
+    const result = await this.#state.updateOne(
+      { _id: id, state: RUNNING, heartbeat },
+      { $set: { owner: OWNER, heartbeat: new Date() } }
+    )
+
+    return result.modifiedCount === 1
+  }
+
+  /**
+   * Applies the steps and marks the migration done. A step that throws leaves the row as it
+   * is: the component does not start, and the next replica to reach a stale claim runs the
+   * migration again from its first step.
+   *
+   * @private
+   */
+  async #run(id, migration) {
+    const total = migration.steps.length
+    const started = Date.now()
+
+    console.info('Applying migration', { migration: id, steps: total })
+
+    const beat = setInterval(() => {
+      this.#state
+        .updateOne({ _id: id }, { $set: { heartbeat: new Date() } })
+        .catch((error) =>
+          console.warn('Migration heartbeat failed', { migration: id, error })
+        )
+    }, HEARTBEAT)
+
+    beat.unref?.()
+
+    let applying = 0
+
+    /*
+     * A backfill takes as long as the collection is large, and every replica of the group waits
+     * out the whole of it. On its own cadence rather than the heartbeat's, which is every five
+     * seconds and would say this a dozen times a minute.
+     */
+    const progress = setInterval(() => {
+      console.info('Migration is still being applied', {
+        migration: id,
+        step: applying,
+        steps: total,
+        elapsed: Date.now() - started
+      })
+    }, PROGRESS)
+
+    progress.unref?.()
+
+    try {
+      for (const step of migration.steps) {
+        applying++
+
+        await this.#step(id, step)
+      }
+
+      await this.#state.updateOne(
+        { _id: id },
+        { $set: { state: DONE, completed: new Date() } }
+      )
+    } finally {
+      clearInterval(beat)
+      clearInterval(progress)
+    }
+
+    console.info('Migration applied', { migration: id, elapsed: Date.now() - started })
+  }
+
+  /** @private */
+  async #step(id, step) {
+    const verbs = Object.keys(step ?? {})
+
+    if (verbs.length !== 1 || !(verbs[0] in STEPS))
+      throw new Error(
+        `Migration '${id}' has a step that is not one of ` +
+          `${Object.keys(STEPS).join(', ')}: ${JSON.stringify(step)}`
+      )
+
+    await STEPS[verbs[0]](this.#collection, step[verbs[0]], id)
+  }
+}
+
+/**
+ * Creates the index the step declares. Where the name is held by an index of a different shape,
+ * that one is dropped: the declaration is what the component is to run against, and refusing
+ * to reconcile is what left a database diverged from its manifest before migrations existed.
+ */
+async function index(collection, { name, keys, ...rest }, id) {
+  if (name === undefined || keys === undefined)
+    throw new Error(`Migration '${id}' declares an index without a name or keys`)
+
+  const spec = Object.fromEntries(
+    Object.entries(keys).map(([field, direction]) => [
+      field,
+      DIRECTIONS[direction] ?? direction
+    ])
+  )
+
+  const options = { name }
+
+  for (const [key, value] of Object.entries(rest)) {
+    if (!(key in OPTIONS))
+      throw new Error(`Migration '${id}' declares an unknown index option '${key}'`)
+
+    options[OPTIONS[key]] = value
+  }
+
+  try {
+    await collection.createIndex(spec, options)
+  } catch (error) {
+    if (!CONFLICTS.includes(error.code)) throw error
+
+    console.info('Recreating an index whose declaration changed', { index: name })
+
+    await collection.dropIndex(name)
+    await collection.createIndex(spec, options)
+  }
+}
+
+async function dropIndex(collection, { name }, id) {
+  if (name === undefined)
+    throw new Error(`Migration '${id}' drops an index without a name`)
+
+  try {
+    await collection.dropIndex(name)
+  } catch (error) {
+    if (error.code !== ERR_INDEX_NOT_FOUND) throw error
+  }
+}
+
+async function update(collection, { filter, update: changeset }, id) {
+  if (changeset === undefined) throw new Error(`Migration '${id}' updates with nothing`)
+
+  const result = await collection.updateMany(filter ?? {}, changeset)
+
+  console.info('Migration updated records', {
+    migration: id,
+    records: result.modifiedCount
+  })
+}
+
+async function remove(collection, { filter }, id) {
+  if (filter === undefined)
+    throw new Error(
+      `Migration '${id}' deletes without a filter; pass {} to mean every record`
+    )
+
+  const result = await collection.deleteMany(filter)
+
+  console.info('Migration deleted records', {
+    migration: id,
+    records: result.deletedCount
+  })
+}
+
+const STEPS = { index, dropIndex, update, delete: remove }
+
+const DIRECTIONS = { asc: 1, desc: -1, hash: 'hashed' }
+
+/** what an index step may say, and what the driver calls it */
+const OPTIONS = {
+  unique: 'unique',
+  sparse: 'sparse',
+  partial: 'partialFilterExpression',
+  ttl: 'expireAfterSeconds'
+}
+
+/**
+ * A component's namespace may not be `system`, so this cannot be a component's own collection;
+ * and MongoDB reserves the `system.` prefix, with a dot, which this is not.
+ */
+export const STATE = 'system_migrations'
+
+const RUNNING = 'running'
+const DONE = 'done'
+
+/** how long a claim outlives its last heartbeat before another replica may take it */
+const LEASE = 30_000
+const HEARTBEAT = 5_000
+const POLL = 1_000
+
+/** how often a run that is taking its time says it is still going, and a wait that it is waiting */
+const PROGRESS = 30_000
+
+/**
+ * Which process holds a claim, and nothing more: it is read by whoever takes an abandoned
+ * one over, and told to whoever reads the collection. Random, because there is no identity
+ * a process is guaranteed to have — a host name says nothing about two replicas on one
+ * machine, and a pid is reused.
+ */
+const OWNER = randomUUID()
+
+const ERR_DUPLICATE_KEY = 11000
+const ERR_INDEX_NOT_FOUND = 27
+const CONFLICTS = [85, 86] // IndexOptionsConflict, IndexKeySpecsConflict