@toa.io/storages.mongodb 1.0.0-alpha.270 → 1.0.0-alpha.273

package/CHANGELOG.md

@@ -3,6 +3,37 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [1.0.0-alpha.273](https://github.com/toa-io/toa/compare/v1.0.0-alpha.272...v1.0.0-alpha.273) (2026-09-02)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.272](https://github.com/toa-io/toa/compare/v1.0.0-alpha.271...v1.0.0-alpha.272) (2026-09-01)
+
+
+* refactor(core)!: pump the outbox in one cycle ([bf590fe](https://github.com/toa-io/toa/commit/bf590fe8ed59c701b5fd1a91ba4874685b79242f))
+* feat(core)!: commit events with the state that produced them ([1eb68cc](https://github.com/toa-io/toa/commit/1eb68cc435dbfa03faa16009fceb866693d22e1a)), closes [#20](https://github.com/toa-io/toa/issues/20)
+
+
+### BREAKING CHANGES
+
+* `Storage.outbox.pending` takes a fourth argument, the id to
+continue from.
+
+Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
+* `event.changeset` is removed; use `origin` and `state`. `State`
+takes an `Outbox` in place of an `Emission`. `difference` is dropped from
+`@toa.io/generic` along with its last caller.
+
+Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
+
+
+
+
+
 # [1.0.0-alpha.270](https://github.com/toa-io/toa/compare/v1.0.0-alpha.269...v1.0.0-alpha.270) (2026-08-31)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/storages.mongodb",
-  "version": "1.0.0-alpha.270",
+  "version": "1.0.0-alpha.273",
   "description": "Toa MongoDB Storage Connector",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,13 +19,13 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/conveyor": "1.0.0-alpha.254",
-    "@toa.io/core": "1.0.0-alpha.270",
-    "@toa.io/generic": "1.0.0-alpha.254",
-    "@toa.io/pointer": "1.0.0-alpha.270",
+    "@toa.io/conveyor": "1.0.0-alpha.273",
+    "@toa.io/core": "1.0.0-alpha.273",
+    "@toa.io/generic": "1.0.0-alpha.273",
+    "@toa.io/pointer": "1.0.0-alpha.273",
     "mongodb": "7.2.0",
-    "openspan": "1.0.0-alpha.270",
+    "openspan": "1.0.0-alpha.272",
     "saslprep": "1.0.3"
   },
-  "gitHead": "94400bdd411b4c25074ffcf58184580ddd060268"
+  "gitHead": "ad3284850ece95ffd5325fd0995707fc144c9c3d"
 }

package/src/client.js

@@ -26,6 +26,25 @@ class Client extends Connector {
    */
   collection
 
+  /**
+   * The outbox rows of this component, absent unless something consumes its events. Created
+   * eagerly beside the entity collection, because a transaction cannot create a collection and
+   * an index build cannot run inside one.
+   *
+   * @public
+   * @type {import('mongodb').Collection | undefined}
+   */
+  outbox
+
+  /**
+   * Whether this deployment can run transactions at all. A standalone mongod cannot, and an
+   * outbox without atomicity is worse than none, so the storage falls back to inline emission.
+   *
+   * @public
+   * @type {boolean}
+   */
+  transactional = false
+
   /**
    * @private
    * @type {Locator}
@@ -44,14 +63,22 @@ class Client extends Connector {
    */
   key
 
+  /**
+   * @private
+   * @type {boolean}
+   */
+  publishes
+
   /**
    * @param {Locator} locator
+   * @param {boolean} [publishes] whether this component publishes anything
    */
-  constructor (locator) {
+  constructor (locator, publishes = false) {
     super()
 
     this.locator = locator
     this.name = locator.lowercase
+    this.publishes = publishes
   }
 
   /**
@@ -76,15 +103,30 @@ class Client extends Connector {
 
     const db = this.instance.client.db(dbname)
 
-    try {
-      this.collection = await db.createCollection(this.name)
-    } catch (e) {
-      if (e.code !== ALREADY_EXISTS) {
-        throw e
-      }
+    this.collection = await collection(db, this.name)
+    this.transactional = await transactional(db)
 
-      this.collection = db.collection(this.name)
-    }
+    if (!this.publishes) return
+
+    if (this.transactional) this.outbox = await collection(db, this.name + OUTBOX)
+    else
+      console.warn('MongoDB is not a replica set; events are emitted inline, without an outbox',
+        { collection: this.name })
+  }
+
+  /**
+   * Runs `fn` in a transaction and answers what it returned. The driver may call `fn` more
+   * than once, so it must not hold state of its own — an outbox row is built by the caller
+   * and reused, and a rolled back attempt leaves nothing behind.
+   *
+   * @public
+   * @template T
+   * @param {(session: import('mongodb').ClientSession) => Promise<T>} fn
+   * @return {Promise<T>}
+   */
+  async transaction (fn) {
+    return this.instance.client.withSession(async (session) =>
+      session.withTransaction(async () => fn(session)))
   }
 
   /**
@@ -155,6 +197,31 @@ function getKey (db, urls) {
   return db + ':' + urls.sort().join(' ')
 }
 
+/**
+ * Concurrent pods race to create the same collection, and losing that race is not an error.
+ */
+async function collection (db, name) {
+  try {
+    return await db.createCollection(name)
+  } catch (e) {
+    if (e.code !== ALREADY_EXISTS) throw e
+
+    return db.collection(name)
+  }
+}
+
+async function transactional (db) {
+  try {
+    const hello = await db.admin().command({ hello: 1 })
+
+    return hello.setName !== undefined || hello.msg === 'isdbgrid'
+  } catch (e) {
+    console.warn('MongoDB transaction support could not be determined', { error: e })
+
+    return false
+  }
+}
+
 /**
  * `monitorCommands` is deliberately absent. It makes the driver materialize every reply
  * eagerly to populate the monitoring event (`CommandSucceededEvent`), which defeats the
@@ -166,5 +233,6 @@ const OPTIONS = {
 }
 
 const ALREADY_EXISTS = 48
+const OUTBOX = '_outbox'
 
 exports.Client = Client

package/src/factory.js

@@ -4,8 +4,8 @@ const { Client } = require('./client')
 const { Storage } = require('./storage')
 
 class Factory {
-  storage (locator, entity) {
-    const client = new Client(locator)
+  storage (locator, entity, options = {}) {
+    const client = new Client(locator, options.outbox === true)
 
     return new Storage(client, entity)
   }

package/src/outbox.js

@@ -0,0 +1,129 @@
+'use strict'
+
+const { console } = require('openspan')
+
+/**
+ * The outbox rows of one component. Its lifecycle is the Client's, so it is not a Connector.
+ *
+ * Core owns what a row means — its id, its lane and when it becomes due. This only writes
+ * them, reads back what is due, and marks what has been published.
+ */
+class Outbox {
+  /** @type {import('mongodb').Collection} */
+  #collection
+
+  #retention
+
+  constructor (collection) {
+    this.#collection = collection
+    this.#retention = retention()
+  }
+
+  /** @param {import('mongodb').ClientSession} session */
+  async insert (row, session) {
+    await this.#collection.insertOne(to(row), { session })
+  }
+
+  /** @param {import('mongodb').ClientSession} session */
+  async insertMany (rows, session) {
+    if (rows.length === 0) return
+
+    await this.#collection.insertMany(rows.map(to), { session })
+  }
+
+  /**
+   * One page of what this replica should publish: due, still unpublished, and in a lane it
+   * owns. In steady state the first page is empty.
+   *
+   * `after` continues from the last id of the page before. Ids are uuid v7, so their order is
+   * the order rows were written and a page is never read twice within a cycle — which matters
+   * because a row stays unpublished in the database until the cycle that sent it marks it.
+   */
+  async pending (lanes, now, limit, after = undefined) {
+    const criteria = { lane: { $in: lanes }, published: false, pending: { $lte: now } }
+
+    if (after !== undefined) criteria._id = { $gt: after }
+
+    const rows = await this.#collection
+      .find(criteria)
+      .sort({ _id: 1 })
+      .limit(limit)
+      .toArray()
+
+    return rows.map(from)
+  }
+
+  /**
+   * One batched write for many events, which is why the ids are held in memory until the
+   * tick rather than updated one by one.
+   */
+  async settle (ids) {
+    if (ids.length === 0) return
+
+    await this.#collection.updateMany({ _id: { $in: ids } },
+      { $set: { published: true, publishedAt: new Date() } })
+  }
+
+  /**
+   * The entity collection's index management does not reach here, so this collection keeps
+   * its own — including pruning, or a later change leaves the old index behind forever.
+   */
+  async index () {
+    const desired = {
+      // holds only what is not published yet, so it stays at in-flight size
+      outbox_pending: {
+        fields: { lane: 1, pending: 1 },
+        options: { name: 'outbox_pending', partialFilterExpression: { published: false } }
+      },
+      // an unpublished row has no `publishedAt`, and the TTL monitor skips those — so a row
+      // that never made it out is never reaped
+      outbox_published_at: {
+        fields: { publishedAt: 1 },
+        options: { name: 'outbox_published_at', expireAfterSeconds: this.#retention }
+      }
+    }
+
+    for (const { fields, options } of Object.values(desired))
+      await this.#collection.createIndex(fields, options)
+        .catch((e) => console.warn('MongoDB outbox index creation failed',
+          { collection: this.#collection.collectionName, name: options.name, error: e }))
+
+    await this.#prune(Object.keys(desired))
+  }
+
+  /** @private */
+  async #prune (desired) {
+    let current
+
+    try {
+      current = await this.#collection.listIndexes().toArray()
+    } catch {
+      return
+    }
+
+    const obsolete = current
+      .map(({ name }) => name)
+      .filter((name) => name !== '_id_' && !desired.includes(name))
+
+    if (obsolete.length === 0) return
+
+    console.info('Removing obsolete outbox indexes',
+      { collection: this.#collection.collectionName, indexes: obsolete.join(', ') })
+
+    await Promise.all(obsolete.map((name) => this.#collection.dropIndex(name)))
+  }
+}
+
+const to = ({ id, ...rest }) => ({ _id: id, ...rest })
+const from = ({ _id, ...rest }) => ({ id: _id, ...rest })
+
+function retention () {
+  const value = Number(process.env.TOA_OUTBOX_RETENTION)
+
+  return Number.isNaN(value) || value < 0 ? RETENTION : value
+}
+
+/** seconds a published row is kept as a change log before the TTL monitor reaps it */
+const RETENTION = 86400
+
+exports.Outbox = Outbox

package/src/storage.js

@@ -4,6 +4,7 @@ const { Connector, exceptions } = require('@toa.io/core')
 const { console } = require('openspan')
 const { translate } = require('./translate')
 const { to, from } = require('./record')
+const { Outbox } = require('./outbox')
 const { ReturnDocument } = require('mongodb')
 
 class Storage extends Connector {
@@ -13,6 +14,12 @@ class Storage extends Connector {
   #collection
   #entity
 
+  /**
+   * @type {Outbox | undefined} absent when nothing consumes this component's events, or when
+   * the deployment cannot run transactions
+   */
+  #outbox
+
   /** @type {Map<string, object>} span options per driver method */
   #spans = new Map()
 
@@ -29,12 +36,25 @@ class Storage extends Connector {
     return this.#collection
   }
 
+  /**
+   * The outbox is offered only where a row can be committed atomically with the entity.
+   * Without that it would be a second write with a window in front of it — worse than the
+   * inline emission it replaces — so the storage simply does not advertise it.
+   */
+  get outbox () {
+    return this.#outbox
+  }
+
   async open () {
     this.#collection = this.#client.collection
 
+    if (this.#client.outbox !== undefined)
+      this.#outbox = new Outbox(this.#client.outbox)
+
     this.#spans.clear()
 
     await this.index()
+    await this.#outbox?.index()
   }
 
   async get (query) {
@@ -84,16 +104,16 @@ class Storage extends Connector {
     return this.#collection.find(criteria, options).stream({ transform: from })
   }
 
-  async add (entity) {
+  async add (entity, session = undefined) {
     const record = to(entity)
 
     const result = await this.command('insertOne', { record },
-      () => this.#collection.insertOne(record))
+      () => this.#collection.insertOne(record, { session }))
 
     return result.acknowledged
   }
 
-  async set (entity) {
+  async set (entity, session = undefined) {
     const criteria = {
       _id: entity.id,
       _version: entity._version - 1
@@ -102,30 +122,52 @@ class Storage extends Connector {
     const record = to(entity)
 
     const result = await this.command('findOneAndReplace', { criteria, record },
-      () => this.#collection.findOneAndReplace(criteria, record))
+      () => this.#collection.findOneAndReplace(criteria, record, { session }))
 
     return result !== null
   }
 
-  async store (entity, attempt = 0) {
+  async store (entity, row = undefined, attempt = 0) {
     try {
-      if (entity._version === 1)
-        return await this.add(entity)
-      else
-        return await this.set(entity)
+      if (row === undefined || this.#outbox === undefined) {
+        if (entity._version === 1)
+          return await this.add(entity)
+        else
+          return await this.set(entity)
+      }
+
+      const committed = await this.#client.transaction(async (session) => {
+        const ok = entity._version === 1
+          ? await this.add(entity, session)
+          : await this.set(entity, session)
+
+        // a lost compare-and-swap must take the row down with it, or a retried transition
+        // leaves a row for a write that never happened
+        if (!ok) {
+          await session.abortTransaction()
+
+          return false
+        }
+
+        await this.#outbox.insert(row, session)
+
+        return true
+      })
+
+      return committed === true
     } catch (error) {
       console.error('MongoDB error', error)
 
       const retry = await retriable(error, attempt)
 
       if (retry)
-        return await this.store(entity, attempt + 1)
+        return await this.store(entity, row, attempt + 1)
       else
         return false
     }
   }
 
-  async massStore (entities, attempt = 0) {
+  async massStore (entities, rows = undefined, attempt = 0) {
     if (entities.length === 0)
       return true
 
@@ -157,13 +199,13 @@ class Storage extends Connector {
         }
     })
 
-    const client = this.#client.instance.client
-
     try {
-      await client.withSession(async (session) => {
-        await session.withTransaction(async () =>
-          await this.command('bulkWrite', { operations: operations.length },
-            async () => await this.#collection.bulkWrite(operations, { session })))
+      await this.#client.transaction(async (session) => {
+        await this.command('bulkWrite', { operations: operations.length },
+          async () => await this.#collection.bulkWrite(operations, { session }))
+
+        if (rows !== undefined && this.#outbox !== undefined)
+          await this.#outbox.insertMany(rows, session)
       })
 
       return true
@@ -173,13 +215,13 @@ class Storage extends Connector {
       const retry = await retriable(error, attempt)
 
       if (retry)
-        return await this.massStore(entities, attempt + 1)
+        return await this.massStore(entities, rows, attempt + 1)
       else
         return false
     }
   }
 
-  async upsert (query, changeset) {
+  async upsert (query, changeset, row = undefined) {
     const { criteria, options } = translate(query)
 
     if (!('_deleted' in changeset) || changeset._deleted === null) {
@@ -192,15 +234,45 @@ class Storage extends Connector {
       $inc: { _version: 1 }
     }
 
-    options.returnDocument = ReturnDocument.AFTER
+    // BEFORE, so that the filter is applied once and atomically and the pre-image comes back
+    // with it — an assignment is the one event whose images are the write's own
+    options.returnDocument = ReturnDocument.BEFORE
+
+    const apply = async (session) => {
+      const found = await this.command('findOneAndUpdate', { criteria, update, options },
+        () => this.#collection.findOneAndUpdate(criteria, update, { ...options, session }))
+
+      if (found === null) return null
+
+      const origin = from(found)
+
+      /*
+       * The post-image is `update` applied to the pre-image, computed rather than read back.
+       * That is exact, not approximate: `$set` on top-level keys is a spread (entity property
+       * names cannot contain dots, so a changeset never carries a path), and `_version` is
+       * incremented by one. It is also a coupling — an operator added to `update` and not
+       * mirrored here diverges silently — which `features/events/outbox.feature` guards.
+       */
+      const state = { ...origin, ...changeset, _version: origin._version + 1 }
+
+      // an assignment's event is the write's own images, so they are filled in here whether
+      // or not the row is going to be committed
+      if (row !== undefined)
+        row.event = { origin, state, ...row.event }
+
+      if (row !== undefined && this.#outbox !== undefined)
+        await this.#outbox.insert(row, session)
+
+      return state
+    }
 
-    const result = await this.command('findOneAndUpdate', { criteria, update, options },
-      () => this.#collection.findOneAndUpdate(criteria, update, options))
+    if (row === undefined || this.#outbox === undefined)
+      return apply(undefined)
 
-    return from(result)
+    return this.#client.transaction(apply)
   }
 
-  async ensure (query, properties, state) {
+  async ensure (query, properties, state, row = undefined) {
     let { criteria, options } = translate(query)
 
     if (query === undefined)
@@ -212,8 +284,19 @@ class Storage extends Connector {
     options.returnDocument = ReturnDocument.AFTER
 
     try {
-      const result = await this.command('findOneAndUpdate', { criteria, update, options },
-        () => this.#collection.findOneAndUpdate(criteria, update, options))
+      const result = row === undefined || this.#outbox === undefined
+        ? await this.command('findOneAndUpdate', { criteria, update, options },
+          () => this.#collection.findOneAndUpdate(criteria, update, options))
+        : await this.#client.transaction(async (session) => {
+          const found = await this.command('findOneAndUpdate', { criteria, update, options },
+            () => this.#collection.findOneAndUpdate(criteria, update, { ...options, session }))
+
+          // only an insert is an event; finding an existing record is not
+          if (found !== null && found._id === state.id)
+            await this.#outbox.insert(row, session)
+
+          return found
+        })
 
       if (result._deleted !== undefined && result._deleted !== null)
         return null