@toa.io/storages.mongodb 1.0.0-alpha.28 → 1.0.0-alpha.283

package/CHANGELOG.md

@@ -0,0 +1,139 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+# [1.0.0-alpha.283](https://github.com/toa-io/toa/compare/v1.0.0-alpha.282...v1.0.0-alpha.283) (2026-09-04)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.282](https://github.com/toa-io/toa/compare/v1.0.0-alpha.281...v1.0.0-alpha.282) (2026-09-03)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.278](https://github.com/toa-io/toa/compare/v1.0.0-alpha.277...v1.0.0-alpha.278) (2026-09-03)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.277](https://github.com/toa-io/toa/compare/v1.0.0-alpha.276...v1.0.0-alpha.277) (2026-09-03)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.274](https://github.com/toa-io/toa/compare/v1.0.0-alpha.273...v1.0.0-alpha.274) (2026-09-02)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.273](https://github.com/toa-io/toa/compare/v1.0.0-alpha.272...v1.0.0-alpha.273) (2026-09-02)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.272](https://github.com/toa-io/toa/compare/v1.0.0-alpha.271...v1.0.0-alpha.272) (2026-09-01)
+
+
+* refactor(core)!: pump the outbox in one cycle ([bf590fe](https://github.com/toa-io/toa/commit/bf590fe8ed59c701b5fd1a91ba4874685b79242f))
+* feat(core)!: commit events with the state that produced them ([1eb68cc](https://github.com/toa-io/toa/commit/1eb68cc435dbfa03faa16009fceb866693d22e1a)), closes [#20](https://github.com/toa-io/toa/issues/20)
+
+
+### BREAKING CHANGES
+
+* `Storage.outbox.pending` takes a fourth argument, the id to
+continue from.
+
+Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
+* `event.changeset` is removed; use `origin` and `state`. `State`
+takes an `Outbox` in place of an `Emission`. `difference` is dropped from
+`@toa.io/generic` along with its last caller.
+
+Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
+
+
+
+
+
+# [1.0.0-alpha.270](https://github.com/toa-io/toa/compare/v1.0.0-alpha.269...v1.0.0-alpha.270) (2026-08-31)
+
+
+### Performance Improvements
+
+* **boot:** stop paying for what a composition does not need to start ([1619c27](https://github.com/toa-io/toa/commit/1619c2743072f4706939ce72f3074e615d26a91e))
+* **storages.mongodb:** time the call instead of monitoring the command ([135f1a9](https://github.com/toa-io/toa/commit/135f1a97c753b1caed9291f85cea0521ef68b0e3))
+
+
+
+
+
+# [1.0.0-alpha.266](https://github.com/toa-io/toa/compare/v1.0.0-alpha.265...v1.0.0-alpha.266) (2026-08-29)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.265](https://github.com/toa-io/toa/compare/v1.0.0-alpha.264...v1.0.0-alpha.265) (2026-08-29)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.264](https://github.com/toa-io/toa/compare/v1.0.0-alpha.263...v1.0.0-alpha.264) (2026-08-29)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.263](https://github.com/toa-io/toa/compare/v1.0.0-alpha.262...v1.0.0-alpha.263) (2026-08-29)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.262](https://github.com/toa-io/toa/compare/v1.0.0-alpha.261...v1.0.0-alpha.262) (2026-08-28)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.259](https://github.com/toa-io/toa/compare/v1.0.0-alpha.258...v1.0.0-alpha.259) (2026-08-24)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.257](https://github.com/toa-io/toa/compare/v1.0.0-alpha.256...v1.0.0-alpha.257) (2026-08-24)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@toa.io/storages.mongodb",
-  "version": "1.0.0-alpha.28",
+  "version": "1.0.0-alpha.283",
+  "type": "module",
   "description": "Toa MongoDB Storage Connector",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,13 +20,13 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/console": "1.0.0-alpha.28",
-    "@toa.io/conveyor": "1.0.0-alpha.28",
-    "@toa.io/core": "1.0.0-alpha.28",
-    "@toa.io/generic": "1.0.0-alpha.28",
-    "@toa.io/pointer": "1.0.0-alpha.28",
-    "mongodb": "6.3.0",
+    "@toa.io/conveyor": "1.0.0-alpha.283",
+    "@toa.io/core": "1.0.0-alpha.283",
+    "@toa.io/generic": "1.0.0-alpha.283",
+    "@toa.io/pointer": "1.0.0-alpha.283",
+    "mongodb": "7.6.0",
+    "openspan": "1.0.0-alpha.277",
     "saslprep": "1.0.3"
   },
-  "gitHead": "89c3b7d1314e357c9e33df8d95f8d46954cd3bcf"
+  "gitHead": "2f52b023bed0ccb2ecdb2b82fe59fbe38bf282e1"
 }

package/readme.md

@@ -0,0 +1,16 @@
+# MongoDB Storage
+
+## Tracing
+
+Commands are recorded as `client` spans within the trace of the current invocation,
+using the [driver command monitoring events](https://www.mongodb.com/docs/drivers/node/current/monitoring/command-monitoring/).
+
+Spans are named `{command} {collection}` and carry `db.*` attributes following the
+[OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/database/mongodb/):
+`db.system`, `db.namespace`, `db.collection.name`, `db.operation.name`.
+
+Commands executed outside of a sampled trace context (e.g. index management on startup)
+and internal driver commands (`hello`, `ping`, authentication) are not recorded.
+
+Monitoring is client-side only and does not affect the MongoDB server. Span recording
+adds no waiting to the query path: exporting is buffered and happens in the background.

package/src/client.js

@@ -1,28 +1,48 @@
-'use strict'
-
 /**
  * @typedef {import('mongodb').MongoClient} MongoClient
  * @typedef {{ count: number, client: MongoClient }} Instance
  * @typedef {import('@toa.io/core').Locator} Locator
  */
 
-const { Connector } = require('@toa.io/core')
-const { resolve } = require('@toa.io/pointer')
-const { ID } = require('./deployment')
-const { MongoClient } = require('mongodb')
+import { console } from 'openspan'
+import { Connector } from '@toa.io/core'
+import { resolve } from '@toa.io/pointer'
+import { ID } from './deployment.js'
+import { MongoClient } from 'mongodb'
 
 /**
  * @type {Record<string, Promise<Instance>>}
  */
 const INSTANCES = {}
 
-class Client extends Connector {
+export class Client extends Connector {
+  name
+
   /**
    * @public
    * @type {import('mongodb').Collection}
    */
   collection
 
+  /**
+   * The outbox rows of this component, absent unless something consumes its events. Created
+   * eagerly beside the entity collection, because a transaction cannot create a collection and
+   * an index build cannot run inside one.
+   *
+   * @public
+   * @type {import('mongodb').Collection | undefined}
+   */
+  outbox
+
+  /**
+   * Whether this deployment can run transactions at all. A standalone mongod cannot, and an
+   * outbox without atomicity is worse than none, so the storage falls back to inline emission.
+   *
+   * @public
+   * @type {boolean}
+   */
+  transactional = false
+
   /**
    * @private
    * @type {Locator}
@@ -41,13 +61,22 @@ class Client extends Connector {
    */
   key
 
+  /**
+   * @private
+   * @type {boolean}
+   */
+  publishes
+
   /**
    * @param {Locator} locator
+   * @param {boolean} [publishes] whether this component publishes anything
    */
-  constructor (locator) {
+  constructor (locator, publishes = false) {
     super()
 
     this.locator = locator
+    this.name = locator.lowercase
+    this.publishes = publishes
   }
 
   /**
@@ -58,26 +87,44 @@ class Client extends Connector {
   async open () {
     const urls = await this.resolveURLs()
     const dbname = this.resolveDB()
-    const collname = this.locator.lowercase
 
     this.key = getKey(dbname, urls)
 
-    INSTANCES[this.key] ??= this.createInstance(urls)
+    try {
+      INSTANCES[this.key] ??= this.createInstance(urls)
+    } catch (error) {
+      console.error('Failed to connect to MongoDB', { urls, error })
+    }
 
     this.instance = await INSTANCES[this.key]
     this.instance.count++
 
     const db = this.instance.client.db(dbname)
 
-    try {
-      this.collection = await db.createCollection(collname)
-    } catch (e) {
-      if (e.code !== ALREADY_EXISTS) {
-        throw e
-      }
+    this.collection = await collection(db, this.name)
+    this.transactional = await transactional(db)
 
-      this.collection = db.collection(collname)
-    }
+    if (!this.publishes) return
+
+    if (this.transactional) this.outbox = await collection(db, this.name + OUTBOX)
+    else
+      console.warn('MongoDB is not a replica set; events are emitted inline, without an outbox',
+        { collection: this.name })
+  }
+
+  /**
+   * Runs `fn` in a transaction and answers what it returned. The driver may call `fn` more
+   * than once, so it must not hold state of its own — an outbox row is built by the caller
+   * and reused, and a rolled back attempt leaves nothing behind.
+   *
+   * @public
+   * @template T
+   * @param {(session: import('mongodb').ClientSession) => Promise<T>} fn
+   * @return {Promise<T>}
+   */
+  async transaction (fn) {
+    return this.instance.client.withSession(async (session) =>
+      session.withTransaction(async () => fn(session)))
   }
 
   /**
@@ -105,7 +152,7 @@ class Client extends Connector {
     const client = new MongoClient(urls.join(','), OPTIONS)
     const hosts = urls.map((str) => new URL(str).host)
 
-    console.info('Connecting to MongoDB:', hosts.join(', '))
+    console.info('Connecting to MongoDB', { address: hosts.join(', ') })
 
     await client.connect()
 
@@ -148,12 +195,40 @@ function getKey (db, urls) {
   return db + ':' + urls.sort().join(' ')
 }
 
+/**
+ * Concurrent pods race to create the same collection, and losing that race is not an error.
+ */
+async function collection (db, name) {
+  try {
+    return await db.createCollection(name)
+  } catch (e) {
+    if (e.code !== ALREADY_EXISTS) throw e
+
+    return db.collection(name)
+  }
+}
+
+async function transactional (db) {
+  try {
+    const hello = await db.admin().command({ hello: 1 })
+
+    return hello.setName !== undefined || hello.msg === 'isdbgrid'
+  } catch (e) {
+    console.warn('MongoDB transaction support could not be determined', { error: e })
+
+    return false
+  }
+}
+
+/**
+ * `monitorCommands` is deliberately absent. It makes the driver materialize every reply
+ * eagerly to populate the monitoring event (`CommandSucceededEvent`), which defeats the
+ * lazy per-document deserialization a cursor exists for — a 100-document batch is then
+ * deserialized twice. `Storage` times its own calls instead.
+ */
 const OPTIONS = {
-  ignoreUndefined: true,
-  connectTimeoutMS: 0,
-  serverSelectionTimeoutMS: 0
+  ignoreUndefined: true
 }
 
 const ALREADY_EXISTS = 48
-
-exports.Client = Client
+const OUTBOX = '_outbox'

package/src/deployment.js

@@ -1,8 +1,6 @@
-'use strict'
+import { createVariables } from '@toa.io/pointer'
 
-const { createVariables } = require('@toa.io/pointer')
-
-const deployment = (instances, annotation) => {
+export const deployment = (instances, annotation) => {
   const requests = instances.map((instance) => createRequest(instance))
   const variables = createVariables(ID, annotation, requests)
 
@@ -16,7 +14,4 @@ function createRequest (instance) {
   }
 }
 
-const ID = 'mongodb'
-
-exports.ID = ID
-exports.deployment = deployment
+export const ID = 'mongodb'

package/src/factory.js

@@ -1,16 +1,10 @@
-'use strict'
+import { Client } from './client.js'
+import { Storage } from './storage.js'
 
-const { Client } = require('./client')
-const { Collection } = require('./collection')
-const { Storage } = require('./storage')
+export class Factory {
+  storage (locator, entity, options = {}) {
+    const client = new Client(locator, options.outbox === true)
 
-class Factory {
-  storage (locator, entity) {
-    const client = new Client(locator)
-    const connection = new Collection(client)
-
-    return new Storage(connection, entity)
+    return new Storage(client, entity)
   }
 }
-
-exports.Factory = Factory

package/src/index.js

@@ -1,7 +1,2 @@
-'use strict'
-
-const { deployment } = require('./deployment')
-const { Factory } = require('./factory')
-
-exports.deployment = deployment
-exports.Factory = Factory
+export { deployment } from './deployment.js'
+export { Factory } from './factory.js'

package/src/outbox.js

@@ -0,0 +1,125 @@
+import { console } from 'openspan'
+
+/**
+ * The outbox rows of one component. Its lifecycle is the Client's, so it is not a Connector.
+ *
+ * Core owns what a row means — its id, its lane and when it becomes due. This only writes
+ * them, reads back what is due, and marks what has been published.
+ */
+export class Outbox {
+  /** @type {import('mongodb').Collection} */
+  #collection
+
+  #retention
+
+  constructor (collection) {
+    this.#collection = collection
+    this.#retention = retention()
+  }
+
+  /** @param {import('mongodb').ClientSession} session */
+  async insert (row, session) {
+    await this.#collection.insertOne(to(row), { session })
+  }
+
+  /** @param {import('mongodb').ClientSession} session */
+  async insertMany (rows, session) {
+    if (rows.length === 0) return
+
+    await this.#collection.insertMany(rows.map(to), { session })
+  }
+
+  /**
+   * One page of what this replica should publish: due, still unpublished, and in a lane it
+   * owns. In steady state the first page is empty.
+   *
+   * `after` continues from the last id of the page before. Ids are uuid v7, so their order is
+   * the order rows were written and a page is never read twice within a cycle — which matters
+   * because a row stays unpublished in the database until the cycle that sent it marks it.
+   */
+  async pending (lanes, now, limit, after = undefined) {
+    const criteria = { lane: { $in: lanes }, published: false, pending: { $lte: now } }
+
+    if (after !== undefined) criteria._id = { $gt: after }
+
+    const rows = await this.#collection
+      .find(criteria)
+      .sort({ _id: 1 })
+      .limit(limit)
+      .toArray()
+
+    return rows.map(from)
+  }
+
+  /**
+   * One batched write for many events, which is why the ids are held in memory until the
+   * tick rather than updated one by one.
+   */
+  async settle (ids) {
+    if (ids.length === 0) return
+
+    await this.#collection.updateMany({ _id: { $in: ids } },
+      { $set: { published: true, publishedAt: new Date() } })
+  }
+
+  /**
+   * The entity collection's index management does not reach here, so this collection keeps
+   * its own — including pruning, or a later change leaves the old index behind forever.
+   */
+  async index () {
+    const desired = {
+      // holds only what is not published yet, so it stays at in-flight size
+      outbox_pending: {
+        fields: { lane: 1, pending: 1 },
+        options: { name: 'outbox_pending', partialFilterExpression: { published: false } }
+      },
+      // an unpublished row has no `publishedAt`, and the TTL monitor skips those — so a row
+      // that never made it out is never reaped
+      outbox_published_at: {
+        fields: { publishedAt: 1 },
+        options: { name: 'outbox_published_at', expireAfterSeconds: this.#retention }
+      }
+    }
+
+    for (const { fields, options } of Object.values(desired))
+      await this.#collection.createIndex(fields, options)
+        .catch((e) => console.warn('MongoDB outbox index creation failed',
+          { collection: this.#collection.collectionName, name: options.name, error: e }))
+
+    await this.#prune(Object.keys(desired))
+  }
+
+  /** @private */
+  async #prune (desired) {
+    let current
+
+    try {
+      current = await this.#collection.listIndexes().toArray()
+    } catch {
+      return
+    }
+
+    const obsolete = current
+      .map(({ name }) => name)
+      .filter((name) => name !== '_id_' && !desired.includes(name))
+
+    if (obsolete.length === 0) return
+
+    console.info('Removing obsolete outbox indexes',
+      { collection: this.#collection.collectionName, indexes: obsolete.join(', ') })
+
+    await Promise.all(obsolete.map((name) => this.#collection.dropIndex(name)))
+  }
+}
+
+const to = ({ id, ...rest }) => ({ _id: id, ...rest })
+const from = ({ _id, ...rest }) => ({ id: _id, ...rest })
+
+function retention () {
+  const value = Number(process.env.TOA_OUTBOX_RETENTION)
+
+  return Number.isNaN(value) || value < 0 ? RETENTION : value
+}
+
+/** seconds a published row is kept as a change log before the TTL monitor reaps it */
+const RETENTION = 86400

package/src/record.js

@@ -1,32 +1,14 @@
-'use strict'
-
-/**
- * @param {toa.core.storages.Record} entity
- * @returns {toa.mongodb.Record}
- */
-const to = (entity) => {
-  const {
-    id,
-    ...rest
-  } = entity
+export function to (entity) {
+  const { id, ...rest } = entity
 
   return /** @type {toa.mongodb.Record} */ { _id: id, ...rest }
 }
 
-/**
- * @param {toa.mongodb.Record} record
- * @returns {toa.core.storages.Record}
- */
-const from = (record) => {
-  if (record === undefined || record === null) return null
+export function from (record) {
+  if (record === undefined || record === null)
+    return null
 
-  const {
-    _id,
-    ...rest
-  } = record
+  const { _id, ...rest } = record
 
   return { id: _id, ...rest }
 }
-
-exports.to = to
-exports.from = from