@toa.io/storages.mongodb 1.0.0-alpha.3 → 1.0.0-alpha.300

package/CHANGELOG.md

@@ -0,0 +1,120 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+# [1.0.0-alpha.300](https://github.com/toa-io/toa/compare/v1.0.0-alpha.299...v1.0.0-alpha.300) (2026-09-11)
+
+* feat(core)!: an operation may ask to run once ([dca616c](https://github.com/toa-io/toa/commit/dca616cb380a7a389e277875d3bd5ca55a6e4d45))
+
+### Bug Fixes
+
+* **extensions:** what a factory remembers does not outlive the tree it was made for ([dd36631](https://github.com/toa-io/toa/commit/dd3663160f500eadfb701bf840ef3741a2d075f0))
+
+### Features
+
+* **core:** an assignment may ask to run once as well ([10fbcbd](https://github.com/toa-io/toa/commit/10fbcbd86a5e82960e65f62c09d0a465ae8d0353))
+
+### BREAKING CHANGES
+
+* `Storage.store`, `.upsert` and `.ensure` take one more
+  argument, and `Storage` has two more members. A connector that ignores them
+  works exactly as it did; a component that asks it for `once` is refused at
+  boot. See migrations/299.md.
+
+
+# [1.0.0-alpha.299](https://github.com/toa-io/toa/compare/v1.0.0-alpha.298...v1.0.0-alpha.299) (2026-09-10)
+
+### Features
+
+* **core:** a record carries the region that wrote it ([13ed187](https://github.com/toa-io/toa/commit/13ed187604f06dec66feec2e26ccf9dae30bb048))
+* **core:** an outbox row is published to destinations ([cfb77b4](https://github.com/toa-io/toa/commit/cfb77b4826ff0f9560769a0996981e143d6525af))
+
+
+# [1.0.0-alpha.298](https://github.com/toa-io/toa/compare/v1.0.0-alpha.297...v1.0.0-alpha.298) (2026-09-08)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.297](https://github.com/toa-io/toa/compare/v1.0.0-alpha.296...v1.0.0-alpha.297) (2026-09-08)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.296](https://github.com/toa-io/toa/compare/v1.0.0-alpha.295...v1.0.0-alpha.296) (2026-09-08)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.295](https://github.com/toa-io/toa/compare/v1.0.0-alpha.294...v1.0.0-alpha.295) (2026-09-07)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.294](https://github.com/toa-io/toa/compare/v1.0.0-alpha.293...v1.0.0-alpha.294) (2026-09-07)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.293](https://github.com/toa-io/toa/compare/v1.0.0-alpha.292...v1.0.0-alpha.293) (2026-09-07)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.292](https://github.com/toa-io/toa/compare/v1.0.0-alpha.291...v1.0.0-alpha.292) (2026-09-07)
+
+### Features
+
+* **definitions:** what a package declares is read from one package ([7052341](https://github.com/toa-io/toa/commit/70523411b5d9e2b204c999aa02365173f3bb518d))
+
+
+# [1.0.0-alpha.291](https://github.com/toa-io/toa/compare/v1.0.0-alpha.290...v1.0.0-alpha.291) (2026-09-07)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.289](https://github.com/toa-io/toa/compare/v1.0.0-alpha.288...v1.0.0-alpha.289) (2026-09-07)
+
+* A deploy moves only what changed, and a component sees none of the runtime's environment (#1073) ([f38e3db](https://github.com/toa-io/toa/commit/f38e3db533f24db866c57bfa1ef294eff1eaf499)), closes [#1073](https://github.com/toa-io/toa/issues/1073) [#1064](https://github.com/toa-io/toa/issues/1064) [#1066](https://github.com/toa-io/toa/issues/1066) [#1067](https://github.com/toa-io/toa/issues/1067) [#1068](https://github.com/toa-io/toa/issues/1068) [#1069](https://github.com/toa-io/toa/issues/1069) [#1071](https://github.com/toa-io/toa/issues/1071) [#1070](https://github.com/toa-io/toa/issues/1070) [#1072](https://github.com/toa-io/toa/issues/1072)
+
+### BREAKING CHANGES
+
+* a component that read `process.env.TOA_*` reads `context` instead;
+  `echo(input)` no longer substitutes from the environment; a bash operation sees no
+  `TOA_*`; images no longer set `USER node` — see migrations/289.md.
+
+
+# [1.0.0-alpha.288](https://github.com/toa-io/toa/compare/v1.0.0-alpha.287...v1.0.0-alpha.288) (2026-09-06)
+
+**Note:** Version bump only for package @toa.io/storages.mongodb
+
+
+
+
+
+# [1.0.0-alpha.287](https://github.com/toa-io/toa/compare/v1.0.0-alpha.286...v1.0.0-alpha.287) (2026-09-06)
+
+### Features
+
+* **mongodb:** a migration says what it is doing ([a917a81](https://github.com/toa-io/toa/commit/a917a81fdc94eb23fd73182c0edf891e6b2df843))

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@toa.io/storages.mongodb",
-  "version": "1.0.0-alpha.3",
+  "version": "1.0.0-alpha.300",
+  "type": "module",
   "description": "Toa MongoDB Storage Connector",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,13 +20,13 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/console": "1.0.0-alpha.3",
-    "@toa.io/conveyor": "1.0.0-alpha.3",
-    "@toa.io/core": "1.0.0-alpha.3",
-    "@toa.io/generic": "1.0.0-alpha.3",
-    "@toa.io/pointer": "1.0.0-alpha.3",
-    "mongodb": "6.3.0",
-    "saslprep": "1.0.3"
+    "@toa.io/conveyor": "1.0.0-alpha.292",
+    "@toa.io/core": "1.0.0-alpha.300",
+    "@toa.io/definitions": "1.0.0-alpha.300",
+    "@toa.io/generic": "1.0.0-alpha.292",
+    "@toa.io/pointer": "1.0.0-alpha.299",
+    "mongodb": "7.6.0",
+    "openspan": "1.0.0-alpha.293"
   },
-  "gitHead": "e36ac7871fc14d15863aaf8f9bbdeace8bdfa9f0"
+  "gitHead": "f91131a9aeb29aa271f177c4055ddcf0cda8c75e"
 }

package/readme.md

@@ -0,0 +1,16 @@
+# MongoDB Storage
+
+## Tracing
+
+Commands are recorded as `client` spans within the trace of the current invocation,
+using the [driver command monitoring events](https://www.mongodb.com/docs/drivers/node/current/monitoring/command-monitoring/).
+
+Spans are named `{command} {collection}` and carry `db.*` attributes following the
+[OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/database/mongodb/):
+`db.system`, `db.namespace`, `db.collection.name`, `db.operation.name`.
+
+Commands executed outside of a sampled trace context (e.g. index management on startup)
+and internal driver commands (`hello`, `ping`, authentication) are not recorded.
+
+Monitoring is client-side only and does not affect the MongoDB server. Span recording
+adds no waiting to the query path: exporting is buffered and happens in the background.

package/src/client.js

@@ -0,0 +1,293 @@
+/**
+ * @typedef {import('mongodb').MongoClient} MongoClient
+ * @typedef {{ count: number, client: MongoClient }} Instance
+ * @typedef {import('@toa.io/core').Locator} Locator
+ */
+
+import { console } from 'openspan'
+import { environment } from '@toa.io/generic'
+import { Connector } from '@toa.io/core'
+import { resolve } from '@toa.io/pointer'
+import { ID } from '@toa.io/definitions/storages.mongodb'
+import { MongoClient } from 'mongodb'
+
+/**
+ * @type {Record<string, Promise<Instance>>}
+ */
+const INSTANCES = {}
+
+export class Client extends Connector {
+  name
+
+  /**
+   * @public
+   * @type {import('mongodb').Collection}
+   */
+  collection
+
+  /**
+   * The outbox rows of this component, absent unless something consumes its events. Created
+   * eagerly beside the entity collection, because a transaction cannot create a collection and
+   * an index build cannot run inside one.
+   *
+   * @public
+   * @type {import('mongodb').Collection | undefined}
+   */
+  outbox
+
+  /**
+   * The calls this component has made good on, absent unless it declares `once` anywhere.
+   * Created eagerly beside the entity collection, for the reason the outbox's is.
+   *
+   * @public
+   * @type {import('mongodb').Collection | undefined}
+   */
+  inbox
+
+  /**
+   * Whether this deployment can run transactions at all. A standalone mongod cannot, and an
+   * outbox without atomicity is worse than none, so the storage falls back to inline emission.
+   *
+   * @public
+   * @type {boolean}
+   */
+  transactional = false
+
+  /**
+   * The database this component's collections live in, which is where the migration state
+   * is kept as well.
+   *
+   * @public
+   * @type {import('mongodb').Db}
+   */
+  db
+
+  /**
+   * @private
+   * @type {Locator}
+   */
+  locator
+
+  /**
+   * @private
+   * @type {Instance}
+   */
+  instance
+
+  /**
+   * @private
+   * @type {string}
+   */
+  key
+
+  /**
+   * @private
+   * @type {boolean}
+   */
+  publishes
+
+  /**
+   * @private
+   * @type {boolean}
+   */
+  claims
+
+  /**
+   * @param {Locator} locator
+   * @param {boolean} [publishes] whether this component publishes anything
+   * @param {boolean} [claims] whether any of its operations declares `once`
+   */
+  constructor(locator, publishes = false, claims = false) {
+    super()
+
+    this.locator = locator
+    this.name = locator.lowercase
+    this.publishes = publishes
+    this.claims = claims
+  }
+
+  /**
+   * @protected
+   * @override
+   * @return {Promise<void>}
+   */
+  async open() {
+    const urls = await this.resolveURLs()
+    const dbname = this.resolveDB()
+
+    this.key = getKey(dbname, urls)
+
+    try {
+      INSTANCES[this.key] ??= this.createInstance(urls)
+    } catch (error) {
+      console.error('Failed to connect to MongoDB', { urls, error })
+    }
+
+    this.instance = await INSTANCES[this.key]
+    this.instance.count++
+
+    const db = this.instance.client.db(dbname)
+
+    this.db = db
+    this.collection = await collection(db, this.name)
+    this.transactional = await transactional(db)
+
+    /*
+     * The outbox may fall back and this may not: inline emission still delivers, where a call
+     * that is not recorded is a call that will be made twice, which is the opposite of what was
+     * asked for. So this refuses rather than warns.
+     */
+    if (this.claims) {
+      if (!this.transactional)
+        throw new Error(
+          `Component '${this.name}' declares 'once', which needs a MongoDB replica set ` +
+            'or a sharded cluster to commit a call with the entity it changed'
+        )
+
+      this.inbox = await collection(db, this.name + INBOX)
+    }
+
+    if (!this.publishes) return
+
+    if (this.transactional) this.outbox = await collection(db, this.name + OUTBOX)
+    else
+      console.warn(
+        'MongoDB is not a replica set; events are emitted inline, without an outbox',
+        { collection: this.name }
+      )
+  }
+
+  /**
+   * Runs `fn` in a transaction and answers what it returned. The driver may call `fn` more
+   * than once, so it must not hold state of its own — an outbox row is built by the caller
+   * and reused, and a rolled back attempt leaves nothing behind.
+   *
+   * @public
+   * @template T
+   * @param {(session: import('mongodb').ClientSession) => Promise<T>} fn
+   * @return {Promise<T>}
+   */
+  async transaction(fn) {
+    return this.instance.client.withSession(async (session) =>
+      session.withTransaction(async () => fn(session))
+    )
+  }
+
+  /**
+   * @protected
+   * @override
+   * @return {Promise<void>}
+   */
+  async close() {
+    /*
+     * What was never counted is not discounted. An `open` that threw between taking the
+     * instance and incrementing it leaves the count one high, and a client nothing ever
+     * closes — which a process that is taken down and built again, as a halt does, would
+     * otherwise leak once per cycle.
+     */
+    if (this.instance === undefined) return
+
+    const instance = this.instance
+
+    this.instance = undefined
+
+    instance.count--
+
+    if (instance.count === 0) {
+      await instance.client.close()
+
+      // another `open` may have taken it in the meantime, and that one is not this one
+      if ((await INSTANCES[this.key]) === instance) delete INSTANCES[this.key]
+    }
+  }
+
+  /**
+   * @private
+   * @param {string[]} urls
+   * @return {Promise<Instance>}
+   */
+  async createInstance(urls) {
+    const client = new MongoClient(urls.join(','), OPTIONS)
+    const hosts = urls.map((str) => new URL(str).host)
+
+    console.info('Connecting to MongoDB', { address: hosts.join(', ') })
+
+    await client.connect()
+
+    return {
+      count: 0,
+      client
+    }
+  }
+
+  /**
+   * @private
+   * @return {Promise<string[]>}
+   */
+  async resolveURLs() {
+    // Toa's own development stack is not on the conventional ports: the applications built on
+    // Toa are, and they share the machine. See CONTRIBUTING.md.
+    if (environment.get('TOA_DEV') === '1') {
+      return ['mongodb://developer:secret@localhost:31020']
+    } else {
+      return await resolve(ID, this.locator.id)
+    }
+  }
+
+  /**
+   * @private
+   * @return {string}
+   */
+  resolveDB() {
+    const context = environment.get('TOA_CONTEXT')
+
+    if (context !== undefined) return context
+
+    if (environment.get('TOA_DEV') === '1') return 'toa-dev'
+
+    throw new Error('Environment variable TOA_CONTEXT is not defined')
+  }
+}
+
+function getKey(db, urls) {
+  return db + ':' + urls.sort().join(' ')
+}
+
+/**
+ * Concurrent pods race to create the same collection, and losing that race is not an error.
+ */
+async function collection(db, name) {
+  try {
+    return await db.createCollection(name)
+  } catch (e) {
+    if (e.code !== ALREADY_EXISTS) throw e
+
+    return db.collection(name)
+  }
+}
+
+async function transactional(db) {
+  try {
+    const hello = await db.admin().command({ hello: 1 })
+
+    return hello.setName !== undefined || hello.msg === 'isdbgrid'
+  } catch (e) {
+    console.warn('MongoDB transaction support could not be determined', { error: e })
+
+    return false
+  }
+}
+
+/**
+ * `monitorCommands` is deliberately absent. It makes the driver materialize every reply
+ * eagerly to populate the monitoring event (`CommandSucceededEvent`), which defeats the
+ * lazy per-document deserialization a cursor exists for — a 100-document batch is then
+ * deserialized twice. `Storage` times its own calls instead.
+ */
+const OPTIONS = {
+  ignoreUndefined: true
+}
+
+const ALREADY_EXISTS = 48
+const OUTBOX = '_outbox'
+const INBOX = '_inbox'

package/src/factory.js

@@ -1,14 +1,10 @@
-'use strict'
+import { Client } from './client.js'
+import { Storage } from './storage.js'
 
-const { Connection } = require('./connection')
-const { Storage } = require('./storage')
+export class Factory {
+  storage(locator, entity, options = {}) {
+    const client = new Client(locator, options.outbox === true, options.inbox === true)
 
-class Factory {
-  storage (locator) {
-    const connection = new Connection(locator)
-
-    return new Storage(connection)
+    return new Storage(client, entity)
   }
 }
-
-exports.Factory = Factory

package/src/inbox.js

@@ -0,0 +1,85 @@
+import { environment } from '@toa.io/generic'
+import { index, prune } from './indexes.js'
+
+/**
+ * The calls a component has already made good on. Its lifecycle is the Client's, so it is not
+ * a Connector.
+ *
+ * Core owns what a record means — its identity and the reply it holds. This only writes it,
+ * inside the transaction the entity is written in, and reads one back.
+ */
+export class Inbox {
+  /** @type {import('mongodb').Collection} */
+  #collection
+
+  #retention
+
+  constructor(collection) {
+    this.#collection = collection
+    this.#retention = retention()
+  }
+
+  /**
+   * What the call under this identity answered, or `null` where it was never made.
+   *
+   * @param {string} id
+   * @returns {Promise<object | null>}
+   */
+  async recall(id) {
+    const record = await this.#collection.findOne(
+      { _id: id },
+      { projection: { reply: 1 } }
+    )
+
+    return record === null ? null : (record.reply ?? {})
+  }
+
+  /**
+   * Writes the call down, in the session the entity is written in. A duplicate key is the
+   * whole mechanism: the call has been made, this transaction changes nothing, and the caller
+   * is answered from what is already there.
+   *
+   * @param {{ id: string, reply: object }} call
+   * @param {import('mongodb').ClientSession} session
+   */
+  async insert(call, session) {
+    await this.#collection.insertOne(
+      { _id: call.id, reply: call.reply ?? {}, at: new Date() },
+      { session }
+    )
+  }
+
+  /**
+   * The runtime's own index, not the component's, so it is declared here rather than in a
+   * migration a component would have to write — including pruning, or a later change leaves
+   * the old index behind forever.
+   */
+  async index() {
+    const desired = {
+      // how long a call is remembered is how long a duplicate of it is caught
+      inbox_at: {
+        fields: { at: 1 },
+        options: { name: 'inbox_at', expireAfterSeconds: this.#retention }
+      }
+    }
+
+    for (const { fields, options } of Object.values(desired))
+      await index(this.#collection, fields, options)
+
+    await prune(this.#collection, Object.keys(desired))
+  }
+}
+
+function retention() {
+  const value = Number(environment.get('TOA_INBOX_RETENTION'))
+
+  return Number.isNaN(value) || value <= 0 ? RETENTION : value
+}
+
+/**
+ * Seconds a call is remembered. An hour, and not the outbox's day: what a duplicate arrives
+ * within is the broker's redelivery, the five attempts `comq` makes of a message, and whatever
+ * a client retries on — minutes. Every call of an operation that declares `once` is a document
+ * here for this long.
+ */
+const RETENTION = 3600

package/src/index.js

@@ -1,7 +1 @@
-'use strict'
-
-const { deployment } = require('./deployment')
-const { Factory } = require('./factory')
-
-exports.deployment = deployment
-exports.Factory = Factory
+export { Factory } from './factory.js'

package/src/indexes.js

@@ -0,0 +1,84 @@
+import { console } from 'openspan'
+
+/**
+ * The indexes the runtime keeps on collections of its own — the outbox's and the inbox's. They
+ * are declared in code rather than in a migration a component would have to write, so making
+ * and pruning them is the same job in both places and is written once.
+ */
+
+/**
+ * The name of one of these is fixed, so changing what it is made of — a retention that changes
+ * `expireAfterSeconds`, say — leaves that name held by an index of the old shape, which MongoDB
+ * refuses to overwrite. The old one is dropped and the declared one made.
+ *
+ * @param {import('mongodb').Collection} collection
+ * @param {object} fields
+ * @param {object} options
+ */
+export async function index(collection, fields, options) {
+  try {
+    await collection.createIndex(fields, options)
+  } catch (e) {
+    if (!CONFLICTS.includes(e.code))
+      return console.warn('MongoDB index creation failed', {
+        collection: collection.collectionName,
+        name: options.name,
+        error: e
+      })
+
+    console.info('Recreating an index whose declaration changed', {
+      collection: collection.collectionName,
+      name: options.name
+    })
+
+    await drop(collection, options.name)
+    await collection.createIndex(fields, options)
+  }
+}
+
+/**
+ * Removes what this collection carries that is no longer declared, so that a change to what the
+ * runtime wants does not leave the shape it wanted before in place forever.
+ *
+ * @param {import('mongodb').Collection} collection
+ * @param {string[]} desired the names that are declared
+ */
+export async function prune(collection, desired) {
+  let current
+
+  try {
+    current = await collection.listIndexes().toArray()
+  } catch {
+    return
+  }
+
+  const obsolete = current
+    .map(({ name }) => name)
+    .filter((name) => name !== '_id_' && !desired.includes(name))
+
+  if (obsolete.length === 0) return
+
+  console.info('Removing obsolete indexes', {
+    collection: collection.collectionName,
+    indexes: obsolete.join(', ')
+  })
+
+  await Promise.all(obsolete.map((name) => drop(collection, name)))
+}
+
+/**
+ * Concurrent replicas prune the same index, and losing that race is not an error.
+ *
+ * @param {import('mongodb').Collection} collection
+ * @param {string} name
+ */
+async function drop(collection, name) {
+  try {
+    await collection.dropIndex(name)
+  } catch (e) {
+    if (e.code !== ERR_INDEX_NOT_FOUND) throw e
+  }
+}
+
+const ERR_INDEX_NOT_FOUND = 27
+const CONFLICTS = [85, 86] // IndexOptionsConflict, IndexKeySpecsConflict