@owlmeans/mongo-resource 0.1.14 → 0.1.16-rc.0

package/src/resource.ts

@@ -1,14 +1,20 @@
 import { appendContextual, assertContext } from '@owlmeans/context'
 import type { BasicContext, Contextual } from '@owlmeans/context'
 import { DEFAULT_DB_ALIAS, DEFAULT_PAGE_SIZE } from './consts.js'
+import { MigrationStage } from '@owlmeans/resource'
 import type { ListCriteria, ResourceMaker, ResourceRecord } from '@owlmeans/resource'
 import type { ServerConfig, ServerContext } from '@owlmeans/server-context'
-import type { MongoDbService, MongoResource } from './types.js'
+import type { MongoDbService, MongoReference, MongoRefOptions, MongoResource, MongoTx } from './types.js'
 import { initializeCollection } from './utils/life-cycle.js'
+import { getDeclaration } from './declarations.js'
 import { ObjectId } from 'mongodb'
 import { MisshapedRecord, RecordExists, UnknownRecordError, UnsupportedArgumentError, RecordUpdateFailed, prepareListOptions } from '@owlmeans/resource'
 import type { JSONSchemaType } from 'ajv'
 import { getSchemaSecureFeilds } from './helper.js'
+import {
+  demarshalRefs, identityCriteria, makeRefMigration, marshalCriteria, marshalReference,
+  refMigrationName
+} from './utils/refs.js'
 
 type Config = ServerConfig
 type Context<C extends Config = Config> = ServerContext<C>
@@ -21,6 +27,18 @@ export const makeMongoResource = <
 ): T => {
   const location = `mongo-resource:${alias}`
 
+  /**
+   * Live view — references may be declared after the resource is built, and the
+   * declarations are module scoped so they survive `reinitializeContext`.
+   */
+  const refs = (): Map<string, MongoReference> => getDeclaration(alias).references
+
+  const demarshal = <Type extends ResourceRecord>(record: Type & { _id?: unknown }): Type => {
+    record.id = record._id instanceof ObjectId ? record._id.toString() : record._id as string
+    delete record._id
+    return demarshalRefs(record, refs())
+  }
+
   const resource: T = appendContextual<T>(alias, {
     get: async (id, field, opts) => {
       const record = await resource.load(id, field, opts)
@@ -37,15 +55,13 @@ export const makeMongoResource = <
         field = field.field
       }
       field = field ?? '_id'
-      const criteria = '_id' === field ? new ObjectId(id) : id
       if (opts?.ttl != null) {
         throw new UnsupportedArgumentError('ttl')
       }
 
-      const record = await resource.collection.findOne({ [field]: criteria })
+      const record = await resource.collection.findOne(identityCriteria(field, id, refs()))
       if (record != null) {
-        record.id = record._id instanceof ObjectId ? record._id.toString() : record._id
-        delete (record as any)._id
+        demarshal(record)
       }
 
       return record
@@ -70,16 +86,14 @@ export const makeMongoResource = <
 
       const original = await resource.get(id, field)
 
-      const criteria = '_id' === field ? new ObjectId(id) : id
-
       const replace = { ...record, _id: new ObjectId(original.id) }
       if (replace.id != null) {
         delete replace.id
       }
 
       const result = await resource.collection.replaceOne(
-        { [field]: criteria },
-        _prepareValues(replace, resource.schema as JSONSchemaType<any>)
+        identityCriteria(field, id, refs()),
+        _prepareValues(replace, resource.schema as JSONSchemaType<any>, refs())
       )
       if (!result.acknowledged) {
         throw new RecordUpdateFailed(`${field}:${id}`)
@@ -128,7 +142,7 @@ export const makeMongoResource = <
       }
       const result = await resource.collection.insertOne({
         ...resource.getDefaults(),
-        ..._prepareValues(record, resource.schema as JSONSchemaType<any>)
+        ..._prepareValues(record, resource.schema as JSONSchemaType<any>, refs())
       })
 
       if (!result.acknowledged) {
@@ -168,9 +182,7 @@ export const makeMongoResource = <
         throw new MisshapedRecord('id')
       }
 
-      const criteria = '_id' === field ? new ObjectId(_id) : _id
-
-      const result = await resource.collection.deleteOne({ [field]: criteria })
+      const result = await resource.collection.deleteOne(identityCriteria(field, _id as string, refs()))
       if (!result.acknowledged || result.deletedCount === 0) {
         return null
       }
@@ -190,7 +202,7 @@ export const makeMongoResource = <
     list: async (criteria, opts) => {
       const options = prepareListOptions(DEFAULT_PAGE_SIZE, criteria, opts)
 
-      criteria = options.criteria
+      criteria = marshalCriteria(options.criteria, refs()) ?? {}
       const pager = options.pager ?? {}
 
       const size = pager?.size ?? DEFAULT_PAGE_SIZE
@@ -218,13 +230,7 @@ export const makeMongoResource = <
       const items = await cursor.toArray()
 
       return {
-        pager, items: items.map(item => {
-          const _item: R = { ...item } as any
-          _item.id = item._id.toString()
-          delete (_item as any)._id
-
-          return _item
-        })
+        pager, items: items.map(item => demarshal({ ...item } as unknown as R))
       }
     },
 
@@ -270,7 +276,35 @@ export const makeMongoResource = <
       resource.indexes = resource.indexes ?? []
       resource.indexes.push({ name, index, options })
       return resource
-    }
+    },
+
+    /**
+     * `migration` and `reference` are `this`-returning in the interface, which an object
+     * literal can't express — hence the member level casts: the implementations return
+     * the closed over `resource`, which is that very object.
+     */
+    migration: ((name: string, apply: (tx: MongoTx) => Promise<void>, stage?: MigrationStage) => {
+      getDeclaration(alias).migrations.register(name, apply, stage)
+      return resource
+    }) as T['migration'],
+
+    migrations: () => getDeclaration(alias).migrations,
+
+    reference: ((field: string, opts?: string | MongoRefOptions) => {
+      const declaration = getDeclaration(alias)
+      const options = typeof opts === 'string' ? { resource: opts } : opts ?? {}
+      declaration.references.set(field, { field, resource: options.resource, noIndex: options.noIndex })
+      /**
+       * The system migration that converts the field's pre-existing string ids. Registered
+       * here rather than at init so it precedes migrations the app declares after its
+       * `reference()` calls — the field's type contract is the foundation those build on.
+       */
+      declaration.migrations.register(refMigrationName(field), makeRefMigration(field), MigrationStage.Pre)
+
+      return resource
+    }) as T['reference'],
+
+    references: () => [...getDeclaration(alias).references.values()]
   } as Partial<T>)
 
   // Explicit collection name override (decoupled from the registration alias, which may
@@ -286,7 +320,9 @@ export const makeMongoResource = <
     await mongo.ready()
     const db = await mongo.db(dbAlias)
     const config = mongo.config(dbAlias)
-    resource.collection = await initializeCollection(db, config, resource as unknown as MongoResource<ResourceRecord>)
+    resource.collection = await initializeCollection(
+      db, config, resource as unknown as MongoResource<ResourceRecord>, context
+    )
   }
 
   resource.reinitializeContext = <Type extends Contextual>(context: BasicContext<Config>) => {
@@ -301,15 +337,27 @@ export const makeMongoResource = <
   return resource
 }
 
-const _prepareValues = <T extends ResourceRecord>(obj: T, schema?: JSONSchemaType<T>): T => {
+const _prepareValues = <T extends ResourceRecord>(
+  obj: T, schema?: JSONSchemaType<T>, refs?: Map<string, MongoReference>
+): T => {
+  /**
+   * Declared references convert independently of the schema — the schema is optional,
+   * and where it exists it declares these fields as strings, whose coercion below would
+   * undo the conversion.
+   */
+  if (refs != null && refs.size > 0) {
+    obj = Object.fromEntries(Object.entries(obj).map(([key, value]) =>
+      refs.has(key) ? [key, marshalReference(key, value)] : [key, value]
+    )) as T
+  }
   // @TODO Validate keys from additional properties in the root
   return schema != null ? Object.fromEntries(Object.entries(obj).map(([key, value]) => {
-    // @TODO What if we reference another table by object id in another field?
-    // How to properly transform them?
-    // What if _id isn't an Object Id?
-    if (key === '_id') {
+    if (key === '_id' && !(value instanceof ObjectId)) {
       return [key, new ObjectId(value as string)]
     }
+    if (key === '_id' || refs?.has(key)) {
+      return [key, value]
+    }
     // A null/undefined value has nothing to coerce — pass it through for any declared
     // type. Without this guard the object-map and array branches below call
     // `Object.entries`/`.map` on `undefined` and throw (and `new Date(null)` would

package/src/types.ts

@@ -1,8 +1,50 @@
-import type { Resource, ResourceRecord, ResourceDbService, DbLocker, ResourceLocker } from '@owlmeans/resource'
+import type {
+  Resource, ResourceRecord, ResourceDbService, DbLocker, ResourceLocker, MigratableResource
+} from '@owlmeans/resource'
 import type { Collection, CreateIndexesOptions, Db, IndexSpecification, MongoClient } from 'mongodb'
 import type { AnySchema } from 'ajv'
 
-export interface MongoResource<T extends ResourceRecord> extends Resource<T>, ResourceLocker<T> {
+/**
+ * What a mongo migration is handed.
+ *
+ * Deliberately not a session or transaction: multi-document transactions require a replica
+ * set, and a standalone `mongod` — the usual development and CI target — rejects them
+ * outright. A migration therefore has to be written to tolerate being interrupted partway,
+ * which in practice means idempotent updates rather than read-then-write.
+ */
+export interface MongoTx {
+  db: Db
+  /** The owning resource's collection. */
+  collection: Collection
+  /** Another registered mongo resource's collection; omit the alias for the owning one. */
+  use: (alias?: string) => Collection
+  /** A collection *name*, for `$lookup.from` and other stages that take one rather than a handle. */
+  ref: (alias?: string) => string
+}
+
+/**
+ * A declared ObjectId reference — a record field that stores another record's id.
+ *
+ * The resource converts the field between the string ids records carry and the `ObjectId`
+ * the collection stores, exactly the way it already does for `_id`: strings in records and
+ * criteria, `ObjectId` on the wire. Declaring a reference also gives the field a mongo
+ * level index and registers the system migration that converts pre-existing string values.
+ */
+export interface MongoReference {
+  /** Top level record property holding the reference (a single id or an array of ids). */
+  field: string
+  /** Alias of the referenced resource. Informational — conversion never resolves it. */
+  resource?: string
+  /** Skip the automatic `{ [field]: 1 }` index. */
+  noIndex?: boolean
+}
+
+export interface MongoRefOptions {
+  resource?: string
+  noIndex?: boolean
+}
+
+export interface MongoResource<T extends ResourceRecord> extends Resource<T>, ResourceLocker<T>, MigratableResource<MongoTx> {
   name?: string
   schema?: AnySchema
   indexes?: Array<{ name: string, index: IndexSpecification, options?: CreateIndexesOptions }>
@@ -10,6 +52,20 @@ export interface MongoResource<T extends ResourceRecord> extends Resource<T>, Re
   db: () => Promise<Db>
   client: () => Promise<MongoClient>
   index: <Type extends MongoResource<T>>(name: string, index: IndexSpecification, options?: CreateIndexesOptions) => Type
+  /**
+   * Declare that a field stores another record's id.
+   *
+   * Chainable and idempotent like {@link MigratableResource.migration}, and stored the same
+   * way — per alias at module scope — because losing the declaration to a context rebuild
+   * would silently stop the string/ObjectId conversion for the field.
+   *
+   * Declare only fields whose values really are mongo ids (assigned from another record's
+   * `id`). Composite keys, external provider ids, DIDs and business slugs must stay
+   * strings — converting them corrupts the collection.
+   */
+  reference: (field: string, opts?: string | MongoRefOptions) => this
+  /** The declared references of this alias. */
+  references: () => MongoReference[]
   getDefaults: () => Partial<T>
 }
 

package/src/utils/index.ts

@@ -3,3 +3,5 @@ export * from './indexes.js'
 export * from './schema.js'
 export * from './name.js'
 export * from './life-cycle.js'
+export * from './migrations.js'
+export * from './refs.js'

package/src/utils/life-cycle.ts

@@ -1,31 +1,131 @@
-import type { DbConfig, ResourceRecord } from '@owlmeans/resource'
-import type { MongoResource } from '../types.js'
-import type { Db, Collection, Document } from 'mongodb'
+import { MigrationStage, runMigrations } from '@owlmeans/resource'
+import type { DbConfig, MigrationReport, ResourceRecord } from '@owlmeans/resource'
+import type { BasicContext } from '@owlmeans/context'
+import type { MongoReference, MongoResource } from '../types.js'
+import type { Db, Collection, Document, IndexSpecification } from 'mongodb'
+import { DEF_MIGRATIONS_COLLECTION } from '../consts.js'
+import { getDeclaration } from '../declarations.js'
 import { mongoCollectionName } from './name.js'
-import { schemaToMongoSchema } from './schema.js'
+import { applyReferenceTypes, schemaToMongoSchema } from './schema.js'
 import { updateIndexes } from './indexes.js'
+import { makeMongoMigrationStore, makeMongoTx } from './migrations.js'
+import { reconcileReferences } from './refs.js'
 
+/**
+ * Bring a resource's collection to the shape its schema declares.
+ *
+ * Order is deliberate, and matches the Postgres counterpart:
+ *
+ *  1. probe for the collection
+ *  2. absent → *baseline* every registered migration; present → run the `pre` ones
+ *     (including the system `$ref:` migrations that convert declared references)
+ *  3. create the collection, or update its validator and indexes
+ *  4. present → run the `post` migrations
+ *  5. reconcile declared references — the second half of their double check: the ledger
+ *     said whether the `$ref:` migration ran, this probes the collection itself and
+ *     converts whatever strings still slipped through
+ *
+ * Step 2 is what stops the two mechanisms colliding. A `pre` migration runs before the
+ * validator is tightened, so it can reshape documents the new validator would reject. On a
+ * collection this call just created there is nothing to reshape — replaying a historical
+ * migration against a collection born in its final shape would at best scan for nothing —
+ * so the migrations are recorded as satisfied instead of run.
+ *
+ * `context` is optional so the existing three-argument call keeps working; without it a
+ * migration's `use`/`ref` can only address the owning resource.
+ */
 export const initializeCollection = async (
-  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>
+  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>,
+  context?: BasicContext<any>
 ): Promise<Collection> => {
-  let _collection: Collection
-
   const name = mongoCollectionName(config, resource)
-  const cursor = db.listCollections({ name })
-  if (!await cursor.hasNext()) {
-    _collection = await createCollection(db, name, resource)
-  } else {
-    _collection = await updateCollection(db, name, resource)
+  const fresh = !await db.listCollections({ name }).hasNext()
+
+  appendReferenceIndexes(resource)
+
+  const migrate = prepareMigrations(db, config, resource, name, context)
+  await migrate(fresh ? { baseline: true } : { stage: MigrationStage.Pre })
+
+  const collection = fresh
+    ? await createCollection(db, name, resource)
+    : await updateCollection(db, name, resource)
+
+  if (!fresh) {
+    await migrate({ stage: MigrationStage.Post })
+    await reconcileReferences(collection, referencesOf(resource), resource.alias)
   }
 
-  return _collection
+  return collection
+}
+
+/** Tolerates hand-built resource objects that predate the reference capability. */
+const referencesOf = (resource: MongoResource<ResourceRecord>): MongoReference[] =>
+  resource.references?.() ?? []
+
+/**
+ * A declared reference is indexed at the mongo level — that's part of its contract. The
+ * index is appended unless the resource already declares one with the same key pattern
+ * (mongo refuses two indexes over identical keys, and the existing one — possibly
+ * unique — wins).
+ */
+const appendReferenceIndexes = (resource: MongoResource<ResourceRecord>): void => {
+  for (const ref of referencesOf(resource)) {
+    if (ref.noIndex === true) {
+      continue
+    }
+    resource.indexes = resource.indexes ?? []
+    const spec = JSON.stringify({ [ref.field]: 1 })
+    const present = resource.indexes.some(index =>
+      JSON.stringify(index.index as IndexSpecification) === spec || index.name === `ref_${ref.field}`
+    )
+    if (!present) {
+      resource.indexes.push({ name: `ref_${ref.field}`, index: { [ref.field]: 1 } })
+    }
+  }
+}
+
+/**
+ * Bind the migration runner to this resource's database, or hand back a no-op when nothing
+ * is registered — the overwhelmingly common case, and one that shouldn't pay for a ledger.
+ */
+const prepareMigrations = (
+  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>, name: string,
+  context?: BasicContext<any>
+): (opts: { stage?: MigrationStage, baseline?: boolean }) => Promise<MigrationReport | null> => {
+  const registry = getDeclaration(resource.alias).migrations
+  if (registry.list().length < 1) {
+    return async () => null
+  }
+
+  const ledgerName = (config.meta as { migrationsCollection?: string } | undefined)?.migrationsCollection
+    ?? DEF_MIGRATIONS_COLLECTION
+  /**
+   * `db.collection()` never round-trips, so taking the handle before the collection exists
+   * is safe — and on the fresh path the transaction is only ever baselined, never used.
+   */
+  const tx = makeMongoTx(db, db.collection(name), config, context as BasicContext<any>, resource.alias)
+  const store = makeMongoMigrationStore(db, tx, ledgerName)
+
+  return async opts => {
+    const report = await runMigrations(resource.alias, registry, store, opts)
+    if (report.applied.length > 0) {
+      console.log(
+        `@owlmeans/mongo-resource: ${name} applied ${report.stage} migrations —`
+        + ` ${report.applied.join(', ')}`
+      )
+    }
+
+    return report
+  }
 }
 
 export const createCollection = async (db: Db, name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
   const collection = await db.createCollection(name, {
     ...(resource.schema != null ? {
       validator: {
-        $jsonSchema: patchJsonSchema(schemaToMongoSchema(resource.schema))
+        $jsonSchema: patchJsonSchema(applyReferenceTypes(
+          schemaToMongoSchema(resource.schema), resource.schema, resource.references()
+        ))
       }
     } : {})
   })
@@ -43,7 +143,9 @@ export const createCollection = async (db: Db, name: string, resource: MongoReso
 
 export const updateCollection = async (db: Db, name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
   if (resource.schema != null) {
-    const $jsonSchema = patchJsonSchema(schemaToMongoSchema(resource.schema))
+    const $jsonSchema = patchJsonSchema(applyReferenceTypes(
+      schemaToMongoSchema(resource.schema), resource.schema, resource.references()
+    ))
     await db.command({ collMod: name, validator: { $jsonSchema } })
   }
 

package/src/utils/migrations.ts

@@ -0,0 +1,171 @@
+import type { BasicContext } from '@owlmeans/context'
+import type { DbConfig, Migration, MigrationStage, MigrationStore, ResourceRecord } from '@owlmeans/resource'
+import type { Collection, Db } from 'mongodb'
+
+import {
+  DEF_MIGRATION_POLL, DEF_MIGRATION_WAIT, DEF_MIGRATIONS_COLLECTION, MONGO_DUPLICATE_KEY
+} from '../consts.js'
+import type { MongoResource, MongoTx } from '../types.js'
+import { mongoCollectionName } from './name.js'
+
+interface LedgerRecord {
+  alias: string
+  name: string
+  stage: MigrationStage
+  checksum: string | null
+  baseline: boolean
+  startedAt: Date
+  /** `null` while a replica is running it — the claim, not the completion. */
+  completedAt: Date | null
+  durationMs: number | null
+}
+
+const isDuplicateKey = (error: unknown): boolean =>
+  (error as { code?: number } | null)?.code === MONGO_DUPLICATE_KEY
+
+/**
+ * A transaction façade over the database. Alias resolution is injected so migrations can
+ * address other resources without importing the context.
+ *
+ * Names are resolved through {@link mongoCollectionName}, not through the other resource's
+ * live `collection` handle, because at migration time that resource may not have
+ * initialized yet — registration order is not dependency order. The name is a pure
+ * function of the config and the alias, so it's knowable either way.
+ */
+export const makeMongoTx = (
+  db: Db, collection: Collection, config: DbConfig, context: BasicContext<any>, self: string
+): MongoTx => {
+  const ref = (alias?: string): string => {
+    if (alias == null || alias === self) {
+      return collection.collectionName
+    }
+
+    return mongoCollectionName(config, context.resource<MongoResource<ResourceRecord>>(alias))
+  }
+
+  return {
+    db,
+    collection,
+    ref,
+    use: alias => alias == null || alias === self ? collection : db.collection(ref(alias))
+  }
+}
+
+const waitForClaim = async (
+  ledger: Collection<LedgerRecord>, alias: string, name: string
+): Promise<void> => {
+  const deadline = Date.now() + DEF_MIGRATION_WAIT
+  while (Date.now() < deadline) {
+    const row = await ledger.findOne({ alias, name })
+    if (row == null) {
+      /** The claim was withdrawn — the other replica's migration failed, and so must this boot. */
+      throw new Error(`${alias}/${name}: abandoned by the replica that claimed it`)
+    }
+    if (row.completedAt != null) {
+      return
+    }
+    await new Promise(resolve => setTimeout(resolve, DEF_MIGRATION_POLL))
+  }
+
+  throw new Error(`${alias}/${name}: still running elsewhere after ${DEF_MIGRATION_WAIT}ms`)
+}
+
+/**
+ * Migration ledger, one collection per database.
+ *
+ * Where the Postgres store commits the ledger row inside the migration's own transaction,
+ * this one can't — see {@link MongoTx}. It uses claim-then-complete instead: the unique
+ * index on `(alias, name)` is the mutual exclusion primitive, so a replica that loses the
+ * race to insert waits for the winner rather than running the same migration twice.
+ */
+export const makeMongoMigrationStore = (
+  db: Db, tx: MongoTx, name: string = DEF_MIGRATIONS_COLLECTION
+): MigrationStore<MongoTx> => {
+  const ledger = (): Collection<LedgerRecord> => db.collection<LedgerRecord>(name)
+
+  return {
+    ensure: async () => {
+      await ledger().createIndex({ alias: 1, name: 1 }, { name: 'alias_name_unique', unique: true })
+    },
+
+    applied: async alias => {
+      /**
+       * Completed rows only. A claim still holding a null `completedAt` belongs to a replica
+       * running right now, and treating it as applied would let this process carry on against
+       * a half-migrated database.
+       */
+      const rows = await ledger().find({ alias, completedAt: { $ne: null } }).toArray()
+
+      return rows.reduce<Record<string, string | null>>((applied, row) => {
+        applied[row.name] = row.checksum
+
+        return applied
+      }, {})
+    },
+
+    baseline: async (alias, migrations) => {
+      const now = new Date()
+      /**
+       * `$setOnInsert` upserts rather than plain inserts: two replicas creating the same
+       * collection at once both baseline, and the second must be a no-op, not a crash.
+       */
+      await ledger().bulkWrite(migrations.map(migration => ({
+        updateOne: {
+          filter: { alias, name: migration.name },
+          update: {
+            $setOnInsert: {
+              alias,
+              name: migration.name,
+              stage: migration.stage,
+              checksum: migration.checksum,
+              baseline: true,
+              startedAt: now,
+              completedAt: now,
+              durationMs: 0
+            }
+          },
+          upsert: true
+        }
+      })), { ordered: false })
+    },
+
+    run: async (alias: string, migration: Migration<MongoTx>) => {
+      const started = Date.now()
+      try {
+        await ledger().insertOne({
+          alias,
+          name: migration.name,
+          stage: migration.stage,
+          checksum: migration.checksum,
+          baseline: false,
+          startedAt: new Date(started),
+          completedAt: null,
+          durationMs: null
+        })
+      } catch (error) {
+        if (!isDuplicateKey(error)) {
+          throw error
+        }
+
+        return await waitForClaim(ledger(), alias, migration.name)
+      }
+
+      try {
+        await migration.apply(tx)
+      } catch (error) {
+        /**
+         * Withdraw the claim so the next boot retries. Postgres rolls its ledger row back
+         * with the migration itself; without a transaction the claim has to be released by
+         * hand, and leaving it would wedge every future boot on a migration that never ran.
+         */
+        await ledger().deleteOne({ alias, name: migration.name }).catch(() => undefined)
+        throw error
+      }
+
+      await ledger().updateOne(
+        { alias, name: migration.name },
+        { $set: { completedAt: new Date(), durationMs: Date.now() - started } }
+      )
+    }
+  }
+}