@owlmeans/mongo-resource 0.1.15 → 0.1.16

package/src/utils/life-cycle.ts

@@ -1,31 +1,131 @@
-import type { DbConfig, ResourceRecord } from '@owlmeans/resource'
-import type { MongoResource } from '../types.js'
-import type { Db, Collection, Document } from 'mongodb'
+import { MigrationStage, runMigrations } from '@owlmeans/resource'
+import type { DbConfig, MigrationReport, ResourceRecord } from '@owlmeans/resource'
+import type { BasicContext } from '@owlmeans/context'
+import type { MongoReference, MongoResource } from '../types.js'
+import type { Db, Collection, Document, IndexSpecification } from 'mongodb'
+import { DEF_MIGRATIONS_COLLECTION } from '../consts.js'
+import { getDeclaration } from '../declarations.js'
 import { mongoCollectionName } from './name.js'
-import { schemaToMongoSchema } from './schema.js'
+import { applyReferenceTypes, schemaToMongoSchema } from './schema.js'
 import { updateIndexes } from './indexes.js'
+import { makeMongoMigrationStore, makeMongoTx } from './migrations.js'
+import { reconcileReferences } from './refs.js'
 
+/**
+ * Bring a resource's collection to the shape its schema declares.
+ *
+ * Order is deliberate, and matches the Postgres counterpart:
+ *
+ *  1. probe for the collection
+ *  2. absent → *baseline* every registered migration; present → run the `pre` ones
+ *     (including the system `$ref:` migrations that convert declared references)
+ *  3. create the collection, or update its validator and indexes
+ *  4. present → run the `post` migrations
+ *  5. reconcile declared references — the second half of their double check: the ledger
+ *     said whether the `$ref:` migration ran, this probes the collection itself and
+ *     converts whatever strings still slipped through
+ *
+ * Step 2 is what stops the two mechanisms colliding. A `pre` migration runs before the
+ * validator is tightened, so it can reshape documents the new validator would reject. On a
+ * collection this call just created there is nothing to reshape — replaying a historical
+ * migration against a collection born in its final shape would at best scan for nothing —
+ * so the migrations are recorded as satisfied instead of run.
+ *
+ * `context` is optional so the existing three-argument call keeps working; without it a
+ * migration's `use`/`ref` can only address the owning resource.
+ */
 export const initializeCollection = async (
-  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>
+  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>,
+  context?: BasicContext<any>
 ): Promise<Collection> => {
-  let _collection: Collection
-
   const name = mongoCollectionName(config, resource)
-  const cursor = db.listCollections({ name })
-  if (!await cursor.hasNext()) {
-    _collection = await createCollection(db, name, resource)
-  } else {
-    _collection = await updateCollection(db, name, resource)
+  const fresh = !await db.listCollections({ name }).hasNext()
+
+  appendReferenceIndexes(resource)
+
+  const migrate = prepareMigrations(db, config, resource, name, context)
+  await migrate(fresh ? { baseline: true } : { stage: MigrationStage.Pre })
+
+  const collection = fresh
+    ? await createCollection(db, name, resource)
+    : await updateCollection(db, name, resource)
+
+  if (!fresh) {
+    await migrate({ stage: MigrationStage.Post })
+    await reconcileReferences(collection, referencesOf(resource), resource.alias)
   }
 
-  return _collection
+  return collection
+}
+
+/** Tolerates hand-built resource objects that predate the reference capability. */
+const referencesOf = (resource: MongoResource<ResourceRecord>): MongoReference[] =>
+  resource.references?.() ?? []
+
+/**
+ * A declared reference is indexed at the mongo level — that's part of its contract. The
+ * index is appended unless the resource already declares one with the same key pattern
+ * (mongo refuses two indexes over identical keys, and the existing one — possibly
+ * unique — wins).
+ */
+const appendReferenceIndexes = (resource: MongoResource<ResourceRecord>): void => {
+  for (const ref of referencesOf(resource)) {
+    if (ref.noIndex === true) {
+      continue
+    }
+    resource.indexes = resource.indexes ?? []
+    const spec = JSON.stringify({ [ref.field]: 1 })
+    const present = resource.indexes.some(index =>
+      JSON.stringify(index.index as IndexSpecification) === spec || index.name === `ref_${ref.field}`
+    )
+    if (!present) {
+      resource.indexes.push({ name: `ref_${ref.field}`, index: { [ref.field]: 1 } })
+    }
+  }
+}
+
+/**
+ * Bind the migration runner to this resource's database, or hand back a no-op when nothing
+ * is registered — the overwhelmingly common case, and one that shouldn't pay for a ledger.
+ */
+const prepareMigrations = (
+  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>, name: string,
+  context?: BasicContext<any>
+): (opts: { stage?: MigrationStage, baseline?: boolean }) => Promise<MigrationReport | null> => {
+  const registry = getDeclaration(resource.alias).migrations
+  if (registry.list().length < 1) {
+    return async () => null
+  }
+
+  const ledgerName = (config.meta as { migrationsCollection?: string } | undefined)?.migrationsCollection
+    ?? DEF_MIGRATIONS_COLLECTION
+  /**
+   * `db.collection()` never round-trips, so taking the handle before the collection exists
+   * is safe — and on the fresh path the transaction is only ever baselined, never used.
+   */
+  const tx = makeMongoTx(db, db.collection(name), config, context as BasicContext<any>, resource.alias)
+  const store = makeMongoMigrationStore(db, tx, ledgerName)
+
+  return async opts => {
+    const report = await runMigrations(resource.alias, registry, store, opts)
+    if (report.applied.length > 0) {
+      console.log(
+        `@owlmeans/mongo-resource: ${name} applied ${report.stage} migrations —`
+        + ` ${report.applied.join(', ')}`
+      )
+    }
+
+    return report
+  }
 }
 
 export const createCollection = async (db: Db, name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
   const collection = await db.createCollection(name, {
     ...(resource.schema != null ? {
       validator: {
-        $jsonSchema: patchJsonSchema(schemaToMongoSchema(resource.schema))
+        $jsonSchema: patchJsonSchema(applyReferenceTypes(
+          schemaToMongoSchema(resource.schema), resource.schema, resource.references()
+        ))
       }
     } : {})
   })
@@ -43,7 +143,9 @@ export const createCollection = async (db: Db, name: string, resource: MongoReso
 
 export const updateCollection = async (db: Db, name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
   if (resource.schema != null) {
-    const $jsonSchema = patchJsonSchema(schemaToMongoSchema(resource.schema))
+    const $jsonSchema = patchJsonSchema(applyReferenceTypes(
+      schemaToMongoSchema(resource.schema), resource.schema, resource.references()
+    ))
     await db.command({ collMod: name, validator: { $jsonSchema } })
   }
 

package/src/utils/migrations.ts

@@ -0,0 +1,171 @@
+import type { BasicContext } from '@owlmeans/context'
+import type { DbConfig, Migration, MigrationStage, MigrationStore, ResourceRecord } from '@owlmeans/resource'
+import type { Collection, Db } from 'mongodb'
+
+import {
+  DEF_MIGRATION_POLL, DEF_MIGRATION_WAIT, DEF_MIGRATIONS_COLLECTION, MONGO_DUPLICATE_KEY
+} from '../consts.js'
+import type { MongoResource, MongoTx } from '../types.js'
+import { mongoCollectionName } from './name.js'
+
+interface LedgerRecord {
+  alias: string
+  name: string
+  stage: MigrationStage
+  checksum: string | null
+  baseline: boolean
+  startedAt: Date
+  /** `null` while a replica is running it — the claim, not the completion. */
+  completedAt: Date | null
+  durationMs: number | null
+}
+
+const isDuplicateKey = (error: unknown): boolean =>
+  (error as { code?: number } | null)?.code === MONGO_DUPLICATE_KEY
+
+/**
+ * A transaction façade over the database. Alias resolution is injected so migrations can
+ * address other resources without importing the context.
+ *
+ * Names are resolved through {@link mongoCollectionName}, not through the other resource's
+ * live `collection` handle, because at migration time that resource may not have
+ * initialized yet — registration order is not dependency order. The name is a pure
+ * function of the config and the alias, so it's knowable either way.
+ */
+export const makeMongoTx = (
+  db: Db, collection: Collection, config: DbConfig, context: BasicContext<any>, self: string
+): MongoTx => {
+  const ref = (alias?: string): string => {
+    if (alias == null || alias === self) {
+      return collection.collectionName
+    }
+
+    return mongoCollectionName(config, context.resource<MongoResource<ResourceRecord>>(alias))
+  }
+
+  return {
+    db,
+    collection,
+    ref,
+    use: alias => alias == null || alias === self ? collection : db.collection(ref(alias))
+  }
+}
+
+const waitForClaim = async (
+  ledger: Collection<LedgerRecord>, alias: string, name: string
+): Promise<void> => {
+  const deadline = Date.now() + DEF_MIGRATION_WAIT
+  while (Date.now() < deadline) {
+    const row = await ledger.findOne({ alias, name })
+    if (row == null) {
+      /** The claim was withdrawn — the other replica's migration failed, and so must this boot. */
+      throw new Error(`${alias}/${name}: abandoned by the replica that claimed it`)
+    }
+    if (row.completedAt != null) {
+      return
+    }
+    await new Promise(resolve => setTimeout(resolve, DEF_MIGRATION_POLL))
+  }
+
+  throw new Error(`${alias}/${name}: still running elsewhere after ${DEF_MIGRATION_WAIT}ms`)
+}
+
+/**
+ * Migration ledger, one collection per database.
+ *
+ * Where the Postgres store commits the ledger row inside the migration's own transaction,
+ * this one can't — see {@link MongoTx}. It uses claim-then-complete instead: the unique
+ * index on `(alias, name)` is the mutual exclusion primitive, so a replica that loses the
+ * race to insert waits for the winner rather than running the same migration twice.
+ */
+export const makeMongoMigrationStore = (
+  db: Db, tx: MongoTx, name: string = DEF_MIGRATIONS_COLLECTION
+): MigrationStore<MongoTx> => {
+  const ledger = (): Collection<LedgerRecord> => db.collection<LedgerRecord>(name)
+
+  return {
+    ensure: async () => {
+      await ledger().createIndex({ alias: 1, name: 1 }, { name: 'alias_name_unique', unique: true })
+    },
+
+    applied: async alias => {
+      /**
+       * Completed rows only. A claim still holding a null `completedAt` belongs to a replica
+       * running right now, and treating it as applied would let this process carry on against
+       * a half-migrated database.
+       */
+      const rows = await ledger().find({ alias, completedAt: { $ne: null } }).toArray()
+
+      return rows.reduce<Record<string, string | null>>((applied, row) => {
+        applied[row.name] = row.checksum
+
+        return applied
+      }, {})
+    },
+
+    baseline: async (alias, migrations) => {
+      const now = new Date()
+      /**
+       * `$setOnInsert` upserts rather than plain inserts: two replicas creating the same
+       * collection at once both baseline, and the second must be a no-op, not a crash.
+       */
+      await ledger().bulkWrite(migrations.map(migration => ({
+        updateOne: {
+          filter: { alias, name: migration.name },
+          update: {
+            $setOnInsert: {
+              alias,
+              name: migration.name,
+              stage: migration.stage,
+              checksum: migration.checksum,
+              baseline: true,
+              startedAt: now,
+              completedAt: now,
+              durationMs: 0
+            }
+          },
+          upsert: true
+        }
+      })), { ordered: false })
+    },
+
+    run: async (alias: string, migration: Migration<MongoTx>) => {
+      const started = Date.now()
+      try {
+        await ledger().insertOne({
+          alias,
+          name: migration.name,
+          stage: migration.stage,
+          checksum: migration.checksum,
+          baseline: false,
+          startedAt: new Date(started),
+          completedAt: null,
+          durationMs: null
+        })
+      } catch (error) {
+        if (!isDuplicateKey(error)) {
+          throw error
+        }
+
+        return await waitForClaim(ledger(), alias, migration.name)
+      }
+
+      try {
+        await migration.apply(tx)
+      } catch (error) {
+        /**
+         * Withdraw the claim so the next boot retries. Postgres rolls its ledger row back
+         * with the migration itself; without a transaction the claim has to be released by
+         * hand, and leaving it would wedge every future boot on a migration that never ran.
+         */
+        await ledger().deleteOne({ alias, name: migration.name }).catch(() => undefined)
+        throw error
+      }
+
+      await ledger().updateOne(
+        { alias, name: migration.name },
+        { $set: { completedAt: new Date(), durationMs: Date.now() - started } }
+      )
+    }
+  }
+}

package/src/utils/refs.ts

@@ -0,0 +1,240 @@
+import { MisshapedRecord } from '@owlmeans/resource'
+import type { ListCriteria } from '@owlmeans/resource'
+import { ObjectId } from 'mongodb'
+import type { Collection, Document } from 'mongodb'
+
+import type { MongoReference, MongoTx } from '../types.js'
+
+/**
+ * The only shape a stored reference is converted from. Deliberately stricter than
+ * `ObjectId.isValid`, which also accepts any 12 character string and would silently
+ * swallow short business keys.
+ */
+const HEX24 = /^[0-9a-fA-F]{24}$/
+
+export const isObjectIdHex = (value: unknown): value is string =>
+  typeof value === 'string' && HEX24.test(value)
+
+/**
+ * Write side of a declared reference: the string id a record carries becomes the
+ * `ObjectId` the collection stores. Arrays convert elementwise.
+ *
+ * Strict on purpose — a declared reference holding something that is not a mongo id is
+ * either a mis-declared field (should never have been a reference) or a bug at the call
+ * site, and storing it as a string would silently reintroduce the mixed type state this
+ * mechanism exists to remove.
+ *
+ * @throws {MisshapedRecord}
+ */
+export const marshalReference = (field: string, value: unknown): unknown => {
+  if (value == null) {
+    return value
+  }
+  if (value instanceof ObjectId) {
+    return value
+  }
+  if (Array.isArray(value)) {
+    return value.map(item => marshalReference(field, item))
+  }
+  if (isObjectIdHex(value)) {
+    return new ObjectId(value)
+  }
+
+  throw new MisshapedRecord(`ref:${field}`)
+}
+
+/** Read side: `ObjectId` back to the string records carry. Tolerates not yet migrated strings. */
+export const demarshalReference = (value: unknown): unknown => {
+  if (value instanceof ObjectId) {
+    return value.toString()
+  }
+  if (Array.isArray(value)) {
+    return value.map(demarshalReference)
+  }
+
+  return value
+}
+
+/** Convert every declared reference of a fetched document back to string ids, in place. */
+export const demarshalRefs = <T extends {}>(record: T, refs: Map<string, MongoReference>): T => {
+  if (refs.size < 1) {
+    return record
+  }
+  for (const field of refs.keys()) {
+    const value = (record as Document)[field]
+    if (value != null) {
+      (record as Document)[field] = demarshalReference(value)
+    }
+  }
+
+  return record
+}
+
+/**
+ * Operators whose operand is never an id — a 24 hex string under `$regex` is a pattern,
+ * not a reference.
+ */
+const OPAQUE_OPERATORS = ['$regex', '$options', '$type', '$size', '$mod', '$exists', '$where']
+
+const LOGICAL_OPERATORS = ['$and', '$or', '$nor']
+
+const marshalCriteriaValue = (value: unknown): unknown => {
+  if (typeof value === 'string') {
+    return isObjectIdHex(value) ? new ObjectId(value) : value
+  }
+  if (Array.isArray(value)) {
+    return value.map(marshalCriteriaValue)
+  }
+  if (value != null && typeof value === 'object' && !(value instanceof ObjectId) && !(value instanceof Date)) {
+    return Object.fromEntries(Object.entries(value).map(([operator, operand]) =>
+      OPAQUE_OPERATORS.includes(operator)
+        ? [operator, operand]
+        : [operator, marshalCriteriaValue(operand)]
+    ))
+  }
+
+  return value
+}
+
+/**
+ * Convert list criteria the way records are converted: values addressed at `_id` or at a
+ * declared reference become `ObjectId`s, and the `id` alias records actually carry is
+ * mapped onto `_id` — documents never store `id`, so before this mapping such criteria
+ * silently matched nothing.
+ *
+ * Tolerant by design: a value that is not 24 hex passes through unconverted. Criteria are
+ * matched against the collection, and against an `ObjectId` typed field a stray string
+ * matches nothing — which is exactly what it matched before the field was converted.
+ */
+export const marshalCriteria = (
+  criteria: ListCriteria | undefined, refs: Map<string, MongoReference>
+): ListCriteria | undefined => {
+  if (criteria == null) {
+    return criteria
+  }
+
+  return Object.fromEntries(Object.entries(criteria).map(([key, value]) => {
+    if (LOGICAL_OPERATORS.includes(key) && Array.isArray(value)) {
+      return [key, value.map(sub => marshalCriteria(sub as ListCriteria, refs))]
+    }
+    if (key === 'id' || key === '_id') {
+      return ['_id', marshalCriteriaValue(value)]
+    }
+    if (refs.has(key)) {
+      return [key, marshalCriteriaValue(value)]
+    }
+
+    return [key, value]
+  })) as ListCriteria
+}
+
+/**
+ * Criteria for addressing a single record by a field — `get`/`load`/`update`/`delete`.
+ *
+ * `_id` keeps its historical strictness (an invalid id throws through the driver). The
+ * `id` alias and declared references convert tolerantly, so a caller probing a reference
+ * with a value that is not a mongo id gets "not found" rather than a throw.
+ */
+export const identityCriteria = (
+  field: string, id: string, refs: Map<string, MongoReference>
+): Document => {
+  if (field === '_id') {
+    return { _id: new ObjectId(id) }
+  }
+  if (field === 'id') {
+    return { _id: isObjectIdHex(id) ? new ObjectId(id) : id }
+  }
+  if (refs.has(field)) {
+    return { [field]: isObjectIdHex(id) ? new ObjectId(id) : id }
+  }
+
+  return { [field]: id }
+}
+
+/**
+ * Name of the system migration that converts a reference field's stored strings.
+ *
+ * The `@1` is the body's version: the shared body below fingerprints identically for
+ * every field, so an edit to it would raise `MigrationConflict` against every ledger on
+ * the next boot. Any semantic change to {@link convertReferenceField} must bump this
+ * suffix instead — the old name stays applied, the new one runs (idempotently) once.
+ */
+export const refMigrationName = (field: string): string => `$ref:${field}@1`
+
+/** Ledger registered body of the system reference migration. */
+export const makeRefMigration = (field: string) => async (tx: MongoTx): Promise<void> => {
+  await convertReferenceField(tx.collection, field)
+}
+
+const convertScalar = (path: string): Document => ({
+  $cond: [
+    {
+      $and: [
+        { $eq: [{ $type: path }, 'string'] },
+        { $regexMatch: { input: path, regex: HEX24 } }
+      ]
+    },
+    { $toObjectId: path },
+    path
+  ]
+})
+
+/**
+ * Convert one reference field's stored string ids to `ObjectId`s — the body of the
+ * system migration and of the boot time reconciliation probe alike.
+ *
+ * Idempotent and interrupt safe: it matches only documents where the field (or one of
+ * its elements) is still a string, converts only values that are actually 24 hex, and
+ * leaves everything else exactly as it was. Safe to run concurrently from several
+ * replicas — a document converts once, the loser's filter no longer matches it.
+ *
+ * Validation is bypassed deliberately: at `Pre` stage the collection still carries the
+ * validator that declares the field a *string*, and after the switch a legacy document
+ * may violate the schema in unrelated ways — either would wedge the boot on a write
+ * that only makes the data more correct. (Bypassing requires the connection's user to
+ * hold the `bypassDocumentValidation` privilege — `dbOwner`/`root` do.)
+ */
+export const convertReferenceField = async (collection: Collection, field: string): Promise<number> => {
+  const result = await collection.updateMany(
+    /** An array valued field matches `$type: 'string'` when any element is a string. */
+    { [field]: { $type: 'string' } },
+    [{
+      $set: {
+        [field]: {
+          $cond: [
+            { $eq: [{ $type: `$${field}` }, 'array'] },
+            { $map: { input: `$${field}`, as: 'ref', in: convertScalar('$$ref') } },
+            convertScalar(`$${field}`)
+          ]
+        }
+      }
+    }],
+    { bypassDocumentValidation: true }
+  )
+
+  return result.modifiedCount
+}
+
+/**
+ * The second half of the double check the reference migration promises: the ledger says
+ * whether the migration ran; this probes whether the collection actually holds no
+ * convertible strings — and repairs it when the two disagree (a restored backup, a
+ * write from a legacy process, a ledger created by hand).
+ */
+export const reconcileReferences = async (
+  collection: Collection, refs: MongoReference[], alias: string
+): Promise<void> => {
+  for (const ref of refs) {
+    const remnant = await collection.findOne(
+      { [ref.field]: { $type: 'string', $regex: HEX24 } },
+      { projection: { _id: 1 } }
+    )
+    if (remnant != null) {
+      const converted = await convertReferenceField(collection, ref.field)
+      console.warn(
+        `@owlmeans/mongo-resource: ${alias}.${ref.field} held string ids outside the migration`
+        + ` ledger — converted ${converted} document(s)`
+      )
+    }
+  }
+}

package/src/utils/schema.ts

@@ -1,6 +1,38 @@
 import type { AnySchema, JSONSchemaType } from 'ajv'
 import type { Document } from 'mongodb'
 
+import type { MongoReference } from '../types.js'
+
+/**
+ * Declared references are stored as `ObjectId`s while the AJV schema — which describes
+ * the records the app exchanges — keeps calling them strings. The collection validator
+ * describes what's stored, so the reference fields are overridden here after the plain
+ * conversion. Nullability and array shape carry over from the declared property.
+ */
+export const applyReferenceTypes = (
+  mongoSchema: Document, schema: AnySchema, refs: MongoReference[]
+): Document => {
+  if (mongoSchema.properties == null || refs.length < 1) {
+    return mongoSchema
+  }
+  const properties: Record<string, JSONSchemaType<unknown>> =
+    (schema as JSONSchemaType<unknown>).properties ?? {}
+  for (const ref of refs) {
+    const declared = properties[ref.field]
+    if (declared == null || mongoSchema.properties[ref.field] == null) {
+      continue
+    }
+    mongoSchema.properties[ref.field] = declared.type === 'array'
+      ? {
+        bsonType: declared.nullable ? ['array', 'null'] : 'array',
+        items: { bsonType: 'objectId' }
+      }
+      : { bsonType: declared.nullable ? ['objectId', 'null'] : 'objectId' }
+  }
+
+  return mongoSchema
+}
+
 export const schemaToMongoSchema = (schema: AnySchema): Document => {
   const _schema = schema as JSONSchemaType<unknown>