@owlmeans/mongo-resource 0.1.18-rc.8 → 0.1.18

package/src/utils/life-cycle.ts

@@ -1,166 +1,152 @@
 import { MigrationStage, runMigrations } from '@owlmeans/resource'
 import type { DbConfig, MigrationReport, ResourceRecord } from '@owlmeans/resource'
 import type { BasicContext } from '@owlmeans/context'
+import { logger } from '@owlmeans/log'
 import type { MongoReference, MongoResource } from '../types.js'
 import type { Db, Collection, Document, IndexSpecification } from 'mongodb'
 import { DEF_MIGRATIONS_COLLECTION } from '../consts.js'
-import { getDeclaration } from '../declarations.js'
+import { mongoDeclarationHelper } from '../declarations.js'
+import type { MongoLifeCycleUtils } from './life-cycle/types.js'
 import { mongoCollectionName } from './name.js'
-import { applyReferenceTypes, schemaToMongoSchema } from './schema.js'
-import { updateIndexes } from './indexes.js'
+import { mongoSchemaUtils } from './schema.js'
+import { mongoIndexUtils } from './indexes.js'
 import { makeMongoMigrationStore, makeMongoTx } from './migrations.js'
-import { reconcileReferences } from './refs.js'
-
-/**
- * Bring a resource's collection to the shape its schema declares.
- *
- * Order is deliberate, and matches the Postgres counterpart:
- *
- *  1. probe for the collection
- *  2. absent → *baseline* every registered migration; present → run the `pre` ones
- *     (including the system `$ref:` migrations that convert declared references)
- *  3. create the collection, or update its validator and indexes
- *  4. present → run the `post` migrations
- *  5. reconcile declared references — the second half of their double check: the ledger
- *     said whether the `$ref:` migration ran, this probes the collection itself and
- *     converts whatever strings still slipped through
- *
- * Step 2 is what stops the two mechanisms colliding. A `pre` migration runs before the
- * validator is tightened, so it can reshape documents the new validator would reject. On a
- * collection this call just created there is nothing to reshape — replaying a historical
- * migration against a collection born in its final shape would at best scan for nothing —
- * so the migrations are recorded as satisfied instead of run.
- *
- * `context` is optional so the existing three-argument call keeps working; without it a
- * migration's `use`/`ref` can only address the owning resource.
- */
-export const initializeCollection = async (
-  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>,
-  context?: BasicContext<any>
-): Promise<Collection> => {
-  const name = mongoCollectionName(config, resource)
-  const fresh = !await db.listCollections({ name }).hasNext()
-
-  appendReferenceIndexes(resource)
-
-  const migrate = prepareMigrations(db, config, resource, name, context)
-  await migrate(fresh ? { baseline: true } : { stage: MigrationStage.Pre })
-
-  const collection = fresh
-    ? await createCollection(db, name, resource)
-    : await updateCollection(db, name, resource)
-
-  if (!fresh) {
-    await migrate({ stage: MigrationStage.Post })
-    await reconcileReferences(collection, referencesOf(resource), resource.alias)
-  }
+import { mongoRefHelper } from './refs.js'
 
-  return collection
-}
+const log = logger('mongo-resource')
 
-/** Tolerates hand-built resource objects that predate the reference capability. */
-const referencesOf = (resource: MongoResource<ResourceRecord>): MongoReference[] =>
-  resource.references?.() ?? []
-
-/**
- * A declared reference is indexed at the mongo level — that's part of its contract. The
- * index is appended unless the resource already declares one with the same key pattern
- * (mongo refuses two indexes over identical keys, and the existing one — possibly
- * unique — wins).
- */
-const appendReferenceIndexes = (resource: MongoResource<ResourceRecord>): void => {
-  for (const ref of referencesOf(resource)) {
-    if (ref.noIndex === true) {
-      continue
-    }
-    resource.indexes = resource.indexes ?? []
-    const spec = JSON.stringify({ [ref.field]: 1 })
-    const present = resource.indexes.some(index =>
-      JSON.stringify(index.index as IndexSpecification) === spec || index.name === `ref_${ref.field}`
-    )
-    if (!present) {
-      resource.indexes.push({ name: `ref_${ref.field}`, index: { [ref.field]: 1 } })
+export const makeMongoLifeCycleUtils = (db: Db): MongoLifeCycleUtils => {
+  const { applyReferenceTypes, schemaToMongoSchema } = mongoSchemaUtils
+
+  const initializeCollection = async (
+    config: DbConfig, resource: MongoResource<ResourceRecord>,
+    context?: BasicContext<any>
+  ): Promise<Collection> => {
+    const name = mongoCollectionName(config, resource)
+    const fresh = !await db.listCollections({ name }).hasNext()
+
+    appendReferenceIndexes(resource)
+
+    const migrate = prepareMigrations(config, resource, name, context)
+    await migrate(fresh ? { baseline: true } : { stage: MigrationStage.Pre })
+
+    const collection = fresh
+      ? await createCollection(name, resource)
+      : await updateCollection(name, resource)
+
+    if (!fresh) {
+      await migrate({ stage: MigrationStage.Post })
+      await mongoRefHelper.reconcileReferences(collection, referencesOf(resource), resource.alias)
     }
-  }
-}
 
-/**
- * Bind the migration runner to this resource's database, or hand back a no-op when nothing
- * is registered — the overwhelmingly common case, and one that shouldn't pay for a ledger.
- */
-const prepareMigrations = (
-  db: Db, config: DbConfig, resource: MongoResource<ResourceRecord>, name: string,
-  context?: BasicContext<any>
-): (opts: { stage?: MigrationStage, baseline?: boolean }) => Promise<MigrationReport | null> => {
-  const registry = getDeclaration(resource.alias).migrations
-  if (registry.list().length < 1) {
-    return async () => null
+    return collection
   }
 
-  const ledgerName = (config.meta as { migrationsCollection?: string } | undefined)?.migrationsCollection
-    ?? DEF_MIGRATIONS_COLLECTION
+  /** Tolerates hand-built resource objects that predate the reference capability. */
+  const referencesOf = (resource: MongoResource<ResourceRecord>): MongoReference[] =>
+    resource.references?.() ?? []
+
   /**
-   * `db.collection()` never round-trips, so taking the handle before the collection exists
-   * is safe — and on the fresh path the transaction is only ever baselined, never used.
+   * A declared reference is indexed at the mongo level — that's part of its contract. The
+   * index is appended unless the resource already declares one with the same key pattern
+   * (mongo refuses two indexes over identical keys, and the existing one — possibly
+   * unique — wins).
    */
-  const tx = makeMongoTx(db, db.collection(name), config, context as BasicContext<any>, resource.alias)
-  const store = makeMongoMigrationStore(db, tx, ledgerName)
-
-  return async opts => {
-    const report = await runMigrations(resource.alias, registry, store, opts)
-    if (report.applied.length > 0) {
-      console.log(
-        `@owlmeans/mongo-resource: ${name} applied ${report.stage} migrations —`
-        + ` ${report.applied.join(', ')}`
+  const appendReferenceIndexes = (resource: MongoResource<ResourceRecord>): void => {
+    for (const ref of referencesOf(resource)) {
+      if (ref.noIndex === true) {
+        continue
+      }
+      resource.indexes = resource.indexes ?? []
+      const spec = JSON.stringify({ [ref.field]: 1 })
+      const present = resource.indexes.some(index =>
+        JSON.stringify(index.index as IndexSpecification) === spec || index.name === `ref_${ref.field}`
       )
+      if (!present) {
+        resource.indexes.push({ name: `ref_${ref.field}`, index: { [ref.field]: 1 } })
+      }
     }
-
-    return report
   }
-}
 
-export const createCollection = async (db: Db, name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
-  const collection = await db.createCollection(name, {
-    ...(resource.schema != null ? {
-      validator: {
-        $jsonSchema: patchJsonSchema(applyReferenceTypes(
-          schemaToMongoSchema(resource.schema), resource.schema, resource.references()
-        ))
+  /**
+   * Bind the migration runner to this resource's database, or hand back a no-op when nothing
+   * is registered — the overwhelmingly common case, and one that shouldn't pay for a ledger.
+   */
+  const prepareMigrations = (
+    config: DbConfig, resource: MongoResource<ResourceRecord>, name: string,
+    context?: BasicContext<any>
+  ): (opts: { stage?: MigrationStage, baseline?: boolean }) => Promise<MigrationReport | null> => {
+    const registry = mongoDeclarationHelper.getDeclaration(resource.alias).migrations
+    if (registry.list().length < 1) {
+      return async () => null
+    }
+
+    const ledgerName = (config.meta as { migrationsCollection?: string } | undefined)?.migrationsCollection
+      ?? DEF_MIGRATIONS_COLLECTION
+    /**
+     * `db.collection()` never round-trips, so taking the handle before the collection exists
+     * is safe — and on the fresh path the transaction is only ever baselined, never used.
+     */
+    const tx = makeMongoTx(db, db.collection(name), config, context as BasicContext<any>, resource.alias)
+    const store = makeMongoMigrationStore(db, tx, ledgerName)
+
+    return async opts => {
+      const report = await runMigrations(resource.alias, registry, store, opts)
+      if (report.applied.length > 0) {
+        log.info('Migrations applied', {
+          resource: resource.alias, collection: name, stage: report.stage, applied: report.applied,
+        }, { event: 'migration.applied' })
       }
-    } : {})
-  })
-
-  if (resource.indexes != null) {
-    await Promise.all(resource.indexes.map(
-      async index => await collection.createIndex(index.index, {
-        name: index.name, ...((index.options != null) ? index.options : {})
-      })
-    ))
+
+      return report
+    }
   }
 
-  return collection
-}
+  const createCollection = async (name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
+    const collection = await db.createCollection(name, {
+      ...(resource.schema != null ? {
+        validator: {
+          $jsonSchema: patchJsonSchema(applyReferenceTypes(
+            schemaToMongoSchema(resource.schema), resource.schema, resource.references()
+          ))
+        }
+      } : {})
+    })
+
+    if (resource.indexes != null) {
+      await Promise.all(resource.indexes.map(
+        async index => await collection.createIndex(index.index, {
+          name: index.name, ...((index.options != null) ? index.options : {})
+        })
+      ))
+    }
 
-export const updateCollection = async (db: Db, name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
-  if (resource.schema != null) {
-    const $jsonSchema = patchJsonSchema(applyReferenceTypes(
-      schemaToMongoSchema(resource.schema), resource.schema, resource.references()
-    ))
-    await db.command({ collMod: name, validator: { $jsonSchema } })
+    return collection
   }
 
-  const collection = db.collection(name)
+  const updateCollection = async (name: string, resource: MongoResource<ResourceRecord>): Promise<Collection> => {
+    if (resource.schema != null) {
+      const $jsonSchema = patchJsonSchema(applyReferenceTypes(
+        schemaToMongoSchema(resource.schema), resource.schema, resource.references()
+      ))
+      await db.command({ collMod: name, validator: { $jsonSchema } })
+    }
 
-  await updateIndexes(collection, resource)
+    const collection = db.collection(name)
 
-  return collection
-}
+    await mongoIndexUtils.updateIndexes(collection, resource)
 
-const patchJsonSchema = (schema: Document): Document => {
-  if (schema.properties != null) {
-    if (schema.properties._id == null) {
-      schema.properties._id = { bsonType: 'objectId' }
+    return collection
+  }
+
+  const patchJsonSchema = (schema: Document): Document => {
+    if (schema.properties != null) {
+      if (schema.properties._id == null) {
+        schema.properties._id = { bsonType: 'objectId' }
+      }
     }
+    return schema
   }
-  return schema
+
+  return { initializeCollection, createCollection, updateCollection }
 }

package/src/utils/migrations.ts

@@ -1,25 +1,11 @@
 import type { BasicContext } from '@owlmeans/context'
-import type { DbConfig, Migration, MigrationStage, MigrationStore, ResourceRecord } from '@owlmeans/resource'
+import type { DbConfig, Migration, MigrationStore, ResourceRecord } from '@owlmeans/resource'
 import type { Collection, Db } from 'mongodb'
 
-import {
-  DEF_MIGRATION_POLL, DEF_MIGRATION_WAIT, DEF_MIGRATIONS_COLLECTION, MONGO_DUPLICATE_KEY
-} from '../consts.js'
+import { DEF_MIGRATION_POLL, DEF_MIGRATION_WAIT, DEF_MIGRATIONS_COLLECTION, MONGO_DUPLICATE_KEY, DEFAULT_DB_ALIAS } from '../consts.js'
 import type { MongoDbService, MongoResource, MongoTx } from '../types.js'
-import { DEFAULT_DB_ALIAS } from '../consts.js'
 import { mongoCollectionName } from './name.js'
-
-interface LedgerRecord {
-  alias: string
-  name: string
-  stage: MigrationStage
-  checksum: string | null
-  baseline: boolean
-  startedAt: Date
-  /** `null` while a replica is running it — the claim, not the completion. */
-  completedAt: Date | null
-  durationMs: number | null
-}
+import type { LedgerRecord } from './types.local.js'
 
 const isDuplicateKey = (error: unknown): boolean =>
   (error as { code?: number } | null)?.code === MONGO_DUPLICATE_KEY

package/src/utils/name.ts

@@ -1,11 +1,6 @@
 import type { DbConfig, ResourceRecord } from '@owlmeans/resource'
 import type { MongoResource } from '../types.js'
-
-// Collection names must use only alphanumeric characters, underscore, and hyphen — no
-// colons or other delimiters. This guard enforces the rule for every collection created
-// through makeMongoResource across all projects (common/internal/app), failing fast at
-// initialization rather than silently creating a badly-named collection.
-const VALID_COLLECTION_NAME = /^[a-zA-Z0-9_-]+$/
+import { VALID_COLLECTION_NAME } from './consts.local.js'
 
 export const mongoCollectionName = (config: DbConfig, resource: MongoResource<ResourceRecord>): string => {
   const name = `${config.resourcePrefix ?? ''}${resource.name ?? resource.alias}`

package/src/utils/refs/types.ts

@@ -0,0 +1,79 @@
+import type { Collection, Document } from 'mongodb'
+
+import type { MongoReference } from '../../types.js'
+
+/** Declared ObjectId references: converting their values, their criteria and their stored data. */
+export interface MongoRefHelper {
+  /** A 24 hex string — the only shape a stored reference is converted from. */
+  isObjectIdHex: (value: unknown) => value is string
+  /**
+   * Write side of a declared reference: the string id a record carries becomes the
+   * `ObjectId` the collection stores. Arrays convert elementwise.
+   *
+   * Strict on purpose — a declared reference holding something that is not a mongo id is
+   * either a mis-declared field (should never have been a reference) or a bug at the call
+   * site, and storing it as a string would silently reintroduce the mixed type state this
+   * mechanism exists to remove.
+   *
+   * @throws {MisshapedRecord}
+   */
+  marshalReference: (field: string, value: unknown) => unknown
+  /** Read side: `ObjectId` back to the string records carry. Tolerates not yet migrated strings. */
+  demarshalReference: (value: unknown) => unknown
+  /** Convert every declared reference of a fetched document back to string ids, in place. */
+  demarshalRefs: <T extends {}>(record: T, refs: Map<string, MongoReference>) => T
+  /**
+   * Convert a mongo filter the way records are converted: values addressed at `_id` or at a
+   * declared reference become `ObjectId`s, and the `id` alias records actually carry is
+   * mapped onto `_id` — documents never store `id`, so before this mapping such criteria
+   * silently matched nothing.
+   *
+   * Tolerant by design: a value that is not 24 hex passes through unconverted. Criteria are
+   * matched against the collection, and against an `ObjectId` typed field a stray string
+   * matches nothing — which is exactly what it matched before the field was converted.
+   *
+   * Operators arrive already in mongo's own vocabulary — `criteriaToFilter` translates
+   * the portable one first, so this pass only has to convert values.
+   */
+  marshalCriteria: (criteria: Document | undefined, refs: Map<string, MongoReference>) => Document | undefined
+  /**
+   * Criteria for addressing a single record by a field — `get`/`load`/`update`/`delete`.
+   *
+   * `_id` keeps its historical strictness (an invalid id throws through the driver). The
+   * `id` alias and declared references convert tolerantly, so a caller probing a reference
+   * with a value that is not a mongo id gets "not found" rather than a throw.
+   */
+  identityCriteria: (field: string, id: string, refs: Map<string, MongoReference>) => Document
+  /**
+   * Name of the system migration that converts a reference field's stored strings.
+   *
+   * The `@1` is the body's version: the shared body (`makeRefMigration`) fingerprints identically
+   * for every field, so an edit to it would raise `MigrationConflict` against every ledger on
+   * the next boot. Any semantic change to {@link MongoRefHelper.convertReferenceField} must bump
+   * this suffix instead — the old name stays applied, the new one runs (idempotently) once.
+   */
+  refMigrationName: (field: string) => string
+  /**
+   * Convert one reference field's stored string ids to `ObjectId`s — the body of the
+   * system migration and of the boot time reconciliation probe alike.
+   *
+   * Idempotent and interrupt safe: it matches only documents where the field (or one of
+   * its elements) is still a string, converts only values that are actually 24 hex, and
+   * leaves everything else exactly as it was. Safe to run concurrently from several
+   * replicas — a document converts once, the loser's filter no longer matches it.
+   *
+   * Validation is bypassed deliberately: at `Pre` stage the collection still carries the
+   * validator that declares the field a *string*, and after the switch a legacy document
+   * may violate the schema in unrelated ways — either would wedge the boot on a write
+   * that only makes the data more correct. (Bypassing requires the connection's user to
+   * hold the `bypassDocumentValidation` privilege — `dbOwner`/`root` do.)
+   */
+  convertReferenceField: (collection: Collection, field: string) => Promise<number>
+  /**
+   * The second half of the double check the reference migration promises: the ledger says
+   * whether the migration ran; this probes whether the collection actually holds no
+   * convertible strings — and repairs it when the two disagree (a restored backup, a
+   * write from a legacy process, a ledger created by hand).
+   */
+  reconcileReferences: (collection: Collection, refs: MongoReference[], alias: string) => Promise<void>
+}