@owlmeans/mongo-resource 0.1.18-rc.8 → 0.1.18

package/src/utils/consts.local.ts

@@ -0,0 +1,35 @@
+/**
+ * Operators mongo speaks natively and that mean the same thing here as they do in SQL and in
+ * the in-memory engine. Everything else in the shared vocabulary is rewritten below into a
+ * mongo expression with the same meaning — a criteria object has to answer identically
+ * whichever store it reaches.
+ */
+export const NATIVE = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin', '$regex'])
+
+/** What the SERVER reports about an index rather than what a declaration asks for — never a
+ *  reason to recreate one. `collation` is here because a collection's default collation is
+ *  reported on every index in it, including the ones declared without any. */
+export const REPORTED_FIELDS = new Set([
+  'v', 'ns', 'name', 'collation', 'textIndexVersion', '2dsphereIndexVersion', 'background'
+])
+
+// Collection names must use only alphanumeric characters, underscore, and hyphen — no
+// colons or other delimiters. This guard enforces the rule for every collection created
+// through makeMongoResource across all projects (common/internal/app), failing fast at
+// initialization rather than silently creating a badly-named collection.
+export const VALID_COLLECTION_NAME = /^[a-zA-Z0-9_-]+$/
+
+/**
+ * The only shape a stored reference is converted from. Deliberately stricter than
+ * `ObjectId.isValid`, which also accepts any 12 character string and would silently
+ * swallow short business keys.
+ */
+export const HEX24 = /^[0-9a-fA-F]{24}$/
+
+/**
+ * Operators whose operand is never an id — a 24 hex string under `$regex` is a pattern,
+ * not a reference.
+ */
+export const OPAQUE_OPERATORS = ['$regex', '$options', '$type', '$size', '$mod', '$exists', '$where']
+
+export const LOGICAL_OPERATORS = ['$and', '$or', '$nor']

package/src/utils/criteria/types.ts

@@ -0,0 +1,28 @@
+import type { Criteria, Sort } from '@owlmeans/resource'
+import type { Document } from 'mongodb'
+
+import type { MongoReference } from '../../types.js'
+
+/** The portable criteria and sort vocabulary, as the documents a collection takes. */
+export interface MongoCriteriaHelper {
+  /**
+   * Translate `Criteria<T>` into the filter a collection takes.
+   *
+   * Two passes: the shared operator vocabulary becomes mongo expressions, then
+   * `mongoRefHelper.marshalCriteria` converts the values addressed at `_id` or at a declared
+   * reference into `ObjectId`s and maps the `id` alias onto `_id`. Both halves are needed — a
+   * criteria object carries string ids and portable operators, a collection stores neither.
+   *
+   * An empty result is an empty filter, which matches everything. Callers that must not act on
+   * "everything" — `purge` — check for it themselves.
+   *
+   * @throws {UnsupportedArgumentError}
+   */
+  criteriaToFilter: (criteria: Criteria<any> | undefined, refs: Map<string, MongoReference>) => Document
+  /**
+   * Translate `Sort<T>[]` into a mongo sort document. A bare field name is ascending, and `id`
+   * addresses `_id` — documents never store an `id` field, so sorting by the name records carry
+   * would silently order by nothing.
+   */
+  sortToMongo: (sort?: Sort<any>[]) => Document | undefined
+}

package/src/utils/criteria.ts

@@ -0,0 +1,235 @@
+import { UnsupportedArgumentError, type Criteria, type Sort } from '@owlmeans/resource'
+import type { Document } from 'mongodb'
+
+import type { MongoReference } from '../types.js'
+import { mongoRefHelper } from './refs.js'
+import { NATIVE } from './consts.local.js'
+import type { MongoCriteriaHelper } from './criteria/types.js'
+
+const escapeRegExp = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+/**
+ * A SQL `LIKE` pattern as an anchored regular expression: `%` is any run, `_` is one
+ * character, and `\` escapes either. The mapping is the one the in-memory engine applies, so
+ * the same pattern selects the same records against a collection.
+ */
+const likeToRegExp = (pattern: string): string => {
+  let source = ''
+  for (let index = 0; index < pattern.length; index++) {
+    const char = pattern[index]!
+    if (char === '\\' && index + 1 < pattern.length) {
+      source += escapeRegExp(pattern[++index]!)
+      continue
+    }
+    if (char === '%') { source += '.*'; continue }
+    if (char === '_') { source += '.'; continue }
+    source += escapeRegExp(char)
+  }
+
+  return `^${source}$`
+}
+
+const asArray = (operand: unknown): unknown[] => Array.isArray(operand) ? operand : [operand]
+
+/**
+ * One operator as the mongo field expression that answers it.
+ *
+ * `$exists` and `$null` both become a comparison against `null`, not mongo's own `$exists`:
+ * the shared vocabulary asks whether a field *has a value*, which the relational and
+ * in-memory stores answer as `IS NULL` / `value == null`. Mongo's `$exists` answers a
+ * different question — whether the key is present — and a key present but null would part
+ * the three stores over one criteria object.
+ *
+ * @throws {UnsupportedArgumentError} on an operator this store cannot answer.
+ */
+const operatorToFilter = (field: string, operator: string, operand: unknown): Document => {
+  if (NATIVE.has(operator)) {
+    return { [operator]: operand }
+  }
+  switch (operator) {
+    case '$exists':
+      return operand === false ? { $eq: null } : { $ne: null }
+    case '$null':
+      return operand === false ? { $ne: null } : { $eq: null }
+    case '$like':
+      return { $regex: likeToRegExp(`${operand}`) }
+    case '$ilike':
+      return { $regex: likeToRegExp(`${operand}`), $options: 'i' }
+    case '$startsWith':
+      return { $regex: `^${escapeRegExp(`${operand}`)}` }
+    case '$endsWith':
+      return { $regex: `${escapeRegExp(`${operand}`)}$` }
+    case '$between':
+      if (!Array.isArray(operand) || operand.length !== 2) {
+        throw new UnsupportedArgumentError(`criteria:$between:${field}`)
+      }
+
+      return { $gte: operand[0], $lte: operand[1] }
+    /** Array membership, mirroring the postgres array operators `@>`, `<@` and `&&`. */
+    case '$contains':
+      return { $all: asArray(operand) }
+    case '$contained':
+      /**
+       * Nothing outside the operand list may appear in the field. The `$ne: null` keeps an
+       * absent field out of the result — `$not` alone is satisfied by a document that has no
+       * such field at all, where the relational store answers NULL and matches nothing.
+       */
+      return { $not: { $elemMatch: { $nin: asArray(operand) } }, $ne: null }
+    case '$overlaps':
+      return { $in: asArray(operand) }
+    default:
+      throw new UnsupportedArgumentError(`criteria-operator:${operator}`)
+  }
+}
+
+/**
+ * An object naming at least one `$` key is a spec, not a value to compare against. A Date and
+ * an array are values even though both are objects.
+ */
+const isOperatorSpec = (value: unknown): value is Record<string, unknown> =>
+  value != null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date)
+  && Object.keys(value).some(key => key.startsWith('$'))
+
+export const createMongoCriteriaHelper = (): MongoCriteriaHelper => {
+  const { marshalCriteria } = mongoRefHelper
+
+  /**
+   * One field's criteria as mongo conditions. Operators that translate into the same mongo key —
+   * `$like` and `$startsWith` both produce `$regex` — are emitted as separate conditions rather
+   * than merged, so neither silently overwrites the other.
+   */
+  const fieldConditions = (field: string, spec: Record<string, unknown>): Document[] => {
+    const merged: Document = {}
+    const conditions: Document[] = []
+
+    for (const [operator, operand] of Object.entries(spec)) {
+      if (operand === undefined) {
+        continue
+      }
+      const fragment = operatorToFilter(field, operator, operand)
+      if (Object.keys(fragment).some(key => key in merged)) {
+        conditions.push({ [field]: fragment })
+        continue
+      }
+      Object.assign(merged, fragment)
+    }
+
+    if (Object.keys(merged).length > 0) {
+      conditions.unshift({ [field]: merged })
+    }
+
+    return conditions
+  }
+
+  /**
+   * Flatten the conditions into one filter document. Mongo reads sibling keys as a conjunction,
+   * so only the ones that would overwrite an earlier key — two `$regex` expressions over the
+   * same field, say — need an explicit `$and`.
+   */
+  const flatten = (conditions: Document[]): Document => {
+    const filter: Document = {}
+    const conflicting: Document[] = []
+
+    for (const condition of conditions) {
+      if (Object.keys(condition).some(key => key in filter)) {
+        conflicting.push(condition)
+        continue
+      }
+      Object.assign(filter, condition)
+    }
+
+    if (conflicting.length > 0) {
+      filter.$and = [...(Array.isArray(filter.$and) ? filter.$and : []), ...conflicting]
+    }
+
+    return filter
+  }
+
+  const build = (criteria: Criteria<any> | undefined): Document[] => {
+    const conditions: Document[] = []
+
+    for (const [key, raw] of Object.entries(criteria ?? {})) {
+      /**
+       * An untouched filter must never empty a list, so `undefined` is skipped rather than
+       * compared. `null` asks for the absence of a value.
+       */
+      if (raw === undefined) {
+        continue
+      }
+
+      if (key === '$and' || key === '$or') {
+        const parts = (Array.isArray(raw) ? raw : [raw])
+          .map(part => build(part as Criteria<any>))
+          .filter(part => part.length > 0)
+          .map(part => flatten(part))
+        if (parts.length > 0) {
+          conditions.push(key === '$and' ? { $and: parts } : { $or: parts })
+        }
+        continue
+      }
+      if (key === '$not') {
+        const inner = build(raw as Criteria<any>)
+        if (inner.length > 0) {
+          /** Mongo has no top level `$not`; `$nor` over a single branch is its negation. */
+          conditions.push({ $nor: [flatten(inner)] })
+        }
+        continue
+      }
+
+      if (raw === null) {
+        conditions.push({ [key]: { $eq: null } })
+        continue
+      }
+      if (isOperatorSpec(raw)) {
+        conditions.push(...fieldConditions(key, raw as Record<string, unknown>))
+        continue
+      }
+      if (Array.isArray(raw)) {
+        /**
+         * A bare array means "any of these", exactly as it does against a relational store.
+         * Exact array equality stays reachable as `{ $eq: [...] }`.
+         */
+        conditions.push({ [key]: { $in: raw } })
+        continue
+      }
+
+      conditions.push({ [key]: raw })
+    }
+
+    return conditions
+  }
+
+  const criteriaToFilter = (
+    criteria: Criteria<any> | undefined, refs: Map<string, MongoReference>
+  ): Document => {
+    const conditions = build(criteria)
+    if (conditions.length < 1) {
+      return {}
+    }
+
+    return marshalCriteria(flatten(conditions), refs) ?? {}
+  }
+
+  const sortToMongo = (sort?: Sort<any>[]): Document | undefined => {
+    if (sort == null || sort.length < 1) {
+      return undefined
+    }
+
+    return sort.reduce<Document>((order, entry) => {
+      const [field, direction]: [string, number] = typeof entry === 'string'
+        ? [entry, 1]
+        : [entry.field, entry.order === 'desc' ? -1 : 1]
+      order[field === 'id' ? '_id' : field] = direction
+
+      return order
+    }, {})
+  }
+
+  return { criteriaToFilter, sortToMongo }
+}
+
+export const mongoCriteriaHelper = createMongoCriteriaHelper()
+
+/** @deprecated compat:factory-refactor — use `mongoCriteriaHelper.criteriaToFilter(…)` */
+export const criteriaToFilter = (criteria: Criteria<any> | undefined, refs: Map<string, MongoReference>): Document =>
+  mongoCriteriaHelper.criteriaToFilter(criteria, refs)

package/src/utils/index.ts

@@ -5,3 +5,4 @@ export * from './name.js'
 export * from './life-cycle.js'
 export * from './migrations.js'
 export * from './refs.js'
+export * from './criteria.js'

package/src/utils/indexes/types.ts

@@ -0,0 +1,20 @@
+import type { ResourceRecord } from '@owlmeans/resource'
+import type { Collection, Document } from 'mongodb'
+
+import type { MongoResource } from '../../types.js'
+
+/** A collection's indexes against the ones its resource declares. */
+export interface MongoIndexUtils {
+  /**
+   * Whether an index the database already holds differs from the one declared.
+   *
+   * Mongo returns an index's options in ITS order, not the declaration's, so comparing the two
+   * documents as JSON reported every index as changed — and every boot then dropped and recreated
+   * every index. That is invisible until two processes boot at once, when the second one's drop
+   * fails with `IndexNotFound` and takes the service down with it. Only the KEY is order-sensitive
+   * (a compound index is its key order); options are a set.
+   */
+  indexChanged: (existing: Document, proposed: Document) => boolean
+  /** Create the declared indexes the collection lacks and recreate the ones that changed. */
+  updateIndexes: (collection: Collection, resource: MongoResource<ResourceRecord>) => Promise<void>
+}

package/src/utils/indexes.ts

@@ -1,38 +1,58 @@
 import type { ResourceRecord } from '@owlmeans/resource'
 import type { MongoResource } from '../types.js'
-import type { Collection } from 'mongodb'
+import type { Collection, Document } from 'mongodb'
+import { logger } from '@owlmeans/log'
+import { REPORTED_FIELDS } from './consts.local.js'
+import type { MongoIndexUtils } from './indexes/types.js'
 
-export const updateIndexes = async (collection: Collection, resource: MongoResource<ResourceRecord>) => {
-  if (resource.indexes != null) {
+const log = logger('mongo-resource')
+
+const options = (spec: Document): [string, unknown][] => Object.entries(spec)
+  .filter(([field]) => field !== 'key' && !REPORTED_FIELDS.has(field))
+  .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)
+
+export const createMongoIndexUtils = (): MongoIndexUtils => {
+  const indexChanged = (existing: Document, proposed: Document): boolean => {
+    // We do not recreate text indexes.
+    if ('weights' in existing) return false
+    if (JSON.stringify(existing.key) !== JSON.stringify(proposed.key)) return true
+
+    return JSON.stringify(options(existing)) !== JSON.stringify(options(proposed))
+  }
+
+  const updateIndexes = async (collection: Collection, resource: MongoResource<ResourceRecord>): Promise<void> => {
+    if (resource.indexes == null) {
+      return
+    }
+    const present = await collection.indexes()
     await Promise.all(resource.indexes.map(
       async index => {
-        const _index = (await collection.indexes()).find(_index => _index.name === index.name)
-        if (_index != null) {
-          const existing = { ..._index }
-          if ('name' in existing) {
-            delete existing.name
-          }
-          if ('v' in existing) {
-            delete existing.v
-          }
-          const propsed = { key: index.index, ...index.options }
-          // We do not recreate text indexes
-          if (JSON.stringify(existing) !== JSON.stringify(propsed) && !('weights' in existing)) {
-            console.info(JSON.stringify(existing))
-            console.info(JSON.stringify(propsed))
-            console.debug(`Recreate index ${resource.alias} : ${index.name}`)
-            await collection.dropIndex(index.name)
-            await collection.createIndex(index.index, {
-              name: index.name, ...((index.options != null) ? index.options : {})
-            })
-          }
-        } else {
-          console.debug(`Create index ${resource.alias} : ${index.name}`)
-          await collection.createIndex(index.index, {
-            name: index.name, ...((index.options != null) ? index.options : {})
-          })
+        const create = async () => await collection.createIndex(index.index, {
+          name: index.name, ...((index.options != null) ? index.options : {})
+        })
+        const existing = present.find(_index => _index.name === index.name)
+        if (existing == null) {
+          log.debug('Create index', { resource: resource.alias, index: index.name })
+          await create()
+          return
         }
+        if (!indexChanged(existing, { key: index.index, ...index.options })) {
+          return
+        }
+        log.debug('Recreate index', { resource: resource.alias, index: index.name })
+        try {
+          await collection.dropIndex(index.name)
+        } catch (e) {
+          // IndexNotFound: another process booting alongside this one dropped it first — it is
+          // recreating the same declaration, so there is nothing here to fail over.
+          if ((e as { code?: number }).code !== 27) throw e
+        }
+        await create()
       }
     ))
   }
+
+  return { indexChanged, updateIndexes }
 }
+
+export const mongoIndexUtils = createMongoIndexUtils()

package/src/utils/life-cycle/types.ts

@@ -0,0 +1,39 @@
+import type { BasicContext } from '@owlmeans/context'
+import type { DbConfig, ResourceRecord } from '@owlmeans/resource'
+import type { Collection } from 'mongodb'
+
+import type { MongoResource } from '../../types.js'
+
+/** A resource collection's initialization over one database. */
+export interface MongoLifeCycleUtils {
+  /**
+   * Bring a resource's collection to the shape its schema declares.
+   *
+   * Order is deliberate, and matches the Postgres counterpart:
+   *
+   *  1. probe for the collection
+   *  2. absent → *baseline* every registered migration; present → run the `pre` ones
+   *     (including the system `$ref:` migrations that convert declared references)
+   *  3. create the collection, or update its validator and indexes
+   *  4. present → run the `post` migrations
+   *  5. reconcile declared references — the second half of their double check: the ledger
+   *     said whether the `$ref:` migration ran, this probes the collection itself and
+   *     converts whatever strings still slipped through
+   *
+   * Step 2 is what stops the two mechanisms colliding. A `pre` migration runs before the
+   * validator is tightened, so it can reshape documents the new validator would reject. On a
+   * collection this call just created there is nothing to reshape — replaying a historical
+   * migration against a collection born in its final shape would at best scan for nothing —
+   * so the migrations are recorded as satisfied instead of run.
+   *
+   * `context` is optional so the existing three-argument call keeps working; without it a
+   * migration's `use`/`ref` can only address the owning resource.
+   */
+  initializeCollection: (
+    config: DbConfig, resource: MongoResource<ResourceRecord>, context?: BasicContext<any>
+  ) => Promise<Collection>
+  /** Create the collection with its validator and its declared indexes. */
+  createCollection: (name: string, resource: MongoResource<ResourceRecord>) => Promise<Collection>
+  /** Replace the validator of an existing collection and bring its indexes up to date. */
+  updateCollection: (name: string, resource: MongoResource<ResourceRecord>) => Promise<Collection>
+}