@toa.io/storages.mongodb 1.0.0-alpha.20 → 1.0.0-alpha.201

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/storages.mongodb",
-  "version": "1.0.0-alpha.20",
+  "version": "1.0.0-alpha.201",
   "description": "Toa MongoDB Storage Connector",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,13 +19,13 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/console": "1.0.0-alpha.20",
-    "@toa.io/conveyor": "1.0.0-alpha.20",
-    "@toa.io/core": "1.0.0-alpha.20",
-    "@toa.io/generic": "1.0.0-alpha.20",
-    "@toa.io/pointer": "1.0.0-alpha.20",
-    "mongodb": "6.3.0",
+    "@toa.io/conveyor": "1.0.0-alpha.173",
+    "@toa.io/core": "1.0.0-alpha.201",
+    "@toa.io/generic": "1.0.0-alpha.173",
+    "@toa.io/pointer": "1.0.0-alpha.200",
+    "mongodb": "7.1.0",
+    "openspan": "1.0.0-alpha.173",
     "saslprep": "1.0.3"
   },
-  "gitHead": "dcde3389a86ab38e377ebc5ab18e5d5a512b85bc"
+  "gitHead": "e4955f36b0f2519516bc4be7c71e55a0e7cfdd44"
 }

package/src/client.js

@@ -6,6 +6,7 @@
  * @typedef {import('@toa.io/core').Locator} Locator
  */
 
+const { console } = require('openspan')
 const { Connector } = require('@toa.io/core')
 const { resolve } = require('@toa.io/pointer')
 const { ID } = require('./deployment')
@@ -17,6 +18,8 @@ const { MongoClient } = require('mongodb')
 const INSTANCES = {}
 
 class Client extends Connector {
+  name
+
   /**
    * @public
    * @type {import('mongodb').Collection}
@@ -48,6 +51,7 @@ class Client extends Connector {
     super()
 
     this.locator = locator
+    this.name = locator.lowercase
   }
 
   /**
@@ -57,17 +61,30 @@ class Client extends Connector {
    */
   async open () {
     const urls = await this.resolveURLs()
-    const db = this.resolveDB()
-    const collection = this.locator.lowercase
+    const dbname = this.resolveDB()
 
-    this.key = getKey(db, urls)
+    this.key = getKey(dbname, urls)
 
-    INSTANCES[this.key] ??= this.createInstance(urls)
+    try {
+      INSTANCES[this.key] ??= this.createInstance(urls)
+    } catch (error) {
+      console.error('Failed to connect to MongoDB', { urls, error })
+    }
 
     this.instance = await INSTANCES[this.key]
     this.instance.count++
 
-    this.collection = this.instance.client.db(db).collection(collection)
+    const db = this.instance.client.db(dbname)
+
+    try {
+      this.collection = await db.createCollection(this.name)
+    } catch (e) {
+      if (e.code !== ALREADY_EXISTS) {
+        throw e
+      }
+
+      this.collection = db.collection(this.name)
+    }
   }
 
   /**
@@ -95,7 +112,7 @@ class Client extends Connector {
     const client = new MongoClient(urls.join(','), OPTIONS)
     const hosts = urls.map((str) => new URL(str).host)
 
-    console.info('Connecting to MongoDB:', hosts.join(', '))
+    console.info('Connecting to MongoDB', { address: hosts.join(', ') })
 
     await client.connect()
 
@@ -139,9 +156,9 @@ function getKey (db, urls) {
 }
 
 const OPTIONS = {
-  ignoreUndefined: true,
-  connectTimeoutMS: 0,
-  serverSelectionTimeoutMS: 0
+  ignoreUndefined: true
 }
 
+const ALREADY_EXISTS = 48
+
 exports.Client = Client

package/src/factory.js

@@ -1,15 +1,13 @@
 'use strict'
 
 const { Client } = require('./client')
-const { Collection } = require('./collection')
 const { Storage } = require('./storage')
 
 class Factory {
   storage (locator, entity) {
     const client = new Client(locator)
-    const connection = new Collection(client)
 
-    return new Storage(connection, entity)
+    return new Storage(client, entity)
   }
 }
 

package/src/record.js

@@ -1,29 +1,16 @@
 'use strict'
 
-/**
- * @param {toa.core.storages.Record} entity
- * @returns {toa.mongodb.Record}
- */
-const to = (entity) => {
-  const {
-    id,
-    ...rest
-  } = entity
+function to (entity) {
+  const { id, ...rest } = entity
 
   return /** @type {toa.mongodb.Record} */ { _id: id, ...rest }
 }
 
-/**
- * @param {toa.mongodb.Record} record
- * @returns {toa.core.storages.Record}
- */
-const from = (record) => {
-  if (record === undefined || record === null) return null
+function from (record) {
+  if (record === undefined || record === null)
+    return null
 
-  const {
-    _id,
-    ...rest
-  } = record
+  const { _id, ...rest } = record
 
   return { id: _id, ...rest }
 }

package/src/storage.js

@@ -1,58 +1,86 @@
 'use strict'
 
-const {
-  Connector,
-  exceptions
-} = require('@toa.io/core')
-
+const { Connector, exceptions } = require('@toa.io/core')
+const { console } = require('openspan')
 const { translate } = require('./translate')
-const {
-  to,
-  from
-} = require('./record')
+const { to, from } = require('./record')
+const { ReturnDocument } = require('mongodb')
 
 class Storage extends Connector {
-  #connection
+  #client
+
+  /** @type {import('mongodb').Collection} */
+  #collection
   #entity
 
-  constructor (connection, entity) {
+  constructor (client, entity) {
     super()
 
-    this.#connection = connection
+    this.#client = client
     this.#entity = entity
 
-    this.depends(connection)
+    this.depends(client)
+  }
+
+  get raw () {
+    return this.#collection
   }
 
   async open () {
+    this.#collection = this.#client.collection
+
     await this.index()
   }
 
   async get (query) {
-    const {
-      criteria,
-      options
-    } = translate(query)
+    const { criteria, options } = translate(query)
+
+    this.debug('findOne', { criteria, options })
 
-    const record = await this.#connection.get(criteria, options)
+    const record = await this.#collection.findOne(criteria, options)
 
     return from(record)
   }
 
   async find (query) {
-    const {
-      criteria,
-      options
-    } = translate(query)
+    const { criteria, options, sample } = translate(query)
 
-    const recordset = await this.#connection.find(criteria, options)
+    if (query?.options?.deleted !== true)
+      criteria._deleted = null
+
+    let cursor
+
+    if (sample === undefined) {
+      this.debug('find', { criteria, options })
+
+      cursor = this.#collection.find(criteria, options)
+    } else {
+      const pipeline = toPipeline(criteria, options, sample)
+
+      this.debug('aggregate', { pipeline })
+
+      cursor = this.#collection.aggregate(pipeline)
+    }
+
+    const recordset = await cursor.toArray()
 
     return recordset.map((item) => from(item))
   }
 
+  async stream (query = undefined) {
+    const { criteria, options } = translate(query)
+
+    this.debug('find (stream)', { criteria, options })
+
+    return this.#collection.find(criteria, options).stream({ transform: from })
+  }
+
   async add (entity) {
     const record = to(entity)
-    const result = await this.#connection.add(record)
+
+    this.debug('insertOne', { record })
+
+    const result = await this.#collection.insertOne(record)
 
     return result.acknowledged
   }
@@ -62,41 +90,69 @@ class Storage extends Connector {
       _id: entity.id,
       _version: entity._version - 1
     }
-    const result = await this.#connection.replace(criteria, to(entity))
+
+    const record = to(entity)
+
+    this.debug('findOneAndReplace', { criteria, record })
+
+    const result = await this.#collection.findOneAndReplace(criteria, record)
 
     return result !== null
   }
 
-  async store (entity) {
+  async store (entity, attempt = 0) {
     try {
-      if (entity._version === 1) {
+      if (entity._version === 1)
         return await this.add(entity)
-      } else {
+      else
         return await this.set(entity)
-      }
     } catch (error) {
-      if (error.code === ERR_DUPLICATE_KEY) {
-
-        const id = error.keyPattern === undefined
-          ? error.message.includes(' index: _id_ ') // AWS DocumentDB
-          : error.keyPattern._id === 1
-
-        if (id) {
-          return false
-        } else {
-          throw new exceptions.DuplicateException()
-        }
-      } else {
-        throw error
-      }
+      const retry = await retriable(error, attempt)
+
+      if (retry)
+        return await this.store(entity, attempt + 1)
+      else
+        return false
+    }
+  }
+
+  async massStore (entities, attempt = 0) {
+    if (entities.length === 0)
+      return true
+
+    const operations = entities.map((entity) => {
+      const record = to(entity)
+
+      if (entity._version === 1)
+        return { insertOne: { document: record } }
+      else
+        return { replaceOne: { filter: { _id: entity.id, _version: entity._version - 1 }, replacement: record } }
+    })
+
+    const client = this.#client.instance.client
+
+    try {
+      await client.withSession(async (session) => {
+        await session.withTransaction(async () => {
+          this.debug('bulkWrite', { operations: operations.length })
+
+          await this.#collection.bulkWrite(operations, { session })
+        })
+      })
+
+      return true
+    } catch (error) {
+      const retry = await retriable(error, attempt)
+
+      if (retry)
+        return await this.massStore(entities, attempt + 1)
+      else
+        return false
     }
   }
 
   async upsert (query, changeset) {
-    const {
-      criteria,
-      options
-    } = translate(query)
+    const { criteria, options } = translate(query)
 
     if (!('_deleted' in changeset) || changeset._deleted === null) {
       delete criteria._deleted
@@ -108,20 +164,50 @@ class Storage extends Connector {
       $inc: { _version: 1 }
     }
 
-    options.returnDocument = 'after'
+    options.returnDocument = ReturnDocument.AFTER
+
+    this.debug('findOneAndUpdate', { criteria, update, options })
 
-    const result = await this.#connection.update(criteria, update, options)
+    const result = await this.#collection.findOneAndUpdate(criteria, update, options)
 
     return from(result)
   }
 
+  async ensure (query, properties, state) {
+    let { criteria, options } = translate(query)
+
+    if (query === undefined)
+      criteria = properties
+
+    const update = { $setOnInsert: to(state) }
+
+    options.upsert = true
+    options.returnDocument = ReturnDocument.AFTER
+
+    console.debug('Database query', { collection: this.#collection.collectionName, method: 'findOneAndUpdate', criteria, update, options })
+
+    try {
+      const result = await this.#collection.findOneAndUpdate(criteria, update, options)
+
+      if (result._deleted !== undefined && result._deleted !== null)
+        return null
+      else
+        return from(result)
+    } catch (error) {
+      if (error.code === ERR_DUPLICATE_KEY)
+        throw new exceptions.DuplicateException(this.#client.name)
+      else
+        throw error
+    }
+  }
+
   async index () {
     const indexes = []
 
     if (this.#entity.unique !== undefined) {
       for (const [name, fields] of Object.entries(this.#entity.unique)) {
-        const sparse = this.checkFields(fields)
-        const unique = await this.uniqueIndex(name, fields, sparse)
+        const optional = this.getOptional(fields)
+        const unique = await this.uniqueIndex(name, fields, optional)
 
         indexes.push(unique)
       }
@@ -131,23 +217,24 @@ class Storage extends Connector {
       for (const [suffix, declaration] of Object.entries(this.#entity.index)) {
         const name = 'index_' + suffix
         const fields = Object.fromEntries(Object.entries(declaration)
-          .map(([name, type]) => [name, INDEX_TYPES[type]]))
+          .map(([name, type]) => [name, INDEX_TYPES[type] ?? type]))
 
-        const sparse = this.checkFields(Object.keys(fields))
+        const optional = this.getOptional(Object.keys(fields))
+        const options = { name, sparse: optional.length > 0 }
 
-        await this.#connection.index(fields, {
-          name,
-          sparse
-        })
+        console.info('Creating index', { fields, options })
+
+        await this.#collection.createIndex(fields, options)
+          .catch((e) => console.warn('MongoDB index creation failed', { collection: this.#collection.collectionName, name, fields, error: e }))
 
         indexes.push(name)
       }
     }
 
-    await this.removeObsolete(indexes)
+    await this.removeObsoleteIndexes(indexes)
   }
 
-  async uniqueIndex (name, properties, sparse = false) {
+  async uniqueIndex (name, properties, optional) {
     const fields = properties.reduce((acc, property) => {
       acc[property] = 1
       return acc
@@ -155,48 +242,80 @@ class Storage extends Connector {
 
     name = 'unique_' + name
 
-    await this.#connection.index(fields, {
-      name,
-      unique: true,
-      sparse
-    })
+    const options = { name, unique: true }
+
+    if (optional.length > 0)
+      options.partialFilterExpression = Object.fromEntries(optional.map((field) => [field, { $exists: true }]))
+
+    console.info('Creating unique index', { name, fields, options })
+
+    await this.#collection.createIndex(fields, options)
+      .catch((e) => console.warn('MongoDB unique index creation failed', 
+        { collection: this.#collection.collectionName, name, fields, error: e }))
 
     return name
   }
 
-  async removeObsolete (desired) {
-    const current = await this.#connection.indexes()
+  async removeObsoleteIndexes (desired) {
+    const current = await this.getCurrentIndexes()
     const obsolete = current.filter((name) => !desired.includes(name))
 
     if (obsolete.length > 0) {
-      console.info(`Remove obsolete indexes: [${obsolete.join(', ')}]`)
+      console.info('Removing obsolete indexes', { collection: this.#collection.collectionName, indexes: obsolete.join(', ') })
+
+      await Promise.all(obsolete.map((name) => this.#collection.dropIndex(name)))
+    }
+  }
+
+  async getCurrentIndexes () {
+    try {
+      const array = await this.#collection.listIndexes().toArray()
 
-      await this.#connection.dropIndexes(obsolete)
+      return array.map(({ name }) => name).filter((name) => name !== '_id_')
+    } catch {
+      return []
     }
   }
 
-  checkFields (fields) {
+  getOptional (fields) {
     const optional = []
 
-    for (const field of fields) {
-      if (!(field in this.#entity.schema.properties)) {
+    for (const field of fields) {      
+      if (!field.includes('.') && !(field in this.#entity.schema.properties))
         throw new Error(`Index field '${field}' is not defined.`)
-      }
 
-      if (!this.#entity.schema.required?.includes(field)) {
+      if (!this.#entity.schema.required?.includes(field))
         optional.push(field)
-      }
     }
 
-    if (optional.length > 0) {
-      console.info(`Index fields [${optional.join(', ')}] are optional, creating sparse index.`)
+    return optional
+  }
 
-      return true
-    } else {
-      return false
-    }
+  debug (method, attributes) {
+    console.debug('MongoDB query', {
+      collection: this.#collection.collectionName,
+      method,
+      ...attributes
+    })
   }
+}
+
+function toPipeline (criteria, options, sample) {
+  const pipeline = []
+
+  if (criteria !== undefined)
+    pipeline.push({ $match: criteria })
 
+  if (sample !== undefined)
+    pipeline.push({ $sample: { size: sample } })
+
+  if (options?.sort !== undefined)
+    pipeline.push({ $sort: options.sort })
+
+  if (options?.projection !== undefined)
+    pipeline.push({ $project: options.projection })
+
+  return pipeline
 }
 
 const INDEX_TYPES = {
@@ -207,4 +326,29 @@ const INDEX_TYPES = {
 
 const ERR_DUPLICATE_KEY = 11000
 
+async function retriable (error, attempt) {
+  if (error.code === ERR_DUPLICATE_KEY) {
+    const id = error.keyPattern === undefined
+      ? error.message.includes(' index: _id_ ') // AWS DocumentDB
+      : error.keyPattern._id === 1
+
+    if (id)
+      return false
+    else
+      throw new exceptions.DuplicateException()
+  } else if (error.cause?.code === 'ECONNREFUSED') {
+    if (attempt === LAST_ATTEMPT)
+      throw error
+
+    const timeout = 1000 + 500 * attempt
+
+    await new Promise((resolve) => setTimeout(resolve, timeout))
+
+    return true
+  } else
+    throw error
+}
+
+const LAST_ATTEMPT = 9
+
 exports.Storage = Storage

package/src/translate.js

@@ -9,18 +9,21 @@ const parse = { ...require('./translate/criteria'), ...require('./translate/opti
 const translate = (query) => {
   const result = {
     criteria: query?.criteria === undefined ? {} : parse.criteria(query.criteria),
-    options: query?.options === undefined ? {} : parse.options(query.options)
+    options: query?.options === undefined ? {} : parse.options(query.options),
+    sample: query?.options?.sample
   }
 
-  if (query?.id !== undefined) {
+  if (query?.id !== undefined)
     result.criteria._id = query.id
-  }
 
-  if (query?.version !== undefined) {
+  if (query?.ids !== undefined)
+    result.criteria._id = { $in: query.ids }
+
+  if (query?.version !== undefined)
     result.criteria._version = query.version
-  }
 
-  result.criteria._deleted = null
+  if (query?.search !== undefined)
+    result.criteria.$text = { $search: query.search }
 
   return result
 }

package/test/record.test.js

@@ -47,19 +47,4 @@ describe('from', () => {
       _version: 0
     })
   })
-
-  it('should not modify argument', () => {
-    /** @type {toa.mongodb.Record} */
-    const record = {
-      _id: '1',
-      _version: 0
-    }
-
-    from(record)
-
-    expect(record).toStrictEqual({
-      _id: '1',
-      _version: 0
-    })
-  })
 })

package/src/collection.js

@@ -1,69 +0,0 @@
-'use strict'
-
-const { Connector } = require('@toa.io/core')
-
-class Collection extends Connector {
-  #client
-  #collection
-
-  constructor (client) {
-    super()
-
-    this.#client = client
-
-    this.depends(client)
-  }
-
-  async open () {
-    this.#collection = this.#client.collection
-  }
-
-  /** @hot */
-  async get (query, options) {
-    return /** @type {toa.mongodb.Record} */ this.#collection.findOne(query, options)
-  }
-
-  /** @hot */
-  async find (query, options) {
-    const cursor = this.#collection.find(query, options)
-
-    return cursor.toArray()
-  }
-
-  /** @hot */
-  async add (record) {
-    return await this.#collection.insertOne(record)
-  }
-
-  /** @hot */
-  async replace (query, record, options) {
-    return await this.#collection.findOneAndReplace(query, record, options)
-  }
-
-  /** @hot */
-  async update (query, update, options) {
-    return this.#collection.findOneAndUpdate(query, update, options)
-  }
-
-  async index (keys, options) {
-    return this.#collection.createIndex(keys, options)
-  }
-
-  async indexes () {
-    try {
-      const array = await this.#collection.listIndexes().toArray()
-
-      return array.map(({ name }) => name).filter((name) => name !== '_id_')
-    } catch {
-      return []
-    }
-  }
-
-  async dropIndexes (names) {
-    const all = names.map((name) => this.#collection.dropIndex(name))
-
-    return Promise.all(all)
-  }
-}
-
-exports.Collection = Collection