@toa.io/storages.mongodb 1.0.0-alpha.2 → 1.0.0-alpha.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toa.io/storages.mongodb",
-  "version": "1.0.0-alpha.2",
+  "version": "1.0.0-alpha.21",
   "description": "Toa MongoDB Storage Connector",
   "author": "temich <tema.gurtovoy@gmail.com>",
   "homepage": "https://github.com/toa-io/toa#readme",
@@ -19,13 +19,13 @@
     "test": "echo \"Error: run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@toa.io/console": "1.0.0-alpha.2",
-    "@toa.io/conveyor": "1.0.0-alpha.2",
-    "@toa.io/core": "1.0.0-alpha.2",
-    "@toa.io/generic": "1.0.0-alpha.2",
-    "@toa.io/pointer": "1.0.0-alpha.2",
+    "@toa.io/console": "1.0.0-alpha.21",
+    "@toa.io/conveyor": "1.0.0-alpha.21",
+    "@toa.io/core": "1.0.0-alpha.21",
+    "@toa.io/generic": "1.0.0-alpha.21",
+    "@toa.io/pointer": "1.0.0-alpha.21",
     "mongodb": "6.3.0",
     "saslprep": "1.0.3"
   },
-  "gitHead": "7688e6e980a65c82ac2e459be4e355eebf406cd0"
+  "gitHead": "da9f4c278f6ab02a28f65c6e25c713c013cbfce9"
 }

package/src/client.js

@@ -0,0 +1,159 @@
+'use strict'
+
+/**
+ * @typedef {import('mongodb').MongoClient} MongoClient
+ * @typedef {{ count: number, client: MongoClient }} Instance
+ * @typedef {import('@toa.io/core').Locator} Locator
+ */
+
+const { Connector } = require('@toa.io/core')
+const { resolve } = require('@toa.io/pointer')
+const { ID } = require('./deployment')
+const { MongoClient } = require('mongodb')
+
+/**
+ * @type {Record<string, Promise<Instance>>}
+ */
+const INSTANCES = {}
+
+class Client extends Connector {
+  /**
+   * @public
+   * @type {import('mongodb').Collection}
+   */
+  collection
+
+  /**
+   * @private
+   * @type {Locator}
+   */
+  locator
+
+  /**
+   * @private
+   * @type {Instance}
+   */
+  instance
+
+  /**
+   * @private
+   * @type {string}
+   */
+  key
+
+  /**
+   * @param {Locator} locator
+   */
+  constructor (locator) {
+    super()
+
+    this.locator = locator
+  }
+
+  /**
+   * @protected
+   * @override
+   * @return {Promise<void>}
+   */
+  async open () {
+    const urls = await this.resolveURLs()
+    const dbname = this.resolveDB()
+    const collname = this.locator.lowercase
+
+    this.key = getKey(dbname, urls)
+
+    INSTANCES[this.key] ??= this.createInstance(urls)
+
+    this.instance = await INSTANCES[this.key]
+    this.instance.count++
+
+    const db = this.instance.client.db(dbname)
+
+    try {
+      this.collection = await db.createCollection(collname)
+    } catch (e) {
+      if (e.code !== ALREADY_EXISTS) {
+        throw e
+      }
+
+      this.collection = db.collection(collname)
+    }
+  }
+
+  /**
+   * @protected
+   * @override
+   * @return {Promise<void>}
+   */
+  async close () {
+    const instance = await INSTANCES[this.key]
+
+    instance.count--
+
+    if (instance.count === 0) {
+      await instance.client.close()
+      delete INSTANCES[this.key]
+    }
+  }
+
+  /**
+   * @private
+   * @param {string[]} urls
+   * @return {Promise<Instance>}
+   */
+  async createInstance (urls) {
+    const client = new MongoClient(urls.join(','), OPTIONS)
+    const hosts = urls.map((str) => new URL(str).host)
+
+    console.info('Connecting to MongoDB:', hosts.join(', '))
+
+    await client.connect()
+
+    return {
+      count: 0,
+      client
+    }
+  }
+
+  /**
+   * @private
+   * @return {Promise<string[]>}
+   */
+  async resolveURLs () {
+    if (process.env.TOA_DEV === '1') {
+      return ['mongodb://developer:secret@localhost']
+    } else {
+      return await resolve(ID, this.locator.id)
+    }
+  }
+
+  /**
+   * @private
+   * @return {string}
+   */
+  resolveDB () {
+    if (process.env.TOA_CONTEXT !== undefined) {
+      return process.env.TOA_CONTEXT
+    }
+
+    if (process.env.TOA_DEV === '1') {
+      return 'toa-dev'
+    }
+
+    throw new Error('Environment variable TOA_CONTEXT is not defined')
+  }
+}
+
+function getKey (db, urls) {
+  return db + ':' + urls.sort().join(' ')
+}
+
+const OPTIONS = {
+  ignoreUndefined: true,
+  connectTimeoutMS: 0,
+  serverSelectionTimeoutMS: 0
+}
+
+const ALREADY_EXISTS = 48
+
+exports.Client = Client

package/src/collection.js

@@ -0,0 +1,69 @@
+'use strict'
+
+const { Connector } = require('@toa.io/core')
+
+class Collection extends Connector {
+  #client
+  #collection
+
+  constructor (client) {
+    super()
+
+    this.#client = client
+
+    this.depends(client)
+  }
+
+  async open () {
+    this.#collection = this.#client.collection
+  }
+
+  /** @hot */
+  async get (query, options) {
+    return /** @type {toa.mongodb.Record} */ this.#collection.findOne(query, options)
+  }
+
+  /** @hot */
+  async find (query, options) {
+    const cursor = this.#collection.find(query, options)
+
+    return cursor.toArray()
+  }
+
+  /** @hot */
+  async add (record) {
+    return await this.#collection.insertOne(record)
+  }
+
+  /** @hot */
+  async replace (query, record, options) {
+    return await this.#collection.findOneAndReplace(query, record, options)
+  }
+
+  /** @hot */
+  async update (query, update, options) {
+    return this.#collection.findOneAndUpdate(query, update, options)
+  }
+
+  async index (keys, options) {
+    return this.#collection.createIndex(keys, options)
+  }
+
+  async indexes () {
+    try {
+      const array = await this.#collection.listIndexes().toArray()
+
+      return array.map(({ name }) => name).filter((name) => name !== '_id_')
+    } catch {
+      return []
+    }
+  }
+
+  async dropIndexes (names) {
+    const all = names.map((name) => this.#collection.dropIndex(name))
+
+    return Promise.all(all)
+  }
+}
+
+exports.Collection = Collection

package/src/factory.js

@@ -1,13 +1,15 @@
 'use strict'
 
-const { Connection } = require('./connection')
+const { Client } = require('./client')
+const { Collection } = require('./collection')
 const { Storage } = require('./storage')
 
 class Factory {
-  storage (locator) {
-    const connection = new Connection(locator)
+  storage (locator, entity) {
+    const client = new Client(locator)
+    const connection = new Collection(client)
 
-    return new Storage(connection)
+    return new Storage(connection, entity)
   }
 }
 

package/src/record.js

@@ -5,9 +5,12 @@
  * @returns {toa.mongodb.Record}
  */
 const to = (entity) => {
-  const { id, _version, ...rest } = entity
+  const {
+    id,
+    ...rest
+  } = entity
 
-  return /** @type {toa.mongodb.Record} */ { _id: id, _version: _version + 1, ...rest }
+  return /** @type {toa.mongodb.Record} */ { _id: id, ...rest }
 }
 
 /**
@@ -17,7 +20,10 @@ const to = (entity) => {
 const from = (record) => {
   if (record === undefined || record === null) return null
 
-  const { _id, ...rest } = record
+  const {
+    _id,
+    ...rest
+  } = record
 
   return { id: _id, ...rest }
 }

package/src/storage.js

@@ -1,30 +1,38 @@
 'use strict'
 
-const { Connector } = require('@toa.io/core')
+const {
+  Connector,
+  exceptions
+} = require('@toa.io/core')
 
 const { translate } = require('./translate')
-const { to, from } = require('./record')
+const {
+  to,
+  from
+} = require('./record')
 
-/**
- * @implements {toa.core.Storage}
- */
 class Storage extends Connector {
-  /** @type {toa.mongodb.Connection} */
   #connection
+  #entity
 
-  /**
-   * @param {toa.mongodb.Connection} connection
-   */
-  constructor (connection) {
+  constructor (connection, entity) {
     super()
 
     this.#connection = connection
+    this.#entity = entity
 
     this.depends(connection)
   }
 
+  async open () {
+    await this.index()
+  }
+
   async get (query) {
-    const { criteria, options } = translate(query)
+    const {
+      criteria,
+      options
+    } = translate(query)
 
     const record = await this.#connection.get(criteria, options)
 
@@ -32,7 +40,11 @@ class Storage extends Connector {
   }
 
   async find (query) {
-    const { criteria, options } = translate(query)
+    const {
+      criteria,
+      options
+    } = translate(query)
+
     const recordset = await this.#connection.find(criteria, options)
 
     return recordset.map((item) => from(item))
@@ -40,35 +52,60 @@ class Storage extends Connector {
 
   async add (entity) {
     const record = to(entity)
+    const result = await this.#connection.add(record)
 
-    return await this.#connection.add(record)
+    return result.acknowledged
   }
 
   async set (entity) {
-    const criteria = { _id: entity.id, _version: entity._version }
+    const criteria = {
+      _id: entity.id,
+      _version: entity._version - 1
+    }
     const result = await this.#connection.replace(criteria, to(entity))
 
     return result !== null
   }
 
   async store (entity) {
-    if (entity._version === 0) return this.add(entity)
-    else return this.set(entity)
+    try {
+      if (entity._version === 1) {
+        return await this.add(entity)
+      } else {
+        return await this.set(entity)
+      }
+    } catch (error) {
+      if (error.code === ERR_DUPLICATE_KEY) {
+
+        const id = error.keyPattern === undefined
+          ? error.message.includes(' index: _id_ ') // AWS DocumentDB
+          : error.keyPattern._id === 1
+
+        if (id) {
+          return false
+        } else {
+          throw new exceptions.DuplicateException()
+        }
+      } else {
+        throw error
+      }
+    }
   }
 
-  async upsert (query, changeset, insert) {
-    const { criteria, options } = translate(query)
-    const update = { $set: { ...changeset }, $inc: { _version: 1 } }
-
-    if (insert !== undefined) {
-      delete insert._version
+  async upsert (query, changeset) {
+    const {
+      criteria,
+      options
+    } = translate(query)
 
-      options.upsert = true
-
-      if (criteria._id !== undefined) insert._id = criteria._id
-      else return null // this shouldn't ever happen
+    if (!('_deleted' in changeset) || changeset._deleted === null) {
+      delete criteria._deleted
+      changeset._deleted = null
+    }
 
-      if (Object.keys(insert) > 0) update.$setOnInsert = insert
+    const update = {
+      $set: { ...changeset },
+      $inc: { _version: 1 }
     }
 
     options.returnDocument = 'after'
@@ -77,6 +114,97 @@ class Storage extends Connector {
 
     return from(result)
   }
+
+  async index () {
+    const indexes = []
+
+    if (this.#entity.unique !== undefined) {
+      for (const [name, fields] of Object.entries(this.#entity.unique)) {
+        const sparse = this.checkFields(fields)
+        const unique = await this.uniqueIndex(name, fields, sparse)
+
+        indexes.push(unique)
+      }
+    }
+
+    if (this.#entity.index !== undefined) {
+      for (const [suffix, declaration] of Object.entries(this.#entity.index)) {
+        const name = 'index_' + suffix
+        const fields = Object.fromEntries(Object.entries(declaration)
+          .map(([name, type]) => [name, INDEX_TYPES[type]]))
+
+        const sparse = this.checkFields(Object.keys(fields))
+
+        await this.#connection.index(fields, {
+          name,
+          sparse
+        })
+
+        indexes.push(name)
+      }
+    }
+
+    await this.removeObsolete(indexes)
+  }
+
+  async uniqueIndex (name, properties, sparse = false) {
+    const fields = properties.reduce((acc, property) => {
+      acc[property] = 1
+      return acc
+    }, {})
+
+    name = 'unique_' + name
+
+    await this.#connection.index(fields, {
+      name,
+      unique: true,
+      sparse
+    })
+
+    return name
+  }
+
+  async removeObsolete (desired) {
+    const current = await this.#connection.indexes()
+    const obsolete = current.filter((name) => !desired.includes(name))
+
+    if (obsolete.length > 0) {
+      console.info(`Remove obsolete indexes: [${obsolete.join(', ')}]`)
+
+      await this.#connection.dropIndexes(obsolete)
+    }
+  }
+
+  checkFields (fields) {
+    const optional = []
+
+    for (const field of fields) {
+      if (!(field in this.#entity.schema.properties)) {
+        throw new Error(`Index field '${field}' is not defined.`)
+      }
+
+      if (!this.#entity.schema.required?.includes(field)) {
+        optional.push(field)
+      }
+    }
+
+    if (optional.length > 0) {
+      console.info(`Index fields [${optional.join(', ')}] are optional, creating sparse index.`)
+
+      return true
+    } else {
+      return false
+    }
+  }
+
 }
 
+const INDEX_TYPES = {
+  'asc': 1,
+  'desc': -1,
+  'hash': 'hashed'
+}
+
+const ERR_DUPLICATE_KEY = 11000
+
 exports.Storage = Storage

package/src/translate.js

@@ -7,12 +7,20 @@ const parse = { ...require('./translate/criteria'), ...require('./translate/opti
  * @returns {{criteria: Object, options: Object}}
  */
 const translate = (query) => {
-  const result = { criteria: {}, options: {} }
+  const result = {
+    criteria: query?.criteria === undefined ? {} : parse.criteria(query.criteria),
+    options: query?.options === undefined ? {} : parse.options(query.options)
+  }
 
-  if (query.criteria !== undefined) result.criteria = parse.criteria(query.criteria)
-  if (query.options !== undefined) result.options = parse.options(query.options)
-  if (query.id !== undefined) result.criteria._id = query.id
-  if (query.version !== undefined) result.criteria._version = query.version
+  if (query?.id !== undefined) {
+    result.criteria._id = query.id
+  }
+
+  if (query?.version !== undefined) {
+    result.criteria._version = query.version
+  }
+
+  result.criteria._deleted = null
 
   return result
 }

package/test/record.test.js

@@ -1,12 +1,17 @@
 'use strict'
 
-const { to, from } = require('../src/record')
-const { random } = require('@toa.io/generic')
+const {
+  to,
+  from
+} = require('../src/record')
 
 describe('to', () => {
   it('should rename id to _id', () => {
     /** @type {toa.core.storages.Record} */
-    const entity = { id: '1', _version: 0 }
+    const entity = {
+      id: '1',
+      _version: 0
+    }
     const record = to(entity)
 
     expect(record).toMatchObject({ _id: '1' })
@@ -14,37 +19,47 @@ describe('to', () => {
 
   it('should not modify argument', () => {
     /** @type {toa.core.storages.Record} */
-    const entity = { id: '1', _version: 0 }
+    const entity = {
+      id: '1',
+      _version: 0
+    }
 
     to(entity)
 
-    expect(entity).toStrictEqual({ id: '1', _version: 0 })
-  })
-
-  it('should increment _version', () => {
-    /** @type {toa.core.storages.Record} */
-    const entity = { id: '1', _version: random() }
-    const record = to(entity)
-
-    expect(record).toMatchObject({ _version: entity._version + 1 })
+    expect(entity).toStrictEqual({
+      id: '1',
+      _version: 0
+    })
   })
 })
 
 describe('from', () => {
   it('should rename _id to id', () => {
     /** @type {toa.mongodb.Record} */
-    const record = { _id: '1', _version: 0 }
+    const record = {
+      _id: '1',
+      _version: 0
+    }
     const entity = from(record)
 
-    expect(entity).toStrictEqual({ id: '1', _version: 0 })
+    expect(entity).toStrictEqual({
+      id: '1',
+      _version: 0
+    })
   })
 
   it('should not modify argument', () => {
     /** @type {toa.mongodb.Record} */
-    const record = { _id: '1', _version: 0 }
+    const record = {
+      _id: '1',
+      _version: 0
+    }
 
     from(record)
 
-    expect(record).toStrictEqual({ _id: '1', _version: 0 })
+    expect(record).toStrictEqual({
+      _id: '1',
+      _version: 0
+    })
   })
 })

package/src/connection.js

@@ -1,103 +0,0 @@
-// noinspection JSCheckFunctionSignatures
-
-'use strict'
-
-const { MongoClient } = require('mongodb')
-const { Connector } = require('@toa.io/core')
-const { resolve } = require('@toa.io/pointer')
-const { Conveyor } = require('@toa.io/conveyor')
-const { ID } = require('./deployment')
-
-class Connection extends Connector {
-  #locator
-  /** @type {import('mongodb').MongoClient} */
-  #client
-  /** @type {import('mongodb').Collection<toa.mongodb.Record>} */
-  #collection
-  /** @type {toa.conveyor.Conveyor<toa.core.storages.Record, boolean>} */
-  #conveyor
-
-  constructor (locator) {
-    super()
-
-    this.#locator = locator
-  }
-
-  async open () {
-    const urls = await this.#resolveURLs()
-    const db = this.#locator.namespace
-    const collection = this.#locator.name
-
-    this.#client = new MongoClient(urls[0], OPTIONS)
-
-    await this.#client.connect()
-
-    this.#collection = this.#client.db(db).collection(collection)
-    this.#conveyor = new Conveyor((objects) => this.addMany(objects))
-
-    console.info(`Storage Mongo '${this.#locator.id}' connected`)
-  }
-
-  async close () {
-    await this.#client?.close()
-
-    console.info(`Storage Mongo '${this.#locator.id}' disconnected`)
-  }
-
-  /** @hot */
-  async get (query, options) {
-    return /** @type {toa.mongodb.Record} */ this.#collection.findOne(query, options)
-  }
-
-  /** @hot */
-  async find (query, options) {
-    const cursor = this.#collection.find(query, options)
-
-    return cursor.toArray()
-  }
-
-  /** @hot */
-  async add (record) {
-    return this.#conveyor.process(record)
-  }
-
-  async addMany (records) {
-    let result
-
-    try {
-      const response = await this.#collection.insertMany(records, { ordered: false })
-
-      result = response.acknowledged
-    } catch (e) {
-      if (e.code === ERR_DUPLICATE_KEY) result = false
-      else throw e
-    }
-
-    return result
-  }
-
-  /** @hot */
-  async replace (query, record, options) {
-    return await this.#collection.findOneAndReplace(query, record, options)
-  }
-
-  /** @hot */
-  async update (query, update, options) {
-    return this.#collection.findOneAndUpdate(query, update, options)
-  }
-
-  async #resolveURLs () {
-    if (process.env.TOA_DEV === '1') return ['mongodb://developer:secret@localhost']
-    else return await resolve(ID, this.#locator.id)
-  }
-}
-
-const OPTIONS = {
-  ignoreUndefined: true,
-  connectTimeoutMS: 0,
-  serverSelectionTimeoutMS: 0
-}
-
-const ERR_DUPLICATE_KEY = 11000
-
-exports.Connection = Connection

package/test/connection.test.js

@@ -1,58 +0,0 @@
-'use strict'
-
-const insertManyMock = jest.fn(() => ({ acknowledged: true }))
-jest.mock('mongodb', () => ({
-  __esModule: true,
-  MongoClient: function () {
-    this.connect = () => {},
-    this.db = () => ({
-      collection: () => ({
-        insertMany: insertManyMock
-      })
-    })
-    return this
-  },
-}))
-jest.mock('@toa.io/pointer', () => ({
-  __esModule: true,
-  resolve: () => ['url'],
-}))
-const { generate } = require('randomstring')
-const { Connection } = require('../src/connection')
-
-
-let connection
-
-beforeEach(async () => {
-  jest.clearAllMocks()
-  connection = new Connection({ id: 1 })
-  await connection.open()
-})
-
-it('should be', () => {
-  expect(Connection).toBeDefined()
-})
-
-it('should insert', async () => {
-  const object = generate()
-
-  await connection.add(object)
-
-  expect(insertManyMock).toHaveBeenCalledWith([object], { ordered: false })
-})
-
-it('should batch insert', async () => {
-  const a = generate()
-  const b = generate()
-  const c = generate()
-
-  await Promise.all([
-    connection.add(a),
-    connection.add(b),
-    connection.add(c)
-  ])
-
-  expect(insertManyMock).toHaveBeenCalledTimes(2)
-  expect(insertManyMock).toHaveBeenNthCalledWith(1, [a], { ordered: false })
-  expect(insertManyMock).toHaveBeenNthCalledWith(2, [b, c], { ordered: false })
-})