@toa.io/storages.mongodb 1.0.0-alpha.26 → 1.0.0-alpha.263
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/package.json +8 -8
- package/readme.md +16 -0
- package/src/client.js +16 -7
- package/src/factory.js +1 -3
- package/src/monitoring.js +87 -0
- package/src/record.js +6 -19
- package/src/storage.js +252 -77
- package/src/translate.js +9 -6
- package/test/monitoring.test.js +99 -0
- package/test/record.test.js +0 -15
- package/test/storage.test.js +73 -0
- package/src/collection.js +0 -69
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Change Log
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
|
+
|
|
6
|
+
# [1.0.0-alpha.263](https://github.com/toa-io/toa/compare/v1.0.0-alpha.262...v1.0.0-alpha.263) (2026-08-29)
|
|
7
|
+
|
|
8
|
+
**Note:** Version bump only for package @toa.io/storages.mongodb
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# [1.0.0-alpha.262](https://github.com/toa-io/toa/compare/v1.0.0-alpha.261...v1.0.0-alpha.262) (2026-08-28)
|
|
15
|
+
|
|
16
|
+
**Note:** Version bump only for package @toa.io/storages.mongodb
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# [1.0.0-alpha.259](https://github.com/toa-io/toa/compare/v1.0.0-alpha.258...v1.0.0-alpha.259) (2026-08-24)
|
|
23
|
+
|
|
24
|
+
**Note:** Version bump only for package @toa.io/storages.mongodb
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# [1.0.0-alpha.257](https://github.com/toa-io/toa/compare/v1.0.0-alpha.256...v1.0.0-alpha.257) (2026-08-24)
|
|
31
|
+
|
|
32
|
+
**Note:** Version bump only for package @toa.io/storages.mongodb
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toa.io/storages.mongodb",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.263",
|
|
4
4
|
"description": "Toa MongoDB Storage Connector",
|
|
5
5
|
"author": "temich <tema.gurtovoy@gmail.com>",
|
|
6
6
|
"homepage": "https://github.com/toa-io/toa#readme",
|
|
@@ -19,13 +19,13 @@
|
|
|
19
19
|
"test": "echo \"Error: run tests from root\" && exit 1"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@toa.io/
|
|
23
|
-
"@toa.io/
|
|
24
|
-
"@toa.io/
|
|
25
|
-
"@toa.io/
|
|
26
|
-
"
|
|
27
|
-
"
|
|
22
|
+
"@toa.io/conveyor": "1.0.0-alpha.254",
|
|
23
|
+
"@toa.io/core": "1.0.0-alpha.262",
|
|
24
|
+
"@toa.io/generic": "1.0.0-alpha.254",
|
|
25
|
+
"@toa.io/pointer": "1.0.0-alpha.263",
|
|
26
|
+
"mongodb": "7.2.0",
|
|
27
|
+
"openspan": "1.0.0-alpha.254",
|
|
28
28
|
"saslprep": "1.0.3"
|
|
29
29
|
},
|
|
30
|
-
"gitHead": "
|
|
30
|
+
"gitHead": "165daa284bb7a75907a593fde359e3553b083417"
|
|
31
31
|
}
|
package/readme.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# MongoDB Storage
|
|
2
|
+
|
|
3
|
+
## Tracing
|
|
4
|
+
|
|
5
|
+
Commands are recorded as `client` spans within the trace of the current invocation,
|
|
6
|
+
using the [driver command monitoring events](https://www.mongodb.com/docs/drivers/node/current/monitoring/command-monitoring/).
|
|
7
|
+
|
|
8
|
+
Spans are named `{command} {collection}` and carry `db.*` attributes following the
|
|
9
|
+
[OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/database/mongodb/):
|
|
10
|
+
`db.system`, `db.namespace`, `db.collection.name`, `db.operation.name`.
|
|
11
|
+
|
|
12
|
+
Commands executed outside of a sampled trace context (e.g. index management on startup)
|
|
13
|
+
and internal driver commands (`hello`, `ping`, authentication) are not recorded.
|
|
14
|
+
|
|
15
|
+
Monitoring is client-side only and does not affect the MongoDB server. Span recording
|
|
16
|
+
adds no waiting to the query path: exporting is buffered and happens in the background.
|
package/src/client.js
CHANGED
|
@@ -6,10 +6,12 @@
|
|
|
6
6
|
* @typedef {import('@toa.io/core').Locator} Locator
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
const { console } = require('openspan')
|
|
9
10
|
const { Connector } = require('@toa.io/core')
|
|
10
11
|
const { resolve } = require('@toa.io/pointer')
|
|
11
12
|
const { ID } = require('./deployment')
|
|
12
13
|
const { MongoClient } = require('mongodb')
|
|
14
|
+
const { monitor } = require('./monitoring')
|
|
13
15
|
|
|
14
16
|
/**
|
|
15
17
|
* @type {Record<string, Promise<Instance>>}
|
|
@@ -17,6 +19,8 @@ const { MongoClient } = require('mongodb')
|
|
|
17
19
|
const INSTANCES = {}
|
|
18
20
|
|
|
19
21
|
class Client extends Connector {
|
|
22
|
+
name
|
|
23
|
+
|
|
20
24
|
/**
|
|
21
25
|
* @public
|
|
22
26
|
* @type {import('mongodb').Collection}
|
|
@@ -48,6 +52,7 @@ class Client extends Connector {
|
|
|
48
52
|
super()
|
|
49
53
|
|
|
50
54
|
this.locator = locator
|
|
55
|
+
this.name = locator.lowercase
|
|
51
56
|
}
|
|
52
57
|
|
|
53
58
|
/**
|
|
@@ -58,11 +63,14 @@ class Client extends Connector {
|
|
|
58
63
|
async open () {
|
|
59
64
|
const urls = await this.resolveURLs()
|
|
60
65
|
const dbname = this.resolveDB()
|
|
61
|
-
const collname = this.locator.lowercase
|
|
62
66
|
|
|
63
67
|
this.key = getKey(dbname, urls)
|
|
64
68
|
|
|
65
|
-
|
|
69
|
+
try {
|
|
70
|
+
INSTANCES[this.key] ??= this.createInstance(urls)
|
|
71
|
+
} catch (error) {
|
|
72
|
+
console.error('Failed to connect to MongoDB', { urls, error })
|
|
73
|
+
}
|
|
66
74
|
|
|
67
75
|
this.instance = await INSTANCES[this.key]
|
|
68
76
|
this.instance.count++
|
|
@@ -70,13 +78,13 @@ class Client extends Connector {
|
|
|
70
78
|
const db = this.instance.client.db(dbname)
|
|
71
79
|
|
|
72
80
|
try {
|
|
73
|
-
this.collection = await db.createCollection(
|
|
81
|
+
this.collection = await db.createCollection(this.name)
|
|
74
82
|
} catch (e) {
|
|
75
83
|
if (e.code !== ALREADY_EXISTS) {
|
|
76
84
|
throw e
|
|
77
85
|
}
|
|
78
86
|
|
|
79
|
-
this.collection = db.collection(
|
|
87
|
+
this.collection = db.collection(this.name)
|
|
80
88
|
}
|
|
81
89
|
}
|
|
82
90
|
|
|
@@ -105,7 +113,9 @@ class Client extends Connector {
|
|
|
105
113
|
const client = new MongoClient(urls.join(','), OPTIONS)
|
|
106
114
|
const hosts = urls.map((str) => new URL(str).host)
|
|
107
115
|
|
|
108
|
-
|
|
116
|
+
monitor(client)
|
|
117
|
+
|
|
118
|
+
console.info('Connecting to MongoDB', { address: hosts.join(', ') })
|
|
109
119
|
|
|
110
120
|
await client.connect()
|
|
111
121
|
|
|
@@ -150,8 +160,7 @@ function getKey (db, urls) {
|
|
|
150
160
|
|
|
151
161
|
const OPTIONS = {
|
|
152
162
|
ignoreUndefined: true,
|
|
153
|
-
|
|
154
|
-
serverSelectionTimeoutMS: 0
|
|
163
|
+
monitorCommands: true
|
|
155
164
|
}
|
|
156
165
|
|
|
157
166
|
const ALREADY_EXISTS = 48
|
package/src/factory.js
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { Client } = require('./client')
|
|
4
|
-
const { Collection } = require('./collection')
|
|
5
4
|
const { Storage } = require('./storage')
|
|
6
5
|
|
|
7
6
|
class Factory {
|
|
8
7
|
storage (locator, entity) {
|
|
9
8
|
const client = new Client(locator)
|
|
10
|
-
const connection = new Collection(client)
|
|
11
9
|
|
|
12
|
-
return new Storage(
|
|
10
|
+
return new Storage(client, entity)
|
|
13
11
|
}
|
|
14
12
|
}
|
|
15
13
|
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const { create, current, record } = require('openspan')
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Records MongoDB commands as client spans using the driver command monitoring events.
|
|
7
|
+
* Requires the client to be created with `monitorCommands: true`.
|
|
8
|
+
*
|
|
9
|
+
* Commands executed outside of a sampled trace context are not recorded.
|
|
10
|
+
*
|
|
11
|
+
* @param {import('mongodb').MongoClient} client
|
|
12
|
+
*/
|
|
13
|
+
function monitor (client) {
|
|
14
|
+
/** @type {Map<number, { context: object, time: number, attributes: object }>} */
|
|
15
|
+
const pending = new Map()
|
|
16
|
+
|
|
17
|
+
client.on('commandStarted', (event) => {
|
|
18
|
+
if (IGNORE.has(event.commandName))
|
|
19
|
+
return
|
|
20
|
+
|
|
21
|
+
const parent = current()
|
|
22
|
+
|
|
23
|
+
if (parent === undefined || !parent.sampled)
|
|
24
|
+
return
|
|
25
|
+
|
|
26
|
+
if (pending.size >= CAPACITY) // should never happen
|
|
27
|
+
pending.delete(pending.keys().next().value)
|
|
28
|
+
|
|
29
|
+
// https://opentelemetry.io/docs/specs/semconv/database/mongodb/
|
|
30
|
+
const attributes = {
|
|
31
|
+
'db.system': 'mongodb',
|
|
32
|
+
'db.namespace': event.databaseName,
|
|
33
|
+
'db.operation.name': event.commandName
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const collection = event.command?.[event.commandName]
|
|
37
|
+
|
|
38
|
+
if (typeof collection === 'string')
|
|
39
|
+
attributes['db.collection.name'] = collection
|
|
40
|
+
|
|
41
|
+
pending.set(event.requestId, {
|
|
42
|
+
context: create(parent),
|
|
43
|
+
time: Date.now(),
|
|
44
|
+
attributes
|
|
45
|
+
})
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
client.on('commandSucceeded', (event) => complete(event))
|
|
49
|
+
client.on('commandFailed', (event) => complete(event, 'error'))
|
|
50
|
+
|
|
51
|
+
function complete (event, status) {
|
|
52
|
+
const started = pending.get(event.requestId)
|
|
53
|
+
|
|
54
|
+
if (started === undefined)
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
pending.delete(event.requestId)
|
|
58
|
+
|
|
59
|
+
const collection = started.attributes['db.collection.name']
|
|
60
|
+
|
|
61
|
+
const span = {
|
|
62
|
+
name: collection === undefined
|
|
63
|
+
? event.commandName
|
|
64
|
+
: `${event.commandName} ${collection}`,
|
|
65
|
+
traceId: started.context.traceId,
|
|
66
|
+
spanId: started.context.spanId,
|
|
67
|
+
parentId: started.context.parentId,
|
|
68
|
+
kind: 'client',
|
|
69
|
+
time: started.time,
|
|
70
|
+
duration: event.duration,
|
|
71
|
+
attributes: started.attributes,
|
|
72
|
+
service: started.context.service
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (status !== undefined)
|
|
76
|
+
span.status = status
|
|
77
|
+
|
|
78
|
+
record(span)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// internal commands, including authentication and connection management
|
|
83
|
+
const IGNORE = new Set(['hello', 'ismaster', 'ping', 'endSessions', 'saslStart', 'saslContinue'])
|
|
84
|
+
|
|
85
|
+
const CAPACITY = 1024
|
|
86
|
+
|
|
87
|
+
exports.monitor = monitor
|
package/src/record.js
CHANGED
|
@@ -1,29 +1,16 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
* @returns {toa.mongodb.Record}
|
|
6
|
-
*/
|
|
7
|
-
const to = (entity) => {
|
|
8
|
-
const {
|
|
9
|
-
id,
|
|
10
|
-
...rest
|
|
11
|
-
} = entity
|
|
3
|
+
function to (entity) {
|
|
4
|
+
const { id, ...rest } = entity
|
|
12
5
|
|
|
13
6
|
return /** @type {toa.mongodb.Record} */ { _id: id, ...rest }
|
|
14
7
|
}
|
|
15
8
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
*/
|
|
20
|
-
const from = (record) => {
|
|
21
|
-
if (record === undefined || record === null) return null
|
|
9
|
+
function from (record) {
|
|
10
|
+
if (record === undefined || record === null)
|
|
11
|
+
return null
|
|
22
12
|
|
|
23
|
-
const {
|
|
24
|
-
_id,
|
|
25
|
-
...rest
|
|
26
|
-
} = record
|
|
13
|
+
const { _id, ...rest } = record
|
|
27
14
|
|
|
28
15
|
return { id: _id, ...rest }
|
|
29
16
|
}
|
package/src/storage.js
CHANGED
|
@@ -1,58 +1,94 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
|
-
const {
|
|
4
|
-
|
|
5
|
-
exceptions
|
|
6
|
-
} = require('@toa.io/core')
|
|
7
|
-
|
|
3
|
+
const { Connector, exceptions } = require('@toa.io/core')
|
|
4
|
+
const { console } = require('openspan')
|
|
8
5
|
const { translate } = require('./translate')
|
|
9
|
-
const {
|
|
10
|
-
|
|
11
|
-
from
|
|
12
|
-
} = require('./record')
|
|
6
|
+
const { to, from } = require('./record')
|
|
7
|
+
const { ReturnDocument } = require('mongodb')
|
|
13
8
|
|
|
14
9
|
class Storage extends Connector {
|
|
15
|
-
#
|
|
10
|
+
#client
|
|
11
|
+
|
|
12
|
+
/** @type {import('mongodb').Collection} */
|
|
13
|
+
#collection
|
|
16
14
|
#entity
|
|
17
15
|
|
|
18
|
-
constructor (
|
|
16
|
+
constructor (client, entity) {
|
|
19
17
|
super()
|
|
20
18
|
|
|
21
|
-
this.#
|
|
19
|
+
this.#client = client
|
|
22
20
|
this.#entity = entity
|
|
23
21
|
|
|
24
|
-
this.depends(
|
|
22
|
+
this.depends(client)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
get raw () {
|
|
26
|
+
return this.#collection
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
async open () {
|
|
30
|
+
this.#collection = this.#client.collection
|
|
31
|
+
|
|
28
32
|
await this.index()
|
|
29
33
|
}
|
|
30
34
|
|
|
31
35
|
async get (query) {
|
|
32
|
-
const {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
+
const { criteria, options } = translate(query)
|
|
37
|
+
|
|
38
|
+
// identity lookups must return deleted records, so that callers
|
|
39
|
+
// can tell a deleted entity from a missing one
|
|
40
|
+
if (query?.id === undefined && query?.options?.deleted !== true)
|
|
41
|
+
criteria._deleted = null
|
|
36
42
|
|
|
37
|
-
|
|
43
|
+
this.debug('findOne', { criteria, options })
|
|
44
|
+
|
|
45
|
+
const record = await this.#collection.findOne(criteria, options)
|
|
38
46
|
|
|
39
47
|
return from(record)
|
|
40
48
|
}
|
|
41
49
|
|
|
42
50
|
async find (query) {
|
|
43
|
-
const {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
51
|
+
const { criteria, options, sample } = translate(query)
|
|
52
|
+
|
|
53
|
+
if (query?.options?.deleted !== true)
|
|
54
|
+
criteria._deleted = null
|
|
55
|
+
|
|
56
|
+
let cursor
|
|
57
|
+
|
|
58
|
+
if (sample === undefined) {
|
|
59
|
+
this.debug('find', { criteria, options })
|
|
60
|
+
|
|
61
|
+
cursor = this.#collection.find(criteria, options)
|
|
62
|
+
} else {
|
|
63
|
+
const pipeline = toPipeline(criteria, options, sample)
|
|
47
64
|
|
|
48
|
-
|
|
65
|
+
this.debug('aggregate', { pipeline })
|
|
66
|
+
|
|
67
|
+
cursor = this.#collection.aggregate(pipeline)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const recordset = await cursor.toArray()
|
|
49
71
|
|
|
50
72
|
return recordset.map((item) => from(item))
|
|
51
73
|
}
|
|
52
74
|
|
|
75
|
+
async stream (query = undefined) {
|
|
76
|
+
const { criteria, options } = translate(query)
|
|
77
|
+
|
|
78
|
+
if (query?.options?.deleted !== true)
|
|
79
|
+
criteria._deleted = null
|
|
80
|
+
|
|
81
|
+
this.debug('find (stream)', { criteria, options })
|
|
82
|
+
|
|
83
|
+
return this.#collection.find(criteria, options).stream({ transform: from })
|
|
84
|
+
}
|
|
85
|
+
|
|
53
86
|
async add (entity) {
|
|
54
87
|
const record = to(entity)
|
|
55
|
-
|
|
88
|
+
|
|
89
|
+
this.debug('insertOne', { record })
|
|
90
|
+
|
|
91
|
+
const result = await this.#collection.insertOne(record)
|
|
56
92
|
|
|
57
93
|
return result.acknowledged
|
|
58
94
|
}
|
|
@@ -62,41 +98,92 @@ class Storage extends Connector {
|
|
|
62
98
|
_id: entity.id,
|
|
63
99
|
_version: entity._version - 1
|
|
64
100
|
}
|
|
65
|
-
|
|
101
|
+
|
|
102
|
+
const record = to(entity)
|
|
103
|
+
|
|
104
|
+
this.debug('findOneAndReplace', { criteria, record })
|
|
105
|
+
|
|
106
|
+
const result = await this.#collection.findOneAndReplace(criteria, record)
|
|
66
107
|
|
|
67
108
|
return result !== null
|
|
68
109
|
}
|
|
69
110
|
|
|
70
|
-
async store (entity) {
|
|
111
|
+
async store (entity, attempt = 0) {
|
|
71
112
|
try {
|
|
72
|
-
if (entity._version === 1)
|
|
113
|
+
if (entity._version === 1)
|
|
73
114
|
return await this.add(entity)
|
|
74
|
-
|
|
115
|
+
else
|
|
75
116
|
return await this.set(entity)
|
|
76
|
-
}
|
|
77
117
|
} catch (error) {
|
|
78
|
-
|
|
118
|
+
console.error('MongoDB error', error)
|
|
119
|
+
|
|
120
|
+
const retry = await retriable(error, attempt)
|
|
121
|
+
|
|
122
|
+
if (retry)
|
|
123
|
+
return await this.store(entity, attempt + 1)
|
|
124
|
+
else
|
|
125
|
+
return false
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async massStore (entities, attempt = 0) {
|
|
130
|
+
if (entities.length === 0)
|
|
131
|
+
return true
|
|
79
132
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
: error.keyPattern._id === 1
|
|
133
|
+
const operations = entities.map((entity) => {
|
|
134
|
+
const record = to(entity)
|
|
83
135
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
136
|
+
if (entity._version === 1) {
|
|
137
|
+
const { _version, ...rest } = record
|
|
138
|
+
|
|
139
|
+
return { // upsert in required when document is deleted
|
|
140
|
+
updateOne: {
|
|
141
|
+
filter: { _id: entity.id },
|
|
142
|
+
update: {
|
|
143
|
+
$set: {
|
|
144
|
+
...rest,
|
|
145
|
+
_deleted: null
|
|
146
|
+
},
|
|
147
|
+
$inc: { _version: 1 },
|
|
148
|
+
},
|
|
149
|
+
upsert: true
|
|
150
|
+
}
|
|
88
151
|
}
|
|
89
|
-
} else
|
|
90
|
-
|
|
91
|
-
|
|
152
|
+
} else
|
|
153
|
+
return {
|
|
154
|
+
replaceOne: {
|
|
155
|
+
filter: { _id: entity.id, _version: entity._version - 1 },
|
|
156
|
+
replacement: record
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
const client = this.#client.instance.client
|
|
162
|
+
|
|
163
|
+
try {
|
|
164
|
+
await client.withSession(async (session) => {
|
|
165
|
+
await session.withTransaction(async () => {
|
|
166
|
+
this.debug('bulkWrite', { operations: operations.length })
|
|
167
|
+
|
|
168
|
+
await this.#collection.bulkWrite(operations, { session })
|
|
169
|
+
})
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
return true
|
|
173
|
+
} catch (error) {
|
|
174
|
+
console.error('MongoDB error', error)
|
|
175
|
+
|
|
176
|
+
const retry = await retriable(error, attempt)
|
|
177
|
+
|
|
178
|
+
if (retry)
|
|
179
|
+
return await this.massStore(entities, attempt + 1)
|
|
180
|
+
else
|
|
181
|
+
return false
|
|
92
182
|
}
|
|
93
183
|
}
|
|
94
184
|
|
|
95
185
|
async upsert (query, changeset) {
|
|
96
|
-
const {
|
|
97
|
-
criteria,
|
|
98
|
-
options
|
|
99
|
-
} = translate(query)
|
|
186
|
+
const { criteria, options } = translate(query)
|
|
100
187
|
|
|
101
188
|
if (!('_deleted' in changeset) || changeset._deleted === null) {
|
|
102
189
|
delete criteria._deleted
|
|
@@ -108,20 +195,50 @@ class Storage extends Connector {
|
|
|
108
195
|
$inc: { _version: 1 }
|
|
109
196
|
}
|
|
110
197
|
|
|
111
|
-
options.returnDocument =
|
|
198
|
+
options.returnDocument = ReturnDocument.AFTER
|
|
199
|
+
|
|
200
|
+
this.debug('findOneAndUpdate', { criteria, update, options })
|
|
112
201
|
|
|
113
|
-
const result = await this.#
|
|
202
|
+
const result = await this.#collection.findOneAndUpdate(criteria, update, options)
|
|
114
203
|
|
|
115
204
|
return from(result)
|
|
116
205
|
}
|
|
117
206
|
|
|
207
|
+
async ensure (query, properties, state) {
|
|
208
|
+
let { criteria, options } = translate(query)
|
|
209
|
+
|
|
210
|
+
if (query === undefined)
|
|
211
|
+
criteria = properties
|
|
212
|
+
|
|
213
|
+
const update = { $setOnInsert: to(state) }
|
|
214
|
+
|
|
215
|
+
options.upsert = true
|
|
216
|
+
options.returnDocument = ReturnDocument.AFTER
|
|
217
|
+
|
|
218
|
+
console.debug('Database query', { collection: this.#collection.collectionName, method: 'findOneAndUpdate', criteria, update, options })
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
const result = await this.#collection.findOneAndUpdate(criteria, update, options)
|
|
222
|
+
|
|
223
|
+
if (result._deleted !== undefined && result._deleted !== null)
|
|
224
|
+
return null
|
|
225
|
+
else
|
|
226
|
+
return from(result)
|
|
227
|
+
} catch (error) {
|
|
228
|
+
if (error.code === ERR_DUPLICATE_KEY)
|
|
229
|
+
throw new exceptions.DuplicateException(this.#client.name)
|
|
230
|
+
else
|
|
231
|
+
throw error
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
118
235
|
async index () {
|
|
119
236
|
const indexes = []
|
|
120
237
|
|
|
121
238
|
if (this.#entity.unique !== undefined) {
|
|
122
239
|
for (const [name, fields] of Object.entries(this.#entity.unique)) {
|
|
123
|
-
const
|
|
124
|
-
const unique = await this.uniqueIndex(name, fields,
|
|
240
|
+
const optional = this.getOptional(fields)
|
|
241
|
+
const unique = await this.uniqueIndex(name, fields, optional)
|
|
125
242
|
|
|
126
243
|
indexes.push(unique)
|
|
127
244
|
}
|
|
@@ -131,23 +248,24 @@ class Storage extends Connector {
|
|
|
131
248
|
for (const [suffix, declaration] of Object.entries(this.#entity.index)) {
|
|
132
249
|
const name = 'index_' + suffix
|
|
133
250
|
const fields = Object.fromEntries(Object.entries(declaration)
|
|
134
|
-
.map(([name, type]) => [name, INDEX_TYPES[type]]))
|
|
251
|
+
.map(([name, type]) => [name, INDEX_TYPES[type] ?? type]))
|
|
135
252
|
|
|
136
|
-
const
|
|
253
|
+
const optional = this.getOptional(Object.keys(fields))
|
|
254
|
+
const options = { name, sparse: optional.length > 0 }
|
|
137
255
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
256
|
+
console.info('Creating index', { fields, options })
|
|
257
|
+
|
|
258
|
+
await this.#collection.createIndex(fields, options)
|
|
259
|
+
.catch((e) => console.warn('MongoDB index creation failed', { collection: this.#collection.collectionName, name, fields, error: e }))
|
|
142
260
|
|
|
143
261
|
indexes.push(name)
|
|
144
262
|
}
|
|
145
263
|
}
|
|
146
264
|
|
|
147
|
-
await this.
|
|
265
|
+
await this.removeObsoleteIndexes(indexes)
|
|
148
266
|
}
|
|
149
267
|
|
|
150
|
-
async uniqueIndex (name, properties,
|
|
268
|
+
async uniqueIndex (name, properties, optional) {
|
|
151
269
|
const fields = properties.reduce((acc, property) => {
|
|
152
270
|
acc[property] = 1
|
|
153
271
|
return acc
|
|
@@ -155,48 +273,80 @@ class Storage extends Connector {
|
|
|
155
273
|
|
|
156
274
|
name = 'unique_' + name
|
|
157
275
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
276
|
+
const options = { name, unique: true }
|
|
277
|
+
|
|
278
|
+
if (optional.length > 0)
|
|
279
|
+
options.partialFilterExpression = Object.fromEntries(optional.map((field) => [field, { $exists: true }]))
|
|
280
|
+
|
|
281
|
+
console.info('Creating unique index', { name, fields, options })
|
|
282
|
+
|
|
283
|
+
await this.#collection.createIndex(fields, options)
|
|
284
|
+
.catch((e) => console.warn('MongoDB unique index creation failed',
|
|
285
|
+
{ collection: this.#collection.collectionName, name, fields, error: e }))
|
|
163
286
|
|
|
164
287
|
return name
|
|
165
288
|
}
|
|
166
289
|
|
|
167
|
-
async
|
|
168
|
-
const current = await this
|
|
290
|
+
async removeObsoleteIndexes (desired) {
|
|
291
|
+
const current = await this.getCurrentIndexes()
|
|
169
292
|
const obsolete = current.filter((name) => !desired.includes(name))
|
|
170
293
|
|
|
171
294
|
if (obsolete.length > 0) {
|
|
172
|
-
console.info(
|
|
295
|
+
console.info('Removing obsolete indexes', { collection: this.#collection.collectionName, indexes: obsolete.join(', ') })
|
|
173
296
|
|
|
174
|
-
await this.#
|
|
297
|
+
await Promise.all(obsolete.map((name) => this.#collection.dropIndex(name)))
|
|
175
298
|
}
|
|
176
299
|
}
|
|
177
300
|
|
|
178
|
-
|
|
301
|
+
async getCurrentIndexes () {
|
|
302
|
+
try {
|
|
303
|
+
const array = await this.#collection.listIndexes().toArray()
|
|
304
|
+
|
|
305
|
+
return array.map(({ name }) => name).filter((name) => name !== '_id_')
|
|
306
|
+
} catch {
|
|
307
|
+
return []
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
getOptional (fields) {
|
|
179
312
|
const optional = []
|
|
180
313
|
|
|
181
|
-
for (const field of fields) {
|
|
182
|
-
if (!(field in this.#entity.schema.properties))
|
|
314
|
+
for (const field of fields) {
|
|
315
|
+
if (!field.includes('.') && !(field in this.#entity.schema.properties))
|
|
183
316
|
throw new Error(`Index field '${field}' is not defined.`)
|
|
184
|
-
}
|
|
185
317
|
|
|
186
|
-
if (!this.#entity.schema.required?.includes(field))
|
|
318
|
+
if (!this.#entity.schema.required?.includes(field))
|
|
187
319
|
optional.push(field)
|
|
188
|
-
}
|
|
189
320
|
}
|
|
190
321
|
|
|
191
|
-
|
|
192
|
-
|
|
322
|
+
return optional
|
|
323
|
+
}
|
|
193
324
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
325
|
+
debug (method, attributes) {
|
|
326
|
+
console.debug('MongoDB query', {
|
|
327
|
+
collection: this.#collection.collectionName,
|
|
328
|
+
method,
|
|
329
|
+
...attributes
|
|
330
|
+
})
|
|
198
331
|
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
function toPipeline (criteria, options, sample) {
|
|
335
|
+
const pipeline = []
|
|
336
|
+
|
|
337
|
+
if (criteria !== undefined)
|
|
338
|
+
pipeline.push({ $match: criteria })
|
|
199
339
|
|
|
340
|
+
if (sample !== undefined)
|
|
341
|
+
pipeline.push({ $sample: { size: sample } })
|
|
342
|
+
|
|
343
|
+
if (options?.sort !== undefined)
|
|
344
|
+
pipeline.push({ $sort: options.sort })
|
|
345
|
+
|
|
346
|
+
if (options?.projection !== undefined)
|
|
347
|
+
pipeline.push({ $project: options.projection })
|
|
348
|
+
|
|
349
|
+
return pipeline
|
|
200
350
|
}
|
|
201
351
|
|
|
202
352
|
const INDEX_TYPES = {
|
|
@@ -207,4 +357,29 @@ const INDEX_TYPES = {
|
|
|
207
357
|
|
|
208
358
|
const ERR_DUPLICATE_KEY = 11000
|
|
209
359
|
|
|
360
|
+
async function retriable (error, attempt) {
|
|
361
|
+
if (error.code === ERR_DUPLICATE_KEY) {
|
|
362
|
+
const id = error.keyPattern === undefined
|
|
363
|
+
? error.message.includes(' index: _id_ ') // AWS DocumentDB
|
|
364
|
+
: error.keyPattern._id === 1
|
|
365
|
+
|
|
366
|
+
if (id)
|
|
367
|
+
return false
|
|
368
|
+
else
|
|
369
|
+
throw new exceptions.DuplicateException()
|
|
370
|
+
} else if (error.cause?.code === 'ECONNREFUSED') {
|
|
371
|
+
if (attempt === LAST_ATTEMPT)
|
|
372
|
+
throw error
|
|
373
|
+
|
|
374
|
+
const timeout = 1000 + 500 * attempt
|
|
375
|
+
|
|
376
|
+
await new Promise((resolve) => setTimeout(resolve, timeout))
|
|
377
|
+
|
|
378
|
+
return true
|
|
379
|
+
} else
|
|
380
|
+
throw error
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
const LAST_ATTEMPT = 9
|
|
384
|
+
|
|
210
385
|
exports.Storage = Storage
|
package/src/translate.js
CHANGED
|
@@ -9,18 +9,21 @@ const parse = { ...require('./translate/criteria'), ...require('./translate/opti
|
|
|
9
9
|
const translate = (query) => {
|
|
10
10
|
const result = {
|
|
11
11
|
criteria: query?.criteria === undefined ? {} : parse.criteria(query.criteria),
|
|
12
|
-
options: query?.options === undefined ? {} : parse.options(query.options)
|
|
12
|
+
options: query?.options === undefined ? {} : parse.options(query.options),
|
|
13
|
+
sample: query?.options?.sample
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
if (query?.id !== undefined)
|
|
16
|
+
if (query?.id !== undefined)
|
|
16
17
|
result.criteria._id = query.id
|
|
17
|
-
}
|
|
18
18
|
|
|
19
|
-
if (query?.
|
|
19
|
+
if (query?.ids !== undefined)
|
|
20
|
+
result.criteria._id = { $in: query.ids }
|
|
21
|
+
|
|
22
|
+
if (query?.version !== undefined)
|
|
20
23
|
result.criteria._version = query.version
|
|
21
|
-
}
|
|
22
24
|
|
|
23
|
-
|
|
25
|
+
if (query?.search !== undefined)
|
|
26
|
+
result.criteria.$text = { $search: query.search }
|
|
24
27
|
|
|
25
28
|
return result
|
|
26
29
|
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const { EventEmitter } = require('node:events')
|
|
4
|
+
const { create, exporting, run, traces } = require('openspan')
|
|
5
|
+
const { monitor } = require('../src/monitoring')
|
|
6
|
+
|
|
7
|
+
let client
|
|
8
|
+
let spans
|
|
9
|
+
|
|
10
|
+
beforeEach(() => {
|
|
11
|
+
client = new EventEmitter()
|
|
12
|
+
spans = []
|
|
13
|
+
|
|
14
|
+
exporting([{ export: (span) => spans.push(span) }])
|
|
15
|
+
monitor(client)
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
afterEach(() => {
|
|
19
|
+
traces()
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
const started = {
|
|
23
|
+
requestId: 1,
|
|
24
|
+
commandName: 'find',
|
|
25
|
+
databaseName: 'toa-dev',
|
|
26
|
+
command: { find: 'pots', filter: {} }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const succeeded = { requestId: 1, commandName: 'find', duration: 1.5 }
|
|
30
|
+
const failed = { requestId: 1, commandName: 'find', duration: 0.5, failure: 'boom' }
|
|
31
|
+
|
|
32
|
+
it('should record a command as a client span', () => {
|
|
33
|
+
const context = create()
|
|
34
|
+
|
|
35
|
+
run(context, () => client.emit('commandStarted', started))
|
|
36
|
+
client.emit('commandSucceeded', succeeded)
|
|
37
|
+
|
|
38
|
+
expect(spans).toHaveLength(1)
|
|
39
|
+
|
|
40
|
+
expect(spans[0]).toMatchObject({
|
|
41
|
+
name: 'find pots',
|
|
42
|
+
traceId: context.traceId,
|
|
43
|
+
parentId: context.spanId,
|
|
44
|
+
kind: 'client',
|
|
45
|
+
duration: 1.5,
|
|
46
|
+
attributes: {
|
|
47
|
+
'db.system': 'mongodb',
|
|
48
|
+
'db.namespace': 'toa-dev',
|
|
49
|
+
'db.operation.name': 'find',
|
|
50
|
+
'db.collection.name': 'pots'
|
|
51
|
+
}
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
expect(spans[0].status).toBeUndefined()
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
it('should record failed commands with error status', () => {
|
|
58
|
+
run(create(), () => client.emit('commandStarted', started))
|
|
59
|
+
client.emit('commandFailed', failed)
|
|
60
|
+
|
|
61
|
+
expect(spans).toHaveLength(1)
|
|
62
|
+
expect(spans[0].status).toBe('error')
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
it('should not record commands outside of a trace context', () => {
|
|
66
|
+
client.emit('commandStarted', started)
|
|
67
|
+
client.emit('commandSucceeded', succeeded)
|
|
68
|
+
|
|
69
|
+
expect(spans).toHaveLength(0)
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('should not record commands of unsampled traces', () => {
|
|
73
|
+
const context = { ...create(), sampled: false }
|
|
74
|
+
|
|
75
|
+
run(context, () => client.emit('commandStarted', started))
|
|
76
|
+
client.emit('commandSucceeded', succeeded)
|
|
77
|
+
|
|
78
|
+
expect(spans).toHaveLength(0)
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
it('should ignore internal commands', () => {
|
|
82
|
+
run(create(), () => {
|
|
83
|
+
client.emit('commandStarted', { ...started, commandName: 'hello', command: { hello: 1 } })
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
client.emit('commandSucceeded', { ...succeeded, commandName: 'hello' })
|
|
87
|
+
|
|
88
|
+
expect(spans).toHaveLength(0)
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
it('should name spans without a collection by the command', () => {
|
|
92
|
+
const start = { ...started, commandName: 'listIndexes', command: { listIndexes: 1 } }
|
|
93
|
+
|
|
94
|
+
run(create(), () => client.emit('commandStarted', start))
|
|
95
|
+
client.emit('commandSucceeded', { ...succeeded, commandName: 'listIndexes' })
|
|
96
|
+
|
|
97
|
+
expect(spans).toHaveLength(1)
|
|
98
|
+
expect(spans[0].name).toBe('listIndexes')
|
|
99
|
+
})
|
package/test/record.test.js
CHANGED
|
@@ -47,19 +47,4 @@ describe('from', () => {
|
|
|
47
47
|
_version: 0
|
|
48
48
|
})
|
|
49
49
|
})
|
|
50
|
-
|
|
51
|
-
it('should not modify argument', () => {
|
|
52
|
-
/** @type {toa.mongodb.Record} */
|
|
53
|
-
const record = {
|
|
54
|
-
_id: '1',
|
|
55
|
-
_version: 0
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
from(record)
|
|
59
|
-
|
|
60
|
-
expect(record).toStrictEqual({
|
|
61
|
-
_id: '1',
|
|
62
|
-
_version: 0
|
|
63
|
-
})
|
|
64
|
-
})
|
|
65
50
|
})
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const { Storage } = require('../src/storage')
|
|
4
|
+
|
|
5
|
+
let collection
|
|
6
|
+
let storage
|
|
7
|
+
|
|
8
|
+
beforeEach(async () => {
|
|
9
|
+
collection = {
|
|
10
|
+
collectionName: 'test',
|
|
11
|
+
findOne: jest.fn(async () => null),
|
|
12
|
+
find: jest.fn(() => ({ stream: () => null }))
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const client = {
|
|
16
|
+
collection,
|
|
17
|
+
link: () => null
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
storage = new Storage(client, { schema: { properties: {} } })
|
|
21
|
+
|
|
22
|
+
await storage.open()
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
describe('get', () => {
|
|
26
|
+
it('should filter deleted', async () => {
|
|
27
|
+
await storage.get({})
|
|
28
|
+
|
|
29
|
+
expect(collection.findOne).toHaveBeenCalledWith({ _deleted: null }, {})
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
it('should filter deleted with sort', async () => {
|
|
33
|
+
await storage.get({ options: { sort: [['_created', 'desc']] } })
|
|
34
|
+
|
|
35
|
+
expect(collection.findOne)
|
|
36
|
+
.toHaveBeenCalledWith({ _deleted: null }, { sort: [['_created', -1]] })
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
it('should not filter deleted if queried by id', async () => {
|
|
40
|
+
const id = 'bcb6780f50e243348cad40ed6b5ef575'
|
|
41
|
+
|
|
42
|
+
await storage.get({ id })
|
|
43
|
+
|
|
44
|
+
expect(collection.findOne).toHaveBeenCalledWith({ _id: id }, {})
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
it('should not filter deleted if requested', async () => {
|
|
48
|
+
await storage.get({ options: { deleted: true } })
|
|
49
|
+
|
|
50
|
+
expect(collection.findOne).toHaveBeenCalledWith({}, {})
|
|
51
|
+
})
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
describe('stream', () => {
|
|
55
|
+
it('should filter deleted', async () => {
|
|
56
|
+
await storage.stream()
|
|
57
|
+
|
|
58
|
+
expect(collection.find).toHaveBeenCalledWith({ _deleted: null }, {})
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('should filter deleted with sort', async () => {
|
|
62
|
+
await storage.stream({ options: { sort: [['_created', 'desc']] } })
|
|
63
|
+
|
|
64
|
+
expect(collection.find)
|
|
65
|
+
.toHaveBeenCalledWith({ _deleted: null }, { sort: [['_created', -1]] })
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
it('should not filter deleted if requested', async () => {
|
|
69
|
+
await storage.stream({ options: { deleted: true } })
|
|
70
|
+
|
|
71
|
+
expect(collection.find).toHaveBeenCalledWith({}, {})
|
|
72
|
+
})
|
|
73
|
+
})
|
package/src/collection.js
DELETED
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
'use strict'
|
|
2
|
-
|
|
3
|
-
const { Connector } = require('@toa.io/core')
|
|
4
|
-
|
|
5
|
-
class Collection extends Connector {
|
|
6
|
-
#client
|
|
7
|
-
#collection
|
|
8
|
-
|
|
9
|
-
constructor (client) {
|
|
10
|
-
super()
|
|
11
|
-
|
|
12
|
-
this.#client = client
|
|
13
|
-
|
|
14
|
-
this.depends(client)
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
async open () {
|
|
18
|
-
this.#collection = this.#client.collection
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/** @hot */
|
|
22
|
-
async get (query, options) {
|
|
23
|
-
return /** @type {toa.mongodb.Record} */ this.#collection.findOne(query, options)
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/** @hot */
|
|
27
|
-
async find (query, options) {
|
|
28
|
-
const cursor = this.#collection.find(query, options)
|
|
29
|
-
|
|
30
|
-
return cursor.toArray()
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/** @hot */
|
|
34
|
-
async add (record) {
|
|
35
|
-
return await this.#collection.insertOne(record)
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/** @hot */
|
|
39
|
-
async replace (query, record, options) {
|
|
40
|
-
return await this.#collection.findOneAndReplace(query, record, options)
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/** @hot */
|
|
44
|
-
async update (query, update, options) {
|
|
45
|
-
return this.#collection.findOneAndUpdate(query, update, options)
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
async index (keys, options) {
|
|
49
|
-
return this.#collection.createIndex(keys, options)
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
async indexes () {
|
|
53
|
-
try {
|
|
54
|
-
const array = await this.#collection.listIndexes().toArray()
|
|
55
|
-
|
|
56
|
-
return array.map(({ name }) => name).filter((name) => name !== '_id_')
|
|
57
|
-
} catch {
|
|
58
|
-
return []
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
async dropIndexes (names) {
|
|
63
|
-
const all = names.map((name) => this.#collection.dropIndex(name))
|
|
64
|
-
|
|
65
|
-
return Promise.all(all)
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
exports.Collection = Collection
|