@toa.io/storages.mongodb 1.0.0-alpha.317 → 1.0.0-alpha.319
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/package.json +4 -4
- package/readme.md +20 -0
- package/src/match.js +103 -0
- package/src/migrations.js +15 -1
- package/src/storage.js +21 -1
- package/src/streams.js +317 -0
- package/test/match.test.js +67 -0
- package/test/storage.test.js +38 -26
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,21 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
# [1.0.0-alpha.319](https://github.com/toa-io/toa/compare/v1.0.0-alpha.318...v1.0.0-alpha.319) (2026-09-27)
|
|
7
|
+
|
|
8
|
+
### Features
|
|
9
|
+
|
|
10
|
+
* **storages.mongodb:** read a set and its changes from a token ([d0a8955](https://github.com/toa-io/toa/commit/d0a8955683b7566e704f7e9cde69f67272c7be1b))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# [1.0.0-alpha.318](https://github.com/toa-io/toa/compare/v1.0.0-alpha.317...v1.0.0-alpha.318) (2026-09-23)
|
|
14
|
+
|
|
15
|
+
**Note:** Version bump only for package @toa.io/storages.mongodb
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
6
21
|
# [1.0.0-alpha.317](https://github.com/toa-io/toa/compare/v1.0.0-alpha.316...v1.0.0-alpha.317) (2026-09-23)
|
|
7
22
|
|
|
8
23
|
**Note:** Version bump only for package @toa.io/storages.mongodb
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toa.io/storages.mongodb",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.319",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Toa MongoDB Storage Connector",
|
|
6
6
|
"author": "temich <tema.gurtovoy@gmail.com>",
|
|
@@ -21,12 +21,12 @@
|
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
23
|
"@toa.io/conveyor": "1.0.0-alpha.311",
|
|
24
|
-
"@toa.io/core": "1.0.0-alpha.
|
|
25
|
-
"@toa.io/definitions": "1.0.0-alpha.
|
|
24
|
+
"@toa.io/core": "1.0.0-alpha.319",
|
|
25
|
+
"@toa.io/definitions": "1.0.0-alpha.319",
|
|
26
26
|
"@toa.io/generic": "1.0.0-alpha.311",
|
|
27
27
|
"@toa.io/pointer": "1.0.0-alpha.311",
|
|
28
28
|
"mongodb": "7.6.0",
|
|
29
29
|
"openspan": "1.0.0-alpha.305"
|
|
30
30
|
},
|
|
31
|
-
"gitHead": "
|
|
31
|
+
"gitHead": "ee4901f2bc12c33fc2098a62f2af0c29838a2c2c"
|
|
32
32
|
}
|
package/readme.md
CHANGED
|
@@ -13,3 +13,23 @@ and internal driver commands (`hello`, `ping`, authentication) are not recorded.
|
|
|
13
13
|
|
|
14
14
|
Monitoring is client-side only and does not affect the MongoDB server. Span recording
|
|
15
15
|
adds no waiting to the query path: exporting is buffered and happens in the background.
|
|
16
|
+
|
|
17
|
+
## Stream tokens
|
|
18
|
+
|
|
19
|
+
A [stream](/documentation/collections.md) ends with a token of changes where two things hold:
|
|
20
|
+
|
|
21
|
+
- **MongoDB runs as a replica set**, which it does wherever there is an outbox.
|
|
22
|
+
- **The collection keeps images** of what a record was before each change, which is how a change
|
|
23
|
+
that takes an entry out of the collection is told from a change outside it. They cost a copy of every
|
|
24
|
+
changed record for as long as the oplog holds it, so a collection keeps them only by a
|
|
25
|
+
[migration](/documentation/component/declaration.md#migrations):
|
|
26
|
+
|
|
27
|
+
```yaml
|
|
28
|
+
# migrations/0003-images.yaml
|
|
29
|
+
- images: true
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
`- images: false` stops keeping them, and a token issued before is then refused with `410`.
|
|
33
|
+
|
|
34
|
+
A token lasts as long as the oplog of the replica set holds the point it names — its window, which
|
|
35
|
+
the size of the oplog and the rate of writes decide.
|
package/src/match.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a record matches a filter `translate` wrote, as MongoDB would decide it. A change
|
|
3
|
+
* stream says a record changed, and whether a reader is to add it or drop it is whether its
|
|
4
|
+
* image after the change matches the reader's criteria, or only its image before.
|
|
5
|
+
*
|
|
6
|
+
* What `translate` writes is all there is to decide: `$and`, `$or`, and a field compared with
|
|
7
|
+
* `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin` or a bare value. A search is not a
|
|
8
|
+
* filter on a record, and is refused.
|
|
9
|
+
*
|
|
10
|
+
* @param {object} record
|
|
11
|
+
* @param {object} filter
|
|
12
|
+
* @returns {boolean}
|
|
13
|
+
*/
|
|
14
|
+
export function match(record, filter) {
|
|
15
|
+
for (const [key, condition] of Object.entries(filter))
|
|
16
|
+
if (!matches(record, key, condition)) return false
|
|
17
|
+
|
|
18
|
+
return true
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function matches(record, key, condition) {
|
|
22
|
+
switch (key) {
|
|
23
|
+
case '$and':
|
|
24
|
+
return condition.every((filter) => match(record, filter))
|
|
25
|
+
case '$or':
|
|
26
|
+
return condition.some((filter) => match(record, filter))
|
|
27
|
+
case '$nor':
|
|
28
|
+
return !condition.some((filter) => match(record, filter))
|
|
29
|
+
default:
|
|
30
|
+
if (key.startsWith('$')) throw new Error(`A change cannot be matched against '${key}'`)
|
|
31
|
+
|
|
32
|
+
return field(read(record, key), condition)
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function field(value, condition) {
|
|
37
|
+
if (!operators(condition)) return OPERATORS.$eq(value, condition)
|
|
38
|
+
|
|
39
|
+
for (const [operator, operand] of Object.entries(condition)) {
|
|
40
|
+
const test = OPERATORS[operator]
|
|
41
|
+
|
|
42
|
+
if (test === undefined) throw new Error(`A change cannot be matched against '${operator}'`)
|
|
43
|
+
if (!test(value, operand)) return false
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return true
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const OPERATORS = {
|
|
50
|
+
$eq: (value, operand) => some(value, (v) => equal(v, operand)),
|
|
51
|
+
$ne: (value, operand) => !OPERATORS.$eq(value, operand),
|
|
52
|
+
$in: (value, operands) => operands.some((operand) => OPERATORS.$eq(value, operand)),
|
|
53
|
+
$nin: (value, operands) => !OPERATORS.$in(value, operands),
|
|
54
|
+
$gt: (value, operand) => some(value, (v) => compare(v, operand) > 0),
|
|
55
|
+
$gte: (value, operand) => some(value, (v) => compare(v, operand) >= 0),
|
|
56
|
+
$lt: (value, operand) => some(value, (v) => compare(v, operand) < 0),
|
|
57
|
+
$lte: (value, operand) => some(value, (v) => compare(v, operand) <= 0)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** An array matches where the array or one of its elements does, as MongoDB has it. */
|
|
61
|
+
function some(value, test) {
|
|
62
|
+
return test(value) || (Array.isArray(value) && value.some(test))
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** `null` is equal to a field that is not there, and a date to a date of the same instant. */
|
|
66
|
+
function equal(value, operand) {
|
|
67
|
+
if (operand === null) return value === null || value === undefined
|
|
68
|
+
if (operand instanceof Date) return value instanceof Date && value.getTime() === operand.getTime()
|
|
69
|
+
|
|
70
|
+
return value === operand
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Values of one type compare, and values of two types are neither greater nor less. */
|
|
74
|
+
function compare(value, operand) {
|
|
75
|
+
if (value instanceof Date && operand instanceof Date) return value.getTime() - operand.getTime()
|
|
76
|
+
if (typeof value === 'number' && typeof operand === 'number') return value - operand
|
|
77
|
+
|
|
78
|
+
if (typeof value === 'string' && typeof operand === 'string')
|
|
79
|
+
return value < operand ? -1 : value > operand ? 1 : 0
|
|
80
|
+
|
|
81
|
+
return NaN
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function operators(condition) {
|
|
85
|
+
if (condition === null || typeof condition !== 'object' || condition instanceof Date) return false
|
|
86
|
+
if (Array.isArray(condition)) return false
|
|
87
|
+
|
|
88
|
+
const keys = Object.keys(condition)
|
|
89
|
+
|
|
90
|
+
return keys.length > 0 && keys.every((key) => key.startsWith('$'))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function read(record, path) {
|
|
94
|
+
let value = record
|
|
95
|
+
|
|
96
|
+
for (const key of path.split('.')) {
|
|
97
|
+
if (value === null || typeof value !== 'object') return undefined
|
|
98
|
+
|
|
99
|
+
value = value[key]
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return value
|
|
103
|
+
}
|
package/src/migrations.js
CHANGED
|
@@ -301,7 +301,21 @@ async function remove(collection, { filter }, id) {
|
|
|
301
301
|
})
|
|
302
302
|
}
|
|
303
303
|
|
|
304
|
-
|
|
304
|
+
/**
|
|
305
|
+
* Keeps, or stops keeping, what a record was before each change and after it, which a stream
|
|
306
|
+
* reads to tell an entry that left a set from a change outside it.
|
|
307
|
+
*/
|
|
308
|
+
async function images(collection, enabled, id) {
|
|
309
|
+
if (typeof enabled !== 'boolean')
|
|
310
|
+
throw new Error(`Migration '${id}' declares images that are neither true nor false`)
|
|
311
|
+
|
|
312
|
+
await collection.s.db.command({
|
|
313
|
+
collMod: collection.collectionName,
|
|
314
|
+
changeStreamPreAndPostImages: { enabled }
|
|
315
|
+
})
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const STEPS = { index, dropIndex, update, delete: remove, images }
|
|
305
319
|
|
|
306
320
|
const DIRECTIONS = { asc: 1, desc: -1, hash: 'hashed' }
|
|
307
321
|
|
package/src/storage.js
CHANGED
|
@@ -5,6 +5,7 @@ import { codec } from './record.js'
|
|
|
5
5
|
import { Inbox } from './inbox.js'
|
|
6
6
|
import { Outbox } from './outbox.js'
|
|
7
7
|
import { Migrations } from './migrations.js'
|
|
8
|
+
import { Streams } from './streams.js'
|
|
8
9
|
import { conflicted, query } from './measurements.js'
|
|
9
10
|
import { ReturnDocument } from 'mongodb'
|
|
10
11
|
|
|
@@ -34,6 +35,9 @@ export class Storage extends Connector {
|
|
|
34
35
|
/** @type {Map<string, object>} span options per driver method */
|
|
35
36
|
#spans = new Map()
|
|
36
37
|
|
|
38
|
+
/** @type {Streams | undefined} what a stream reads, and where it continues from */
|
|
39
|
+
#streams
|
|
40
|
+
|
|
37
41
|
/** how a record is written and read back, which depends on what the entity declares */
|
|
38
42
|
#to
|
|
39
43
|
#from
|
|
@@ -108,6 +112,22 @@ export class Storage extends Connector {
|
|
|
108
112
|
|
|
109
113
|
await this.#outbox?.index()
|
|
110
114
|
await this.#inbox?.index()
|
|
115
|
+
|
|
116
|
+
this.#streams = new Streams(
|
|
117
|
+
this.#collection,
|
|
118
|
+
() => this.#client.instance.client,
|
|
119
|
+
this.#from,
|
|
120
|
+
this.#client.transactional === true && (await this.#images())
|
|
121
|
+
)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Whether the collection keeps what a record was before a change, which a migration says. */
|
|
125
|
+
async #images() {
|
|
126
|
+
const collection = await this.#client.db
|
|
127
|
+
.listCollections({ name: this.#collection.collectionName })
|
|
128
|
+
.next()
|
|
129
|
+
|
|
130
|
+
return collection?.options?.changeStreamPreAndPostImages?.enabled === true
|
|
111
131
|
}
|
|
112
132
|
|
|
113
133
|
async get(query) {
|
|
@@ -160,7 +180,7 @@ export class Storage extends Connector {
|
|
|
160
180
|
|
|
161
181
|
this.debug('find (stream)', { criteria, options })
|
|
162
182
|
|
|
163
|
-
return this.#
|
|
183
|
+
return this.#streams.stream({ criteria, options }, query?.options?.token)
|
|
164
184
|
}
|
|
165
185
|
|
|
166
186
|
async add(entity, session = undefined) {
|
package/src/streams.js
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { Readable } from 'node:stream'
|
|
3
|
+
import { Timestamp } from 'mongodb'
|
|
4
|
+
import { exceptions, parts } from '@toa.io/core'
|
|
5
|
+
import { match } from './match.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A collection read as a stream, and what changed in it since a token.
|
|
9
|
+
*
|
|
10
|
+
* A token is a position in the history MongoDB keeps of the writes committed to the collection —
|
|
11
|
+
* a change stream's resume token — so what it continues from is the order writes committed in.
|
|
12
|
+
* A first read takes the position before it reads anything: what committed before it is in what
|
|
13
|
+
* the read finds, and what committed after it is in the next read. A page is read after the last
|
|
14
|
+
* `_id` of the one before it, ordered by `_id`, which never changes, so an entry stays on one side
|
|
15
|
+
* of a page boundary for the whole read.
|
|
16
|
+
*
|
|
17
|
+
* Where MongoDB keeps no such history — a standalone server, or a collection that keeps no images
|
|
18
|
+
* of a record before a change — pages are still read, and a complete read ends with a `null`
|
|
19
|
+
* token: there is nothing to continue from.
|
|
20
|
+
*/
|
|
21
|
+
export class Streams {
|
|
22
|
+
/** @type {import('mongodb').Collection} */
|
|
23
|
+
#collection
|
|
24
|
+
|
|
25
|
+
/** @type {() => import('mongodb').MongoClient} the client sessions are started on */
|
|
26
|
+
#client
|
|
27
|
+
|
|
28
|
+
/** @type {(record: object) => object} */
|
|
29
|
+
#from
|
|
30
|
+
|
|
31
|
+
/** whether a position can be taken and read from */
|
|
32
|
+
#history
|
|
33
|
+
|
|
34
|
+
constructor(collection, client, from, history) {
|
|
35
|
+
this.#collection = collection
|
|
36
|
+
this.#client = client
|
|
37
|
+
this.#from = from
|
|
38
|
+
this.#history = history
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {{ criteria: object, options: object }} translated
|
|
43
|
+
* @param {string} [given] the token the read continues from
|
|
44
|
+
* @returns {Promise<Readable>}
|
|
45
|
+
*/
|
|
46
|
+
async stream(translated, given) {
|
|
47
|
+
const { criteria, options } = translated
|
|
48
|
+
const hash = digest(criteria)
|
|
49
|
+
|
|
50
|
+
if (given === undefined) return await this.#first(criteria, options, hash)
|
|
51
|
+
|
|
52
|
+
const token = decode(given)
|
|
53
|
+
|
|
54
|
+
if (token.h !== hash)
|
|
55
|
+
throw new exceptions.QuerySyntaxException('The token was issued for other criteria')
|
|
56
|
+
|
|
57
|
+
if (token.id !== undefined) return await this.#page(criteria, options, hash, token)
|
|
58
|
+
if (token.p === null || !this.#history) throw lost()
|
|
59
|
+
|
|
60
|
+
return await this.#changes(criteria, options, hash, token)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async #first(criteria, options, hash) {
|
|
64
|
+
const session = this.#client().startSession({ causalConsistency: true })
|
|
65
|
+
|
|
66
|
+
try {
|
|
67
|
+
const position = this.#history ? await this.#position(session) : null
|
|
68
|
+
|
|
69
|
+
return await this.#read(criteria, options, hash, session, position, undefined)
|
|
70
|
+
} catch (exception) {
|
|
71
|
+
await session.endSession()
|
|
72
|
+
|
|
73
|
+
throw exception
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async #page(criteria, options, hash, token) {
|
|
78
|
+
if (token.p !== null && !this.#history) throw lost()
|
|
79
|
+
|
|
80
|
+
const session = this.#client().startSession({ causalConsistency: true })
|
|
81
|
+
|
|
82
|
+
if (token.t !== null) session.advanceOperationTime(new Timestamp(BigInt(token.t)))
|
|
83
|
+
|
|
84
|
+
const position = token.p === null ? null : { p: token.p, t: token.t }
|
|
85
|
+
|
|
86
|
+
try {
|
|
87
|
+
return await this.#read(criteria, options, hash, session, position, token.id)
|
|
88
|
+
} catch (exception) {
|
|
89
|
+
await session.endSession()
|
|
90
|
+
|
|
91
|
+
throw exception
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The position: the resume token of a change stream opened with a batch of none. Its
|
|
97
|
+
* operation time is where a page is read at, so that a member of the replica set behind it
|
|
98
|
+
* waits until it has replicated it.
|
|
99
|
+
*/
|
|
100
|
+
async #position(session) {
|
|
101
|
+
const db = this.#collection.s.db
|
|
102
|
+
const name = this.#collection.collectionName
|
|
103
|
+
|
|
104
|
+
const reply = await db.command(
|
|
105
|
+
{ aggregate: name, pipeline: [{ $changeStream: {} }], cursor: { batchSize: 0 } },
|
|
106
|
+
{ session }
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
const { id, postBatchResumeToken } = reply.cursor
|
|
110
|
+
|
|
111
|
+
if (!id.isZero()) await db.command({ killCursors: name, cursors: [id] }, { session })
|
|
112
|
+
|
|
113
|
+
return { p: postBatchResumeToken._data, t: session.operationTime.toString() }
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* The collection, or a page of it. A read that pages is ordered by `_id` and continues after the
|
|
118
|
+
* last one; one that does not is read in the order the query states.
|
|
119
|
+
*/
|
|
120
|
+
async #read(criteria, options, hash, session, position, after) {
|
|
121
|
+
const limit = options.limit
|
|
122
|
+
const filter = after === undefined ? criteria : { $and: [criteria, { _id: { $gt: after } }] }
|
|
123
|
+
|
|
124
|
+
const read = { ...options, session, readConcern: { level: 'majority' } }
|
|
125
|
+
|
|
126
|
+
if (limit !== undefined) read.sort = { _id: 1 }
|
|
127
|
+
|
|
128
|
+
const cursor = this.#collection.find(filter, read)
|
|
129
|
+
|
|
130
|
+
// what refuses the read refuses it before a part is sent, where it is still an answer
|
|
131
|
+
await cursor.hasNext()
|
|
132
|
+
|
|
133
|
+
const from = this.#from
|
|
134
|
+
|
|
135
|
+
async function* yielding() {
|
|
136
|
+
let count = 0
|
|
137
|
+
let last
|
|
138
|
+
|
|
139
|
+
try {
|
|
140
|
+
for await (const record of cursor) {
|
|
141
|
+
count++
|
|
142
|
+
last = record._id
|
|
143
|
+
|
|
144
|
+
yield parts.entry(from(record))
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (limit !== undefined && count === limit)
|
|
148
|
+
yield parts.token(encode({ v: VERSION, p: position?.p ?? null, t: position?.t ?? null, id: last, h: hash }))
|
|
149
|
+
else yield parts.token(position === null ? null : encode({ v: VERSION, p: position.p, h: hash }))
|
|
150
|
+
} finally {
|
|
151
|
+
await cursor.close()
|
|
152
|
+
await session.endSession()
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return Readable.from(yielding())
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* What committed after the token that concerns the collection: a change whose image after it matches
|
|
161
|
+
* the criteria is an entry, and one whose image before it alone does is a removal. A change
|
|
162
|
+
* that concerns neither is left out by the pipeline, so a reader learns no id it cannot read.
|
|
163
|
+
*/
|
|
164
|
+
async #changes(criteria, options, hash, token) {
|
|
165
|
+
const pipeline = [
|
|
166
|
+
{
|
|
167
|
+
$match: {
|
|
168
|
+
$or: [
|
|
169
|
+
prefix(criteria, 'fullDocument.'),
|
|
170
|
+
prefix(criteria, 'fullDocumentBeforeChange.'),
|
|
171
|
+
{ operationType: { $in: ENDINGS } }
|
|
172
|
+
]
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
]
|
|
176
|
+
|
|
177
|
+
const stream = this.#collection.watch(pipeline, {
|
|
178
|
+
startAfter: { _data: token.p },
|
|
179
|
+
fullDocument: 'required',
|
|
180
|
+
fullDocumentBeforeChange: 'required'
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
const limit = options.limit
|
|
184
|
+
|
|
185
|
+
let first
|
|
186
|
+
|
|
187
|
+
try {
|
|
188
|
+
first = await stream.tryNext()
|
|
189
|
+
} catch (exception) {
|
|
190
|
+
await stream.close()
|
|
191
|
+
|
|
192
|
+
throw refusal(exception)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const from = this.#from
|
|
196
|
+
|
|
197
|
+
async function* yielding() {
|
|
198
|
+
let count = 0
|
|
199
|
+
let event = first
|
|
200
|
+
|
|
201
|
+
try {
|
|
202
|
+
while (event !== null) {
|
|
203
|
+
if (ENDINGS.includes(event.operationType)) throw lost()
|
|
204
|
+
|
|
205
|
+
const part = classify(event, criteria, from)
|
|
206
|
+
|
|
207
|
+
if (part !== undefined) {
|
|
208
|
+
count++
|
|
209
|
+
|
|
210
|
+
yield part
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
if (limit !== undefined && count >= limit) break
|
|
214
|
+
|
|
215
|
+
event = await stream.tryNext().catch((exception) => {
|
|
216
|
+
throw refusal(exception)
|
|
217
|
+
})
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
yield parts.token(encode({ v: VERSION, p: stream.resumeToken._data, h: hash }))
|
|
221
|
+
} finally {
|
|
222
|
+
await stream.close()
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
return Readable.from(yielding())
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function classify(event, criteria, from) {
|
|
231
|
+
const after = event.fullDocument
|
|
232
|
+
const before = event.fullDocumentBeforeChange
|
|
233
|
+
|
|
234
|
+
if (after !== undefined && after !== null && match(after, criteria)) return parts.entry(from(after))
|
|
235
|
+
|
|
236
|
+
if (before !== undefined && before !== null && match(before, criteria))
|
|
237
|
+
return parts.removed(String(event.documentKey._id))
|
|
238
|
+
|
|
239
|
+
return undefined
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* The criteria against an image of a change stream event. A key that is an operator carries
|
|
244
|
+
* filters, and every other key is a field of the record.
|
|
245
|
+
*/
|
|
246
|
+
function prefix(filter, path) {
|
|
247
|
+
const result = {}
|
|
248
|
+
|
|
249
|
+
for (const [key, value] of Object.entries(filter))
|
|
250
|
+
if (LOGICAL.includes(key)) result[key] = value.map((filter) => prefix(filter, path))
|
|
251
|
+
else if (key.startsWith('$'))
|
|
252
|
+
throw new exceptions.QuerySyntaxException(`What changed in a collection cannot be read by '${key}'`)
|
|
253
|
+
else result[path + key] = value
|
|
254
|
+
|
|
255
|
+
return result
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function refusal(exception) {
|
|
259
|
+
if (LOST.includes(exception?.code) || exception?.hasErrorLabel?.('NonResumableChangeStreamError'))
|
|
260
|
+
return lost()
|
|
261
|
+
|
|
262
|
+
return exception
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function lost() {
|
|
266
|
+
return new exceptions.StateHistoryException('The token names a point the storage no longer holds')
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* The criteria a token was issued for: a read that continues from it under other criteria would
|
|
271
|
+
* answer the changes to one collection as if they were those of another.
|
|
272
|
+
*/
|
|
273
|
+
function digest(criteria) {
|
|
274
|
+
return createHash('sha1').update(JSON.stringify(criteria)).digest('base64url').slice(0, 16)
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function encode(token) {
|
|
278
|
+
return Buffer.from(JSON.stringify(token)).toString('base64url')
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** A token this storage did not write, or wrote in another format, names nothing it holds. */
|
|
282
|
+
function decode(given) {
|
|
283
|
+
let token
|
|
284
|
+
|
|
285
|
+
try {
|
|
286
|
+
token = JSON.parse(Buffer.from(given, 'base64url').toString())
|
|
287
|
+
} catch {
|
|
288
|
+
throw lost()
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const valid =
|
|
292
|
+
token !== null &&
|
|
293
|
+
typeof token === 'object' &&
|
|
294
|
+
token.v === VERSION &&
|
|
295
|
+
typeof token.h === 'string' &&
|
|
296
|
+
(typeof token.p === 'string' || token.p === null) &&
|
|
297
|
+
(token.id === undefined || typeof token.id === 'string') &&
|
|
298
|
+
(token.id === undefined || token.t === null || typeof token.t === 'string')
|
|
299
|
+
|
|
300
|
+
if (!valid) throw lost()
|
|
301
|
+
|
|
302
|
+
return token
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** the format of a token, which a storage that reads another refuses */
|
|
306
|
+
const VERSION = 1
|
|
307
|
+
|
|
308
|
+
const LOGICAL = ['$and', '$or', '$nor']
|
|
309
|
+
|
|
310
|
+
/** what ends a change stream: the collection it follows is gone or renamed */
|
|
311
|
+
const ENDINGS = ['drop', 'rename', 'dropDatabase', 'invalidate']
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* ChangeStreamHistoryLost, ChangeStreamFatalError (a resume token that is not in the oplog), and
|
|
315
|
+
* NoMatchingDocument (an image that has expired).
|
|
316
|
+
*/
|
|
317
|
+
const LOST = [286, 280, 47]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, it } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
|
|
4
|
+
import { match } from '../src/match.js'
|
|
5
|
+
|
|
6
|
+
const record = {
|
|
7
|
+
_id: 'a1',
|
|
8
|
+
owner: 'alice',
|
|
9
|
+
rank: 5,
|
|
10
|
+
tags: ['red', 'green'],
|
|
11
|
+
DELETED: null,
|
|
12
|
+
CREATED: new Date(1000)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe('match', () => {
|
|
16
|
+
it('should match a bare value as equality', () => {
|
|
17
|
+
assert.equal(match(record, { owner: 'alice' }), true)
|
|
18
|
+
assert.equal(match(record, { owner: 'bob' }), false)
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
it('should match null against a field that is null or not there', () => {
|
|
22
|
+
assert.equal(match(record, { DELETED: null }), true)
|
|
23
|
+
assert.equal(match(record, { missing: null }), true)
|
|
24
|
+
assert.equal(match({ ...record, DELETED: new Date(1) }, { DELETED: null }), false)
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
it('should compare values of one type', () => {
|
|
28
|
+
assert.equal(match(record, { rank: { $gt: 4 } }), true)
|
|
29
|
+
assert.equal(match(record, { rank: { $gte: 5, $lt: 6 } }), true)
|
|
30
|
+
assert.equal(match(record, { rank: { $lte: 4 } }), false)
|
|
31
|
+
assert.equal(match(record, { owner: { $gt: 'a' } }), true)
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
it('should compare values of two types as neither greater nor less', () => {
|
|
35
|
+
assert.equal(match(record, { rank: { $gt: '1' } }), false)
|
|
36
|
+
assert.equal(match(record, { rank: { $lt: '9' } }), false)
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
it('should compare dates by their instant', () => {
|
|
40
|
+
assert.equal(match(record, { CREATED: new Date(1000) }), true)
|
|
41
|
+
assert.equal(match(record, { CREATED: { $gt: new Date(999) } }), true)
|
|
42
|
+
assert.equal(match(record, { CREATED: { $eq: new Date(1001) } }), false)
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
it('should match a list of values', () => {
|
|
46
|
+
assert.equal(match(record, { owner: { $in: ['bob', 'alice'] } }), true)
|
|
47
|
+
assert.equal(match(record, { owner: { $nin: ['bob', 'alice'] } }), false)
|
|
48
|
+
assert.equal(match(record, { owner: { $ne: 'bob' } }), true)
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it('should match an array where one of its elements does', () => {
|
|
52
|
+
assert.equal(match(record, { tags: 'red' }), true)
|
|
53
|
+
assert.equal(match(record, { tags: { $in: ['blue', 'green'] } }), true)
|
|
54
|
+
assert.equal(match(record, { tags: { $ne: 'red' } }), false)
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
it('should combine filters', () => {
|
|
58
|
+
assert.equal(match(record, { $and: [{ owner: 'alice' }, { rank: { $gt: 1 } }] }), true)
|
|
59
|
+
assert.equal(match(record, { $or: [{ owner: 'bob' }, { rank: { $gt: 1 } }] }), true)
|
|
60
|
+
assert.equal(match(record, { $or: [{ owner: 'bob' }, { rank: { $gt: 9 } }] }), false)
|
|
61
|
+
assert.equal(match(record, { owner: 'alice', DELETED: null, _id: 'a1' }), true)
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('should refuse a search', () => {
|
|
65
|
+
assert.throws(() => match(record, { $text: { $search: 'x' } }))
|
|
66
|
+
})
|
|
67
|
+
})
|
package/test/storage.test.js
CHANGED
|
@@ -12,20 +12,30 @@ beforeEach(async () => {
|
|
|
12
12
|
collection = {
|
|
13
13
|
collectionName: 'test',
|
|
14
14
|
findOne: mock.fn(async () => null),
|
|
15
|
-
find: mock.fn(() => (
|
|
15
|
+
find: mock.fn(() => cursor()),
|
|
16
16
|
updateMany: mock.fn(async () => ({ modifiedCount: 0 })),
|
|
17
17
|
updateOne: mock.fn(async () => ({ upsertedCount: 0, modifiedCount: 0 }))
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
db = { collection: mock.fn(() => null) }
|
|
21
21
|
|
|
22
|
-
const
|
|
22
|
+
const session = { endSession: async () => undefined }
|
|
23
|
+
const instance = { client: { startSession: () => session } }
|
|
24
|
+
const client = { collection, db, instance, link: () => null }
|
|
23
25
|
|
|
24
26
|
storage = new Storage(client, { schema: { properties: {} } })
|
|
25
27
|
|
|
26
28
|
await storage.open()
|
|
27
29
|
})
|
|
28
30
|
|
|
31
|
+
function cursor() {
|
|
32
|
+
return {
|
|
33
|
+
hasNext: async () => false,
|
|
34
|
+
close: async () => undefined,
|
|
35
|
+
async *[Symbol.asyncIterator]() {}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
29
39
|
describe('open', () => {
|
|
30
40
|
it('should not touch the migrations record where the entity declares none', () => {
|
|
31
41
|
assert.equal(db.collection.mock.callCount(), 0)
|
|
@@ -89,43 +99,45 @@ describe('get', () => {
|
|
|
89
99
|
})
|
|
90
100
|
|
|
91
101
|
describe('stream', () => {
|
|
102
|
+
const read = () => collection.find.mock.calls[0].arguments
|
|
103
|
+
|
|
92
104
|
it('should filter deleted', async () => {
|
|
93
105
|
await storage.stream()
|
|
94
106
|
|
|
95
|
-
assert.
|
|
96
|
-
collection.find.mock.calls.some(
|
|
97
|
-
(call) =>
|
|
98
|
-
call.arguments.length === 2 &&
|
|
99
|
-
isDeepStrictEqual(call.arguments[0], { DELETED: null }) &&
|
|
100
|
-
isDeepStrictEqual(call.arguments[1], {})
|
|
101
|
-
)
|
|
102
|
-
)
|
|
107
|
+
assert.deepStrictEqual(read()[0], { DELETED: null })
|
|
103
108
|
})
|
|
104
109
|
|
|
105
110
|
it('should filter deleted with sort', async () => {
|
|
106
111
|
await storage.stream({ options: { sort: [['CREATED', 'desc']] } })
|
|
107
112
|
|
|
108
|
-
assert.
|
|
109
|
-
|
|
110
|
-
(call) =>
|
|
111
|
-
call.arguments.length === 2 &&
|
|
112
|
-
isDeepStrictEqual(call.arguments[0], { DELETED: null }) &&
|
|
113
|
-
isDeepStrictEqual(call.arguments[1], { sort: [['CREATED', -1]] })
|
|
114
|
-
)
|
|
115
|
-
)
|
|
113
|
+
assert.deepStrictEqual(read()[0], { DELETED: null })
|
|
114
|
+
assert.deepStrictEqual(read()[1].sort, [['CREATED', -1]])
|
|
116
115
|
})
|
|
117
116
|
|
|
118
117
|
it('should not filter deleted if requested', async () => {
|
|
119
118
|
await storage.stream({ options: { deleted: true } })
|
|
120
119
|
|
|
121
|
-
assert.
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
120
|
+
assert.deepStrictEqual(read()[0], {})
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
it('should read what the majority of the replica set holds', async () => {
|
|
124
|
+
await storage.stream()
|
|
125
|
+
|
|
126
|
+
assert.deepStrictEqual(read()[1].readConcern, { level: 'majority' })
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it('should end the set with no position where the storage keeps no history', async () => {
|
|
130
|
+
const stream = await storage.stream()
|
|
131
|
+
const parts = await stream.toArray()
|
|
132
|
+
|
|
133
|
+
assert.deepStrictEqual(parts, [{ token: null }])
|
|
134
|
+
})
|
|
135
|
+
|
|
136
|
+
it('should read a page ordered by id', async () => {
|
|
137
|
+
await storage.stream({ options: { limit: 2 } })
|
|
138
|
+
|
|
139
|
+
assert.deepStrictEqual(read()[1].sort, { _id: 1 })
|
|
140
|
+
assert.equal(read()[1].limit, 2)
|
|
129
141
|
})
|
|
130
142
|
})
|
|
131
143
|
|