@toa.io/storages.mongodb 1.0.0-alpha.286 → 1.0.0-alpha.288
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/package.json +4 -4
- package/src/migrations.js +81 -8
- package/src/record.js +81 -0
- package/src/storage.js +36 -22
- package/src/system.js +44 -0
- package/src/translate/criteria.js +20 -9
- package/src/translate.js +3 -2
- package/test/migrations.test.js +56 -1
- package/test/record.test.js +91 -1
- package/test/storage.test.js +11 -1
- package/test/translate.test.js +41 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Change Log
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
|
+
|
|
6
|
+
# [1.0.0-alpha.288](https://github.com/toa-io/toa/compare/v1.0.0-alpha.287...v1.0.0-alpha.288) (2026-09-06)
|
|
7
|
+
|
|
8
|
+
**Note:** Version bump only for package @toa.io/storages.mongodb
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# [1.0.0-alpha.287](https://github.com/toa-io/toa/compare/v1.0.0-alpha.286...v1.0.0-alpha.287) (2026-09-06)
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
* **mongodb:** a migration says what it is doing ([a917a81](https://github.com/toa-io/toa/commit/a917a81fdc94eb23fd73182c0edf891e6b2df843))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toa.io/storages.mongodb",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.288",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Toa MongoDB Storage Connector",
|
|
6
6
|
"author": "temich <tema.gurtovoy@gmail.com>",
|
|
@@ -21,12 +21,12 @@
|
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
23
|
"@toa.io/conveyor": "1.0.0-alpha.286",
|
|
24
|
-
"@toa.io/core": "1.0.0-alpha.
|
|
24
|
+
"@toa.io/core": "1.0.0-alpha.288",
|
|
25
25
|
"@toa.io/generic": "1.0.0-alpha.286",
|
|
26
|
-
"@toa.io/pointer": "1.0.0-alpha.
|
|
26
|
+
"@toa.io/pointer": "1.0.0-alpha.287",
|
|
27
27
|
"mongodb": "7.6.0",
|
|
28
28
|
"openspan": "1.0.0-alpha.286",
|
|
29
29
|
"saslprep": "1.0.3"
|
|
30
30
|
},
|
|
31
|
-
"gitHead": "
|
|
31
|
+
"gitHead": "d3d696f0a4e287c3c297ac81028d600ad7574112"
|
|
32
32
|
}
|
package/src/migrations.js
CHANGED
|
@@ -37,26 +37,66 @@ export class Migrations {
|
|
|
37
37
|
* against what an earlier one leaves behind.
|
|
38
38
|
*/
|
|
39
39
|
async run() {
|
|
40
|
-
|
|
40
|
+
let applied = 0
|
|
41
|
+
|
|
42
|
+
for (const migration of this.#list) if (await this.#apply(migration)) applied++
|
|
43
|
+
|
|
44
|
+
// debug, because the ordinary start has nothing to report: every migration was applied
|
|
45
|
+
// long ago by whoever started first. But a run that says nothing at all cannot be told
|
|
46
|
+
// from one that never looked
|
|
47
|
+
console.debug('Migrations checked', {
|
|
48
|
+
collection: this.#collection.collectionName,
|
|
49
|
+
declared: this.#list.length,
|
|
50
|
+
applied
|
|
51
|
+
})
|
|
41
52
|
}
|
|
42
53
|
|
|
43
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Answers whether this replica was the one that applied it.
|
|
56
|
+
*
|
|
57
|
+
* @private
|
|
58
|
+
*/
|
|
44
59
|
async #apply(migration) {
|
|
45
60
|
const id = `${this.#collection.collectionName}:${migration.id}`
|
|
46
61
|
|
|
62
|
+
/** when the wait for another replica was last reported */
|
|
63
|
+
let announced
|
|
64
|
+
|
|
47
65
|
while (true) {
|
|
48
|
-
if (await this.#claim(id))
|
|
66
|
+
if (await this.#claim(id)) {
|
|
67
|
+
await this.#run(id, migration)
|
|
68
|
+
|
|
69
|
+
return true
|
|
70
|
+
}
|
|
49
71
|
|
|
50
72
|
const row = await this.#state.findOne({ _id: id })
|
|
51
73
|
|
|
52
74
|
// removed between the claim and the read; whoever did that wants it applied again
|
|
53
75
|
if (row === null) continue
|
|
54
76
|
|
|
55
|
-
if (row.state === DONE)
|
|
77
|
+
if (row.state === DONE) {
|
|
78
|
+
console.debug('Migration was applied already', { migration: id })
|
|
79
|
+
|
|
80
|
+
return false
|
|
81
|
+
}
|
|
56
82
|
|
|
57
83
|
const stale = Date.now() - row.heartbeat.getTime() > LEASE
|
|
58
84
|
|
|
59
85
|
if (!stale) {
|
|
86
|
+
/*
|
|
87
|
+
* The component does not serve until this returns, so a replica waiting here is a pod
|
|
88
|
+
* that is simply not up, with nothing anywhere saying why. On its own cadence rather
|
|
89
|
+
* than the poll's, which is a second.
|
|
90
|
+
*/
|
|
91
|
+
if (announced === undefined || Date.now() - announced > PROGRESS) {
|
|
92
|
+
announced = Date.now()
|
|
93
|
+
|
|
94
|
+
console.info('Waiting for another replica to apply a migration', {
|
|
95
|
+
migration: id,
|
|
96
|
+
owner: row.owner
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
|
|
60
100
|
await sleep(POLL)
|
|
61
101
|
|
|
62
102
|
continue
|
|
@@ -68,7 +108,11 @@ export class Migrations {
|
|
|
68
108
|
heartbeat: row.heartbeat
|
|
69
109
|
})
|
|
70
110
|
|
|
71
|
-
if (await this.#steal(id, row.heartbeat))
|
|
111
|
+
if (await this.#steal(id, row.heartbeat)) {
|
|
112
|
+
await this.#run(id, migration)
|
|
113
|
+
|
|
114
|
+
return true
|
|
115
|
+
}
|
|
72
116
|
}
|
|
73
117
|
}
|
|
74
118
|
|
|
@@ -118,7 +162,10 @@ export class Migrations {
|
|
|
118
162
|
* @private
|
|
119
163
|
*/
|
|
120
164
|
async #run(id, migration) {
|
|
121
|
-
|
|
165
|
+
const total = migration.steps.length
|
|
166
|
+
const started = Date.now()
|
|
167
|
+
|
|
168
|
+
console.info('Applying migration', { migration: id, steps: total })
|
|
122
169
|
|
|
123
170
|
const beat = setInterval(() => {
|
|
124
171
|
this.#state
|
|
@@ -130,8 +177,30 @@ export class Migrations {
|
|
|
130
177
|
|
|
131
178
|
beat.unref?.()
|
|
132
179
|
|
|
180
|
+
let applying = 0
|
|
181
|
+
|
|
182
|
+
/*
|
|
183
|
+
* A backfill takes as long as the collection is large, and every replica of the group waits
|
|
184
|
+
* out the whole of it. On its own cadence rather than the heartbeat's, which is every five
|
|
185
|
+
* seconds and would say this a dozen times a minute.
|
|
186
|
+
*/
|
|
187
|
+
const progress = setInterval(() => {
|
|
188
|
+
console.info('Migration is still being applied', {
|
|
189
|
+
migration: id,
|
|
190
|
+
step: applying,
|
|
191
|
+
steps: total,
|
|
192
|
+
elapsed: Date.now() - started
|
|
193
|
+
})
|
|
194
|
+
}, PROGRESS)
|
|
195
|
+
|
|
196
|
+
progress.unref?.()
|
|
197
|
+
|
|
133
198
|
try {
|
|
134
|
-
for (const step of migration.steps)
|
|
199
|
+
for (const step of migration.steps) {
|
|
200
|
+
applying++
|
|
201
|
+
|
|
202
|
+
await this.#step(id, step)
|
|
203
|
+
}
|
|
135
204
|
|
|
136
205
|
await this.#state.updateOne(
|
|
137
206
|
{ _id: id },
|
|
@@ -139,9 +208,10 @@ export class Migrations {
|
|
|
139
208
|
)
|
|
140
209
|
} finally {
|
|
141
210
|
clearInterval(beat)
|
|
211
|
+
clearInterval(progress)
|
|
142
212
|
}
|
|
143
213
|
|
|
144
|
-
console.info('Migration applied', { migration: id })
|
|
214
|
+
console.info('Migration applied', { migration: id, elapsed: Date.now() - started })
|
|
145
215
|
}
|
|
146
216
|
|
|
147
217
|
/** @private */
|
|
@@ -257,6 +327,9 @@ const LEASE = 30_000
|
|
|
257
327
|
const HEARTBEAT = 5_000
|
|
258
328
|
const POLL = 1_000
|
|
259
329
|
|
|
330
|
+
/** how often a run that is taking its time says it is still going, and a wait that it is waiting */
|
|
331
|
+
const PROGRESS = 30_000
|
|
332
|
+
|
|
260
333
|
/**
|
|
261
334
|
* Which process holds a claim, and nothing more: it is read by whoever takes an abandoned
|
|
262
335
|
* one over, and told to whoever reads the collection. Random, because there is no identity
|
package/src/record.js
CHANGED
|
@@ -11,3 +11,84 @@ export function from(record) {
|
|
|
11
11
|
|
|
12
12
|
return { id: _id, ...rest }
|
|
13
13
|
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* How this collection's records are written and read back.
|
|
17
|
+
*
|
|
18
|
+
* A property the entity declares as a moment is held as a BSON date rather than as what the
|
|
19
|
+
* entity carries. It is what the TTL monitor reads — it skips a field that is not a date,
|
|
20
|
+
* silently — and what sorts and compares as a moment rather than as text or as a number that
|
|
21
|
+
* happens to be one. The entity keeps what it declared, so nothing outside this connector
|
|
22
|
+
* learns a type only one storage has.
|
|
23
|
+
*
|
|
24
|
+
* Two ways to say it, because they differ in what userland holds rather than in what is
|
|
25
|
+
* stored: `{ string, date-time }` for an application that carries ISO strings, and
|
|
26
|
+
* `{ integer, epoch-millis }` for one that carries what `Date.now()` answers. The system
|
|
27
|
+
* timestamps of every record are the second.
|
|
28
|
+
*
|
|
29
|
+
* Top-level properties only. One nested inside an object or an array is left as it is, and
|
|
30
|
+
* `norm` refuses it, so that the declaration does not mean two things.
|
|
31
|
+
*
|
|
32
|
+
* A component that declares no moment holds the plain pair, and pays nothing for any of this.
|
|
33
|
+
*/
|
|
34
|
+
export function codec(properties) {
|
|
35
|
+
/** @type {Array<[string, (value: Date) => unknown]>} */
|
|
36
|
+
const read = []
|
|
37
|
+
|
|
38
|
+
for (const [name, schema] of Object.entries(properties ?? {})) {
|
|
39
|
+
const cast = READ[schema?.format]
|
|
40
|
+
|
|
41
|
+
if (cast !== undefined && schema.type === TYPES[schema.format]) read.push([name, cast])
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const dates = read.map(([name]) => name)
|
|
45
|
+
|
|
46
|
+
if (dates.length === 0) return { to, from, dates }
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
dates,
|
|
50
|
+
|
|
51
|
+
// one way in for both: `Date` takes the milliseconds and the ISO string alike
|
|
52
|
+
to: (entity) => convert(to(entity), dates.map((name) => [name, date])),
|
|
53
|
+
|
|
54
|
+
from: (record) => {
|
|
55
|
+
const state = from(record)
|
|
56
|
+
|
|
57
|
+
return state === null ? null : convert(state, read)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Writes into an object either `to` or `from` has just made, so nothing the caller holds is
|
|
64
|
+
* touched.
|
|
65
|
+
*
|
|
66
|
+
* @private
|
|
67
|
+
*/
|
|
68
|
+
function convert(record, casts) {
|
|
69
|
+
for (const [name, cast] of casts) {
|
|
70
|
+
const value = record[name]
|
|
71
|
+
|
|
72
|
+
// absent, or the null a property that has not been written to holds
|
|
73
|
+
if (value === undefined || value === null) continue
|
|
74
|
+
|
|
75
|
+
record[name] = cast(value)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return record
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const date = (value) => new Date(value)
|
|
82
|
+
|
|
83
|
+
/*
|
|
84
|
+
* What a record written before the property was declared a moment holds is what it was given,
|
|
85
|
+
* and it is answered as it is: a collection is converted by a migration rather than by every
|
|
86
|
+
* read that finds one.
|
|
87
|
+
*/
|
|
88
|
+
const READ = {
|
|
89
|
+
'date-time': (value) => (value instanceof Date ? value.toISOString() : value),
|
|
90
|
+
'epoch-millis': (value) => (value instanceof Date ? value.getTime() : value)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** What each format is written on, so that one said of the wrong type is not acted on. */
|
|
94
|
+
const TYPES = { 'date-time': 'string', 'epoch-millis': 'integer' }
|
package/src/storage.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { Connector, exceptions } from '@toa.io/core'
|
|
2
2
|
import { console } from 'openspan'
|
|
3
3
|
import { translate } from './translate.js'
|
|
4
|
-
import {
|
|
4
|
+
import { codec } from './record.js'
|
|
5
5
|
import { Outbox } from './outbox.js'
|
|
6
6
|
import { Migrations } from './migrations.js'
|
|
7
|
+
import { SYSTEM } from './system.js'
|
|
7
8
|
import { ReturnDocument } from 'mongodb'
|
|
8
9
|
|
|
9
10
|
export class Storage extends Connector {
|
|
@@ -25,12 +26,25 @@ export class Storage extends Connector {
|
|
|
25
26
|
/** @type {Map<string, object>} span options per driver method */
|
|
26
27
|
#spans = new Map()
|
|
27
28
|
|
|
29
|
+
/** how a record is written and read back, which depends on what the entity declares */
|
|
30
|
+
#to
|
|
31
|
+
#from
|
|
32
|
+
|
|
33
|
+
/** properties held as BSON dates, so that a criterion against one is one too */
|
|
34
|
+
#dates
|
|
35
|
+
|
|
28
36
|
constructor(client, entity) {
|
|
29
37
|
super()
|
|
30
38
|
|
|
31
39
|
this.#client = client
|
|
32
40
|
this.#entity = entity
|
|
33
41
|
|
|
42
|
+
const { to, from, dates } = codec(entity?.properties)
|
|
43
|
+
|
|
44
|
+
this.#to = to
|
|
45
|
+
this.#from = from
|
|
46
|
+
this.#dates = dates
|
|
47
|
+
|
|
34
48
|
this.depends(client)
|
|
35
49
|
}
|
|
36
50
|
|
|
@@ -59,19 +73,19 @@ export class Storage extends Connector {
|
|
|
59
73
|
|
|
60
74
|
this.#spans.clear()
|
|
61
75
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
76
|
+
// the runtime's own come first: what a component declares is written against a collection
|
|
77
|
+
// whose system properties are already what this release holds them as
|
|
78
|
+
this.#migrations = new Migrations(this.#client.db, this.#collection, [
|
|
79
|
+
...SYSTEM,
|
|
80
|
+
...(this.#entity.migrations ?? [])
|
|
81
|
+
])
|
|
68
82
|
|
|
69
|
-
await this.#migrations
|
|
83
|
+
await this.#migrations.run()
|
|
70
84
|
await this.#outbox?.index()
|
|
71
85
|
}
|
|
72
86
|
|
|
73
87
|
async get(query) {
|
|
74
|
-
const { criteria, options } = translate(query)
|
|
88
|
+
const { criteria, options } = translate(query, this.#dates)
|
|
75
89
|
|
|
76
90
|
// identity lookups must return deleted records, so that callers
|
|
77
91
|
// can tell a deleted entity from a missing one
|
|
@@ -82,11 +96,11 @@ export class Storage extends Connector {
|
|
|
82
96
|
this.#collection.findOne(criteria, options)
|
|
83
97
|
)
|
|
84
98
|
|
|
85
|
-
return from(record)
|
|
99
|
+
return this.#from(record)
|
|
86
100
|
}
|
|
87
101
|
|
|
88
102
|
async find(query) {
|
|
89
|
-
const { criteria, options, sample } = translate(query)
|
|
103
|
+
const { criteria, options, sample } = translate(query, this.#dates)
|
|
90
104
|
|
|
91
105
|
if (query?.options?.deleted !== true) criteria.DELETED = null
|
|
92
106
|
|
|
@@ -99,7 +113,7 @@ export class Storage extends Connector {
|
|
|
99
113
|
)
|
|
100
114
|
: await this.aggregate(criteria, options, sample)
|
|
101
115
|
|
|
102
|
-
return recordset.map((item) => from(item))
|
|
116
|
+
return recordset.map((item) => this.#from(item))
|
|
103
117
|
}
|
|
104
118
|
|
|
105
119
|
/** @private */
|
|
@@ -114,17 +128,17 @@ export class Storage extends Connector {
|
|
|
114
128
|
}
|
|
115
129
|
|
|
116
130
|
async stream(query = undefined) {
|
|
117
|
-
const { criteria, options } = translate(query)
|
|
131
|
+
const { criteria, options } = translate(query, this.#dates)
|
|
118
132
|
|
|
119
133
|
if (query?.options?.deleted !== true) criteria.DELETED = null
|
|
120
134
|
|
|
121
135
|
this.debug('find (stream)', { criteria, options })
|
|
122
136
|
|
|
123
|
-
return this.#collection.find(criteria, options).stream({ transform: from })
|
|
137
|
+
return this.#collection.find(criteria, options).stream({ transform: this.#from })
|
|
124
138
|
}
|
|
125
139
|
|
|
126
140
|
async add(entity, session = undefined) {
|
|
127
|
-
const record = to(entity)
|
|
141
|
+
const record = this.#to(entity)
|
|
128
142
|
|
|
129
143
|
const result = await this.command('insertOne', { record }, () =>
|
|
130
144
|
this.#collection.insertOne(record, { session })
|
|
@@ -139,7 +153,7 @@ export class Storage extends Connector {
|
|
|
139
153
|
VERSION: entity.VERSION - 1
|
|
140
154
|
}
|
|
141
155
|
|
|
142
|
-
const record = to(entity)
|
|
156
|
+
const record = this.#to(entity)
|
|
143
157
|
|
|
144
158
|
const result = await this.command('findOneAndReplace', { criteria, record }, () =>
|
|
145
159
|
this.#collection.findOneAndReplace(criteria, record, { session })
|
|
@@ -189,7 +203,7 @@ export class Storage extends Connector {
|
|
|
189
203
|
if (entities.length === 0) return true
|
|
190
204
|
|
|
191
205
|
const operations = entities.map((entity) => {
|
|
192
|
-
const record = to(entity)
|
|
206
|
+
const record = this.#to(entity)
|
|
193
207
|
|
|
194
208
|
if (entity.VERSION === 1) {
|
|
195
209
|
const { VERSION, ...rest } = record
|
|
@@ -241,7 +255,7 @@ export class Storage extends Connector {
|
|
|
241
255
|
}
|
|
242
256
|
|
|
243
257
|
async upsert(query, changeset, row = undefined) {
|
|
244
|
-
const { criteria, options } = translate(query)
|
|
258
|
+
const { criteria, options } = translate(query, this.#dates)
|
|
245
259
|
|
|
246
260
|
if (!('DELETED' in changeset) || changeset.DELETED === null) {
|
|
247
261
|
delete criteria.DELETED
|
|
@@ -266,7 +280,7 @@ export class Storage extends Connector {
|
|
|
266
280
|
|
|
267
281
|
if (found === null) return null
|
|
268
282
|
|
|
269
|
-
const origin = from(found)
|
|
283
|
+
const origin = this.#from(found)
|
|
270
284
|
|
|
271
285
|
/*
|
|
272
286
|
* The post-image is `update` applied to the pre-image, computed rather than read back.
|
|
@@ -293,11 +307,11 @@ export class Storage extends Connector {
|
|
|
293
307
|
}
|
|
294
308
|
|
|
295
309
|
async ensure(query, properties, state, row = undefined) {
|
|
296
|
-
let { criteria, options } = translate(query)
|
|
310
|
+
let { criteria, options } = translate(query, this.#dates)
|
|
297
311
|
|
|
298
312
|
if (query === undefined) criteria = properties
|
|
299
313
|
|
|
300
|
-
const update = { $setOnInsert: to(state) }
|
|
314
|
+
const update = { $setOnInsert: this.#to(state) }
|
|
301
315
|
|
|
302
316
|
options.upsert = true
|
|
303
317
|
options.returnDocument = ReturnDocument.AFTER
|
|
@@ -327,7 +341,7 @@ export class Storage extends Connector {
|
|
|
327
341
|
})
|
|
328
342
|
|
|
329
343
|
if (result.DELETED !== undefined && result.DELETED !== null) return null
|
|
330
|
-
else return from(result)
|
|
344
|
+
else return this.#from(result)
|
|
331
345
|
} catch (error) {
|
|
332
346
|
if (error.code === ERR_DUPLICATE_KEY)
|
|
333
347
|
throw new exceptions.DuplicateException(this.#client.name)
|
package/src/system.js
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migrations the runtime owns, applied to every collection ahead of whatever the component
|
|
3
|
+
* declares.
|
|
4
|
+
*
|
|
5
|
+
* These convert what the runtime itself writes, so they are declared here rather than in a
|
|
6
|
+
* migration every component would have to write — the same reason the outbox keeps its own
|
|
7
|
+
* indexes. A component cannot write them: it does not own the system properties, and a
|
|
8
|
+
* prototype's migrations are not inherited.
|
|
9
|
+
*
|
|
10
|
+
* They are applied once for the database, recorded beside the component's own, and named so
|
|
11
|
+
* that the two can never collide.
|
|
12
|
+
*/
|
|
13
|
+
export const SYSTEM = [
|
|
14
|
+
{
|
|
15
|
+
id: 'system:0001-epoch-millis',
|
|
16
|
+
steps: [
|
|
17
|
+
{
|
|
18
|
+
/*
|
|
19
|
+
* The system timestamps became dates: what sorts, compares and expires as a moment
|
|
20
|
+
* rather than as a number that happens to be one. A record written before this holds
|
|
21
|
+
* them as milliseconds, and a collection holding both would sort every number ahead of
|
|
22
|
+
* every date — so the release that brings this stops the deployment first.
|
|
23
|
+
*
|
|
24
|
+
* `CREATED` says whether a record has been converted, because every record has one.
|
|
25
|
+
* Null passes through both conversions, which is what an undeleted record's `DELETED`
|
|
26
|
+
* is; and the widening is not decoration — MongoDB makes a date from a long and
|
|
27
|
+
* refuses one from an int, and a small enough millisecond is stored as an int.
|
|
28
|
+
*/
|
|
29
|
+
update: {
|
|
30
|
+
filter: { CREATED: { $type: 'number' } },
|
|
31
|
+
update: [
|
|
32
|
+
{
|
|
33
|
+
$set: {
|
|
34
|
+
CREATED: { $toDate: { $toLong: '$CREATED' } },
|
|
35
|
+
UPDATED: { $toDate: { $toLong: '$UPDATED' } },
|
|
36
|
+
DELETED: { $toDate: { $toLong: '$DELETED' } }
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
]
|
|
43
|
+
}
|
|
44
|
+
]
|
|
@@ -2,13 +2,14 @@ import { rename } from './rename.js'
|
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* @param {import('@toa.io/core/types').storages.Node} node
|
|
5
|
+
* @param {string[]} dates properties held as BSON dates
|
|
5
6
|
* @returns {import('mongodb').Filter}
|
|
6
7
|
*/
|
|
7
|
-
export const criteria = (node) => {
|
|
8
|
+
export const criteria = (node, dates = NONE) => {
|
|
8
9
|
if (TYPES[node.type] === undefined)
|
|
9
10
|
throw new Error(`AST parse error: unknown node type '${node.type}'`)
|
|
10
11
|
|
|
11
|
-
return TYPES[node.type](node)
|
|
12
|
+
return TYPES[node.type](node, dates)
|
|
12
13
|
}
|
|
13
14
|
|
|
14
15
|
const OPERATORS = {
|
|
@@ -32,23 +33,33 @@ const OPERATORS = {
|
|
|
32
33
|
|
|
33
34
|
const TYPES = {}
|
|
34
35
|
|
|
35
|
-
TYPES.LOGIC = (expression) => {
|
|
36
|
-
const left = criteria(expression.left)
|
|
37
|
-
const right = criteria(expression.right)
|
|
36
|
+
TYPES.LOGIC = (expression, dates) => {
|
|
37
|
+
const left = criteria(expression.left, dates)
|
|
38
|
+
const right = criteria(expression.right, dates)
|
|
38
39
|
|
|
39
40
|
return { [OPERATORS.LOGIC[expression.operator]]: [left, right] }
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
TYPES.COMPARISON = (expression) => {
|
|
43
|
-
const left = criteria(expression.left)
|
|
44
|
-
const right = criteria(expression.right)
|
|
43
|
+
TYPES.COMPARISON = (expression, dates) => {
|
|
44
|
+
const left = criteria(expression.left, dates)
|
|
45
|
+
const right = criteria(expression.right, dates)
|
|
45
46
|
const operator = OPERATORS.COMPARISON[expression.operator]
|
|
46
47
|
|
|
47
48
|
if (operator === undefined)
|
|
48
49
|
throw new Error(`AST parse error: unknown operator '${expression.operator}'`)
|
|
49
50
|
|
|
50
|
-
|
|
51
|
+
// the record holds a date where the criterion carries the string the entity declares, and a
|
|
52
|
+
// string compared against a date is a criterion that quietly matches nothing
|
|
53
|
+
const value = dates.includes(left) ? date(right) : right
|
|
54
|
+
|
|
55
|
+
return { [left]: { [operator]: value } }
|
|
51
56
|
}
|
|
52
57
|
|
|
53
58
|
TYPES.SELECTOR = (expression) => rename(expression.selector)
|
|
54
59
|
TYPES.VALUE = (expression) => expression.value
|
|
60
|
+
|
|
61
|
+
/** `=in=` and `=out=` carry a list, and every other operator one value. */
|
|
62
|
+
const date = (value) =>
|
|
63
|
+
Array.isArray(value) ? value.map(date) : value === null ? null : new Date(value)
|
|
64
|
+
|
|
65
|
+
const NONE = []
|
package/src/translate.js
CHANGED
|
@@ -5,11 +5,12 @@ const parse = { ..._criteria, ..._options }
|
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* @param {import('@toa.io/core/types').storages.Query} query
|
|
8
|
+
* @param {string[]} [dates] properties held as BSON dates, so a criterion against one is one too
|
|
8
9
|
* @returns {{criteria: Object, options: Object}}
|
|
9
10
|
*/
|
|
10
|
-
export const translate = (query) => {
|
|
11
|
+
export const translate = (query, dates) => {
|
|
11
12
|
const result = {
|
|
12
|
-
criteria: query?.criteria === undefined ? {} : parse.criteria(query.criteria),
|
|
13
|
+
criteria: query?.criteria === undefined ? {} : parse.criteria(query.criteria, dates),
|
|
13
14
|
options: query?.options === undefined ? {} : parse.options(query.options),
|
|
14
15
|
sample: query?.options?.sample
|
|
15
16
|
}
|
package/test/migrations.test.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { describe, it, beforeEach, mock } from 'node:test'
|
|
1
|
+
import { describe, it, beforeEach, afterEach, mock } from 'node:test'
|
|
2
|
+
import { console } from 'openspan'
|
|
2
3
|
import assert from 'node:assert/strict'
|
|
3
4
|
|
|
4
5
|
import { Migrations, STATE } from '../src/migrations.js'
|
|
@@ -331,3 +332,57 @@ describe('state', () => {
|
|
|
331
332
|
assert.equal(rows.size, 1)
|
|
332
333
|
})
|
|
333
334
|
})
|
|
335
|
+
|
|
336
|
+
describe('logging', () => {
|
|
337
|
+
let info
|
|
338
|
+
let debug
|
|
339
|
+
|
|
340
|
+
const messages = (spy) => spy.mock.calls.map((call) => call.arguments[0])
|
|
341
|
+
|
|
342
|
+
beforeEach(() => {
|
|
343
|
+
info = mock.method(console, 'info', () => undefined)
|
|
344
|
+
debug = mock.method(console, 'debug', () => undefined)
|
|
345
|
+
})
|
|
346
|
+
|
|
347
|
+
afterEach(() => {
|
|
348
|
+
info.mock.restore()
|
|
349
|
+
debug.mock.restore()
|
|
350
|
+
})
|
|
351
|
+
|
|
352
|
+
it('should report a migration it applies', async () => {
|
|
353
|
+
await run([{ id: '0001', steps: [{ update: { update: {} } }] }])
|
|
354
|
+
|
|
355
|
+
assert.ok(messages(info).includes('Applying migration'))
|
|
356
|
+
assert.ok(messages(info).includes('Migration applied'))
|
|
357
|
+
})
|
|
358
|
+
|
|
359
|
+
it('should say at debug that a migration was applied already', async () => {
|
|
360
|
+
const list = [{ id: '0001', steps: [{ update: { update: {} } }] }]
|
|
361
|
+
|
|
362
|
+
await run(list)
|
|
363
|
+
|
|
364
|
+
info.mock.resetCalls()
|
|
365
|
+
debug.mock.resetCalls()
|
|
366
|
+
|
|
367
|
+
await run(list)
|
|
368
|
+
|
|
369
|
+
assert.deepStrictEqual(messages(info), [], 'the ordinary start has nothing to report')
|
|
370
|
+
assert.ok(messages(debug).includes('Migration was applied already'))
|
|
371
|
+
})
|
|
372
|
+
|
|
373
|
+
it('should say how many of the declared ones it applied', async () => {
|
|
374
|
+
await run([
|
|
375
|
+
{ id: '0001', steps: [{ update: { update: {} } }] },
|
|
376
|
+
{ id: '0002', steps: [{ update: { update: {} } }] }
|
|
377
|
+
])
|
|
378
|
+
|
|
379
|
+
const [message, attributes] = debug.mock.calls.at(-1).arguments
|
|
380
|
+
|
|
381
|
+
assert.strictEqual(message, 'Migrations checked')
|
|
382
|
+
assert.deepStrictEqual(attributes, {
|
|
383
|
+
collection: 'test_one',
|
|
384
|
+
declared: 2,
|
|
385
|
+
applied: 2
|
|
386
|
+
})
|
|
387
|
+
})
|
|
388
|
+
})
|
package/test/record.test.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, it } from 'node:test'
|
|
2
2
|
import assert from 'node:assert/strict'
|
|
3
3
|
|
|
4
|
-
import { to, from } from '../src/record.js'
|
|
4
|
+
import { to, from, codec } from '../src/record.js'
|
|
5
5
|
|
|
6
6
|
describe('to', () => {
|
|
7
7
|
it('should rename id to _id', () => {
|
|
@@ -46,3 +46,93 @@ describe('from', () => {
|
|
|
46
46
|
})
|
|
47
47
|
})
|
|
48
48
|
})
|
|
49
|
+
|
|
50
|
+
describe('codec', () => {
|
|
51
|
+
const properties = {
|
|
52
|
+
settled: { type: 'string', format: 'date-time' },
|
|
53
|
+
endpoint: { type: 'string' }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
it('should hold a plain record where the entity declares no date', () => {
|
|
57
|
+
const { to: write, from: read, dates } = codec({ endpoint: { type: 'string' } })
|
|
58
|
+
|
|
59
|
+
assert.deepStrictEqual(dates, [])
|
|
60
|
+
assert.strictEqual(write, to)
|
|
61
|
+
assert.strictEqual(read, from)
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('should write a date-time property as a date', () => {
|
|
65
|
+
const { to: write } = codec(properties)
|
|
66
|
+
const record = write({ id: '1', settled: '2026-09-05T10:00:00.000Z', endpoint: 'a.b.c' })
|
|
67
|
+
|
|
68
|
+
assert.ok(record.settled instanceof Date)
|
|
69
|
+
assert.strictEqual(record.settled.toISOString(), '2026-09-05T10:00:00.000Z')
|
|
70
|
+
assert.strictEqual(record.endpoint, 'a.b.c', 'and leaves every other property alone')
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
it('should read a date back as the string the entity carries', () => {
|
|
74
|
+
const { from: read } = codec(properties)
|
|
75
|
+
const entity = read({ _id: '1', settled: new Date('2026-09-05T10:00:00.000Z') })
|
|
76
|
+
|
|
77
|
+
assert.strictEqual(entity.settled, '2026-09-05T10:00:00.000Z')
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
it('should leave a property that has not been written to', () => {
|
|
81
|
+
const { to: write, from: read } = codec(properties)
|
|
82
|
+
|
|
83
|
+
assert.strictEqual(write({ id: '1', settled: null }).settled, null)
|
|
84
|
+
assert.strictEqual(read({ _id: '1' }).settled, undefined)
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('should read a record written before the property was declared a date', () => {
|
|
88
|
+
const { from: read } = codec(properties)
|
|
89
|
+
|
|
90
|
+
assert.strictEqual(read({ _id: '1', settled: 'not a date' }).settled, 'not a date')
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
it('should not modify argument', () => {
|
|
94
|
+
const { to: write } = codec(properties)
|
|
95
|
+
const entity = { id: '1', settled: '2026-09-05T10:00:00.000Z' }
|
|
96
|
+
|
|
97
|
+
write(entity)
|
|
98
|
+
|
|
99
|
+
assert.strictEqual(entity.settled, '2026-09-05T10:00:00.000Z')
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
it('should ignore a format said of the wrong type', () => {
|
|
103
|
+
assert.deepStrictEqual(codec({ at: { type: 'string', format: 'epoch-millis' } }).dates, [])
|
|
104
|
+
assert.deepStrictEqual(codec({ at: { type: 'integer', format: 'date-time' } }).dates, [])
|
|
105
|
+
})
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
describe('codec, epoch-millis', () => {
|
|
109
|
+
const properties = {
|
|
110
|
+
DELETED: { type: 'integer', format: 'epoch-millis' },
|
|
111
|
+
VERSION: { type: 'integer' }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const millis = 1788698804112
|
|
115
|
+
|
|
116
|
+
it('should write milliseconds as a date', () => {
|
|
117
|
+
const record = codec(properties).to({ id: '1', DELETED: millis, VERSION: 2 })
|
|
118
|
+
|
|
119
|
+
assert.ok(record.DELETED instanceof Date)
|
|
120
|
+
assert.strictEqual(record.DELETED.getTime(), millis)
|
|
121
|
+
assert.strictEqual(record.VERSION, 2, 'and leaves a plain integer alone')
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
it('should read a date back as milliseconds', () => {
|
|
125
|
+
const entity = codec(properties).from({ _id: '1', DELETED: new Date(millis) })
|
|
126
|
+
|
|
127
|
+
assert.strictEqual(entity.DELETED, millis)
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
it('should leave the null of a record that is not deleted', () => {
|
|
131
|
+
assert.strictEqual(codec(properties).to({ id: '1', DELETED: null }).DELETED, null)
|
|
132
|
+
assert.strictEqual(codec(properties).from({ _id: '1', DELETED: null }).DELETED, null)
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
it('should read a record written before the property was a date', () => {
|
|
136
|
+
assert.strictEqual(codec(properties).from({ _id: '1', DELETED: millis }).DELETED, millis)
|
|
137
|
+
})
|
|
138
|
+
})
|
package/test/storage.test.js
CHANGED
|
@@ -11,11 +11,21 @@ beforeEach(async () => {
|
|
|
11
11
|
collection = {
|
|
12
12
|
collectionName: 'test',
|
|
13
13
|
findOne: mock.fn(async () => null),
|
|
14
|
-
find: mock.fn(() => ({ stream: () => null }))
|
|
14
|
+
find: mock.fn(() => ({ stream: () => null })),
|
|
15
|
+
updateMany: mock.fn(async () => ({ modifiedCount: 0 }))
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// the runtime's own migrations are applied to every collection, so a storage needs the
|
|
19
|
+
// database its record of them lives in
|
|
20
|
+
const state = {
|
|
21
|
+
insertOne: mock.fn(async () => ({})),
|
|
22
|
+
findOne: mock.fn(async () => null),
|
|
23
|
+
updateOne: mock.fn(async () => ({ modifiedCount: 1 }))
|
|
15
24
|
}
|
|
16
25
|
|
|
17
26
|
const client = {
|
|
18
27
|
collection,
|
|
28
|
+
db: { collection: () => state },
|
|
19
29
|
link: () => null
|
|
20
30
|
}
|
|
21
31
|
|
package/test/translate.test.js
CHANGED
|
@@ -42,3 +42,44 @@ describe('options', () => {
|
|
|
42
42
|
assert.deepStrictEqual(query.options.projection, { a: 1, b: 1, c: 1, _id: 1 })
|
|
43
43
|
})
|
|
44
44
|
})
|
|
45
|
+
|
|
46
|
+
describe('dates', () => {
|
|
47
|
+
const comparison = (selector, operator, value) => ({
|
|
48
|
+
type: 'COMPARISON',
|
|
49
|
+
operator,
|
|
50
|
+
left: { type: 'SELECTOR', selector },
|
|
51
|
+
right: { type: 'VALUE', value }
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
it('should compare a date property against a date', () => {
|
|
55
|
+
const criteria = comparison('settled', '<', '2026-09-05T10:00:00.000Z')
|
|
56
|
+
const query = translate({ criteria }, ['settled'])
|
|
57
|
+
|
|
58
|
+
assert.ok(query.criteria.settled.$lt instanceof Date)
|
|
59
|
+
assert.strictEqual(
|
|
60
|
+
query.criteria.settled.$lt.toISOString(),
|
|
61
|
+
'2026-09-05T10:00:00.000Z'
|
|
62
|
+
)
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
it('should convert every value of a list', () => {
|
|
66
|
+
const criteria = comparison('settled', '=in=', ['2026-09-05T10:00:00.000Z'])
|
|
67
|
+
const query = translate({ criteria }, ['settled'])
|
|
68
|
+
|
|
69
|
+
assert.ok(query.criteria.settled.$in[0] instanceof Date)
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('should leave a property that is not a date', () => {
|
|
73
|
+
const criteria = comparison('endpoint', '==', 'a.b.c')
|
|
74
|
+
const query = translate({ criteria }, ['settled'])
|
|
75
|
+
|
|
76
|
+
assert.deepStrictEqual(query.criteria, { endpoint: { $eq: 'a.b.c' } })
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
it('should leave every criterion where the entity declares no date', () => {
|
|
80
|
+
const criteria = comparison('settled', '<', '2026-09-05T10:00:00.000Z')
|
|
81
|
+
const query = translate({ criteria })
|
|
82
|
+
|
|
83
|
+
assert.strictEqual(query.criteria.settled.$lt, '2026-09-05T10:00:00.000Z')
|
|
84
|
+
})
|
|
85
|
+
})
|