@toa.io/storages.mongodb 1.0.0-alpha.33 → 1.0.0-alpha.331
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +365 -0
- package/package.json +10 -9
- package/readme.md +38 -0
- package/src/client.js +181 -40
- package/src/factory.js +6 -12
- package/src/inbox.js +117 -0
- package/src/index.js +1 -7
- package/src/indexes.js +84 -0
- package/src/match.js +108 -0
- package/src/measurements.js +39 -0
- package/src/migrations.js +357 -0
- package/src/outbox.js +131 -0
- package/src/record.js +97 -8
- package/src/storage.js +554 -121
- package/src/streams.js +400 -0
- package/src/translate/criteria.js +25 -18
- package/src/translate/options.js +20 -11
- package/src/translate/rename.js +1 -5
- package/src/translate.js +14 -15
- package/test/match.test.js +70 -0
- package/test/migrations.test.js +388 -0
- package/test/query/criteria.fixtures.js +2 -7
- package/test/query/criteria.test.js +5 -4
- package/test/record.test.js +142 -15
- package/test/storage.test.js +319 -0
- package/test/streams.test.js +60 -0
- package/test/translate.test.js +59 -7
- package/types/connection.d.ts +18 -12
- package/types/pointer.d.ts +4 -6
- package/types/record.d.ts +5 -7
- package/src/collection.js +0 -71
- package/src/deployment.js +0 -22
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { registry } from 'openspan'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What this storage measures. `provider` is which database it is, so that a deployment running more
|
|
5
|
+
* than one can be read a kind at a time. No database label: the database is the context, which
|
|
6
|
+
* every series already carries as its resource.
|
|
7
|
+
*/
|
|
8
|
+
const meters = registry()
|
|
9
|
+
|
|
10
|
+
/** Prometheus' own ladder, in seconds: from five milliseconds to ten seconds. */
|
|
11
|
+
const DURATIONS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
|
|
12
|
+
|
|
13
|
+
const PROVIDER = 'mongodb'
|
|
14
|
+
|
|
15
|
+
const duration = meters.histogram(
|
|
16
|
+
'toa.storage.query.duration',
|
|
17
|
+
{ buckets: DURATIONS, unit: 's' },
|
|
18
|
+
{ provider: null, collection: null, operation: null }
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A compare-and-swap that lost. Whether it is retried or raised is the operation's to declare, and
|
|
23
|
+
* the retried one succeeds — so without this it is a duration that grew and nothing else.
|
|
24
|
+
*/
|
|
25
|
+
const conflicts = meters.counter('toa.storage.conflicts', {
|
|
26
|
+
provider: null,
|
|
27
|
+
collection: null
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
export function query(collection, operation) {
|
|
31
|
+
return {
|
|
32
|
+
histogram: duration,
|
|
33
|
+
labels: { provider: PROVIDER, collection, operation }
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function conflicted(collection) {
|
|
38
|
+
conflicts.add(1, { provider: PROVIDER, collection })
|
|
39
|
+
}
|
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import { setTimeout as sleep } from 'node:timers/promises'
|
|
3
|
+
|
|
4
|
+
import { console } from 'openspan'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Applies a component's migrations to its collection, in the order they were declared, and
|
|
8
|
+
* records each one so that it is applied once for the database rather than once per replica.
|
|
9
|
+
*
|
|
10
|
+
* The record is also the lock: the row that says a migration ran is inserted before it runs,
|
|
11
|
+
* and MongoDB refuses the second insert of the same `_id`. Nothing else is needed to make the
|
|
12
|
+
* group agree, which is why this works where there is no Redis.
|
|
13
|
+
*/
|
|
14
|
+
export class Migrations {
|
|
15
|
+
/** @type {import('mongodb').Collection} */
|
|
16
|
+
#collection
|
|
17
|
+
|
|
18
|
+
/** @type {import('mongodb').Collection} */
|
|
19
|
+
#state
|
|
20
|
+
|
|
21
|
+
/** @type {Array<{ id: string, steps: object[] }>} */
|
|
22
|
+
#list
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {import('mongodb').Db} db
|
|
26
|
+
* @param {import('mongodb').Collection} collection the entity's own
|
|
27
|
+
* @param {Array<{ id: string, steps: object[] }>} list
|
|
28
|
+
*/
|
|
29
|
+
constructor(db, collection, list) {
|
|
30
|
+
this.#collection = collection
|
|
31
|
+
this.#state = db.collection(STATE)
|
|
32
|
+
this.#list = list
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* A migration is applied only after the one before it, because a later one is written
|
|
37
|
+
* against what an earlier one leaves behind.
|
|
38
|
+
*/
|
|
39
|
+
async run() {
|
|
40
|
+
let applied = 0
|
|
41
|
+
|
|
42
|
+
for (const migration of this.#list) if (await this.#apply(migration)) applied++
|
|
43
|
+
|
|
44
|
+
// debug, because the ordinary start has nothing to report: every migration was applied
|
|
45
|
+
// long ago by whoever started first. But a run that says nothing at all cannot be told
|
|
46
|
+
// from one that never looked
|
|
47
|
+
console.debug('Migrations checked', {
|
|
48
|
+
collection: this.#collection.collectionName,
|
|
49
|
+
declared: this.#list.length,
|
|
50
|
+
applied
|
|
51
|
+
})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Answers whether this replica was the one that applied it.
|
|
56
|
+
*
|
|
57
|
+
* @private
|
|
58
|
+
*/
|
|
59
|
+
async #apply(migration) {
|
|
60
|
+
const id = `${this.#collection.collectionName}:${migration.id}`
|
|
61
|
+
|
|
62
|
+
/** when the wait for another replica was last reported */
|
|
63
|
+
let announced
|
|
64
|
+
|
|
65
|
+
while (true) {
|
|
66
|
+
if (await this.#claim(id)) {
|
|
67
|
+
await this.#run(id, migration)
|
|
68
|
+
|
|
69
|
+
return true
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const row = await this.#state.findOne({ _id: id })
|
|
73
|
+
|
|
74
|
+
// removed between the claim and the read; whoever did that wants it applied again
|
|
75
|
+
if (row === null) continue
|
|
76
|
+
|
|
77
|
+
if (row.state === DONE) {
|
|
78
|
+
console.debug('Migration was applied already', { migration: id })
|
|
79
|
+
|
|
80
|
+
return false
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const stale = Date.now() - row.heartbeat.getTime() > LEASE
|
|
84
|
+
|
|
85
|
+
if (!stale) {
|
|
86
|
+
/*
|
|
87
|
+
* The component does not serve until this returns, so a replica waiting here is a pod
|
|
88
|
+
* that is simply not up, with nothing anywhere saying why. On its own cadence rather
|
|
89
|
+
* than the poll's, which is a second.
|
|
90
|
+
*/
|
|
91
|
+
if (announced === undefined || Date.now() - announced > PROGRESS) {
|
|
92
|
+
announced = Date.now()
|
|
93
|
+
|
|
94
|
+
console.info('Waiting for another replica to apply a migration', {
|
|
95
|
+
migration: id,
|
|
96
|
+
owner: row.owner
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
await sleep(POLL)
|
|
101
|
+
|
|
102
|
+
continue
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
console.warn('Taking over an abandoned migration', {
|
|
106
|
+
migration: id,
|
|
107
|
+
owner: row.owner,
|
|
108
|
+
heartbeat: row.heartbeat
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
if (await this.#steal(id, row.heartbeat)) {
|
|
112
|
+
await this.#run(id, migration)
|
|
113
|
+
|
|
114
|
+
return true
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Writes the row that says this replica is applying the migration. Answers whether it won.
|
|
121
|
+
*
|
|
122
|
+
* @private
|
|
123
|
+
*/
|
|
124
|
+
async #claim(id) {
|
|
125
|
+
try {
|
|
126
|
+
await this.#state.insertOne({
|
|
127
|
+
_id: id,
|
|
128
|
+
state: RUNNING,
|
|
129
|
+
owner: OWNER,
|
|
130
|
+
started: new Date(),
|
|
131
|
+
heartbeat: new Date()
|
|
132
|
+
})
|
|
133
|
+
|
|
134
|
+
return true
|
|
135
|
+
} catch (error) {
|
|
136
|
+
if (error.code === ERR_DUPLICATE_KEY) return false
|
|
137
|
+
|
|
138
|
+
throw error
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Takes a claim whose owner stopped saying it was alive. The heartbeat it read is part of
|
|
144
|
+
* the criteria, so only one of several waiting replicas takes it.
|
|
145
|
+
*
|
|
146
|
+
* @private
|
|
147
|
+
*/
|
|
148
|
+
async #steal(id, heartbeat) {
|
|
149
|
+
const result = await this.#state.updateOne(
|
|
150
|
+
{ _id: id, state: RUNNING, heartbeat },
|
|
151
|
+
{ $set: { owner: OWNER, heartbeat: new Date() } }
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
return result.modifiedCount === 1
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Applies the steps and marks the migration done. A step that throws leaves the row as it
|
|
159
|
+
* is: the component does not start, and the next replica to reach a stale claim runs the
|
|
160
|
+
* migration again from its first step.
|
|
161
|
+
*
|
|
162
|
+
* @private
|
|
163
|
+
*/
|
|
164
|
+
async #run(id, migration) {
|
|
165
|
+
const total = migration.steps.length
|
|
166
|
+
const started = Date.now()
|
|
167
|
+
|
|
168
|
+
console.info('Applying migration', { migration: id, steps: total })
|
|
169
|
+
|
|
170
|
+
const beat = setInterval(() => {
|
|
171
|
+
this.#state
|
|
172
|
+
.updateOne({ _id: id }, { $set: { heartbeat: new Date() } })
|
|
173
|
+
.catch((error) =>
|
|
174
|
+
console.warn('Migration heartbeat failed', { migration: id, error })
|
|
175
|
+
)
|
|
176
|
+
}, HEARTBEAT)
|
|
177
|
+
|
|
178
|
+
beat.unref?.()
|
|
179
|
+
|
|
180
|
+
let applying = 0
|
|
181
|
+
|
|
182
|
+
/*
|
|
183
|
+
* A backfill takes as long as the collection is large, and every replica of the group waits
|
|
184
|
+
* out the whole of it. On its own cadence rather than the heartbeat's, which is every five
|
|
185
|
+
* seconds and would say this a dozen times a minute.
|
|
186
|
+
*/
|
|
187
|
+
const progress = setInterval(() => {
|
|
188
|
+
console.info('Migration is still being applied', {
|
|
189
|
+
migration: id,
|
|
190
|
+
step: applying,
|
|
191
|
+
steps: total,
|
|
192
|
+
elapsed: Date.now() - started
|
|
193
|
+
})
|
|
194
|
+
}, PROGRESS)
|
|
195
|
+
|
|
196
|
+
progress.unref?.()
|
|
197
|
+
|
|
198
|
+
try {
|
|
199
|
+
for (const step of migration.steps) {
|
|
200
|
+
applying++
|
|
201
|
+
|
|
202
|
+
await this.#step(id, step)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
await this.#state.updateOne(
|
|
206
|
+
{ _id: id },
|
|
207
|
+
{ $set: { state: DONE, completed: new Date() } }
|
|
208
|
+
)
|
|
209
|
+
} finally {
|
|
210
|
+
clearInterval(beat)
|
|
211
|
+
clearInterval(progress)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
console.info('Migration applied', { migration: id, elapsed: Date.now() - started })
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** @private */
|
|
218
|
+
async #step(id, step) {
|
|
219
|
+
const verbs = Object.keys(step ?? {})
|
|
220
|
+
|
|
221
|
+
if (verbs.length !== 1 || !(verbs[0] in STEPS))
|
|
222
|
+
throw new Error(
|
|
223
|
+
`Migration '${id}' has a step that is not one of ` +
|
|
224
|
+
`${Object.keys(STEPS).join(', ')}: ${JSON.stringify(step)}`
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
await STEPS[verbs[0]](this.#collection, step[verbs[0]], id)
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Creates the index the step declares. Where the name is held by an index of a different shape,
|
|
233
|
+
* that one is dropped: the declaration is what the component is to run against, and refusing
|
|
234
|
+
* to reconcile is what left a database diverged from its manifest before migrations existed.
|
|
235
|
+
*/
|
|
236
|
+
async function index(collection, { name, keys, ...rest }, id) {
|
|
237
|
+
if (name === undefined || keys === undefined)
|
|
238
|
+
throw new Error(`Migration '${id}' declares an index without a name or keys`)
|
|
239
|
+
|
|
240
|
+
const spec = Object.fromEntries(
|
|
241
|
+
Object.entries(keys).map(([field, direction]) => [
|
|
242
|
+
field,
|
|
243
|
+
DIRECTIONS[direction] ?? direction
|
|
244
|
+
])
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
const options = { name }
|
|
248
|
+
|
|
249
|
+
for (const [key, value] of Object.entries(rest)) {
|
|
250
|
+
if (!(key in OPTIONS))
|
|
251
|
+
throw new Error(`Migration '${id}' declares an unknown index option '${key}'`)
|
|
252
|
+
|
|
253
|
+
options[OPTIONS[key]] = value
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
try {
|
|
257
|
+
await collection.createIndex(spec, options)
|
|
258
|
+
} catch (error) {
|
|
259
|
+
if (!CONFLICTS.includes(error.code)) throw error
|
|
260
|
+
|
|
261
|
+
console.info('Recreating an index whose declaration changed', { index: name })
|
|
262
|
+
|
|
263
|
+
await collection.dropIndex(name)
|
|
264
|
+
await collection.createIndex(spec, options)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function dropIndex(collection, { name }, id) {
|
|
269
|
+
if (name === undefined)
|
|
270
|
+
throw new Error(`Migration '${id}' drops an index without a name`)
|
|
271
|
+
|
|
272
|
+
try {
|
|
273
|
+
await collection.dropIndex(name)
|
|
274
|
+
} catch (error) {
|
|
275
|
+
if (error.code !== ERR_INDEX_NOT_FOUND) throw error
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
async function update(collection, { filter, update: changeset }, id) {
|
|
280
|
+
if (changeset === undefined) throw new Error(`Migration '${id}' updates with nothing`)
|
|
281
|
+
|
|
282
|
+
const result = await collection.updateMany(filter ?? {}, changeset)
|
|
283
|
+
|
|
284
|
+
console.info('Migration updated records', {
|
|
285
|
+
migration: id,
|
|
286
|
+
records: result.modifiedCount
|
|
287
|
+
})
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
async function remove(collection, { filter }, id) {
|
|
291
|
+
if (filter === undefined)
|
|
292
|
+
throw new Error(
|
|
293
|
+
`Migration '${id}' deletes without a filter; pass {} to mean every record`
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
const result = await collection.deleteMany(filter)
|
|
297
|
+
|
|
298
|
+
console.info('Migration deleted records', {
|
|
299
|
+
migration: id,
|
|
300
|
+
records: result.deletedCount
|
|
301
|
+
})
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Keeps, or stops keeping, what a record was before each change and after it, which a stream
|
|
306
|
+
* reads to tell an entry that left a set from a change outside it.
|
|
307
|
+
*/
|
|
308
|
+
async function images(collection, enabled, id) {
|
|
309
|
+
if (typeof enabled !== 'boolean')
|
|
310
|
+
throw new Error(`Migration '${id}' declares images that are neither true nor false`)
|
|
311
|
+
|
|
312
|
+
await collection.s.db.command({
|
|
313
|
+
collMod: collection.collectionName,
|
|
314
|
+
changeStreamPreAndPostImages: { enabled }
|
|
315
|
+
})
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const STEPS = { index, dropIndex, update, delete: remove, images }
|
|
319
|
+
|
|
320
|
+
const DIRECTIONS = { asc: 1, desc: -1, hash: 'hashed' }
|
|
321
|
+
|
|
322
|
+
/** what an index step may say, and what the driver calls it */
|
|
323
|
+
const OPTIONS = {
|
|
324
|
+
unique: 'unique',
|
|
325
|
+
sparse: 'sparse',
|
|
326
|
+
partial: 'partialFilterExpression',
|
|
327
|
+
ttl: 'expireAfterSeconds'
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* A component's namespace may not be `system`, so this cannot be a component's own collection;
|
|
332
|
+
* and MongoDB reserves the `system.` prefix, with a dot, which this is not.
|
|
333
|
+
*/
|
|
334
|
+
export const STATE = 'system_migrations'
|
|
335
|
+
|
|
336
|
+
const RUNNING = 'running'
|
|
337
|
+
const DONE = 'done'
|
|
338
|
+
|
|
339
|
+
/** how long a claim outlives its last heartbeat before another replica may take it */
|
|
340
|
+
const LEASE = 30_000
|
|
341
|
+
const HEARTBEAT = 5_000
|
|
342
|
+
const POLL = 1_000
|
|
343
|
+
|
|
344
|
+
/** how often a run that is taking its time says it is still going, and a wait that it is waiting */
|
|
345
|
+
const PROGRESS = 30_000
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Which process holds a claim, and nothing more: it is read by whoever takes an abandoned
|
|
349
|
+
* one over, and told to whoever reads the collection. Random, because there is no identity
|
|
350
|
+
* a process is guaranteed to have — a host name says nothing about two replicas on one
|
|
351
|
+
* machine, and a pid is reused.
|
|
352
|
+
*/
|
|
353
|
+
const OWNER = randomUUID()
|
|
354
|
+
|
|
355
|
+
const ERR_DUPLICATE_KEY = 11000
|
|
356
|
+
const ERR_INDEX_NOT_FOUND = 27
|
|
357
|
+
const CONFLICTS = [85, 86] // IndexOptionsConflict, IndexKeySpecsConflict
|
package/src/outbox.js
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { environment } from '@toa.io/generic'
|
|
2
|
+
import { index, prune } from './indexes.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The outbox rows of one component. Its lifecycle is the Client's, so it is not a Connector.
|
|
6
|
+
*
|
|
7
|
+
* Core owns what a row means — its id, its lane and when it becomes due. This only writes
|
|
8
|
+
* them, reads back what is due, and marks what has been published.
|
|
9
|
+
*/
|
|
10
|
+
export class Outbox {
|
|
11
|
+
/** @type {import('mongodb').Collection} */
|
|
12
|
+
#collection
|
|
13
|
+
|
|
14
|
+
#retention
|
|
15
|
+
|
|
16
|
+
constructor(collection) {
|
|
17
|
+
this.#collection = collection
|
|
18
|
+
this.#retention = retention()
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** @param {import('mongodb').ClientSession} session */
|
|
22
|
+
async insert(row, session) {
|
|
23
|
+
await this.#collection.insertOne(to(row), { session })
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** @param {import('mongodb').ClientSession} session */
|
|
27
|
+
async insertMany(rows, session) {
|
|
28
|
+
if (rows.length === 0) return
|
|
29
|
+
|
|
30
|
+
await this.#collection.insertMany(rows.map(to), { session })
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* One page of what this replica should publish: due, still unpublished, and in a lane it
|
|
35
|
+
* owns. In steady state the first page is empty.
|
|
36
|
+
*
|
|
37
|
+
* `after` continues from the last id of the page before. Ids are uuid v7, so their order is
|
|
38
|
+
* the order rows were written and a page is never read twice within a cycle — which matters
|
|
39
|
+
* because a row stays unpublished in the database until the cycle that sent it marks it.
|
|
40
|
+
*/
|
|
41
|
+
async pending(lanes, now, limit, after = undefined) {
|
|
42
|
+
const criteria = { lane: { $in: lanes }, published: false, pending: { $lte: now } }
|
|
43
|
+
|
|
44
|
+
if (after !== undefined) criteria._id = { $gt: after }
|
|
45
|
+
|
|
46
|
+
const rows = await this.#collection
|
|
47
|
+
.find(criteria)
|
|
48
|
+
.sort({ _id: 1 })
|
|
49
|
+
.limit(limit)
|
|
50
|
+
.toArray()
|
|
51
|
+
|
|
52
|
+
return rows.map(from)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Takes those destinations out of what those rows are outstanding for, and marks published
|
|
57
|
+
* the ones left outstanding for nothing — which is what `pending` selects on and what the
|
|
58
|
+
* TTL reaps by, so neither changes.
|
|
59
|
+
*
|
|
60
|
+
* One batched write for many rows, which is why the ids are held in memory until the tick
|
|
61
|
+
* rather than updated one by one. A row written before this property existed reads as
|
|
62
|
+
* outstanding for `events` alone, which is all there was.
|
|
63
|
+
*/
|
|
64
|
+
async settle(ids, destinations) {
|
|
65
|
+
if (ids.length === 0) return
|
|
66
|
+
|
|
67
|
+
await this.#collection.updateMany({ _id: { $in: ids } }, [
|
|
68
|
+
{
|
|
69
|
+
$set: {
|
|
70
|
+
outstanding: {
|
|
71
|
+
$setDifference: [{ $ifNull: ['$outstanding', [EVENTS]] }, destinations]
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
$set: {
|
|
77
|
+
published: { $eq: [{ $size: '$outstanding' }, 0] },
|
|
78
|
+
// written once, when the last destination lands: an unpublished row has none, and
|
|
79
|
+
// the TTL monitor skips a document that lacks the field it expires by
|
|
80
|
+
publishedAt: {
|
|
81
|
+
$cond: [{ $eq: [{ $size: '$outstanding' }, 0] }, '$$NOW', '$publishedAt']
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
])
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* These are the runtime's indexes, not the component's, so they are declared here rather
|
|
90
|
+
* than in a migration a component would have to write — including pruning, or a later
|
|
91
|
+
* change leaves the old index behind forever.
|
|
92
|
+
*/
|
|
93
|
+
async index() {
|
|
94
|
+
const desired = {
|
|
95
|
+
// holds only what is not published yet, so it stays at in-flight size
|
|
96
|
+
outbox_pending: {
|
|
97
|
+
fields: { lane: 1, pending: 1 },
|
|
98
|
+
options: { name: 'outbox_pending', partialFilterExpression: { published: false } }
|
|
99
|
+
},
|
|
100
|
+
// an unpublished row has no `publishedAt`, and the TTL monitor skips those — so a row
|
|
101
|
+
// that never made it out is never reaped
|
|
102
|
+
outbox_published_at: {
|
|
103
|
+
fields: { publishedAt: 1 },
|
|
104
|
+
options: { name: 'outbox_published_at', expireAfterSeconds: this.#retention }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
for (const { fields, options } of Object.values(desired))
|
|
109
|
+
await index(this.#collection, fields, options)
|
|
110
|
+
|
|
111
|
+
await prune(this.#collection, Object.keys(desired))
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const to = ({ id, ...rest }) => ({ _id: id, ...rest })
|
|
116
|
+
|
|
117
|
+
// a row written before destinations existed is outstanding for the events, which is all a row
|
|
118
|
+
// was ever published to then
|
|
119
|
+
const from = ({ _id, ...rest }) => ({ id: _id, outstanding: [EVENTS], ...rest })
|
|
120
|
+
|
|
121
|
+
function retention() {
|
|
122
|
+
const value = Number(environment.get('TOA_OUTBOX_RETENTION'))
|
|
123
|
+
|
|
124
|
+
return Number.isNaN(value) || value < 0 ? RETENTION : value
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** what a component's own events are outstanding for; `Emission.name` */
|
|
128
|
+
const EVENTS = 'events'
|
|
129
|
+
|
|
130
|
+
/** seconds a published row is kept as a change log before the TTL monitor reaps it */
|
|
131
|
+
const RETENTION = 86400
|
package/src/record.js
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
function to (entity) {
|
|
1
|
+
export function to(entity) {
|
|
4
2
|
const { id, ...rest } = entity
|
|
5
3
|
|
|
6
4
|
return /** @type {toa.mongodb.Record} */ { _id: id, ...rest }
|
|
7
5
|
}
|
|
8
6
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Renamed in the object the driver decoded, which is a new one for every document; an entity
|
|
9
|
+
* copies a state before an operation that writes changes it. A copy costs ten times the rename,
|
|
10
|
+
* and what it would keep is only `id` in front, which a reply does not promise.
|
|
11
|
+
*/
|
|
12
|
+
export function from(record) {
|
|
13
|
+
if (record === undefined || record === null) return null
|
|
12
14
|
|
|
13
15
|
record.id = record._id
|
|
14
16
|
delete record._id
|
|
@@ -16,5 +18,92 @@ function from (record) {
|
|
|
16
18
|
return record
|
|
17
19
|
}
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
+
/**
|
|
22
|
+
* How this collection's records are written and read back.
|
|
23
|
+
*
|
|
24
|
+
* A property the entity declares as a moment is held as a BSON date rather than as what the
|
|
25
|
+
* entity carries. It is what the TTL monitor reads — it skips a field that is not a date,
|
|
26
|
+
* silently — and what sorts and compares as a moment rather than as text or as a number that
|
|
27
|
+
* happens to be one. The entity keeps what it declared, so nothing outside this connector
|
|
28
|
+
* learns a type only one storage has.
|
|
29
|
+
*
|
|
30
|
+
* Two ways to say it, because they differ in what userland holds rather than in what is
|
|
31
|
+
* stored: `{ string, date-time }` for an application that carries ISO strings, and
|
|
32
|
+
* `{ integer, epoch-millis }` for one that carries what `Date.now()` answers. The system
|
|
33
|
+
* timestamps of every record are the second.
|
|
34
|
+
*
|
|
35
|
+
* Top-level properties only. One nested inside an object or an array is left as it is, and
|
|
36
|
+
* `norm` refuses it, so that the declaration does not mean two things.
|
|
37
|
+
*
|
|
38
|
+
* A component that declares no moment holds the plain pair, and pays nothing for any of this.
|
|
39
|
+
*/
|
|
40
|
+
export function codec(properties) {
|
|
41
|
+
/** @type {Array<[string, (value: Date) => unknown]>} */
|
|
42
|
+
const read = []
|
|
43
|
+
|
|
44
|
+
for (const [name, schema] of Object.entries(properties ?? {})) {
|
|
45
|
+
const cast = READ[schema?.format]
|
|
46
|
+
|
|
47
|
+
if (cast !== undefined && schema.type === TYPES[schema.format])
|
|
48
|
+
read.push([name, cast])
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const dates = read.map(([name]) => name)
|
|
52
|
+
|
|
53
|
+
if (dates.length === 0) return { to, from, changes, dates }
|
|
54
|
+
|
|
55
|
+
const write = dates.map((name) => [name, date])
|
|
56
|
+
|
|
57
|
+
return {
|
|
58
|
+
dates,
|
|
59
|
+
|
|
60
|
+
// one way in for both: `Date` takes the milliseconds and the ISO string alike
|
|
61
|
+
to: (entity) => convert(to(entity), write),
|
|
62
|
+
|
|
63
|
+
// what an assignment sets, which is the same properties with no record around them
|
|
64
|
+
changes: (changeset) => convert({ ...changeset }, write),
|
|
65
|
+
|
|
66
|
+
from: (record) => {
|
|
67
|
+
const state = from(record)
|
|
68
|
+
|
|
69
|
+
return state === null ? null : convert(state, read)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Writes into an object `to`, `from` or `changes` has just made, so nothing the caller holds is
|
|
76
|
+
* touched.
|
|
77
|
+
*
|
|
78
|
+
* @private
|
|
79
|
+
*/
|
|
80
|
+
function convert(record, casts) {
|
|
81
|
+
for (const [name, cast] of casts) {
|
|
82
|
+
const value = record[name]
|
|
83
|
+
|
|
84
|
+
// absent, or the null a property that has not been written to holds
|
|
85
|
+
if (value === undefined || value === null) continue
|
|
86
|
+
|
|
87
|
+
record[name] = cast(value)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return record
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const date = (value) => new Date(value)
|
|
94
|
+
|
|
95
|
+
/** A changeset of an entity that declares no moment is set as it is. */
|
|
96
|
+
const changes = (changeset) => changeset
|
|
97
|
+
|
|
98
|
+
/*
|
|
99
|
+
* What a record written before the property was declared a moment holds is what it was given,
|
|
100
|
+
* and it is answered as it is: a collection is converted by a migration rather than by every
|
|
101
|
+
* read that finds one.
|
|
102
|
+
*/
|
|
103
|
+
const READ = {
|
|
104
|
+
'date-time': (value) => (value instanceof Date ? value.toISOString() : value),
|
|
105
|
+
'epoch-millis': (value) => (value instanceof Date ? value.getTime() : value)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** What each format is written on, so that one said of the wrong type is not acted on. */
|
|
109
|
+
const TYPES = { 'date-time': 'string', 'epoch-millis': 'integer' }
|