@tinyshed/tinystore 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +262 -0
  3. package/bin/tinystore.js +50 -0
  4. package/dist/background.d.ts +30 -0
  5. package/dist/background.js +93 -0
  6. package/dist/blobs.d.ts +98 -0
  7. package/dist/blobs.js +299 -0
  8. package/dist/cancel.d.ts +14 -0
  9. package/dist/cancel.js +22 -0
  10. package/dist/clock.d.ts +21 -0
  11. package/dist/clock.js +35 -0
  12. package/dist/config.d.ts +60 -0
  13. package/dist/config.js +417 -0
  14. package/dist/connection.d.ts +78 -0
  15. package/dist/connection.js +405 -0
  16. package/dist/console.d.ts +48 -0
  17. package/dist/console.js +195 -0
  18. package/dist/errors.d.ts +103 -0
  19. package/dist/errors.js +158 -0
  20. package/dist/handles.d.ts +28 -0
  21. package/dist/handles.js +37 -0
  22. package/dist/index.d.ts +21 -0
  23. package/dist/index.js +7 -0
  24. package/dist/jobs.d.ts +304 -0
  25. package/dist/jobs.js +684 -0
  26. package/dist/kv.d.ts +283 -0
  27. package/dist/kv.js +815 -0
  28. package/dist/limiter.d.ts +45 -0
  29. package/dist/limiter.js +68 -0
  30. package/dist/logger.d.ts +96 -0
  31. package/dist/logger.js +243 -0
  32. package/dist/metrics.d.ts +249 -0
  33. package/dist/metrics.js +610 -0
  34. package/dist/once.d.ts +31 -0
  35. package/dist/once.js +116 -0
  36. package/dist/quota.d.ts +44 -0
  37. package/dist/quota.js +98 -0
  38. package/dist/records.d.ts +182 -0
  39. package/dist/records.js +346 -0
  40. package/dist/runtime/bun.d.ts +2 -0
  41. package/dist/runtime/bun.js +164 -0
  42. package/dist/runtime/node.d.ts +2 -0
  43. package/dist/runtime/node.js +157 -0
  44. package/dist/runtime/shared.d.ts +25 -0
  45. package/dist/runtime/shared.js +45 -0
  46. package/dist/runtime.d.ts +48 -0
  47. package/dist/runtime.js +4 -0
  48. package/dist/schema.d.ts +39 -0
  49. package/dist/schema.js +21 -0
  50. package/dist/session.d.ts +120 -0
  51. package/dist/session.js +546 -0
  52. package/dist/sql.d.ts +121 -0
  53. package/dist/sql.js +342 -0
  54. package/dist/store.d.ts +103 -0
  55. package/dist/store.js +153 -0
  56. package/dist/time.d.ts +35 -0
  57. package/dist/time.js +44 -0
  58. package/dist/trace.d.ts +18 -0
  59. package/dist/trace.js +22 -0
  60. package/dist/wire/codec.d.ts +90 -0
  61. package/dist/wire/codec.js +383 -0
  62. package/dist/wire/frame.d.ts +41 -0
  63. package/dist/wire/frame.js +102 -0
  64. package/dist/wire/messages.d.ts +1403 -0
  65. package/dist/wire/messages.js +472 -0
  66. package/dist/wire/msgpack.d.ts +93 -0
  67. package/dist/wire/msgpack.js +680 -0
  68. package/package.json +40 -0
  69. package/src/background.ts +108 -0
  70. package/src/blobs.ts +437 -0
  71. package/src/cancel.ts +26 -0
  72. package/src/clock.ts +45 -0
  73. package/src/config.ts +501 -0
  74. package/src/connection.ts +512 -0
  75. package/src/console.ts +228 -0
  76. package/src/errors.ts +211 -0
  77. package/src/handles.ts +68 -0
  78. package/src/index.ts +92 -0
  79. package/src/jobs.ts +963 -0
  80. package/src/kv.ts +1093 -0
  81. package/src/limiter.ts +99 -0
  82. package/src/logger.ts +326 -0
  83. package/src/metrics.ts +896 -0
  84. package/src/once.ts +142 -0
  85. package/src/quota.ts +137 -0
  86. package/src/records.ts +537 -0
  87. package/src/runtime/bun.ts +190 -0
  88. package/src/runtime/node.ts +177 -0
  89. package/src/runtime/shared.ts +61 -0
  90. package/src/runtime.ts +57 -0
  91. package/src/schema.ts +69 -0
  92. package/src/session.ts +665 -0
  93. package/src/sql.ts +478 -0
  94. package/src/store.ts +219 -0
  95. package/src/time.ts +87 -0
  96. package/src/trace.ts +32 -0
  97. package/src/wire/codec.ts +445 -0
  98. package/src/wire/frame.ts +139 -0
  99. package/src/wire/messages.ts +562 -0
  100. package/src/wire/msgpack.ts +712 -0
package/src/jobs.ts ADDED
@@ -0,0 +1,963 @@
1
+ // Work that runs at its time, in jobs.db, as Go's jobs package keeps it:
2
+ // queues ordered by time, leases, retries and repeats. A handle opens with its
3
+ // first call; a work loop is the server's own Work, its jobs handed over the
4
+ // connection and settled by what the handler returns.
5
+
6
+ import { currentSignal } from './cancel.ts'
7
+ import { type Connection, download, type Link } from './connection.ts'
8
+ import { CancelledError, CorruptError, errorOf, InvalidError } from './errors.ts'
9
+ import { checkName, handleOn, type Page } from './handles.ts'
10
+ import { check, isSchema, type StandardSchemaV1 } from './schema.ts'
11
+ import { LostError, watch } from './session.ts'
12
+ import type { Database, SqlBatch } from './sql.ts'
13
+ import { type Duration, dateOf, ms, type Time, unixMs } from './time.ts'
14
+ import {
15
+ JobsAnswer,
16
+ JobsBatch,
17
+ JobsChange,
18
+ JobsEntry,
19
+ JobsHeld,
20
+ JobsKept,
21
+ JobsKey,
22
+ JobsLease,
23
+ JobsOutcome,
24
+ JobsOutcomes,
25
+ JobsPage,
26
+ JobsQuery,
27
+ JobsQueue,
28
+ JobsSettled,
29
+ JobsWorkers,
30
+ methods,
31
+ } from './wire/messages.ts'
32
+
33
+ export interface QueueOptions {
34
+ /** how long a claim holds a job before it is given back: 30 s */
35
+ lease?: Duration
36
+ /** the attempts after which a job fails for good: 10 */
37
+ maxAttempts?: number
38
+ /** a retry's wait, doubling from first to most: a second to an hour */
39
+ backoff?: { first?: Duration; most?: Duration }
40
+ /** the jobs a queue holds before it refuses the next with LimitError: ten million */
41
+ maxWaiting?: number
42
+ /** how long a failed job is kept for Get and Scan: seven days */
43
+ keepFailed?: Duration
44
+ /** remembers a done job's key this long, so that enqueuing it again adds nothing */
45
+ keepDone?: Duration
46
+ /** the jobs that may run at once, across every worker of the store: no bound */
47
+ maxRunning?: number
48
+ /**
49
+ * keeps the queue in this database's file instead of jobs.db, so that a
50
+ * batch of the database commits a job with its rows: `queue.withTx(tx)`
51
+ */
52
+ in?: Database
53
+ }
54
+
55
+ /** When a job runs again: cron text in a zone by its name, a daily time, or every so often. */
56
+ export type Repeat =
57
+ | { cron: string; zone: string }
58
+ | { daily: string; zone: string }
59
+ | { every: Duration }
60
+
61
+ export interface EnqueueOptions {
62
+ /** the job's time; one past runs now */
63
+ at?: Time
64
+ after?: Duration
65
+ /** names one job: enqueuing it again adds nothing and can bring it forward, never back */
66
+ key?: string
67
+ /** needs a key, since only a key stops it */
68
+ repeat?: Repeat
69
+ }
70
+
71
+ /** done while keepDone keeps its key; cancelled only a watcher sees */
72
+ export type JobState = 'waiting' | 'running' | 'failed' | 'done' | 'cancelled'
73
+
74
+ export interface JobEntry<T> {
75
+ key: string
76
+ value: T
77
+ /** the time it runs for */
78
+ at: Date
79
+ /** its attempts, a running one included */
80
+ attempt: number
81
+ state: JobState
82
+ /** the jobs that run before a waiting one, up to 10,000; get and watch count it, scan leaves it 0 */
83
+ ahead: number
84
+ /** what a running job's handler last reported with job.progress */
85
+ progress: unknown
86
+ /** when the last run a handler finished began: acknowledged, retried, failed or snoozed */
87
+ ran: Date | undefined
88
+ /** how many milliseconds that run took */
89
+ took: number | undefined
90
+ /** its last failure */
91
+ error: string | undefined
92
+ /** a repeating job's cron text and zone */
93
+ repeat: string | undefined
94
+ }
95
+
96
+ export interface WorkOptions {
97
+ /** the jobs in this process's hands at once: 1, the one order a queue promises */
98
+ workers?: number
99
+ /** how long a job may be in the handler's hands, after which its attempt fails: a minute */
100
+ timeout?: Duration
101
+ /** stops taking jobs: the ones in hand finish first, and work returns */
102
+ signal?: AbortSignal
103
+ /** returns once no job is due and none runs, as a test wants it */
104
+ untilIdle?: boolean
105
+ }
106
+
107
+ type Settlement =
108
+ | { how: 'ack' }
109
+ | { how: 'retry'; error: string; at?: Time; after?: Duration }
110
+ | { how: 'fail'; error: string }
111
+ | { how: 'snooze'; at?: Time; after?: Duration }
112
+
113
+ const how = { ack: 1, retry: 2, fail: 3, snooze: 4, extend: 5, progress: 6 } as const
114
+
115
+ /** the JSON a handler reports of its job, at most 4 KiB of it */
116
+ function progressText(progress: unknown): string {
117
+ let text: string | undefined
118
+ try {
119
+ text = JSON.stringify(progress)
120
+ } catch (err) {
121
+ throw new InvalidError(`a job's progress JSON cannot write: ${(err as Error).message}`)
122
+ }
123
+ if (text === undefined) {
124
+ throw new InvalidError("a job's progress JSON cannot write: undefined, a function or a symbol")
125
+ }
126
+ if (new TextEncoder().encode(text).length > 4096) {
127
+ throw new InvalidError("a job's progress is past 4 KiB of JSON")
128
+ }
129
+ return text
130
+ }
131
+
132
+ /**
133
+ * Sends what a handler reports of its job: the latest, one send at a time
134
+ * and at most one each 100 ms, so that a handler reporting each chunk it
135
+ * reads costs the connection ten frames a second.
136
+ */
137
+ class Reporter {
138
+ readonly #send: (text: string) => Promise<void>
139
+ #latest: string | undefined
140
+ #sending: Promise<void> | undefined
141
+ #timer: ReturnType<typeof setTimeout> | undefined
142
+ #sent = 0
143
+ #stopped = false
144
+
145
+ constructor(send: (text: string) => Promise<void>) {
146
+ this.#send = send
147
+ }
148
+
149
+ report(text: string): void {
150
+ this.#latest = text
151
+ this.#flush()
152
+ }
153
+
154
+ #flush(): void {
155
+ if (this.#stopped || this.#sending !== undefined || this.#latest === undefined) {
156
+ return
157
+ }
158
+ const wait = this.#sent + 100 - Date.now()
159
+ if (wait > 0) {
160
+ this.#timer ??= setTimeout(() => {
161
+ this.#timer = undefined
162
+ this.#flush()
163
+ }, wait)
164
+ return
165
+ }
166
+ const text = this.#latest
167
+ this.#latest = undefined
168
+ this.#sent = Date.now()
169
+ this.#sending = this.#send(text)
170
+ .catch(() => {})
171
+ .finally(() => {
172
+ this.#sending = undefined
173
+ this.#flush()
174
+ })
175
+ }
176
+
177
+ /** Drops what waits and lets what is being sent finish, before the job's outcome follows it. */
178
+ async stop(): Promise<void> {
179
+ this.#stopped = true
180
+ clearTimeout(this.#timer)
181
+ await this.#sending
182
+ }
183
+ }
184
+
185
+ /** Calls a method on the connection a job is held on, answering its body. */
186
+ type Call = (method: number, body: Uint8Array) => Promise<Uint8Array>
187
+
188
+ /**
189
+ * The steps of a held job's run, which the server keeps under the number the
190
+ * job was held by: a step's kept answer, or the step run and its answer kept.
191
+ */
192
+ class Steps {
193
+ readonly #call: Call
194
+ readonly #job: number
195
+
196
+ constructor(call: Call, job: number) {
197
+ this.#call = call
198
+ this.#job = job
199
+ }
200
+
201
+ async run<R>(name: string, fn: () => R | Promise<R>): Promise<R> {
202
+ const kept = JobsKept.decode(
203
+ await this.#call(methods['jobs.step'], JobsAnswer.encode({ job: this.#job, name })),
204
+ )
205
+ if (kept.found === true) {
206
+ return JSON.parse(kept.answer ?? 'null') as R
207
+ }
208
+ const answer = await fn()
209
+ let text: string
210
+ try {
211
+ text = JSON.stringify(answer) ?? 'null'
212
+ } catch (err) {
213
+ throw new InvalidError(
214
+ `step ${name} answered what JSON cannot write: ${(err as Error).message}`,
215
+ )
216
+ }
217
+ await this.#call(
218
+ methods['jobs.keep'],
219
+ JobsAnswer.encode({ job: this.#job, name, answer: text }),
220
+ )
221
+ return answer
222
+ }
223
+ }
224
+
225
+ /**
226
+ * A job in a work loop's handler. Returning acknowledges it and throwing
227
+ * retries it, unless the handler said otherwise with retry, fail or snooze;
228
+ * the last of those it calls is how the job settles.
229
+ */
230
+ export class Job<T> {
231
+ readonly key: string
232
+ readonly value: T
233
+ /** the time it ran for */
234
+ readonly at: Date
235
+ /** its attempts, this one included */
236
+ readonly attempt: number
237
+ /**
238
+ * aborts when the job's timeout passes, its loop stops, or cancel takes it,
239
+ * whose reason is then a CancelledError: the handler should stop too
240
+ */
241
+ readonly signal: AbortSignal
242
+ settlement: Settlement | undefined
243
+ readonly #reporter: Reporter | undefined
244
+ readonly #steps: Steps | undefined
245
+
246
+ constructor(
247
+ key: string,
248
+ value: T,
249
+ at: Date,
250
+ attempt: number,
251
+ signal: AbortSignal,
252
+ reporter?: Reporter,
253
+ steps?: Steps,
254
+ ) {
255
+ this.key = key
256
+ this.value = value
257
+ this.at = at
258
+ this.attempt = attempt
259
+ this.signal = signal
260
+ this.#reporter = reporter
261
+ this.#steps = steps
262
+ }
263
+
264
+ /**
265
+ * Runs fn once in the job's run and keeps what it answers, as JSON: an
266
+ * attempt after a retry, a lost lease or a restart gets the kept answer
267
+ * back without running fn again. A step whose attempt ends before its
268
+ * answer is kept runs again, so what fn does outside the store should bear
269
+ * doing twice. A name is the step's within the run, which a loop numbers:
270
+ * 'model:1', 'tool:1', 'model:2'. An answer is at most 1 MiB of JSON.
271
+ *
272
+ * const hits = await job.step('search', () => search(job.value.query))
273
+ */
274
+ step<R>(name: string, fn: () => R | Promise<R>): Promise<R> {
275
+ if (this.#steps === undefined) {
276
+ return Promise.reject(new InvalidError('a job no work loop handed over keeps no steps'))
277
+ }
278
+ return this.#steps.run(name, fn)
279
+ }
280
+
281
+ /**
282
+ * Reports how far the job got, any JSON within 4 KiB: get and watch show
283
+ * the latest until the job is settled. It does not wait for the server.
284
+ *
285
+ * await transcode({ signal: job.signal, onProgress: p => job.progress(p) })
286
+ */
287
+ progress(progress: unknown): void {
288
+ this.#reporter?.report(progressText(progress))
289
+ }
290
+
291
+ /** Fails this attempt: the job runs again after its backoff, or at the time given. */
292
+ retry(error?: unknown, when?: { at?: Time; after?: Duration }): void {
293
+ this.settlement = { how: 'retry', error: reasonOf(error), ...when }
294
+ }
295
+
296
+ /** Fails the job for good, keeping it for keepFailed with its reason. */
297
+ fail(error?: unknown): void {
298
+ this.settlement = { how: 'fail', error: reasonOf(error) }
299
+ }
300
+
301
+ /** Moves the job to another time without counting the attempt. */
302
+ snooze(when: { at?: Time; after?: Duration }): void {
303
+ this.settlement = { how: 'snooze', ...when }
304
+ }
305
+ }
306
+
307
+ /** A job a claim leased, which its caller settles itself before the lease ends. */
308
+ export class ClaimedJob<T> {
309
+ readonly key: string
310
+ readonly value: T
311
+ readonly at: Date
312
+ readonly attempt: number
313
+ readonly #settle: (outcome: Parameters<typeof JobsOutcome.encode>[0]) => Promise<void>
314
+ readonly #job: number
315
+ readonly #steps: Steps
316
+
317
+ constructor(
318
+ held: ReturnType<typeof JobsHeld.decode>,
319
+ value: T,
320
+ settle: (outcome: Parameters<typeof JobsOutcome.encode>[0]) => Promise<void>,
321
+ call: Call,
322
+ ) {
323
+ this.key = held.key ?? ''
324
+ this.value = value
325
+ this.at = new Date(held.at ?? 0)
326
+ this.attempt = held.attempt ?? 0
327
+ this.#job = held.job ?? 0
328
+ this.#settle = settle
329
+ this.#steps = new Steps(call, this.#job)
330
+ }
331
+
332
+ /** Runs fn once in the job's run and keeps its answer, as a work loop's job.step does. */
333
+ step<R>(name: string, fn: () => R | Promise<R>): Promise<R> {
334
+ return this.#steps.run(name, fn)
335
+ }
336
+
337
+ /** The job is done. */
338
+ ack(): Promise<void> {
339
+ return this.#settle({ job: this.#job, how: how.ack })
340
+ }
341
+
342
+ retry(error?: unknown, when?: { at?: Time; after?: Duration }): Promise<void> {
343
+ return this.#settle({ job: this.#job, how: how.retry, err: reasonOf(error), ...timing(when) })
344
+ }
345
+
346
+ fail(error?: unknown): Promise<void> {
347
+ return this.#settle({ job: this.#job, how: how.fail, err: reasonOf(error) })
348
+ }
349
+
350
+ snooze(when: { at?: Time; after?: Duration }): Promise<void> {
351
+ return this.#settle({ job: this.#job, how: how.snooze, ...timing(when) })
352
+ }
353
+
354
+ /** Holds the job this much longer from now. */
355
+ extend(d: Duration): Promise<void> {
356
+ return this.#settle({ job: this.#job, how: how.extend, after: ms(d) })
357
+ }
358
+
359
+ /** Reports how far the job got, any JSON within 4 KiB, which get and watch show until it is settled. */
360
+ progress(progress: unknown): Promise<void> {
361
+ return this.#settle({ job: this.#job, how: how.progress, progress: progressText(progress) })
362
+ }
363
+ }
364
+
365
+ function reasonOf(error: unknown): string {
366
+ if (error === undefined) {
367
+ return 'the handler gave no reason'
368
+ }
369
+ return error instanceof Error ? error.message : String(error)
370
+ }
371
+
372
+ function timing(when: { at?: Time; after?: Duration } | undefined): {
373
+ at?: number
374
+ after?: number
375
+ } {
376
+ if (when?.at !== undefined) {
377
+ return { at: unixMs(when.at) }
378
+ }
379
+ if (when?.after !== undefined) {
380
+ return { after: ms(when.after) }
381
+ }
382
+ return {}
383
+ }
384
+
385
+ /** Encodes a queue's values as JSON, the text every language reads, and back. */
386
+ interface Values<T> {
387
+ encode(value: T): string
388
+ decode(text: string): T | Promise<T>
389
+ }
390
+
391
+ const jsonValues: Values<unknown> = {
392
+ encode: value => {
393
+ let text: string | undefined
394
+ try {
395
+ text = JSON.stringify(value)
396
+ } catch (err) {
397
+ throw new InvalidError(`a job's value JSON cannot write: ${(err as Error).message}`)
398
+ }
399
+ if (text === undefined) {
400
+ throw new InvalidError("a job's value JSON cannot write: undefined, a function or a symbol")
401
+ }
402
+ return text
403
+ },
404
+ decode: text => {
405
+ try {
406
+ return JSON.parse(text)
407
+ } catch (err) {
408
+ throw new CorruptError(`a job's value is not JSON any more: ${(err as Error).message}`)
409
+ }
410
+ },
411
+ }
412
+
413
+ function schemaValues<S extends StandardSchemaV1>(schema: S): Values<unknown> {
414
+ return {
415
+ encode: jsonValues.encode,
416
+ decode: async text => {
417
+ const checked = await check(schema, await jsonValues.decode(text))
418
+ if ('issues' in checked) {
419
+ throw new CorruptError(`a job's value does not meet the queue's schema: ${checked.issues}`)
420
+ }
421
+ return checked.value
422
+ },
423
+ }
424
+ }
425
+
426
+ function repeatOf(r: Repeat): Parameters<typeof JobsQueue.encode>[0]['schedule'] {
427
+ if ('every' in r) {
428
+ return { every: ms(r.every) }
429
+ }
430
+ if ('daily' in r) {
431
+ const time = /^(\d{1,2}):(\d{2})$/.exec(r.daily)
432
+ if (time === null || Number(time[1]) > 23 || Number(time[2]) > 59) {
433
+ throw new InvalidError(`the daily time ${JSON.stringify(r.daily)}; write it as 03:10`)
434
+ }
435
+ return { cron: `${Number(time[2])} ${Number(time[1])} * * *`, zone: r.zone }
436
+ }
437
+ return { cron: r.cron, zone: r.zone }
438
+ }
439
+
440
+ export class Jobs {
441
+ readonly #link: Link
442
+
443
+ constructor(link: Link) {
444
+ this.#link = link
445
+ }
446
+
447
+ /** A queue of JSON values of type T. */
448
+ queue<T = unknown>(name: string, options?: QueueOptions): Queue<T>
449
+ /** A queue whose values each claim checks, with zod's, valibot's or another Standard Schema. */
450
+ queue<S extends StandardSchemaV1>(
451
+ name: string,
452
+ schema: S,
453
+ options?: QueueOptions,
454
+ ): Queue<StandardSchemaV1.InferOutput<S>>
455
+ queue(
456
+ name: string,
457
+ of?: StandardSchemaV1 | QueueOptions,
458
+ options?: QueueOptions,
459
+ ): Queue<unknown> {
460
+ checkName(name, 'queue')
461
+ let values = jsonValues
462
+ if (isSchema(of)) {
463
+ values = schemaValues(of)
464
+ } else if (of !== undefined) {
465
+ options = of
466
+ }
467
+ const open = JobsQueue.encode(queueFields(name, options))
468
+ return new Queue(this.#link, name, open, values, options?.in)
469
+ }
470
+
471
+ /**
472
+ * A schedule: a queue of one job under the schedule's name, which repeats
473
+ * as it says. work runs it; cancel stops it.
474
+ */
475
+ schedule(name: string, repeat: Repeat, options?: QueueOptions): Queue<null> {
476
+ checkName(name, 'schedule')
477
+ const open = JobsQueue.encode({ ...queueFields(name, options), schedule: repeatOf(repeat) })
478
+ const nothing = { encode: () => '{}', decode: () => null }
479
+ return new Queue(this.#link, name, open, nothing, options?.in)
480
+ }
481
+ }
482
+
483
+ function queueFields(
484
+ name: string,
485
+ options: QueueOptions | undefined,
486
+ ): Parameters<typeof JobsQueue.encode>[0] {
487
+ return {
488
+ name,
489
+ lease: options?.lease === undefined ? undefined : ms(options.lease),
490
+ maxAttempts: options?.maxAttempts,
491
+ backoffFirst: options?.backoff?.first === undefined ? undefined : ms(options.backoff.first),
492
+ backoffMost: options?.backoff?.most === undefined ? undefined : ms(options.backoff.most),
493
+ maxWaiting: options?.maxWaiting,
494
+ keepFailed: options?.keepFailed === undefined ? undefined : ms(options.keepFailed),
495
+ keepDone: options?.keepDone === undefined ? undefined : ms(options.keepDone),
496
+ maxRunning: options?.maxRunning,
497
+ in: options?.in?.name,
498
+ }
499
+ }
500
+
501
+ /** A job's fields as jobs.enqueue and a batch carry them. */
502
+ function jobFields<T>(values: Values<T>, value: T, options: EnqueueOptions | undefined) {
503
+ return {
504
+ value: values.encode(value),
505
+ key: options?.key,
506
+ at: options?.at === undefined ? undefined : unixMs(options.at),
507
+ after: options?.after === undefined ? undefined : ms(options.after),
508
+ repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
509
+ }
510
+ }
511
+
512
+ /** A queue's enqueue inside a batch of the database it lives in. */
513
+ export class QueueTx<T> {
514
+ readonly #tx: SqlBatch
515
+ readonly #open: Uint8Array
516
+ readonly #values: Values<T>
517
+
518
+ constructor(tx: SqlBatch, open: Uint8Array, values: Values<T>) {
519
+ this.#tx = tx
520
+ this.#open = open
521
+ this.#values = values
522
+ }
523
+
524
+ /** Adds a job to the batch; it settles once the batch has committed. */
525
+ enqueue(value: T, options?: EnqueueOptions): Promise<void> {
526
+ return this.#tx.enqueue(this.#open, jobFields(this.#values, value, options))
527
+ }
528
+ }
529
+
530
+ const states: Record<number, JobState> = {
531
+ 1: 'waiting',
532
+ 2: 'running',
533
+ 3: 'failed',
534
+ 4: 'done',
535
+ 5: 'cancelled',
536
+ }
537
+
538
+ export class Queue<T> {
539
+ readonly name: string
540
+ readonly #link: Link
541
+ readonly #open: Uint8Array
542
+ readonly #values: Values<T>
543
+ readonly #in: Database | undefined
544
+
545
+ constructor(
546
+ link: Link,
547
+ name: string,
548
+ open: Uint8Array,
549
+ values: Values<T>,
550
+ inDatabase: Database | undefined,
551
+ ) {
552
+ this.#link = link
553
+ this.name = name
554
+ this.#open = open
555
+ this.#values = values as Values<T>
556
+ this.#in = inDatabase
557
+ }
558
+
559
+ /** The queue's handle, its database opened first when it lives in one, as the server needs. */
560
+ async #handle(connection: Connection): Promise<number> {
561
+ await this.#in?.handle(connection)
562
+ return handleOn(connection, methods['jobs.open'], this.#open)
563
+ }
564
+
565
+ /**
566
+ * The queue's enqueue inside a batch of the database it lives in, which
567
+ * it was opened `in`: the job commits with the batch's rows or not at all.
568
+ *
569
+ * await db.batch(tx => {
570
+ * tx.exec`update notes set body = ${body} where id = ${id}`
571
+ * index.withTx(tx).enqueue({ id })
572
+ * })
573
+ */
574
+ withTx(tx: SqlBatch): QueueTx<T> {
575
+ if (this.#in === undefined || this.#in.name !== tx.database) {
576
+ const lives = this.#in === undefined ? 'jobs.db' : `sql ${this.#in.name}`
577
+ throw new InvalidError(
578
+ `the queue ${this.name} lives in ${lives}, not in sql ${tx.database}: ` +
579
+ "open it with { in: db } to enqueue in that database's batches",
580
+ )
581
+ }
582
+ return new QueueTx(tx, this.#open, this.#values)
583
+ }
584
+
585
+ /** Adds a job; it returns once the job is in the file. */
586
+ enqueue(value: T, options?: EnqueueOptions): Promise<void> {
587
+ return this.enqueueAll([{ value, ...options }])
588
+ }
589
+
590
+ /** Adds jobs in one transaction, all or none: a refused one names itself as `call`. */
591
+ async enqueueAll(jobs: readonly ({ value: T } & EnqueueOptions)[]): Promise<void> {
592
+ const encoded = jobs.map(job => jobFields(this.#values, job.value, job))
593
+ await this.#link.run('write', async connection => {
594
+ const handle = await this.#handle(connection)
595
+ await connection.session.call(
596
+ methods['jobs.enqueue'],
597
+ JobsBatch.encode({ handle, jobs: encoded }),
598
+ )
599
+ })
600
+ }
601
+
602
+ /**
603
+ * Changes a job that waits or failed: its value and, when the options
604
+ * say, its time or repeat. A job a worker holds, done or absent is
605
+ * ConflictError.
606
+ */
607
+ async update(key: string, value: T, options?: Omit<EnqueueOptions, 'key'>): Promise<void> {
608
+ const change = {
609
+ value: this.#values.encode(value),
610
+ key,
611
+ at: options?.at === undefined ? undefined : unixMs(options.at),
612
+ after: options?.after === undefined ? undefined : ms(options.after),
613
+ repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
614
+ }
615
+ await this.#link.run('write', async connection => {
616
+ const handle = await this.#handle(connection)
617
+ await connection.session.call(
618
+ methods['jobs.update'],
619
+ JobsChange.encode({ ...change, handle }),
620
+ )
621
+ })
622
+ }
623
+
624
+ /**
625
+ * Removes the job under key, whether it waits, runs or failed, and says
626
+ * whether there was one. A running job's handler sees job.signal abort,
627
+ * and what it returns settles nothing.
628
+ */
629
+ cancel(key: string): Promise<boolean> {
630
+ return this.#link.run('write', async connection => {
631
+ const handle = await this.#handle(connection)
632
+ const body = await connection.session.call(
633
+ methods['jobs.cancel'],
634
+ JobsKey.encode({ handle, key }),
635
+ )
636
+ return JobsEntry.decode(body).found === true
637
+ })
638
+ }
639
+
640
+ /**
641
+ * The job a key names, waiting, running or failed, or done while keepDone
642
+ * keeps its key; undefined when none is.
643
+ */
644
+ async get(key: string): Promise<JobEntry<T> | undefined> {
645
+ const entry = await this.#link.run('read', async connection => {
646
+ const handle = await this.#handle(connection)
647
+ return JobsEntry.decode(
648
+ await connection.session.call(methods['jobs.get'], JobsKey.encode({ handle, key })),
649
+ )
650
+ })
651
+ return entry.found === true ? this.#entryOf(entry) : undefined
652
+ }
653
+
654
+ async #entryOf(entry: ReturnType<typeof JobsEntry.decode>): Promise<JobEntry<T>> {
655
+ const state = states[entry.state ?? 1] ?? 'waiting'
656
+ return {
657
+ key: entry.key ?? '',
658
+ value:
659
+ state === 'done' && entry.value === undefined
660
+ ? (undefined as T)
661
+ : await this.#values.decode(entry.value ?? 'null'),
662
+ at: new Date(entry.at ?? 0),
663
+ attempt: entry.attempt ?? 0,
664
+ state,
665
+ ahead: entry.ahead ?? 0,
666
+ progress: entry.progress === undefined ? undefined : JSON.parse(entry.progress),
667
+ ran: dateOf(entry.ran),
668
+ took: entry.ran === undefined ? undefined : (entry.took ?? 0),
669
+ error: entry.err,
670
+ repeat: entry.repeat,
671
+ }
672
+ }
673
+
674
+ /**
675
+ * Yields the job under key as it is, then again each time its state,
676
+ * place, attempt, time, progress or error changes, until it ends: done,
677
+ * failed or cancelled, the last entry it yields. A key that names no job
678
+ * yields nothing. Leaving the loop ends the watch; a connection lost is
679
+ * connected again, and the watch goes on from the job as it is then.
680
+ *
681
+ * for await (const s of videos.watch(id)) send(s.state, s.ahead, s.progress)
682
+ */
683
+ async *watch(key: string): AsyncGenerator<JobEntry<T>> {
684
+ let last: string | undefined
685
+ for (;;) {
686
+ const stream = await this.#link.run('read', async connection => {
687
+ const handle = await this.#handle(connection)
688
+ const opened = await connection.session.open(
689
+ methods['jobs.watch'],
690
+ JobsKey.encode({ handle, key }),
691
+ true,
692
+ )
693
+ await opened.next()
694
+ return opened
695
+ })
696
+ const unwatch = watch(currentSignal(), stream)
697
+ try {
698
+ for (;;) {
699
+ const event = await stream.next()
700
+ stream.consumed(event.body.length)
701
+ if (event.end) {
702
+ return
703
+ }
704
+ const entry = await this.#entryOf(JobsEntry.decode(event.body))
705
+ const seen = JSON.stringify([
706
+ entry.state,
707
+ entry.ahead,
708
+ entry.attempt,
709
+ entry.at,
710
+ entry.progress,
711
+ entry.error,
712
+ ])
713
+ if (seen !== last) {
714
+ last = seen
715
+ yield entry
716
+ }
717
+ }
718
+ } catch (err) {
719
+ if (!(err instanceof LostError)) {
720
+ throw err
721
+ }
722
+ } finally {
723
+ unwatch()
724
+ stream.cancel()
725
+ }
726
+ }
727
+ }
728
+
729
+ /**
730
+ * A page of the jobs under a prefix, in the byte order of their keys; with
731
+ * state 'failed' and no prefix, the failed jobs, the last failed first.
732
+ */
733
+ async scan(query?: { prefix?: string; state?: 'failed'; after?: string; limit?: number }) {
734
+ const got = await this.#link.run('read', async connection => {
735
+ const handle = await this.#handle(connection)
736
+ const body = JobsQuery.encode({
737
+ handle,
738
+ prefix: query?.prefix,
739
+ state: query?.state === 'failed' ? 3 : undefined,
740
+ after: query?.after,
741
+ limit: query?.limit,
742
+ })
743
+ return download(connection, methods['jobs.scan'], body)
744
+ })
745
+ const items: JobEntry<T>[] = []
746
+ for (const item of got.items) {
747
+ items.push(await this.#entryOf(JobsEntry.decode(item)))
748
+ }
749
+ const page = JobsPage.decode(got.trailer)
750
+ return { items, next: page.more === true ? (page.after ?? '') : undefined } satisfies Page<
751
+ JobEntry<T>,
752
+ string
753
+ >
754
+ }
755
+
756
+ /** Walks scan's pages, holding no snapshot between them. */
757
+ async *all(query?: {
758
+ prefix?: string
759
+ state?: 'failed'
760
+ limit?: number
761
+ }): AsyncGenerator<JobEntry<T>> {
762
+ let after: string | undefined
763
+ for (;;) {
764
+ const page = await this.scan({ ...query, ...(after === undefined ? {} : { after }) })
765
+ yield* page.items
766
+ if (page.next === undefined) {
767
+ return
768
+ }
769
+ after = page.next
770
+ }
771
+ }
772
+
773
+ /**
774
+ * Leases the next due job, or says there is none, without waiting. The
775
+ * claim lives on this connection: settle it before its lease ends.
776
+ */
777
+ async claim(options?: { lease?: Duration }): Promise<ClaimedJob<T> | undefined> {
778
+ return this.#link.run('write', async connection => {
779
+ const handle = await this.#handle(connection)
780
+ const lease = options?.lease === undefined ? undefined : ms(options.lease)
781
+ const held = JobsHeld.decode(
782
+ await connection.session.call(methods['jobs.claim'], JobsLease.encode({ handle, lease })),
783
+ )
784
+ if (held.found !== true) {
785
+ return undefined
786
+ }
787
+ const value = await this.#values.decode(held.value ?? 'null')
788
+ return new ClaimedJob(
789
+ held,
790
+ value,
791
+ async outcome => {
792
+ const body = await connection.session.call(
793
+ methods['jobs.settle'],
794
+ JobsOutcomes.encode({ outcomes: [outcome] }),
795
+ )
796
+ const refused = JobsSettled.decode(body).settled?.[0]
797
+ if (refused !== undefined && refused !== null) {
798
+ throw errorOf(refused.code ?? 'internal', refused.message ?? '', refused.what ?? {})
799
+ }
800
+ },
801
+ (method, body) => connection.session.call(method, body),
802
+ )
803
+ })
804
+ }
805
+
806
+ /**
807
+ * Runs the queue's jobs as they come due, workers at once, until the
808
+ * signal aborts: the server's own Work loop claims ahead and extends
809
+ * leases, and nothing polls. A handler's return acknowledges its job and a
810
+ * throw retries it. A connection lost takes the jobs in hand with it, as a
811
+ * process that died would, and the loop connects again.
812
+ */
813
+ async work(
814
+ handler: (job: Job<T>) => void | Promise<void>,
815
+ options: WorkOptions = {},
816
+ ): Promise<void> {
817
+ for (;;) {
818
+ options.signal?.throwIfAborted()
819
+ try {
820
+ await this.#link.run(
821
+ 'read',
822
+ connection => this.#workOn(connection, handler, options),
823
+ options.signal,
824
+ )
825
+ if (options.untilIdle === true || options.signal?.aborted === true) {
826
+ return
827
+ }
828
+ } catch (err) {
829
+ if (!(err instanceof LostError) || options.signal?.aborted === true) {
830
+ throw err
831
+ }
832
+ }
833
+ }
834
+ }
835
+
836
+ async #workOn(
837
+ connection: Connection,
838
+ handler: (job: Job<T>) => void | Promise<void>,
839
+ options: WorkOptions,
840
+ ) {
841
+ const handle = await this.#handle(connection)
842
+ const timeout = options.timeout === undefined ? 60_000 : ms(options.timeout)
843
+ const stream = await connection.session.open(
844
+ methods['jobs.work'],
845
+ JobsWorkers.encode({
846
+ handle,
847
+ workers: options.workers,
848
+ timeout,
849
+ untilIdle: options.untilIdle || undefined,
850
+ cancels: true,
851
+ }),
852
+ false,
853
+ )
854
+ const stopping = new AbortController()
855
+ const stop = () => stopping.abort()
856
+ options.signal?.addEventListener('abort', stop, { once: true })
857
+ const inHand = new Set<Promise<void>>()
858
+ const cancels = new Map<number, AbortController>()
859
+ let ended = false
860
+ const endOurSide = async () => {
861
+ if (!ended) {
862
+ ended = true
863
+ await Promise.allSettled(inHand)
864
+ await stream.send(new Uint8Array(0), true).catch(() => {})
865
+ }
866
+ }
867
+ stopping.signal.addEventListener('abort', () => void endOurSide(), { once: true })
868
+ try {
869
+ await stream.next()
870
+ for (;;) {
871
+ const event = await stream.next()
872
+ stream.consumed(event.body.length)
873
+ if (event.end) {
874
+ return
875
+ }
876
+ const held = JobsHeld.decode(event.body)
877
+ if (held.cancelled === true) {
878
+ cancels.get(held.job ?? 0)?.abort(new CancelledError('cancel took the job while it ran'))
879
+ continue
880
+ }
881
+ const cancel = new AbortController()
882
+ cancels.set(held.job ?? 0, cancel)
883
+ const steps = new Steps(
884
+ (method, body) => connection.session.call(method, body),
885
+ held.job ?? 0,
886
+ )
887
+ const running = this.#run(
888
+ stream,
889
+ held,
890
+ steps,
891
+ handler,
892
+ timeout,
893
+ stopping.signal,
894
+ cancel.signal,
895
+ ).finally(() => cancels.delete(held.job ?? 0))
896
+ inHand.add(running)
897
+ running.finally(() => inHand.delete(running))
898
+ }
899
+ } finally {
900
+ options.signal?.removeEventListener('abort', stop)
901
+ stopping.abort()
902
+ }
903
+ }
904
+
905
+ // runs one job's handler and sends the outcome it settled by; a handler
906
+ // stopped by the loop's end gives its job back without counting the
907
+ // attempt, and one cancel took settles nothing, so it sends nothing
908
+ async #run(
909
+ stream: Awaited<ReturnType<Connection['session']['open']>>,
910
+ held: ReturnType<typeof JobsHeld.decode>,
911
+ steps: Steps,
912
+ handler: (job: Job<T>) => void | Promise<void>,
913
+ timeout: number,
914
+ stopping: AbortSignal,
915
+ cancelled: AbortSignal,
916
+ ): Promise<void> {
917
+ const signal = AbortSignal.any([stopping, AbortSignal.timeout(timeout), cancelled])
918
+ const number = held.job ?? 0
919
+ const reporter = new Reporter(progress =>
920
+ stream.send(JobsOutcome.encode({ job: number, how: how.progress, progress }), false),
921
+ )
922
+ let outcome: Parameters<typeof JobsOutcome.encode>[0]
923
+ try {
924
+ const job = new Job(
925
+ held.key ?? '',
926
+ await this.#values.decode(held.value ?? 'null'),
927
+ new Date(held.at ?? 0),
928
+ held.attempt ?? 0,
929
+ signal,
930
+ reporter,
931
+ steps,
932
+ )
933
+ try {
934
+ await handler(job)
935
+ outcome = outcomeOf(number, job.settlement ?? { how: 'ack' })
936
+ } catch (err) {
937
+ outcome =
938
+ stopping.aborted && signal.aborted
939
+ ? { job: number, how: how.snooze, at: held.at ?? Date.now() }
940
+ : outcomeOf(number, job.settlement ?? { how: 'retry', error: reasonOf(err) })
941
+ }
942
+ } catch (err) {
943
+ outcome = { job: number, how: how.fail, err: `the value no longer reads: ${reasonOf(err)}` }
944
+ }
945
+ await reporter.stop()
946
+ if (!cancelled.aborted) {
947
+ await stream.send(JobsOutcome.encode(outcome), false).catch(() => {})
948
+ }
949
+ }
950
+ }
951
+
952
+ function outcomeOf(job: number, s: Settlement): Parameters<typeof JobsOutcome.encode>[0] {
953
+ switch (s.how) {
954
+ case 'ack':
955
+ return { job, how: how.ack }
956
+ case 'retry':
957
+ return { job, how: how.retry, err: s.error, ...timing(s) }
958
+ case 'fail':
959
+ return { job, how: how.fail, err: s.error }
960
+ case 'snooze':
961
+ return { job, how: how.snooze, ...timing(s) }
962
+ }
963
+ }