@tinyshed/tinystore 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +262 -0
  3. package/bin/tinystore.js +50 -0
  4. package/dist/background.d.ts +30 -0
  5. package/dist/background.js +93 -0
  6. package/dist/blobs.d.ts +98 -0
  7. package/dist/blobs.js +299 -0
  8. package/dist/cancel.d.ts +14 -0
  9. package/dist/cancel.js +22 -0
  10. package/dist/clock.d.ts +21 -0
  11. package/dist/clock.js +35 -0
  12. package/dist/config.d.ts +60 -0
  13. package/dist/config.js +417 -0
  14. package/dist/connection.d.ts +78 -0
  15. package/dist/connection.js +405 -0
  16. package/dist/console.d.ts +48 -0
  17. package/dist/console.js +195 -0
  18. package/dist/errors.d.ts +103 -0
  19. package/dist/errors.js +158 -0
  20. package/dist/handles.d.ts +28 -0
  21. package/dist/handles.js +37 -0
  22. package/dist/index.d.ts +21 -0
  23. package/dist/index.js +7 -0
  24. package/dist/jobs.d.ts +304 -0
  25. package/dist/jobs.js +684 -0
  26. package/dist/kv.d.ts +283 -0
  27. package/dist/kv.js +815 -0
  28. package/dist/limiter.d.ts +45 -0
  29. package/dist/limiter.js +68 -0
  30. package/dist/logger.d.ts +96 -0
  31. package/dist/logger.js +243 -0
  32. package/dist/metrics.d.ts +249 -0
  33. package/dist/metrics.js +610 -0
  34. package/dist/once.d.ts +31 -0
  35. package/dist/once.js +116 -0
  36. package/dist/quota.d.ts +44 -0
  37. package/dist/quota.js +98 -0
  38. package/dist/records.d.ts +182 -0
  39. package/dist/records.js +346 -0
  40. package/dist/runtime/bun.d.ts +2 -0
  41. package/dist/runtime/bun.js +164 -0
  42. package/dist/runtime/node.d.ts +2 -0
  43. package/dist/runtime/node.js +157 -0
  44. package/dist/runtime/shared.d.ts +25 -0
  45. package/dist/runtime/shared.js +45 -0
  46. package/dist/runtime.d.ts +48 -0
  47. package/dist/runtime.js +4 -0
  48. package/dist/schema.d.ts +39 -0
  49. package/dist/schema.js +21 -0
  50. package/dist/session.d.ts +120 -0
  51. package/dist/session.js +546 -0
  52. package/dist/sql.d.ts +121 -0
  53. package/dist/sql.js +342 -0
  54. package/dist/store.d.ts +103 -0
  55. package/dist/store.js +153 -0
  56. package/dist/time.d.ts +35 -0
  57. package/dist/time.js +44 -0
  58. package/dist/trace.d.ts +18 -0
  59. package/dist/trace.js +22 -0
  60. package/dist/wire/codec.d.ts +90 -0
  61. package/dist/wire/codec.js +383 -0
  62. package/dist/wire/frame.d.ts +41 -0
  63. package/dist/wire/frame.js +102 -0
  64. package/dist/wire/messages.d.ts +1403 -0
  65. package/dist/wire/messages.js +472 -0
  66. package/dist/wire/msgpack.d.ts +93 -0
  67. package/dist/wire/msgpack.js +680 -0
  68. package/package.json +40 -0
  69. package/src/background.ts +108 -0
  70. package/src/blobs.ts +437 -0
  71. package/src/cancel.ts +26 -0
  72. package/src/clock.ts +45 -0
  73. package/src/config.ts +501 -0
  74. package/src/connection.ts +512 -0
  75. package/src/console.ts +228 -0
  76. package/src/errors.ts +211 -0
  77. package/src/handles.ts +68 -0
  78. package/src/index.ts +92 -0
  79. package/src/jobs.ts +963 -0
  80. package/src/kv.ts +1093 -0
  81. package/src/limiter.ts +99 -0
  82. package/src/logger.ts +326 -0
  83. package/src/metrics.ts +896 -0
  84. package/src/once.ts +142 -0
  85. package/src/quota.ts +137 -0
  86. package/src/records.ts +537 -0
  87. package/src/runtime/bun.ts +190 -0
  88. package/src/runtime/node.ts +177 -0
  89. package/src/runtime/shared.ts +61 -0
  90. package/src/runtime.ts +57 -0
  91. package/src/schema.ts +69 -0
  92. package/src/session.ts +665 -0
  93. package/src/sql.ts +478 -0
  94. package/src/store.ts +219 -0
  95. package/src/time.ts +87 -0
  96. package/src/trace.ts +32 -0
  97. package/src/wire/codec.ts +445 -0
  98. package/src/wire/frame.ts +139 -0
  99. package/src/wire/messages.ts +562 -0
  100. package/src/wire/msgpack.ts +712 -0
package/dist/jobs.js ADDED
@@ -0,0 +1,684 @@
1
+ // Work that runs at its time, in jobs.db, as Go's jobs package keeps it:
2
+ // queues ordered by time, leases, retries and repeats. A handle opens with its
3
+ // first call; a work loop is the server's own Work, its jobs handed over the
4
+ // connection and settled by what the handler returns.
5
+ import { currentSignal } from './cancel.js';
6
+ import { download } from './connection.js';
7
+ import { CancelledError, CorruptError, errorOf, InvalidError } from './errors.js';
8
+ import { checkName, handleOn } from './handles.js';
9
+ import { check, isSchema } from './schema.js';
10
+ import { LostError, watch } from './session.js';
11
+ import { dateOf, ms, unixMs } from './time.js';
12
+ import { JobsAnswer, JobsBatch, JobsChange, JobsEntry, JobsHeld, JobsKept, JobsKey, JobsLease, JobsOutcome, JobsOutcomes, JobsPage, JobsQuery, JobsQueue, JobsSettled, JobsWorkers, methods, } from './wire/messages.js';
13
+ const how = { ack: 1, retry: 2, fail: 3, snooze: 4, extend: 5, progress: 6 };
14
+ /** the JSON a handler reports of its job, at most 4 KiB of it */
15
+ function progressText(progress) {
16
+ let text;
17
+ try {
18
+ text = JSON.stringify(progress);
19
+ }
20
+ catch (err) {
21
+ throw new InvalidError(`a job's progress JSON cannot write: ${err.message}`);
22
+ }
23
+ if (text === undefined) {
24
+ throw new InvalidError("a job's progress JSON cannot write: undefined, a function or a symbol");
25
+ }
26
+ if (new TextEncoder().encode(text).length > 4096) {
27
+ throw new InvalidError("a job's progress is past 4 KiB of JSON");
28
+ }
29
+ return text;
30
+ }
31
+ /**
32
+ * Sends what a handler reports of its job: the latest, one send at a time
33
+ * and at most one each 100 ms, so that a handler reporting each chunk it
34
+ * reads costs the connection ten frames a second.
35
+ */
36
+ class Reporter {
37
+ #send;
38
+ #latest;
39
+ #sending;
40
+ #timer;
41
+ #sent = 0;
42
+ #stopped = false;
43
+ constructor(send) {
44
+ this.#send = send;
45
+ }
46
+ report(text) {
47
+ this.#latest = text;
48
+ this.#flush();
49
+ }
50
+ #flush() {
51
+ if (this.#stopped || this.#sending !== undefined || this.#latest === undefined) {
52
+ return;
53
+ }
54
+ const wait = this.#sent + 100 - Date.now();
55
+ if (wait > 0) {
56
+ this.#timer ??= setTimeout(() => {
57
+ this.#timer = undefined;
58
+ this.#flush();
59
+ }, wait);
60
+ return;
61
+ }
62
+ const text = this.#latest;
63
+ this.#latest = undefined;
64
+ this.#sent = Date.now();
65
+ this.#sending = this.#send(text)
66
+ .catch(() => { })
67
+ .finally(() => {
68
+ this.#sending = undefined;
69
+ this.#flush();
70
+ });
71
+ }
72
+ /** Drops what waits and lets what is being sent finish, before the job's outcome follows it. */
73
+ async stop() {
74
+ this.#stopped = true;
75
+ clearTimeout(this.#timer);
76
+ await this.#sending;
77
+ }
78
+ }
79
+ /**
80
+ * The steps of a held job's run, which the server keeps under the number the
81
+ * job was held by: a step's kept answer, or the step run and its answer kept.
82
+ */
83
+ class Steps {
84
+ #call;
85
+ #job;
86
+ constructor(call, job) {
87
+ this.#call = call;
88
+ this.#job = job;
89
+ }
90
+ async run(name, fn) {
91
+ const kept = JobsKept.decode(await this.#call(methods['jobs.step'], JobsAnswer.encode({ job: this.#job, name })));
92
+ if (kept.found === true) {
93
+ return JSON.parse(kept.answer ?? 'null');
94
+ }
95
+ const answer = await fn();
96
+ let text;
97
+ try {
98
+ text = JSON.stringify(answer) ?? 'null';
99
+ }
100
+ catch (err) {
101
+ throw new InvalidError(`step ${name} answered what JSON cannot write: ${err.message}`);
102
+ }
103
+ await this.#call(methods['jobs.keep'], JobsAnswer.encode({ job: this.#job, name, answer: text }));
104
+ return answer;
105
+ }
106
+ }
107
+ /**
108
+ * A job in a work loop's handler. Returning acknowledges it and throwing
109
+ * retries it, unless the handler said otherwise with retry, fail or snooze;
110
+ * the last of those it calls is how the job settles.
111
+ */
112
+ export class Job {
113
+ key;
114
+ value;
115
+ /** the time it ran for */
116
+ at;
117
+ /** its attempts, this one included */
118
+ attempt;
119
+ /**
120
+ * aborts when the job's timeout passes, its loop stops, or cancel takes it,
121
+ * whose reason is then a CancelledError: the handler should stop too
122
+ */
123
+ signal;
124
+ settlement;
125
+ #reporter;
126
+ #steps;
127
+ constructor(key, value, at, attempt, signal, reporter, steps) {
128
+ this.key = key;
129
+ this.value = value;
130
+ this.at = at;
131
+ this.attempt = attempt;
132
+ this.signal = signal;
133
+ this.#reporter = reporter;
134
+ this.#steps = steps;
135
+ }
136
+ /**
137
+ * Runs fn once in the job's run and keeps what it answers, as JSON: an
138
+ * attempt after a retry, a lost lease or a restart gets the kept answer
139
+ * back without running fn again. A step whose attempt ends before its
140
+ * answer is kept runs again, so what fn does outside the store should bear
141
+ * doing twice. A name is the step's within the run, which a loop numbers:
142
+ * 'model:1', 'tool:1', 'model:2'. An answer is at most 1 MiB of JSON.
143
+ *
144
+ * const hits = await job.step('search', () => search(job.value.query))
145
+ */
146
+ step(name, fn) {
147
+ if (this.#steps === undefined) {
148
+ return Promise.reject(new InvalidError('a job no work loop handed over keeps no steps'));
149
+ }
150
+ return this.#steps.run(name, fn);
151
+ }
152
+ /**
153
+ * Reports how far the job got, any JSON within 4 KiB: get and watch show
154
+ * the latest until the job is settled. It does not wait for the server.
155
+ *
156
+ * await transcode({ signal: job.signal, onProgress: p => job.progress(p) })
157
+ */
158
+ progress(progress) {
159
+ this.#reporter?.report(progressText(progress));
160
+ }
161
+ /** Fails this attempt: the job runs again after its backoff, or at the time given. */
162
+ retry(error, when) {
163
+ this.settlement = { how: 'retry', error: reasonOf(error), ...when };
164
+ }
165
+ /** Fails the job for good, keeping it for keepFailed with its reason. */
166
+ fail(error) {
167
+ this.settlement = { how: 'fail', error: reasonOf(error) };
168
+ }
169
+ /** Moves the job to another time without counting the attempt. */
170
+ snooze(when) {
171
+ this.settlement = { how: 'snooze', ...when };
172
+ }
173
+ }
174
+ /** A job a claim leased, which its caller settles itself before the lease ends. */
175
+ export class ClaimedJob {
176
+ key;
177
+ value;
178
+ at;
179
+ attempt;
180
+ #settle;
181
+ #job;
182
+ #steps;
183
+ constructor(held, value, settle, call) {
184
+ this.key = held.key ?? '';
185
+ this.value = value;
186
+ this.at = new Date(held.at ?? 0);
187
+ this.attempt = held.attempt ?? 0;
188
+ this.#job = held.job ?? 0;
189
+ this.#settle = settle;
190
+ this.#steps = new Steps(call, this.#job);
191
+ }
192
+ /** Runs fn once in the job's run and keeps its answer, as a work loop's job.step does. */
193
+ step(name, fn) {
194
+ return this.#steps.run(name, fn);
195
+ }
196
+ /** The job is done. */
197
+ ack() {
198
+ return this.#settle({ job: this.#job, how: how.ack });
199
+ }
200
+ retry(error, when) {
201
+ return this.#settle({ job: this.#job, how: how.retry, err: reasonOf(error), ...timing(when) });
202
+ }
203
+ fail(error) {
204
+ return this.#settle({ job: this.#job, how: how.fail, err: reasonOf(error) });
205
+ }
206
+ snooze(when) {
207
+ return this.#settle({ job: this.#job, how: how.snooze, ...timing(when) });
208
+ }
209
+ /** Holds the job this much longer from now. */
210
+ extend(d) {
211
+ return this.#settle({ job: this.#job, how: how.extend, after: ms(d) });
212
+ }
213
+ /** Reports how far the job got, any JSON within 4 KiB, which get and watch show until it is settled. */
214
+ progress(progress) {
215
+ return this.#settle({ job: this.#job, how: how.progress, progress: progressText(progress) });
216
+ }
217
+ }
218
+ function reasonOf(error) {
219
+ if (error === undefined) {
220
+ return 'the handler gave no reason';
221
+ }
222
+ return error instanceof Error ? error.message : String(error);
223
+ }
224
+ function timing(when) {
225
+ if (when?.at !== undefined) {
226
+ return { at: unixMs(when.at) };
227
+ }
228
+ if (when?.after !== undefined) {
229
+ return { after: ms(when.after) };
230
+ }
231
+ return {};
232
+ }
233
+ const jsonValues = {
234
+ encode: value => {
235
+ let text;
236
+ try {
237
+ text = JSON.stringify(value);
238
+ }
239
+ catch (err) {
240
+ throw new InvalidError(`a job's value JSON cannot write: ${err.message}`);
241
+ }
242
+ if (text === undefined) {
243
+ throw new InvalidError("a job's value JSON cannot write: undefined, a function or a symbol");
244
+ }
245
+ return text;
246
+ },
247
+ decode: text => {
248
+ try {
249
+ return JSON.parse(text);
250
+ }
251
+ catch (err) {
252
+ throw new CorruptError(`a job's value is not JSON any more: ${err.message}`);
253
+ }
254
+ },
255
+ };
256
+ function schemaValues(schema) {
257
+ return {
258
+ encode: jsonValues.encode,
259
+ decode: async (text) => {
260
+ const checked = await check(schema, await jsonValues.decode(text));
261
+ if ('issues' in checked) {
262
+ throw new CorruptError(`a job's value does not meet the queue's schema: ${checked.issues}`);
263
+ }
264
+ return checked.value;
265
+ },
266
+ };
267
+ }
268
+ function repeatOf(r) {
269
+ if ('every' in r) {
270
+ return { every: ms(r.every) };
271
+ }
272
+ if ('daily' in r) {
273
+ const time = /^(\d{1,2}):(\d{2})$/.exec(r.daily);
274
+ if (time === null || Number(time[1]) > 23 || Number(time[2]) > 59) {
275
+ throw new InvalidError(`the daily time ${JSON.stringify(r.daily)}; write it as 03:10`);
276
+ }
277
+ return { cron: `${Number(time[2])} ${Number(time[1])} * * *`, zone: r.zone };
278
+ }
279
+ return { cron: r.cron, zone: r.zone };
280
+ }
281
+ export class Jobs {
282
+ #link;
283
+ constructor(link) {
284
+ this.#link = link;
285
+ }
286
+ queue(name, of, options) {
287
+ checkName(name, 'queue');
288
+ let values = jsonValues;
289
+ if (isSchema(of)) {
290
+ values = schemaValues(of);
291
+ }
292
+ else if (of !== undefined) {
293
+ options = of;
294
+ }
295
+ const open = JobsQueue.encode(queueFields(name, options));
296
+ return new Queue(this.#link, name, open, values, options?.in);
297
+ }
298
+ /**
299
+ * A schedule: a queue of one job under the schedule's name, which repeats
300
+ * as it says. work runs it; cancel stops it.
301
+ */
302
+ schedule(name, repeat, options) {
303
+ checkName(name, 'schedule');
304
+ const open = JobsQueue.encode({ ...queueFields(name, options), schedule: repeatOf(repeat) });
305
+ const nothing = { encode: () => '{}', decode: () => null };
306
+ return new Queue(this.#link, name, open, nothing, options?.in);
307
+ }
308
+ }
309
+ function queueFields(name, options) {
310
+ return {
311
+ name,
312
+ lease: options?.lease === undefined ? undefined : ms(options.lease),
313
+ maxAttempts: options?.maxAttempts,
314
+ backoffFirst: options?.backoff?.first === undefined ? undefined : ms(options.backoff.first),
315
+ backoffMost: options?.backoff?.most === undefined ? undefined : ms(options.backoff.most),
316
+ maxWaiting: options?.maxWaiting,
317
+ keepFailed: options?.keepFailed === undefined ? undefined : ms(options.keepFailed),
318
+ keepDone: options?.keepDone === undefined ? undefined : ms(options.keepDone),
319
+ maxRunning: options?.maxRunning,
320
+ in: options?.in?.name,
321
+ };
322
+ }
323
+ /** A job's fields as jobs.enqueue and a batch carry them. */
324
+ function jobFields(values, value, options) {
325
+ return {
326
+ value: values.encode(value),
327
+ key: options?.key,
328
+ at: options?.at === undefined ? undefined : unixMs(options.at),
329
+ after: options?.after === undefined ? undefined : ms(options.after),
330
+ repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
331
+ };
332
+ }
333
+ /** A queue's enqueue inside a batch of the database it lives in. */
334
+ export class QueueTx {
335
+ #tx;
336
+ #open;
337
+ #values;
338
+ constructor(tx, open, values) {
339
+ this.#tx = tx;
340
+ this.#open = open;
341
+ this.#values = values;
342
+ }
343
+ /** Adds a job to the batch; it settles once the batch has committed. */
344
+ enqueue(value, options) {
345
+ return this.#tx.enqueue(this.#open, jobFields(this.#values, value, options));
346
+ }
347
+ }
348
+ const states = {
349
+ 1: 'waiting',
350
+ 2: 'running',
351
+ 3: 'failed',
352
+ 4: 'done',
353
+ 5: 'cancelled',
354
+ };
355
+ export class Queue {
356
+ name;
357
+ #link;
358
+ #open;
359
+ #values;
360
+ #in;
361
+ constructor(link, name, open, values, inDatabase) {
362
+ this.#link = link;
363
+ this.name = name;
364
+ this.#open = open;
365
+ this.#values = values;
366
+ this.#in = inDatabase;
367
+ }
368
+ /** The queue's handle, its database opened first when it lives in one, as the server needs. */
369
+ async #handle(connection) {
370
+ await this.#in?.handle(connection);
371
+ return handleOn(connection, methods['jobs.open'], this.#open);
372
+ }
373
+ /**
374
+ * The queue's enqueue inside a batch of the database it lives in, which
375
+ * it was opened `in`: the job commits with the batch's rows or not at all.
376
+ *
377
+ * await db.batch(tx => {
378
+ * tx.exec`update notes set body = ${body} where id = ${id}`
379
+ * index.withTx(tx).enqueue({ id })
380
+ * })
381
+ */
382
+ withTx(tx) {
383
+ if (this.#in === undefined || this.#in.name !== tx.database) {
384
+ const lives = this.#in === undefined ? 'jobs.db' : `sql ${this.#in.name}`;
385
+ throw new InvalidError(`the queue ${this.name} lives in ${lives}, not in sql ${tx.database}: ` +
386
+ "open it with { in: db } to enqueue in that database's batches");
387
+ }
388
+ return new QueueTx(tx, this.#open, this.#values);
389
+ }
390
+ /** Adds a job; it returns once the job is in the file. */
391
+ enqueue(value, options) {
392
+ return this.enqueueAll([{ value, ...options }]);
393
+ }
394
+ /** Adds jobs in one transaction, all or none: a refused one names itself as `call`. */
395
+ async enqueueAll(jobs) {
396
+ const encoded = jobs.map(job => jobFields(this.#values, job.value, job));
397
+ await this.#link.run('write', async (connection) => {
398
+ const handle = await this.#handle(connection);
399
+ await connection.session.call(methods['jobs.enqueue'], JobsBatch.encode({ handle, jobs: encoded }));
400
+ });
401
+ }
402
+ /**
403
+ * Changes a job that waits or failed: its value and, when the options
404
+ * say, its time or repeat. A job a worker holds, done or absent is
405
+ * ConflictError.
406
+ */
407
+ async update(key, value, options) {
408
+ const change = {
409
+ value: this.#values.encode(value),
410
+ key,
411
+ at: options?.at === undefined ? undefined : unixMs(options.at),
412
+ after: options?.after === undefined ? undefined : ms(options.after),
413
+ repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
414
+ };
415
+ await this.#link.run('write', async (connection) => {
416
+ const handle = await this.#handle(connection);
417
+ await connection.session.call(methods['jobs.update'], JobsChange.encode({ ...change, handle }));
418
+ });
419
+ }
420
+ /**
421
+ * Removes the job under key, whether it waits, runs or failed, and says
422
+ * whether there was one. A running job's handler sees job.signal abort,
423
+ * and what it returns settles nothing.
424
+ */
425
+ cancel(key) {
426
+ return this.#link.run('write', async (connection) => {
427
+ const handle = await this.#handle(connection);
428
+ const body = await connection.session.call(methods['jobs.cancel'], JobsKey.encode({ handle, key }));
429
+ return JobsEntry.decode(body).found === true;
430
+ });
431
+ }
432
+ /**
433
+ * The job a key names, waiting, running or failed, or done while keepDone
434
+ * keeps its key; undefined when none is.
435
+ */
436
+ async get(key) {
437
+ const entry = await this.#link.run('read', async (connection) => {
438
+ const handle = await this.#handle(connection);
439
+ return JobsEntry.decode(await connection.session.call(methods['jobs.get'], JobsKey.encode({ handle, key })));
440
+ });
441
+ return entry.found === true ? this.#entryOf(entry) : undefined;
442
+ }
443
+ async #entryOf(entry) {
444
+ const state = states[entry.state ?? 1] ?? 'waiting';
445
+ return {
446
+ key: entry.key ?? '',
447
+ value: state === 'done' && entry.value === undefined
448
+ ? undefined
449
+ : await this.#values.decode(entry.value ?? 'null'),
450
+ at: new Date(entry.at ?? 0),
451
+ attempt: entry.attempt ?? 0,
452
+ state,
453
+ ahead: entry.ahead ?? 0,
454
+ progress: entry.progress === undefined ? undefined : JSON.parse(entry.progress),
455
+ ran: dateOf(entry.ran),
456
+ took: entry.ran === undefined ? undefined : (entry.took ?? 0),
457
+ error: entry.err,
458
+ repeat: entry.repeat,
459
+ };
460
+ }
461
+ /**
462
+ * Yields the job under key as it is, then again each time its state,
463
+ * place, attempt, time, progress or error changes, until it ends: done,
464
+ * failed or cancelled, the last entry it yields. A key that names no job
465
+ * yields nothing. Leaving the loop ends the watch; a connection lost is
466
+ * connected again, and the watch goes on from the job as it is then.
467
+ *
468
+ * for await (const s of videos.watch(id)) send(s.state, s.ahead, s.progress)
469
+ */
470
+ async *watch(key) {
471
+ let last;
472
+ for (;;) {
473
+ const stream = await this.#link.run('read', async (connection) => {
474
+ const handle = await this.#handle(connection);
475
+ const opened = await connection.session.open(methods['jobs.watch'], JobsKey.encode({ handle, key }), true);
476
+ await opened.next();
477
+ return opened;
478
+ });
479
+ const unwatch = watch(currentSignal(), stream);
480
+ try {
481
+ for (;;) {
482
+ const event = await stream.next();
483
+ stream.consumed(event.body.length);
484
+ if (event.end) {
485
+ return;
486
+ }
487
+ const entry = await this.#entryOf(JobsEntry.decode(event.body));
488
+ const seen = JSON.stringify([
489
+ entry.state,
490
+ entry.ahead,
491
+ entry.attempt,
492
+ entry.at,
493
+ entry.progress,
494
+ entry.error,
495
+ ]);
496
+ if (seen !== last) {
497
+ last = seen;
498
+ yield entry;
499
+ }
500
+ }
501
+ }
502
+ catch (err) {
503
+ if (!(err instanceof LostError)) {
504
+ throw err;
505
+ }
506
+ }
507
+ finally {
508
+ unwatch();
509
+ stream.cancel();
510
+ }
511
+ }
512
+ }
513
+ /**
514
+ * A page of the jobs under a prefix, in the byte order of their keys; with
515
+ * state 'failed' and no prefix, the failed jobs, the last failed first.
516
+ */
517
+ async scan(query) {
518
+ const got = await this.#link.run('read', async (connection) => {
519
+ const handle = await this.#handle(connection);
520
+ const body = JobsQuery.encode({
521
+ handle,
522
+ prefix: query?.prefix,
523
+ state: query?.state === 'failed' ? 3 : undefined,
524
+ after: query?.after,
525
+ limit: query?.limit,
526
+ });
527
+ return download(connection, methods['jobs.scan'], body);
528
+ });
529
+ const items = [];
530
+ for (const item of got.items) {
531
+ items.push(await this.#entryOf(JobsEntry.decode(item)));
532
+ }
533
+ const page = JobsPage.decode(got.trailer);
534
+ return { items, next: page.more === true ? (page.after ?? '') : undefined };
535
+ }
536
+ /** Walks scan's pages, holding no snapshot between them. */
537
+ async *all(query) {
538
+ let after;
539
+ for (;;) {
540
+ const page = await this.scan({ ...query, ...(after === undefined ? {} : { after }) });
541
+ yield* page.items;
542
+ if (page.next === undefined) {
543
+ return;
544
+ }
545
+ after = page.next;
546
+ }
547
+ }
548
+ /**
549
+ * Leases the next due job, or says there is none, without waiting. The
550
+ * claim lives on this connection: settle it before its lease ends.
551
+ */
552
+ async claim(options) {
553
+ return this.#link.run('write', async (connection) => {
554
+ const handle = await this.#handle(connection);
555
+ const lease = options?.lease === undefined ? undefined : ms(options.lease);
556
+ const held = JobsHeld.decode(await connection.session.call(methods['jobs.claim'], JobsLease.encode({ handle, lease })));
557
+ if (held.found !== true) {
558
+ return undefined;
559
+ }
560
+ const value = await this.#values.decode(held.value ?? 'null');
561
+ return new ClaimedJob(held, value, async (outcome) => {
562
+ const body = await connection.session.call(methods['jobs.settle'], JobsOutcomes.encode({ outcomes: [outcome] }));
563
+ const refused = JobsSettled.decode(body).settled?.[0];
564
+ if (refused !== undefined && refused !== null) {
565
+ throw errorOf(refused.code ?? 'internal', refused.message ?? '', refused.what ?? {});
566
+ }
567
+ }, (method, body) => connection.session.call(method, body));
568
+ });
569
+ }
570
+ /**
571
+ * Runs the queue's jobs as they come due, workers at once, until the
572
+ * signal aborts: the server's own Work loop claims ahead and extends
573
+ * leases, and nothing polls. A handler's return acknowledges its job and a
574
+ * throw retries it. A connection lost takes the jobs in hand with it, as a
575
+ * process that died would, and the loop connects again.
576
+ */
577
+ async work(handler, options = {}) {
578
+ for (;;) {
579
+ options.signal?.throwIfAborted();
580
+ try {
581
+ await this.#link.run('read', connection => this.#workOn(connection, handler, options), options.signal);
582
+ if (options.untilIdle === true || options.signal?.aborted === true) {
583
+ return;
584
+ }
585
+ }
586
+ catch (err) {
587
+ if (!(err instanceof LostError) || options.signal?.aborted === true) {
588
+ throw err;
589
+ }
590
+ }
591
+ }
592
+ }
593
+ async #workOn(connection, handler, options) {
594
+ const handle = await this.#handle(connection);
595
+ const timeout = options.timeout === undefined ? 60_000 : ms(options.timeout);
596
+ const stream = await connection.session.open(methods['jobs.work'], JobsWorkers.encode({
597
+ handle,
598
+ workers: options.workers,
599
+ timeout,
600
+ untilIdle: options.untilIdle || undefined,
601
+ cancels: true,
602
+ }), false);
603
+ const stopping = new AbortController();
604
+ const stop = () => stopping.abort();
605
+ options.signal?.addEventListener('abort', stop, { once: true });
606
+ const inHand = new Set();
607
+ const cancels = new Map();
608
+ let ended = false;
609
+ const endOurSide = async () => {
610
+ if (!ended) {
611
+ ended = true;
612
+ await Promise.allSettled(inHand);
613
+ await stream.send(new Uint8Array(0), true).catch(() => { });
614
+ }
615
+ };
616
+ stopping.signal.addEventListener('abort', () => void endOurSide(), { once: true });
617
+ try {
618
+ await stream.next();
619
+ for (;;) {
620
+ const event = await stream.next();
621
+ stream.consumed(event.body.length);
622
+ if (event.end) {
623
+ return;
624
+ }
625
+ const held = JobsHeld.decode(event.body);
626
+ if (held.cancelled === true) {
627
+ cancels.get(held.job ?? 0)?.abort(new CancelledError('cancel took the job while it ran'));
628
+ continue;
629
+ }
630
+ const cancel = new AbortController();
631
+ cancels.set(held.job ?? 0, cancel);
632
+ const steps = new Steps((method, body) => connection.session.call(method, body), held.job ?? 0);
633
+ const running = this.#run(stream, held, steps, handler, timeout, stopping.signal, cancel.signal).finally(() => cancels.delete(held.job ?? 0));
634
+ inHand.add(running);
635
+ running.finally(() => inHand.delete(running));
636
+ }
637
+ }
638
+ finally {
639
+ options.signal?.removeEventListener('abort', stop);
640
+ stopping.abort();
641
+ }
642
+ }
643
+ // runs one job's handler and sends the outcome it settled by; a handler
644
+ // stopped by the loop's end gives its job back without counting the
645
+ // attempt, and one cancel took settles nothing, so it sends nothing
646
+ async #run(stream, held, steps, handler, timeout, stopping, cancelled) {
647
+ const signal = AbortSignal.any([stopping, AbortSignal.timeout(timeout), cancelled]);
648
+ const number = held.job ?? 0;
649
+ const reporter = new Reporter(progress => stream.send(JobsOutcome.encode({ job: number, how: how.progress, progress }), false));
650
+ let outcome;
651
+ try {
652
+ const job = new Job(held.key ?? '', await this.#values.decode(held.value ?? 'null'), new Date(held.at ?? 0), held.attempt ?? 0, signal, reporter, steps);
653
+ try {
654
+ await handler(job);
655
+ outcome = outcomeOf(number, job.settlement ?? { how: 'ack' });
656
+ }
657
+ catch (err) {
658
+ outcome =
659
+ stopping.aborted && signal.aborted
660
+ ? { job: number, how: how.snooze, at: held.at ?? Date.now() }
661
+ : outcomeOf(number, job.settlement ?? { how: 'retry', error: reasonOf(err) });
662
+ }
663
+ }
664
+ catch (err) {
665
+ outcome = { job: number, how: how.fail, err: `the value no longer reads: ${reasonOf(err)}` };
666
+ }
667
+ await reporter.stop();
668
+ if (!cancelled.aborted) {
669
+ await stream.send(JobsOutcome.encode(outcome), false).catch(() => { });
670
+ }
671
+ }
672
+ }
673
+ function outcomeOf(job, s) {
674
+ switch (s.how) {
675
+ case 'ack':
676
+ return { job, how: how.ack };
677
+ case 'retry':
678
+ return { job, how: how.retry, err: s.error, ...timing(s) };
679
+ case 'fail':
680
+ return { job, how: how.fail, err: s.error };
681
+ case 'snooze':
682
+ return { job, how: how.snooze, ...timing(s) };
683
+ }
684
+ }