@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/dist/jobs.js CHANGED
@@ -1,30 +1,126 @@
1
- // Work that runs at its time, in jobs.db, as Go's jobs package keeps it:
2
- // queues ordered by time, leases, retries and repeats. A handle opens with its
3
- // first call; a work loop is the server's own Work, its jobs handed over the
4
- // connection and settled by what the handler returns.
5
- import { currentSignal } from './cancel.js';
6
- import { download } from './connection.js';
7
- import { CancelledError, CorruptError, errorOf, InvalidError } from './errors.js';
8
- import { checkName, handleOn } from './handles.js';
9
- import { check, isSchema } from './schema.js';
10
- import { LostError, watch } from './session.js';
11
- import { dateOf, ms, unixMs } from './time.js';
12
- import { JobsAnswer, JobsBatch, JobsChange, JobsEntry, JobsHeld, JobsKept, JobsKey, JobsLease, JobsOutcome, JobsOutcomes, JobsPage, JobsQuery, JobsQueue, JobsSettled, JobsWorkers, methods, } from './wire/messages.js';
13
- const how = { ack: 1, retry: 2, fail: 3, snooze: 4, extend: 5, progress: 6 };
14
- /** the JSON a handler reports of its job, at most 4 KiB of it */
15
- function progressText(progress) {
16
- let text;
17
- try {
18
- text = JSON.stringify(progress);
1
+ // Work an application must do later, or now but outside the request that
2
+ // asked for it, as docs/guides/jobs/README.md has it: queues of JSON values run in the
3
+ // order of their time, ids that find a job again, repeats, schedules and
4
+ // limits. A worker's handlers run here while the server's loop claims the
5
+ // queue's jobs and hands each over on one stream, the answers going back on
6
+ // it. A handle opens with its first call on a connection.
7
+ import { FailureLog, messageOf } from './background.js';
8
+ import { CancelledError, CorruptError, InvalidError, UnavailableError } from './errors.js';
9
+ import { checkName, handleOn, viaLink } from './handles.js';
10
+ import { rateOf } from './limits.js';
11
+ import { sayOwn } from './logger.js';
12
+ import { check } from './schema.js';
13
+ import { ConnectionLostError } from './session.js';
14
+ import { ms, unixMs } from './time.js';
15
+ import { JobsAnswer, JobsCall, JobsChanged, JobsHeld, JobsId, JobsJob, JobsKept, JobsList, JobsPage, JobsQueueOpen, JobsScheduleOpen, JobsStep, JobsTx, JobsWork, methods, } from './wire/protocol.js';
16
+ /** The handlers a client's worker runs at once at most, as the server bounds them. */
17
+ const mostHandlers = 1024;
18
+ /** The pause before a worker dials again after a failure, doubling up to the last. */
19
+ const firstPause = 100;
20
+ const lastPause = 30_000;
21
+ /** How a run ends otherwise than done, made by run.retry, run.snooze or run.fail: its handler returns it. */
22
+ export class Answer {
23
+ #how;
24
+ /** Answers come from a run. */
25
+ constructor(how) {
26
+ this.#how = how;
27
+ }
28
+ toString() {
29
+ return `run.${this.#how}(…)`;
19
30
  }
20
- catch (err) {
21
- throw new InvalidError(`a job's progress JSON cannot write: ${err.message}`);
31
+ }
32
+ /** This call on a job: what it is, its signal, its steps and progress, and the answers its handler returns. */
33
+ export class Run {
34
+ /** the job's id, undefined for one added without */
35
+ id;
36
+ /** when the run was due, which "thirty days ago" counts from even when it starts late */
37
+ at;
38
+ /** this run's number among the job's attempts, the first being 1 */
39
+ attempt;
40
+ group;
41
+ /**
42
+ * aborts when the run passes its timeout, or when a cancel takes the job,
43
+ * whose reason is then a CancelledError: the handler should stop too
44
+ */
45
+ signal;
46
+ #steps;
47
+ #reporter;
48
+ #made;
49
+ /** Runs come to a handler from its worker. */
50
+ constructor(parts) {
51
+ this.id = parts.held.id;
52
+ this.at = new Date(parts.held.at ?? 0);
53
+ this.attempt = parts.held.attempt ?? 0;
54
+ this.group = parts.held.group;
55
+ this.signal = parts.signal;
56
+ this.#steps = parts.steps;
57
+ this.#reporter = parts.reporter;
58
+ this.#made = parts.made;
22
59
  }
23
- if (text === undefined) {
24
- throw new InvalidError("a job's progress JSON cannot write: undefined, a function or a symbol");
60
+ /**
61
+ * Runs fn once in the job's run and keeps its answer, as JSON, across the
62
+ * job's attempts: the attempt after a retry, a lost worker or a restart
63
+ * gets the answer back without running fn again. A step whose attempt
64
+ * ends before its answer is kept runs again, so what fn does outside the
65
+ * store should bear doing twice. A name is the step's within its job,
66
+ * numbered in a loop: 'model:1', 'tool:1'.
67
+ *
68
+ * const found = await run.step('search', () => search(question.text))
69
+ */
70
+ step(name, fn) {
71
+ return this.#steps.run(name, fn);
72
+ }
73
+ /**
74
+ * Shows how far the job got, any JSON within 4 KiB, to get: the latest
75
+ * goes, ten a second at most, and nothing waits for the server.
76
+ */
77
+ setProgress(progress) {
78
+ this.#reporter.report(progressText(progress));
79
+ }
80
+ /** Runs the job again after its backoff, or at the time given, the run counted as an attempt: return it. */
81
+ retry(when) {
82
+ return this.#make('retry', when === undefined ? {} : timing(when));
83
+ }
84
+ /** Runs the job again at the time given, the run not counted: return it. */
85
+ snooze(when) {
86
+ return this.#make('snooze', timing(when));
87
+ }
88
+ /** Fails the job for good now, its reason kept as its error: return it. */
89
+ fail(reason) {
90
+ return this.#make('fail', { error: messageOf(reason) });
91
+ }
92
+ #make(how, fields) {
93
+ const answer = new Answer(how);
94
+ this.#made.set(answer, { ...fields, how });
95
+ return answer;
96
+ }
97
+ }
98
+ /** A retry's or a snooze's time: a Date at it, a span from now on the server's clock. */
99
+ function timing(when) {
100
+ return when instanceof Date ? { at: unixMs(when) } : { delay: ms(when) };
101
+ }
102
+ /** How a handler's return ends its run: an answer it made, done for anything else, and a failed run for an answer made and not returned. */
103
+ function answerOf(returned, made) {
104
+ if (returned instanceof Answer) {
105
+ return (made.get(returned) ?? {
106
+ how: 'retry',
107
+ error: 'the handler returned an answer another run made',
108
+ });
109
+ }
110
+ const [unreturned] = made.keys();
111
+ if (unreturned !== undefined) {
112
+ return {
113
+ how: 'retry',
114
+ error: `the handler made ${unreturned} and returned something else: return the answer that ends the run`,
115
+ };
25
116
  }
117
+ return { how: 'done' };
118
+ }
119
+ /** The JSON a handler reports of its job, 4 KiB of it at most. */
120
+ function progressText(progress) {
121
+ const text = jsonText(progress, "a job's progress");
26
122
  if (new TextEncoder().encode(text).length > 4096) {
27
- throw new InvalidError("a job's progress is past 4 KiB of JSON");
123
+ throw new InvalidError("a job's progress past 4 KiB of JSON");
28
124
  }
29
125
  return text;
30
126
  }
@@ -69,476 +165,261 @@ class Reporter {
69
165
  this.#flush();
70
166
  });
71
167
  }
72
- /** Drops what waits and lets what is being sent finish, before the job's outcome follows it. */
168
+ /** Drops what waits and lets what is being sent finish, before the run's answer follows it. */
73
169
  async stop() {
74
170
  this.#stopped = true;
75
171
  clearTimeout(this.#timer);
76
172
  await this.#sending;
77
173
  }
78
174
  }
79
- /**
80
- * The steps of a held job's run, which the server keeps under the number the
81
- * job was held by: a step's kept answer, or the step run and its answer kept.
82
- */
175
+ /** The steps of a held job's run, which the server keeps under the run's number on its connection. */
83
176
  class Steps {
84
177
  #call;
85
- #job;
86
- constructor(call, job) {
178
+ #run;
179
+ #cancelled;
180
+ constructor(call, run, cancelled) {
87
181
  this.#call = call;
88
- this.#job = job;
182
+ this.#run = run;
183
+ this.#cancelled = cancelled;
89
184
  }
90
185
  async run(name, fn) {
91
- const kept = JobsKept.decode(await this.#call(methods['jobs.step'], JobsAnswer.encode({ job: this.#job, name })));
186
+ this.#cancelled.throwIfAborted();
187
+ const asked = JobsStep.encode({ run: this.#run, name });
188
+ const kept = JobsKept.decode(await this.#call(methods['jobs.step'], asked));
92
189
  if (kept.found === true) {
93
190
  return JSON.parse(kept.answer ?? 'null');
94
191
  }
95
192
  const answer = await fn();
96
- let text;
97
- try {
98
- text = JSON.stringify(answer) ?? 'null';
99
- }
100
- catch (err) {
101
- throw new InvalidError(`step ${name} answered what JSON cannot write: ${err.message}`);
102
- }
103
- await this.#call(methods['jobs.keep'], JobsAnswer.encode({ job: this.#job, name, answer: text }));
193
+ const text = jsonText(answer, `step ${JSON.stringify(name)}'s answer`);
194
+ await this.#call(methods['jobs.keep'], JobsStep.encode({ run: this.#run, name, answer: text }));
104
195
  return answer;
105
196
  }
106
197
  }
107
- /**
108
- * A job in a work loop's handler. Returning acknowledges it and throwing
109
- * retries it, unless the handler said otherwise with retry, fail or snooze;
110
- * the last of those it calls is how the job settles.
111
- */
112
- export class Job {
113
- key;
114
- value;
115
- /** the time it ran for */
116
- at;
117
- /** its attempts, this one included */
118
- attempt;
119
- /**
120
- * aborts when the job's timeout passes, its loop stops, or cancel takes it,
121
- * whose reason is then a CancelledError: the handler should stop too
122
- */
123
- signal;
124
- settlement;
125
- #reporter;
126
- #steps;
127
- constructor(key, value, at, attempt, signal, reporter, steps) {
128
- this.key = key;
129
- this.value = value;
130
- this.at = at;
131
- this.attempt = attempt;
132
- this.signal = signal;
133
- this.#reporter = reporter;
134
- this.#steps = steps;
135
- }
136
- /**
137
- * Runs fn once in the job's run and keeps what it answers, as JSON: an
138
- * attempt after a retry, a lost lease or a restart gets the kept answer
139
- * back without running fn again. A step whose attempt ends before its
140
- * answer is kept runs again, so what fn does outside the store should bear
141
- * doing twice. A name is the step's within the run, which a loop numbers:
142
- * 'model:1', 'tool:1', 'model:2'. An answer is at most 1 MiB of JSON.
143
- *
144
- * const hits = await job.step('search', () => search(job.value.query))
145
- */
146
- step(name, fn) {
147
- if (this.#steps === undefined) {
148
- return Promise.reject(new InvalidError('a job no work loop handed over keeps no steps'));
149
- }
150
- return this.#steps.run(name, fn);
151
- }
152
- /**
153
- * Reports how far the job got, any JSON within 4 KiB: get and watch show
154
- * the latest until the job is settled. It does not wait for the server.
155
- *
156
- * await transcode({ signal: job.signal, onProgress: p => job.progress(p) })
157
- */
158
- progress(progress) {
159
- this.#reporter?.report(progressText(progress));
160
- }
161
- /** Fails this attempt: the job runs again after its backoff, or at the time given. */
162
- retry(error, when) {
163
- this.settlement = { how: 'retry', error: reasonOf(error), ...when };
164
- }
165
- /** Fails the job for good, keeping it for keepFailed with its reason. */
166
- fail(error) {
167
- this.settlement = { how: 'fail', error: reasonOf(error) };
168
- }
169
- /** Moves the job to another time without counting the attempt. */
170
- snooze(when) {
171
- this.settlement = { how: 'snooze', ...when };
172
- }
173
- }
174
- /** A job a claim leased, which its caller settles itself before the lease ends. */
175
- export class ClaimedJob {
176
- key;
177
- value;
178
- at;
179
- attempt;
180
- #settle;
181
- #job;
182
- #steps;
183
- constructor(held, value, settle, call) {
184
- this.key = held.key ?? '';
185
- this.value = value;
186
- this.at = new Date(held.at ?? 0);
187
- this.attempt = held.attempt ?? 0;
188
- this.#job = held.job ?? 0;
189
- this.#settle = settle;
190
- this.#steps = new Steps(call, this.#job);
191
- }
192
- /** Runs fn once in the job's run and keeps its answer, as a work loop's job.step does. */
193
- step(name, fn) {
194
- return this.#steps.run(name, fn);
195
- }
196
- /** The job is done. */
197
- ack() {
198
- return this.#settle({ job: this.#job, how: how.ack });
199
- }
200
- retry(error, when) {
201
- return this.#settle({ job: this.#job, how: how.retry, err: reasonOf(error), ...timing(when) });
202
- }
203
- fail(error) {
204
- return this.#settle({ job: this.#job, how: how.fail, err: reasonOf(error) });
205
- }
206
- snooze(when) {
207
- return this.#settle({ job: this.#job, how: how.snooze, ...timing(when) });
208
- }
209
- /** Holds the job this much longer from now. */
210
- extend(d) {
211
- return this.#settle({ job: this.#job, how: how.extend, after: ms(d) });
212
- }
213
- /** Reports how far the job got, any JSON within 4 KiB, which get and watch show until it is settled. */
214
- progress(progress) {
215
- return this.#settle({ job: this.#job, how: how.progress, progress: progressText(progress) });
216
- }
217
- }
218
- function reasonOf(error) {
219
- if (error === undefined) {
220
- return 'the handler gave no reason';
198
+ /** A value's JSON, the text every language reads; undefined writes as null, as a step's answer may be nothing. */
199
+ function jsonText(value, what) {
200
+ let text;
201
+ try {
202
+ text = JSON.stringify(value);
221
203
  }
222
- return error instanceof Error ? error.message : String(error);
223
- }
224
- function timing(when) {
225
- if (when?.at !== undefined) {
226
- return { at: unixMs(when.at) };
204
+ catch (err) {
205
+ throw new InvalidError(`${what} JSON cannot write: ${messageOf(err)}`);
227
206
  }
228
- if (when?.after !== undefined) {
229
- return { after: ms(when.after) };
207
+ if (text === undefined) {
208
+ if (value === undefined) {
209
+ return 'null';
210
+ }
211
+ throw new InvalidError(`${what} JSON cannot write: a function or a symbol`);
230
212
  }
231
- return {};
213
+ return text;
232
214
  }
233
- const jsonValues = {
234
- encode: value => {
235
- let text;
236
- try {
237
- text = JSON.stringify(value);
238
- }
239
- catch (err) {
240
- throw new InvalidError(`a job's value JSON cannot write: ${err.message}`);
241
- }
242
- if (text === undefined) {
243
- throw new InvalidError("a job's value JSON cannot write: undefined, a function or a symbol");
244
- }
245
- return text;
246
- },
247
- decode: text => {
248
- try {
249
- return JSON.parse(text);
250
- }
251
- catch (err) {
252
- throw new CorruptError(`a job's value is not JSON any more: ${err.message}`);
253
- }
254
- },
255
- };
256
- function schemaValues(schema) {
215
+ function valuesOf(schema) {
257
216
  return {
258
- encode: jsonValues.encode,
217
+ encode: value => {
218
+ if (value === undefined) {
219
+ throw new InvalidError('a value of undefined, which JSON cannot write: give null');
220
+ }
221
+ return jsonText(value, 'a value');
222
+ },
259
223
  decode: async (text) => {
260
- const checked = await check(schema, await jsonValues.decode(text));
224
+ let value;
225
+ try {
226
+ value = JSON.parse(text);
227
+ }
228
+ catch (err) {
229
+ throw new CorruptError(`a value that is not JSON any more: ${messageOf(err)}`);
230
+ }
231
+ if (schema === undefined) {
232
+ return value;
233
+ }
234
+ const checked = await check(schema, value);
261
235
  if ('issues' in checked) {
262
- throw new CorruptError(`a job's value does not meet the queue's schema: ${checked.issues}`);
236
+ throw new CorruptError(`a value that no longer meets the queue's schema: ${checked.issues}`);
263
237
  }
264
238
  return checked.value;
265
239
  },
266
240
  };
267
241
  }
268
- function repeatOf(r) {
269
- if ('every' in r) {
270
- return { every: ms(r.every) };
242
+ function callOf(source, method, body, idempotence) {
243
+ return source.via.call(source.openMethod, source.open, method, body, idempotence);
244
+ }
245
+ async function getJob(source, id) {
246
+ const body = await callOf(source, methods['jobs.get'], handle => JobsId.encode({ handle, id }), 'read');
247
+ const found = JobsJob.decode(body);
248
+ return found.found === true ? jobOf(source, found) : undefined;
249
+ }
250
+ async function cancelJob(source, id) {
251
+ const body = await callOf(source, methods['jobs.cancel'], handle => JobsId.encode({ handle, id }), 'write');
252
+ return JobsChanged.decode(body).changed === true;
253
+ }
254
+ /** A job as the server sent it, its value read as the queue reads one. */
255
+ async function jobOf(source, found) {
256
+ let value;
257
+ try {
258
+ value = await source.values.decode(found.value ?? 'null');
271
259
  }
272
- if ('daily' in r) {
273
- const time = /^(\d{1,2}):(\d{2})$/.exec(r.daily);
274
- if (time === null || Number(time[1]) > 23 || Number(time[2]) > 59) {
275
- throw new InvalidError(`the daily time ${JSON.stringify(r.daily)}; write it as 03:10`);
276
- }
277
- return { cron: `${Number(time[2])} ${Number(time[1])} * * *`, zone: r.zone };
260
+ catch (err) {
261
+ throw new CorruptError(`${source.describe}: id ${JSON.stringify(found.id ?? '')}: ${messageOf(err)}`);
278
262
  }
279
- return { cron: r.cron, zone: r.zone };
263
+ return {
264
+ id: found.id,
265
+ value,
266
+ state: (found.state ?? 'waiting'),
267
+ at: new Date(found.at ?? 0),
268
+ attempt: found.attempt ?? 0,
269
+ ahead: found.ahead ?? 0,
270
+ progress: found.progress === undefined ? undefined : JSON.parse(found.progress),
271
+ error: found.error,
272
+ group: found.group,
273
+ repeat: found.repeat,
274
+ lastRun: found.startedAt === undefined
275
+ ? undefined
276
+ : {
277
+ startedAt: new Date(found.startedAt),
278
+ endedAt: new Date(found.endedAt ?? found.startedAt),
279
+ },
280
+ };
280
281
  }
281
- export class Jobs {
282
- #link;
283
- constructor(link) {
284
- this.#link = link;
285
- }
286
- queue(name, of, options) {
287
- checkName(name, 'queue');
288
- let values = jsonValues;
289
- if (isSchema(of)) {
290
- values = schemaValues(of);
282
+ /**
283
+ * Follows the job under id over the server's watch, until the stream's last
284
+ * DATA. A lost connection or a server going away watches again, and the
285
+ * first report of the new watch is left out when it shows what was seen.
286
+ */
287
+ async function* watchJob(source, id) {
288
+ let last;
289
+ for (;;) {
290
+ const stream = await source.link.run('read', connection => openWatch(connection, source, id));
291
+ try {
292
+ for (;;) {
293
+ const event = await stream.next();
294
+ stream.consumed(event.body.length);
295
+ if (event.end) {
296
+ return;
297
+ }
298
+ const found = JobsJob.decode(event.body);
299
+ if (last === undefined || !showsTheSame(last, found)) {
300
+ last = found;
301
+ yield await jobOf(source, found);
302
+ }
303
+ }
291
304
  }
292
- else if (of !== undefined) {
293
- options = of;
305
+ catch (err) {
306
+ if (!(err instanceof ConnectionLostError || err instanceof UnavailableError)) {
307
+ throw err;
308
+ }
309
+ }
310
+ finally {
311
+ stream.cancel();
294
312
  }
295
- const open = JobsQueue.encode(queueFields(name, options));
296
- return new Queue(this.#link, name, open, values, options?.in);
297
- }
298
- /**
299
- * A schedule: a queue of one job under the schedule's name, which repeats
300
- * as it says. work runs it; cancel stops it.
301
- */
302
- schedule(name, repeat, options) {
303
- checkName(name, 'schedule');
304
- const open = JobsQueue.encode({ ...queueFields(name, options), schedule: repeatOf(repeat) });
305
- const nothing = { encode: () => '{}', decode: () => null };
306
- return new Queue(this.#link, name, open, nothing, options?.in);
307
313
  }
308
314
  }
309
- function queueFields(name, options) {
310
- return {
311
- name,
312
- lease: options?.lease === undefined ? undefined : ms(options.lease),
313
- maxAttempts: options?.maxAttempts,
314
- backoffFirst: options?.backoff?.first === undefined ? undefined : ms(options.backoff.first),
315
- backoffMost: options?.backoff?.most === undefined ? undefined : ms(options.backoff.most),
316
- maxWaiting: options?.maxWaiting,
317
- keepFailed: options?.keepFailed === undefined ? undefined : ms(options.keepFailed),
318
- keepDone: options?.keepDone === undefined ? undefined : ms(options.keepDone),
319
- maxRunning: options?.maxRunning,
320
- in: options?.in?.name,
321
- };
315
+ /** Opens a watch of the job under id, once its RESPONSE came. */
316
+ async function openWatch(connection, source, id) {
317
+ const handle = await handleOn(connection, source.openMethod, source.open);
318
+ const stream = await connection.session.open(methods['jobs.watch'], JobsId.encode({ handle, id }), true);
319
+ await stream.next();
320
+ return stream;
322
321
  }
323
- /** A job's fields as jobs.enqueue and a batch carry them. */
324
- function jobFields(values, value, options) {
325
- return {
326
- value: values.encode(value),
327
- key: options?.key,
328
- at: options?.at === undefined ? undefined : unixMs(options.at),
329
- after: options?.after === undefined ? undefined : ms(options.after),
330
- repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
331
- };
322
+ /** Whether two reports show a watcher the same: the job's state, place, attempt, time, progress and error. */
323
+ function showsTheSame(a, b) {
324
+ return (a.state === b.state &&
325
+ a.ahead === b.ahead &&
326
+ a.attempt === b.attempt &&
327
+ a.at === b.at &&
328
+ a.progress === b.progress &&
329
+ a.error === b.error);
332
330
  }
333
- /** A queue's enqueue inside a batch of the database it lives in. */
334
- export class QueueTx {
335
- #tx;
336
- #open;
337
- #values;
338
- constructor(tx, open, values) {
339
- this.#tx = tx;
340
- this.#open = open;
341
- this.#values = values;
342
- }
343
- /** Adds a job to the batch; it settles once the batch has committed. */
344
- enqueue(value, options) {
345
- return this.#tx.enqueue(this.#open, jobFields(this.#values, value, options));
346
- }
347
- }
348
- const states = {
349
- 1: 'waiting',
350
- 2: 'running',
351
- 3: 'failed',
352
- 4: 'done',
353
- 5: 'cancelled',
354
- };
331
+ // a queue's source and name, kept out of its public shape, for a database's transaction
332
+ const queues = new WeakMap();
333
+ /** A queue of jobs of one type, run in the order of their time: open it with `store.queue(name)`. */
355
334
  export class Queue {
356
335
  name;
357
- #link;
358
- #open;
359
- #values;
360
- #in;
361
- constructor(link, name, open, values, inDatabase) {
362
- this.#link = link;
336
+ #source;
337
+ /** Queues come from `store.queue` and `db.queue`. */
338
+ constructor(source, name) {
339
+ this.#source = source;
363
340
  this.name = name;
364
- this.#open = open;
365
- this.#values = values;
366
- this.#in = inDatabase;
367
- }
368
- /** The queue's handle, its database opened first when it lives in one, as the server needs. */
369
- async #handle(connection) {
370
- await this.#in?.handle(connection);
371
- return handleOn(connection, methods['jobs.open'], this.#open);
341
+ queues.set(this, { name, source: source });
372
342
  }
373
343
  /**
374
- * The queue's enqueue inside a batch of the database it lives in, which
375
- * it was opened `in`: the job commits with the batch's rows or not at all.
344
+ * Adds a job unless its id is taken: by a job scheduled, waiting, running
345
+ * or failed, or done while the queue's dedupe keeps it. It returns once
346
+ * the job is on disk, and says whether it added one; a job without an id
347
+ * is always added.
376
348
  *
377
- * await db.batch(tx => {
378
- * tx.exec`update notes set body = ${body} where id = ${id}`
379
- * index.withTx(tx).enqueue({ id })
380
- * })
349
+ * await reminders.add({ userId: 42, text: 'Call mom' }, { delay: '1h' })
381
350
  */
382
- withTx(tx) {
383
- if (this.#in === undefined || this.#in.name !== tx.database) {
384
- const lives = this.#in === undefined ? 'jobs.db' : `sql ${this.#in.name}`;
385
- throw new InvalidError(`the queue ${this.name} lives in ${lives}, not in sql ${tx.database}: ` +
386
- "open it with { in: db } to enqueue in that database's batches");
387
- }
388
- return new QueueTx(tx, this.#open, this.#values);
389
- }
390
- /** Adds a job; it returns once the job is in the file. */
391
- enqueue(value, options) {
392
- return this.enqueueAll([{ value, ...options }]);
393
- }
394
- /** Adds jobs in one transaction, all or none: a refused one names itself as `call`. */
395
- async enqueueAll(jobs) {
396
- const encoded = jobs.map(job => jobFields(this.#values, job.value, job));
397
- await this.#link.run('write', async (connection) => {
398
- const handle = await this.#handle(connection);
399
- await connection.session.call(methods['jobs.enqueue'], JobsBatch.encode({ handle, jobs: encoded }));
400
- });
351
+ async add(value, options = {}) {
352
+ const fields = {
353
+ ...jobFields(options),
354
+ id: options.id,
355
+ value: this.#source.values.encode(value),
356
+ };
357
+ return (await this.#changed(methods['jobs.add'], fields)) === true;
401
358
  }
402
359
  /**
403
- * Changes a job that waits or failed: its value and, when the options
404
- * say, its time or repeat. A job a worker holds, done or absent is
405
- * ConflictError.
360
+ * Makes the id's job this value at this time, whatever it was: one waiting
361
+ * is made new, a failed one starts again, and a running one runs once more
362
+ * after this run, with this value.
363
+ *
364
+ * await checks.set(`check:${check.id}`, check, { delay: '25h' })
406
365
  */
407
- async update(key, value, options) {
408
- const change = {
409
- value: this.#values.encode(value),
410
- key,
411
- at: options?.at === undefined ? undefined : unixMs(options.at),
412
- after: options?.after === undefined ? undefined : ms(options.after),
413
- repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
414
- };
415
- await this.#link.run('write', async (connection) => {
416
- const handle = await this.#handle(connection);
417
- await connection.session.call(methods['jobs.update'], JobsChange.encode({ ...change, handle }));
418
- });
366
+ async set(id, value, options = {}) {
367
+ const fields = { ...jobFields(options), id, value: this.#source.values.encode(value) };
368
+ await callOf(this.#source, methods['jobs.set'], handle => JobsCall.encode({ ...fields, handle }), 'write');
419
369
  }
420
370
  /**
421
- * Removes the job under key, whether it waits, runs or failed, and says
422
- * whether there was one. A running job's handler sees job.signal abort,
423
- * and what it returns settles nothing.
371
+ * Changes the id's job when it has not started: its value, and its time,
372
+ * group or repeat when the options give them. It says whether it did: a
373
+ * job that runs, ran or never was answers false.
424
374
  */
425
- cancel(key) {
426
- return this.#link.run('write', async (connection) => {
427
- const handle = await this.#handle(connection);
428
- const body = await connection.session.call(methods['jobs.cancel'], JobsKey.encode({ handle, key }));
429
- return JobsEntry.decode(body).found === true;
430
- });
375
+ async update(id, value, options = {}) {
376
+ const fields = { ...jobFields(options), id, value: this.#source.values.encode(value) };
377
+ return (await this.#changed(methods['jobs.update'], fields)) === true;
431
378
  }
432
379
  /**
433
- * The job a key names, waiting, running or failed, or done while keepDone
434
- * keeps its key; undefined when none is.
380
+ * Takes the job under id, whatever its state, and says whether there was
381
+ * one. A running one's handler sees its signal abort, and what it returns
382
+ * then settles nothing; a repeating one stops.
435
383
  */
436
- async get(key) {
437
- const entry = await this.#link.run('read', async (connection) => {
438
- const handle = await this.#handle(connection);
439
- return JobsEntry.decode(await connection.session.call(methods['jobs.get'], JobsKey.encode({ handle, key })));
440
- });
441
- return entry.found === true ? this.#entryOf(entry) : undefined;
442
- }
443
- async #entryOf(entry) {
444
- const state = states[entry.state ?? 1] ?? 'waiting';
445
- return {
446
- key: entry.key ?? '',
447
- value: state === 'done' && entry.value === undefined
448
- ? undefined
449
- : await this.#values.decode(entry.value ?? 'null'),
450
- at: new Date(entry.at ?? 0),
451
- attempt: entry.attempt ?? 0,
452
- state,
453
- ahead: entry.ahead ?? 0,
454
- progress: entry.progress === undefined ? undefined : JSON.parse(entry.progress),
455
- ran: dateOf(entry.ran),
456
- took: entry.ran === undefined ? undefined : (entry.took ?? 0),
457
- error: entry.err,
458
- repeat: entry.repeat,
459
- };
384
+ cancel(id) {
385
+ return cancelJob(this.#source, id);
460
386
  }
461
387
  /**
462
- * Yields the job under key as it is, then again each time its state,
463
- * place, attempt, time, progress or error changes, until it ends: done,
464
- * failed or cancelled, the last entry it yields. A key that names no job
465
- * yields nothing. Leaving the loop ends the watch; a connection lost is
466
- * connected again, and the watch goes on from the job as it is then.
467
- *
468
- * for await (const s of videos.watch(id)) send(s.state, s.ahead, s.progress)
388
+ * Where the job under id is: scheduled, waiting and how many jobs are
389
+ * ahead of it, running and its progress, failed and why, or done while
390
+ * the queue's dedupe keeps its id; undefined for an id with no job.
469
391
  */
470
- async *watch(key) {
471
- let last;
472
- for (;;) {
473
- const stream = await this.#link.run('read', async (connection) => {
474
- const handle = await this.#handle(connection);
475
- const opened = await connection.session.open(methods['jobs.watch'], JobsKey.encode({ handle, key }), true);
476
- await opened.next();
477
- return opened;
478
- });
479
- const unwatch = watch(currentSignal(), stream);
480
- try {
481
- for (;;) {
482
- const event = await stream.next();
483
- stream.consumed(event.body.length);
484
- if (event.end) {
485
- return;
486
- }
487
- const entry = await this.#entryOf(JobsEntry.decode(event.body));
488
- const seen = JSON.stringify([
489
- entry.state,
490
- entry.ahead,
491
- entry.attempt,
492
- entry.at,
493
- entry.progress,
494
- entry.error,
495
- ]);
496
- if (seen !== last) {
497
- last = seen;
498
- yield entry;
499
- }
500
- }
501
- }
502
- catch (err) {
503
- if (!(err instanceof LostError)) {
504
- throw err;
505
- }
506
- }
507
- finally {
508
- unwatch();
509
- stream.cancel();
510
- }
511
- }
392
+ get(id) {
393
+ return getJob(this.#source, id);
512
394
  }
513
395
  /**
514
- * A page of the jobs under a prefix, in the byte order of their keys; with
515
- * state 'failed' and no prefix, the failed jobs, the last failed first.
396
+ * The job under id as it is, then again each time its state, place,
397
+ * attempt, time, progress or error changes; it ends with the job: done,
398
+ * failed, or cancelled, a state only a watcher sees. An id with no job
399
+ * yields nothing. A watch whose connection is lost watches again on the
400
+ * next; a job that ended meanwhile ends it without its last state.
401
+ *
402
+ * for await (const job of videos.watch(id)) send(job.state, job.ahead, job.progress)
516
403
  */
517
- async scan(query) {
518
- const got = await this.#link.run('read', async (connection) => {
519
- const handle = await this.#handle(connection);
520
- const body = JobsQuery.encode({
521
- handle,
522
- prefix: query?.prefix,
523
- state: query?.state === 'failed' ? 3 : undefined,
524
- after: query?.after,
525
- limit: query?.limit,
526
- });
527
- return download(connection, methods['jobs.scan'], body);
528
- });
529
- const items = [];
530
- for (const item of got.items) {
531
- items.push(await this.#entryOf(JobsEntry.decode(item)));
404
+ watch(id) {
405
+ return watchJob(this.#source, id);
406
+ }
407
+ /** A page of the queue's jobs whose ids start with a prefix, or of its failed ones. */
408
+ async list(options = {}) {
409
+ const body = await callOf(this.#source, methods['jobs.list'], handle => JobsList.encode({ handle, ...options }), 'read');
410
+ const page = JobsPage.decode(body);
411
+ const jobs = [];
412
+ for (const found of page.jobs ?? []) {
413
+ jobs.push(await jobOf(this.#source, found));
532
414
  }
533
- const page = JobsPage.decode(got.trailer);
534
- return { items, next: page.more === true ? (page.after ?? '') : undefined };
415
+ return { jobs, next: page.next };
535
416
  }
536
- /** Walks scan's pages, holding no snapshot between them. */
537
- async *all(query) {
417
+ /** Every job the options name, a page at a time, holding nothing between pages. */
418
+ async *all(options = {}) {
538
419
  let after;
539
420
  for (;;) {
540
- const page = await this.scan({ ...query, ...(after === undefined ? {} : { after }) });
541
- yield* page.items;
421
+ const page = await this.list({ ...options, after });
422
+ yield* page.jobs;
542
423
  if (page.next === undefined) {
543
424
  return;
544
425
  }
@@ -546,139 +427,438 @@ export class Queue {
546
427
  }
547
428
  }
548
429
  /**
549
- * Leases the next due job, or says there is none, without waiting. The
550
- * claim lives on this connection: settle it before its lease ends.
430
+ * Runs handler on each job as it falls due, as many at once as the
431
+ * queue's concurrency lets, one when it sets none, and returns the running
432
+ * worker at once: stop it at shutdown, as store.close() does. A worker
433
+ * whose connection is lost reaches the store again, the jobs it held
434
+ * running again as a dead worker's would.
435
+ *
436
+ * const worker = reminders.work(async ({ userId, text }) => push(userId, text))
551
437
  */
552
- async claim(options) {
553
- return this.#link.run('write', async (connection) => {
554
- const handle = await this.#handle(connection);
555
- const lease = options?.lease === undefined ? undefined : ms(options.lease);
556
- const held = JobsHeld.decode(await connection.session.call(methods['jobs.claim'], JobsLease.encode({ handle, lease })));
557
- if (held.found !== true) {
558
- return undefined;
559
- }
560
- const value = await this.#values.decode(held.value ?? 'null');
561
- return new ClaimedJob(held, value, async (outcome) => {
562
- const body = await connection.session.call(methods['jobs.settle'], JobsOutcomes.encode({ outcomes: [outcome] }));
563
- const refused = JobsSettled.decode(body).settled?.[0];
564
- if (refused !== undefined && refused !== null) {
565
- throw errorOf(refused.code ?? 'internal', refused.message ?? '', refused.what ?? {});
566
- }
567
- }, (method, body) => connection.session.call(method, body));
568
- });
438
+ work(handler, options = {}) {
439
+ const concurrency = handlersOf(options, this.#source.describe);
440
+ return new Worker(working => keepWorking({ source: this.#source, handler, concurrency, working }), this.#source.workers);
569
441
  }
570
442
  /**
571
- * Runs the queue's jobs as they come due, workers at once, until the
572
- * signal aborts: the server's own Work loop claims ahead and extends
573
- * leases, and nothing polls. A handler's return acknowledges its job and a
574
- * throw retries it. A connection lost takes the jobs in hand with it, as a
575
- * process that died would, and the loop connects again.
443
+ * Runs what is due when it starts and what falls due meanwhile, then says
444
+ * how many jobs ran; it never waits for a later job. Tests, scripts and
445
+ * commands use it.
576
446
  */
577
- async work(handler, options = {}) {
578
- for (;;) {
579
- options.signal?.throwIfAborted();
580
- try {
581
- await this.#link.run('read', connection => this.#workOn(connection, handler, options), options.signal);
582
- if (options.untilIdle === true || options.signal?.aborted === true) {
583
- return;
584
- }
585
- }
586
- catch (err) {
587
- if (!(err instanceof LostError) || options.signal?.aborted === true) {
588
- throw err;
589
- }
447
+ runDue(handler, options = {}) {
448
+ return runDue(this.#source, handler, handlersOf(options, this.#source.describe));
449
+ }
450
+ async #changed(method, fields) {
451
+ const body = await callOf(this.#source, method, handle => JobsCall.encode({ ...fields, handle }), 'write');
452
+ return JobsChanged.decode(body).changed;
453
+ }
454
+ }
455
+ /**
456
+ * A queue's calls inside a database's transaction, from `tx.with(queue)`: a
457
+ * job it adds is there only if the transaction commits, and the queue's
458
+ * workers find it once it has. A worker, a watch and a page work outside.
459
+ */
460
+ export class TxQueue {
461
+ name;
462
+ #queue;
463
+ /** Comes from `tx.with(queue)`. */
464
+ constructor(queue) {
465
+ this.#queue = queue;
466
+ this.name = queue.name;
467
+ }
468
+ /** Adds a job unless its id is taken, as `queue.add` does, with the transaction. */
469
+ add(value, options = {}) {
470
+ return this.#queue.add(value, options);
471
+ }
472
+ set(id, value, options = {}) {
473
+ return this.#queue.set(id, value, options);
474
+ }
475
+ update(id, value, options = {}) {
476
+ return this.#queue.update(id, value, options);
477
+ }
478
+ cancel(id) {
479
+ return this.#queue.cancel(id);
480
+ }
481
+ /** The job under id as the transaction sees it, its own writes among it. */
482
+ get(id) {
483
+ return this.#queue.get(id);
484
+ }
485
+ }
486
+ /**
487
+ * A queue's writes inside the store's transaction, from `tx.with(queue)` in
488
+ * `store.tx`: each is kept for the commit, which makes them all or none, so
489
+ * what one did is known once the transaction has committed.
490
+ *
491
+ * await store.tx(async tx => {
492
+ * for (const member of members) await tx.with(pushes).add({ member, message }, { id: `${message}:${member}` })
493
+ * })
494
+ */
495
+ export class StoreTxQueue {
496
+ name;
497
+ #source;
498
+ #keep;
499
+ /** Comes from `tx.with(queue)` in `store.tx`. */
500
+ constructor(parts, keep) {
501
+ this.name = parts.name;
502
+ this.#source = parts.source;
503
+ this.#keep = keep;
504
+ }
505
+ /** Adds a job unless its id is taken, with the commit. */
506
+ async add(value, options = {}) {
507
+ this.#write('jobs.add', { ...jobFields(options), id: options.id }, value);
508
+ }
509
+ /** Makes the id's job this value at this time, whatever it was, with the commit. */
510
+ async set(id, value, options = {}) {
511
+ this.#write('jobs.set', { ...jobFields(options), id }, value);
512
+ }
513
+ /** Changes the id's job when it has not started, with the commit. */
514
+ async update(id, value, options = {}) {
515
+ this.#write('jobs.update', { ...jobFields(options), id }, value);
516
+ }
517
+ /** Takes the job under id, whatever its state, with the commit. */
518
+ async cancel(id) {
519
+ this.#keep({ source: this.#source, method: methods['jobs.cancel'], id });
520
+ }
521
+ #write(method, fields, value) {
522
+ const call = { ...fields, value: this.#source.values.encode(value) };
523
+ this.#keep({ source: this.#source, method: methods[method], call });
524
+ }
525
+ }
526
+ /** Sends a transaction's writes of jobs as one jobs.tx: all of them are made, or none. */
527
+ export async function commitJobs(link, ops) {
528
+ await link.run('write', async (connection) => {
529
+ const writes = [];
530
+ for (const op of ops) {
531
+ const handle = await handleOn(connection, op.source.openMethod, op.source.open);
532
+ writes.push({
533
+ method: op.method,
534
+ call: op.call === undefined ? undefined : { ...op.call, handle },
535
+ id: op.id === undefined ? undefined : { handle, id: op.id },
536
+ });
537
+ }
538
+ await connection.session.call(methods['jobs.tx'], JobsTx.encode({ writes }));
539
+ });
540
+ }
541
+ /** A queue's parts, for a database's transaction that takes it in; undefined for anything else. */
542
+ export function queueParts(handle) {
543
+ return queues.get(handle);
544
+ }
545
+ /** The queue's calls inside a transaction, going through its stream. */
546
+ export function queueVia(parts, via) {
547
+ return new TxQueue(new Queue({ ...parts.source, via }, parts.name));
548
+ }
549
+ /** A repeat the code owns: one repeating job under its name, and the worker that runs it. */
550
+ export class Schedule {
551
+ name;
552
+ #source;
553
+ #handler;
554
+ #worker;
555
+ /** Schedules come from `store.schedule`, started. */
556
+ constructor(source, name, handler) {
557
+ this.#source = source;
558
+ this.name = name;
559
+ this.#handler = (_, run) => handler(run);
560
+ const worked = { source, handler: this.#handler, concurrency: undefined };
561
+ this.#worker = new Worker(working => keepWorking({ ...worked, working }), source.workers);
562
+ }
563
+ /** When it runs next and how its last run went: what a page of cron jobs shows. */
564
+ get() {
565
+ return getJob(this.#source, this.name);
566
+ }
567
+ /** Runs the schedule's job when it is due, as a queue's runDue does. */
568
+ runDue() {
569
+ return runDue(this.#source, this.#handler, undefined);
570
+ }
571
+ /** Stops the schedule's worker, as a worker's stop does. */
572
+ stop() {
573
+ return this.#worker.stop();
574
+ }
575
+ /** Removes the schedule's job, until the program opens it again. */
576
+ cancel() {
577
+ return cancelJob(this.#source, this.name);
578
+ }
579
+ }
580
+ /** What a queue's open sends, checked before anything leaves. */
581
+ /**
582
+ * A queue, its open checked before anything leaves; `home` is the database
583
+ * whose file keeps it, its open naming the database's handle on each
584
+ * connection.
585
+ */
586
+ export function openQueue(link, workers, name, options, home) {
587
+ checkName(name, 'queue');
588
+ const describe = home === undefined ? `jobs queue ${name}` : `${home.describe}: jobs queue ${name}`;
589
+ const concurrency = typeof options.concurrency === 'number' ? { total: options.concurrency } : options.concurrency;
590
+ const fields = {
591
+ name,
592
+ attempts: whole(options.attempts, describe, 'attempts'),
593
+ backoff: backoffOf(options.backoff),
594
+ timeout: msOf(options.timeout),
595
+ concurrency: concurrency === undefined
596
+ ? undefined
597
+ : {
598
+ total: whole(concurrency.total, describe, 'a concurrency'),
599
+ group: whole(concurrency.group, describe, "a group's concurrency"),
600
+ },
601
+ rate: options.rate === undefined ? undefined : rateOf(options.rate),
602
+ dedupe: msOf(options.dedupe),
603
+ keep: msOf(options.keep),
604
+ maxWaiting: whole(options.maxWaiting, describe, 'a maxWaiting'),
605
+ };
606
+ const bytes = JobsQueueOpen.encode(fields);
607
+ const open = home === undefined
608
+ ? bytes
609
+ : home.keep(methods['jobs.queue.open'], bytes, async (connection) => JobsQueueOpen.encode({ ...fields, database: await home.handle(connection) }));
610
+ const source = {
611
+ link,
612
+ describe,
613
+ openMethod: methods['jobs.queue.open'],
614
+ open,
615
+ home,
616
+ via: home === undefined ? viaLink(link) : home.via(`jobs queue ${name}`),
617
+ values: valuesOf(options.schema),
618
+ timeout: options.timeout === undefined ? 60_000 : ms(options.timeout),
619
+ workers,
620
+ };
621
+ return new Queue(source, name);
622
+ }
623
+ /** A schedule, its open checked before anything leaves, and its worker started. */
624
+ export function openSchedule(link, workers, name, when, handler, options = {}) {
625
+ checkName(name, 'schedule');
626
+ const describe = `jobs schedule ${name}`;
627
+ const repeat = 'cron' in when
628
+ ? { cron: when.cron, timeZone: zoneOf(when.timeZone, describe) }
629
+ : { every: ms(when.every) };
630
+ const open = JobsScheduleOpen.encode({
631
+ name,
632
+ ...repeat,
633
+ attempts: whole(options.attempts, describe, 'attempts'),
634
+ backoff: backoffOf(options.backoff),
635
+ timeout: msOf(options.timeout),
636
+ });
637
+ const source = {
638
+ link,
639
+ describe,
640
+ openMethod: methods['jobs.schedule.open'],
641
+ open,
642
+ via: viaLink(link),
643
+ values: valuesOf(undefined),
644
+ timeout: options.timeout === undefined ? 60_000 : ms(options.timeout),
645
+ workers,
646
+ };
647
+ return new Schedule(source, name, handler);
648
+ }
649
+ /** A cron's zone, which an hour means nothing without: undefined, a user's never set, is refused. */
650
+ function zoneOf(zone, describe) {
651
+ if (typeof zone !== 'string' || zone === '') {
652
+ throw new InvalidError(`${describe}: a cron needs a timeZone, 'UTC' among them`);
653
+ }
654
+ return zone;
655
+ }
656
+ function jobFields(options) {
657
+ return {
658
+ at: options.at === undefined ? undefined : unixMs(options.at),
659
+ delay: msOf(options.delay),
660
+ group: options.group,
661
+ every: msOf(options.every),
662
+ cron: options.cron,
663
+ timeZone: options.timeZone,
664
+ };
665
+ }
666
+ function msOf(d) {
667
+ return d === undefined ? undefined : ms(d);
668
+ }
669
+ function backoffOf(backoff) {
670
+ return backoff === undefined ? undefined : { initial: ms(backoff.initial), max: ms(backoff.max) };
671
+ }
672
+ /** A count the server takes as a whole number; one that is none is refused here, naming what it was. */
673
+ function whole(n, describe, what) {
674
+ if (n !== undefined && (!Number.isSafeInteger(n) || n < 0)) {
675
+ throw new InvalidError(`${describe}: ${what} of ${n}, which is no whole number`);
676
+ }
677
+ return n;
678
+ }
679
+ function handlersOf(options, describe) {
680
+ const n = options.concurrency;
681
+ if (n !== undefined && (!Number.isSafeInteger(n) || n < 1 || n > mostHandlers)) {
682
+ throw new InvalidError(`${describe}: a worker of ${n} handlers, not 1 to ${mostHandlers}`);
683
+ }
684
+ return n;
685
+ }
686
+ /** A worker running a queue's handlers here while the server's loop claims their jobs. */
687
+ export class Worker {
688
+ #working = { stopping: false, stream: undefined, wake: undefined };
689
+ #done;
690
+ /** Workers come from a queue's work and from store.schedule, running. */
691
+ constructor(run, workers) {
692
+ workers.add(this);
693
+ this.#done = run(this.#working).finally(() => workers.delete(this));
694
+ }
695
+ /**
696
+ * Takes no job more and waits for the handlers under way, each bounded by
697
+ * its timeout: the server gives back the jobs it held for them, and ends
698
+ * the worker once their answers are written.
699
+ */
700
+ stop() {
701
+ const working = this.#working;
702
+ if (!working.stopping) {
703
+ working.stopping = true;
704
+ working.stream?.send(JobsAnswer.encode({ how: 'stop' }), false).catch(() => { });
705
+ working.wake?.();
706
+ }
707
+ return this.#done;
708
+ }
709
+ [Symbol.asyncDispose]() {
710
+ return this.stop();
711
+ }
712
+ }
713
+ /**
714
+ * Runs a worker until it stops: a stream on the link's connection, and
715
+ * another once a connection is lost or a server going away has ended one.
716
+ * What fails between is said in TinyStore's own lines, as background work's
717
+ * failures are.
718
+ */
719
+ async function keepWorking(worked) {
720
+ const { source, working } = worked;
721
+ const failures = new FailureLog(`${source.describe} worker`, sayOwn);
722
+ let pause = firstPause;
723
+ while (!working.stopping) {
724
+ try {
725
+ await source.link.run('read', connection => WorkStream.serve(connection, worked, false));
726
+ failures.succeeded();
727
+ pause = firstPause;
728
+ }
729
+ catch (err) {
730
+ if (working.stopping) {
731
+ return;
590
732
  }
733
+ failures.failed(err);
734
+ pause = Math.min(pause * 2, lastPause);
591
735
  }
736
+ await pausing(pause, working);
592
737
  }
593
- async #workOn(connection, handler, options) {
594
- const handle = await this.#handle(connection);
595
- const timeout = options.timeout === undefined ? 60_000 : ms(options.timeout);
596
- const stream = await connection.session.open(methods['jobs.work'], JobsWorkers.encode({
738
+ }
739
+ /** Runs a stream that ends once nothing is due, and says how many jobs its handlers finished. */
740
+ async function runDue(source, handler, concurrency) {
741
+ const working = { stopping: false, stream: undefined, wake: undefined };
742
+ let finished = 0;
743
+ await source.link.run('read', async (connection) => {
744
+ finished += await WorkStream.serve(connection, { source, handler, concurrency, working }, true);
745
+ });
746
+ return finished;
747
+ }
748
+ /** Waits ms, or until the worker stops. */
749
+ function pausing(ms, working) {
750
+ if (working.stopping) {
751
+ return Promise.resolve();
752
+ }
753
+ return new Promise(resolve => {
754
+ const done = () => {
755
+ clearTimeout(timer);
756
+ working.wake = undefined;
757
+ resolve();
758
+ };
759
+ const timer = setTimeout(done, ms);
760
+ working.wake = done;
761
+ });
762
+ }
763
+ /** A worker's stream on one connection: the jobs the server hands over, and the handlers they run here. */
764
+ class WorkStream {
765
+ #worked;
766
+ #connection;
767
+ #stream;
768
+ /** the runs under way, by number, and what cancels each */
769
+ #runs = new Map();
770
+ #finished = 0;
771
+ constructor(worked, connection, stream) {
772
+ this.#worked = worked;
773
+ this.#connection = connection;
774
+ this.#stream = stream;
775
+ }
776
+ /**
777
+ * Opens a worker's stream on a connection and runs it to its end, which
778
+ * the server sends: after a stop or a GOAWAY once the jobs in hand are
779
+ * answered, or once nothing is due when it ends when idle. It says how
780
+ * many jobs its handlers finished.
781
+ */
782
+ static async serve(connection, worked, untilIdle) {
783
+ const handle = await handleOn(connection, worked.source.openMethod, worked.source.open);
784
+ const asked = JobsWork.encode({
597
785
  handle,
598
- workers: options.workers,
599
- timeout,
600
- untilIdle: options.untilIdle || undefined,
601
- cancels: true,
602
- }), false);
603
- const stopping = new AbortController();
604
- const stop = () => stopping.abort();
605
- options.signal?.addEventListener('abort', stop, { once: true });
606
- const inHand = new Set();
607
- const cancels = new Map();
608
- let ended = false;
609
- const endOurSide = async () => {
610
- if (!ended) {
611
- ended = true;
612
- await Promise.allSettled(inHand);
613
- await stream.send(new Uint8Array(0), true).catch(() => { });
786
+ concurrency: worked.concurrency,
787
+ untilIdle: untilIdle || undefined,
788
+ });
789
+ const stream = await connection.session.open(methods['jobs.work'], asked, false);
790
+ const { working } = worked;
791
+ working.stream = stream;
792
+ if (working.stopping) {
793
+ stream.send(JobsAnswer.encode({ how: 'stop' }), false).catch(() => { });
794
+ }
795
+ try {
796
+ return await new WorkStream(worked, connection, stream).#serve();
797
+ }
798
+ finally {
799
+ if (working.stream === stream) {
800
+ working.stream = undefined;
614
801
  }
615
- };
616
- stopping.signal.addEventListener('abort', () => void endOurSide(), { once: true });
802
+ }
803
+ }
804
+ async #serve() {
617
805
  try {
618
- await stream.next();
619
- for (;;) {
620
- const event = await stream.next();
621
- stream.consumed(event.body.length);
806
+ const started = await this.#stream.next();
807
+ while (!started.end) {
808
+ const event = await this.#stream.next();
809
+ this.#stream.consumed(event.body.length);
622
810
  if (event.end) {
623
- return;
624
- }
625
- const held = JobsHeld.decode(event.body);
626
- if (held.cancelled === true) {
627
- cancels.get(held.job ?? 0)?.abort(new CancelledError('cancel took the job while it ran'));
628
- continue;
811
+ break;
629
812
  }
630
- const cancel = new AbortController();
631
- cancels.set(held.job ?? 0, cancel);
632
- const steps = new Steps((method, body) => connection.session.call(method, body), held.job ?? 0);
633
- const running = this.#run(stream, held, steps, handler, timeout, stopping.signal, cancel.signal).finally(() => cancels.delete(held.job ?? 0));
634
- inHand.add(running);
635
- running.finally(() => inHand.delete(running));
813
+ this.#take(JobsHeld.decode(event.body));
636
814
  }
815
+ return this.#finished;
637
816
  }
638
- finally {
639
- options.signal?.removeEventListener('abort', stop);
640
- stopping.abort();
817
+ catch (err) {
818
+ for (const cancel of this.#runs.values()) {
819
+ cancel.abort(err);
820
+ }
821
+ throw err;
641
822
  }
642
823
  }
643
- // runs one job's handler and sends the outcome it settled by; a handler
644
- // stopped by the loop's end gives its job back without counting the
645
- // attempt, and one cancel took settles nothing, so it sends nothing
646
- async #run(stream, held, steps, handler, timeout, stopping, cancelled) {
647
- const signal = AbortSignal.any([stopping, AbortSignal.timeout(timeout), cancelled]);
648
- const number = held.job ?? 0;
649
- const reporter = new Reporter(progress => stream.send(JobsOutcome.encode({ job: number, how: how.progress, progress }), false));
650
- let outcome;
824
+ /** A job the server handed over, run at once, since it sends no more than the handlers; or a cancel of one. */
825
+ #take(held) {
826
+ const run = held.run ?? 0;
827
+ if (held.cancelled === true) {
828
+ this.#runs.get(run)?.abort(new CancelledError('a cancel took the job while it ran'));
829
+ this.#runs.delete(run);
830
+ return;
831
+ }
832
+ const cancel = new AbortController();
833
+ this.#runs.set(run, cancel);
834
+ void this.#run(held, cancel).finally(() => this.#runs.delete(run));
835
+ }
836
+ /** Runs one job's handler and sends the answer it ended with; a job a cancel took is the server's, and gets none. */
837
+ async #run(held, cancel) {
838
+ const { source, handler } = this.#worked;
839
+ const run = held.run ?? 0;
840
+ const signal = AbortSignal.any([cancel.signal, AbortSignal.timeout(source.timeout)]);
841
+ const reporter = new Reporter(progress => this.#stream.send(JobsAnswer.encode({ run, how: 'progress', progress }), false));
842
+ let answer;
651
843
  try {
652
- const job = new Job(held.key ?? '', await this.#values.decode(held.value ?? 'null'), new Date(held.at ?? 0), held.attempt ?? 0, signal, reporter, steps);
844
+ const value = await source.values.decode(held.value ?? 'null');
845
+ const made = new Map();
846
+ const steps = new Steps((method, body) => this.#connection.session.call(method, body), run, cancel.signal);
653
847
  try {
654
- await handler(job);
655
- outcome = outcomeOf(number, job.settlement ?? { how: 'ack' });
848
+ answer = answerOf(await handler(value, new Run({ held, signal, steps, reporter, made })), made);
656
849
  }
657
850
  catch (err) {
658
- outcome =
659
- stopping.aborted && signal.aborted
660
- ? { job: number, how: how.snooze, at: held.at ?? Date.now() }
661
- : outcomeOf(number, job.settlement ?? { how: 'retry', error: reasonOf(err) });
851
+ answer = { how: 'retry', error: messageOf(err) };
662
852
  }
663
853
  }
664
854
  catch (err) {
665
- outcome = { job: number, how: how.fail, err: `the value no longer reads: ${reasonOf(err)}` };
855
+ answer = { how: 'fail', error: `its value no longer reads: ${messageOf(err)}` };
666
856
  }
667
857
  await reporter.stop();
668
- if (!cancelled.aborted) {
669
- await stream.send(JobsOutcome.encode(outcome), false).catch(() => { });
858
+ if (cancel.signal.aborted) {
859
+ return;
670
860
  }
671
- }
672
- }
673
- function outcomeOf(job, s) {
674
- switch (s.how) {
675
- case 'ack':
676
- return { job, how: how.ack };
677
- case 'retry':
678
- return { job, how: how.retry, err: s.error, ...timing(s) };
679
- case 'fail':
680
- return { job, how: how.fail, err: s.error };
681
- case 'snooze':
682
- return { job, how: how.snooze, ...timing(s) };
861
+ this.#finished++;
862
+ await this.#stream.send(JobsAnswer.encode({ run, ...answer }), false).catch(() => { });
683
863
  }
684
864
  }