@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/src/jobs.ts
CHANGED
|
@@ -1,130 +1,294 @@
|
|
|
1
|
-
// Work
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import type
|
|
13
|
-
import {
|
|
1
|
+
// Work an application must do later, or now but outside the request that
|
|
2
|
+
// asked for it, as docs/guides/jobs/README.md has it: queues of JSON values run in the
|
|
3
|
+
// order of their time, ids that find a job again, repeats, schedules and
|
|
4
|
+
// limits. A worker's handlers run here while the server's loop claims the
|
|
5
|
+
// queue's jobs and hands each over on one stream, the answers going back on
|
|
6
|
+
// it. A handle opens with its first call on a connection.
|
|
7
|
+
|
|
8
|
+
import { FailureLog, messageOf } from './background.ts'
|
|
9
|
+
import type { Connection, Idempotence, Link } from './connection.ts'
|
|
10
|
+
import { CancelledError, CorruptError, InvalidError, UnavailableError } from './errors.ts'
|
|
11
|
+
import { checkName, type Home, handleOn, type Open, type Via, viaLink } from './handles.ts'
|
|
12
|
+
import { type Rate, rateOf } from './limits.ts'
|
|
13
|
+
import { sayOwn } from './logger.ts'
|
|
14
|
+
import { check, type StandardSchemaV1 } from './schema.ts'
|
|
15
|
+
import { ConnectionLostError, type Stream } from './session.ts'
|
|
16
|
+
import { type Duration, ms, type Time, unixMs } from './time.ts'
|
|
17
|
+
import type { Read, Written } from './wire/codec.ts'
|
|
14
18
|
import {
|
|
15
19
|
JobsAnswer,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
JobsEntry,
|
|
20
|
+
JobsCall,
|
|
21
|
+
JobsChanged,
|
|
19
22
|
JobsHeld,
|
|
23
|
+
JobsId,
|
|
24
|
+
JobsJob,
|
|
20
25
|
JobsKept,
|
|
21
|
-
|
|
22
|
-
JobsLease,
|
|
23
|
-
JobsOutcome,
|
|
24
|
-
JobsOutcomes,
|
|
26
|
+
JobsList,
|
|
25
27
|
JobsPage,
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
JobsQueueOpen,
|
|
29
|
+
JobsScheduleOpen,
|
|
30
|
+
JobsStep,
|
|
31
|
+
JobsTx,
|
|
32
|
+
JobsWork,
|
|
30
33
|
methods,
|
|
31
|
-
} from './wire/
|
|
34
|
+
} from './wire/protocol.ts'
|
|
35
|
+
|
|
36
|
+
/** How many jobs of a queue run at once: in all, across every worker of the store, and in each group. */
|
|
37
|
+
export interface Concurrency {
|
|
38
|
+
total?: number | undefined
|
|
39
|
+
/** at most this many jobs of one group at once: one customer's backlog holds back no other */
|
|
40
|
+
group?: number | undefined
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** The wait before a retry: initial, doubling each time up to max, a tenth longer or shorter at random. */
|
|
44
|
+
export interface Backoff {
|
|
45
|
+
initial: Duration
|
|
46
|
+
max: Duration
|
|
47
|
+
}
|
|
32
48
|
|
|
33
49
|
export interface QueueOptions {
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
|
|
50
|
-
* batch of the database commits a job with its rows: `queue.withTx(tx)`
|
|
51
|
-
*/
|
|
52
|
-
in?: Database
|
|
50
|
+
/** runs a job gets before it fails for good, the first counted: 10 */
|
|
51
|
+
attempts?: number | undefined
|
|
52
|
+
/** 1s doubling up to 1h */
|
|
53
|
+
backoff?: Backoff | undefined
|
|
54
|
+
/** how long one run may take before its signal aborts: a minute */
|
|
55
|
+
timeout?: Duration | undefined
|
|
56
|
+
/** jobs running at once across every worker of the store: 8, or { total: 8, group: 2 }; one a worker unless given */
|
|
57
|
+
concurrency?: number | Concurrency | undefined
|
|
58
|
+
/** jobs started in any span: '30/s' */
|
|
59
|
+
rate?: Rate | undefined
|
|
60
|
+
/** how long a done job's id stays taken, so that an add of it adds nothing */
|
|
61
|
+
dedupe?: Duration | undefined
|
|
62
|
+
/** how long a failed job stays, to be found and started again: a week */
|
|
63
|
+
keep?: Duration | undefined
|
|
64
|
+
/** jobs that may wait; an add past it is LimitError: ten million */
|
|
65
|
+
maxWaiting?: number | undefined
|
|
53
66
|
}
|
|
54
67
|
|
|
55
|
-
/**
|
|
56
|
-
export
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
68
|
+
/** A job's time, group and repeat, as an add, a set and an update take them. */
|
|
69
|
+
export interface JobOptions {
|
|
70
|
+
/** runs at this time, one past now; not beside delay */
|
|
71
|
+
at?: Time | undefined
|
|
72
|
+
/** runs this long from now */
|
|
73
|
+
delay?: Duration | undefined
|
|
74
|
+
/** the group whose running jobs the queue's concurrency.group bounds */
|
|
75
|
+
group?: string | undefined
|
|
76
|
+
/** repeats every span, each id at a phase of its own within it; needs an id */
|
|
77
|
+
every?: Duration | undefined
|
|
78
|
+
/** repeats by five cron fields, or @daily and its kind, on timeZone's wall clock; needs an id */
|
|
79
|
+
cron?: string | undefined
|
|
80
|
+
/** an IANA name, 'UTC' among them, which a cron needs: one given as undefined is InvalidError, never UTC */
|
|
81
|
+
timeZone?: string | undefined
|
|
82
|
+
}
|
|
60
83
|
|
|
61
|
-
export interface
|
|
62
|
-
/** the job's
|
|
63
|
-
|
|
64
|
-
after?: Duration
|
|
65
|
-
/** names one job: enqueuing it again adds nothing and can bring it forward, never back */
|
|
66
|
-
key?: string
|
|
67
|
-
/** needs a key, since only a key stops it */
|
|
68
|
-
repeat?: Repeat
|
|
84
|
+
export interface AddOptions extends JobOptions {
|
|
85
|
+
/** the job's id, 1 to 1024 bytes: what finds it again, and adds it once */
|
|
86
|
+
id?: string | undefined
|
|
69
87
|
}
|
|
70
88
|
|
|
71
|
-
/** done while
|
|
72
|
-
export type JobState = 'waiting' | 'running' | '
|
|
89
|
+
/** done while dedupe keeps its id; cancelled only a watcher sees */
|
|
90
|
+
export type JobState = 'scheduled' | 'waiting' | 'running' | 'done' | 'failed' | 'cancelled'
|
|
73
91
|
|
|
74
|
-
|
|
75
|
-
|
|
92
|
+
/** A job as its queue holds it. */
|
|
93
|
+
export interface Job<T> {
|
|
94
|
+
id: string | undefined
|
|
76
95
|
value: T
|
|
77
|
-
|
|
96
|
+
state: JobState
|
|
97
|
+
/** when it runs next; for one done or failed, when its last run was for */
|
|
78
98
|
at: Date
|
|
79
99
|
/** its attempts, a running one included */
|
|
80
100
|
attempt: number
|
|
81
|
-
|
|
82
|
-
/** the jobs that run before a waiting one, up to 10,000; get and watch count it, scan leaves it 0 */
|
|
101
|
+
/** the jobs that run before a waiting one, up to 10,000 */
|
|
83
102
|
ahead: number
|
|
84
|
-
/** what
|
|
103
|
+
/** what its handler last reported of a running job */
|
|
85
104
|
progress: unknown
|
|
86
|
-
/**
|
|
87
|
-
ran: Date | undefined
|
|
88
|
-
/** how many milliseconds that run took */
|
|
89
|
-
took: number | undefined
|
|
90
|
-
/** its last failure */
|
|
105
|
+
/** why its last run failed */
|
|
91
106
|
error: string | undefined
|
|
92
|
-
|
|
107
|
+
group: string | undefined
|
|
108
|
+
/** a repeating job's repeat as it keeps it: '@every 30s +6178ms', '10 3 * * * Europe/Berlin' */
|
|
93
109
|
repeat: string | undefined
|
|
110
|
+
/** when the last run a handler finished started and ended */
|
|
111
|
+
lastRun: { startedAt: Date; endedAt: Date } | undefined
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface ListOptions {
|
|
115
|
+
/** ids that start with it, in their byte order: end it with a separator, 'chat:4:' */
|
|
116
|
+
prefix?: string | undefined
|
|
117
|
+
/** the jobs in this state alone; 'failed' with no prefix lists the last failed first */
|
|
118
|
+
state?: 'scheduled' | 'waiting' | 'running' | 'failed' | undefined
|
|
119
|
+
/** where the page before ended: its next */
|
|
120
|
+
after?: string | undefined
|
|
121
|
+
/** jobs a page holds at most: 100, 1000 at most */
|
|
122
|
+
limit?: number | undefined
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** A page of a queue's jobs, and where the next starts when there is one. */
|
|
126
|
+
export interface JobPage<T> {
|
|
127
|
+
jobs: Job<T>[]
|
|
128
|
+
next: string | undefined
|
|
94
129
|
}
|
|
95
130
|
|
|
96
131
|
export interface WorkOptions {
|
|
97
|
-
/** the
|
|
98
|
-
|
|
99
|
-
/** how long a job may be in the handler's hands, after which its attempt fails: a minute */
|
|
100
|
-
timeout?: Duration
|
|
101
|
-
/** stops taking jobs: the ones in hand finish first, and work returns */
|
|
102
|
-
signal?: AbortSignal
|
|
103
|
-
/** returns once no job is due and none runs, as a test wants it */
|
|
104
|
-
untilIdle?: boolean
|
|
132
|
+
/** the handlers this worker runs at once, under the queue's bound: its total, or one */
|
|
133
|
+
concurrency?: number | undefined
|
|
105
134
|
}
|
|
106
135
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
136
|
+
/**
|
|
137
|
+
* What runs a job: it gets the job's value and its run. Returning ends the
|
|
138
|
+
* job done, throwing runs it again after its backoff, and returning what
|
|
139
|
+
* run.retry, run.snooze or run.fail made ends it so.
|
|
140
|
+
*/
|
|
141
|
+
export type Handler<T> = (value: T, run: Run) => unknown
|
|
112
142
|
|
|
113
|
-
|
|
143
|
+
/** When a schedule runs: every span, or by a cron on a time zone's wall clock. */
|
|
144
|
+
export type ScheduleWhen = { every: Duration } | { cron: string; timeZone: string }
|
|
114
145
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
146
|
+
export interface ScheduleOptions {
|
|
147
|
+
/** runs a time gets before it fails, the first counted: 10; the schedule goes on */
|
|
148
|
+
attempts?: number | undefined
|
|
149
|
+
backoff?: Backoff | undefined
|
|
150
|
+
/** how long one run may take before its signal aborts: a minute */
|
|
151
|
+
timeout?: Duration | undefined
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** A run's answer as the stream carries it. */
|
|
155
|
+
type AnswerFields = Omit<Written<typeof JobsAnswer.fields>, 'run'>
|
|
156
|
+
|
|
157
|
+
/** The handlers a client's worker runs at once at most, as the server bounds them. */
|
|
158
|
+
const mostHandlers = 1024
|
|
159
|
+
|
|
160
|
+
/** The pause before a worker dials again after a failure, doubling up to the last. */
|
|
161
|
+
const firstPause = 100
|
|
162
|
+
const lastPause = 30_000
|
|
163
|
+
|
|
164
|
+
/** How a run ends otherwise than done, made by run.retry, run.snooze or run.fail: its handler returns it. */
|
|
165
|
+
export class Answer {
|
|
166
|
+
readonly #how: string
|
|
167
|
+
|
|
168
|
+
/** Answers come from a run. */
|
|
169
|
+
constructor(how: string) {
|
|
170
|
+
this.#how = how
|
|
122
171
|
}
|
|
123
|
-
|
|
124
|
-
|
|
172
|
+
|
|
173
|
+
toString(): string {
|
|
174
|
+
return `run.${this.#how}(…)`
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** What a run is made of, which its worker gives it. */
|
|
179
|
+
interface RunParts {
|
|
180
|
+
held: Read<typeof JobsHeld.fields>
|
|
181
|
+
signal: AbortSignal
|
|
182
|
+
steps: Steps
|
|
183
|
+
reporter: Reporter
|
|
184
|
+
/** the answers the run made, which the worker reads what the handler returned against */
|
|
185
|
+
made: Map<Answer, AnswerFields>
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** This call on a job: what it is, its signal, its steps and progress, and the answers its handler returns. */
|
|
189
|
+
export class Run {
|
|
190
|
+
/** the job's id, undefined for one added without */
|
|
191
|
+
readonly id: string | undefined
|
|
192
|
+
/** when the run was due, which "thirty days ago" counts from even when it starts late */
|
|
193
|
+
readonly at: Date
|
|
194
|
+
/** this run's number among the job's attempts, the first being 1 */
|
|
195
|
+
readonly attempt: number
|
|
196
|
+
readonly group: string | undefined
|
|
197
|
+
/**
|
|
198
|
+
* aborts when the run passes its timeout, or when a cancel takes the job,
|
|
199
|
+
* whose reason is then a CancelledError: the handler should stop too
|
|
200
|
+
*/
|
|
201
|
+
readonly signal: AbortSignal
|
|
202
|
+
readonly #steps: Steps
|
|
203
|
+
readonly #reporter: Reporter
|
|
204
|
+
readonly #made: Map<Answer, AnswerFields>
|
|
205
|
+
|
|
206
|
+
/** Runs come to a handler from its worker. */
|
|
207
|
+
constructor(parts: RunParts) {
|
|
208
|
+
this.id = parts.held.id
|
|
209
|
+
this.at = new Date(parts.held.at ?? 0)
|
|
210
|
+
this.attempt = parts.held.attempt ?? 0
|
|
211
|
+
this.group = parts.held.group
|
|
212
|
+
this.signal = parts.signal
|
|
213
|
+
this.#steps = parts.steps
|
|
214
|
+
this.#reporter = parts.reporter
|
|
215
|
+
this.#made = parts.made
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Runs fn once in the job's run and keeps its answer, as JSON, across the
|
|
220
|
+
* job's attempts: the attempt after a retry, a lost worker or a restart
|
|
221
|
+
* gets the answer back without running fn again. A step whose attempt
|
|
222
|
+
* ends before its answer is kept runs again, so what fn does outside the
|
|
223
|
+
* store should bear doing twice. A name is the step's within its job,
|
|
224
|
+
* numbered in a loop: 'model:1', 'tool:1'.
|
|
225
|
+
*
|
|
226
|
+
* const found = await run.step('search', () => search(question.text))
|
|
227
|
+
*/
|
|
228
|
+
step<R>(name: string, fn: () => R | Promise<R>): Promise<R> {
|
|
229
|
+
return this.#steps.run(name, fn)
|
|
125
230
|
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Shows how far the job got, any JSON within 4 KiB, to get: the latest
|
|
234
|
+
* goes, ten a second at most, and nothing waits for the server.
|
|
235
|
+
*/
|
|
236
|
+
setProgress(progress: unknown): void {
|
|
237
|
+
this.#reporter.report(progressText(progress))
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Runs the job again after its backoff, or at the time given, the run counted as an attempt: return it. */
|
|
241
|
+
retry(when?: Duration | Date): Answer {
|
|
242
|
+
return this.#make('retry', when === undefined ? {} : timing(when))
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** Runs the job again at the time given, the run not counted: return it. */
|
|
246
|
+
snooze(when: Duration | Date): Answer {
|
|
247
|
+
return this.#make('snooze', timing(when))
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Fails the job for good now, its reason kept as its error: return it. */
|
|
251
|
+
fail(reason: string | Error): Answer {
|
|
252
|
+
return this.#make('fail', { error: messageOf(reason) })
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
#make(how: 'retry' | 'snooze' | 'fail', fields: AnswerFields): Answer {
|
|
256
|
+
const answer = new Answer(how)
|
|
257
|
+
this.#made.set(answer, { ...fields, how })
|
|
258
|
+
return answer
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** A retry's or a snooze's time: a Date at it, a span from now on the server's clock. */
|
|
263
|
+
function timing(when: Duration | Date): AnswerFields {
|
|
264
|
+
return when instanceof Date ? { at: unixMs(when) } : { delay: ms(when) }
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** How a handler's return ends its run: an answer it made, done for anything else, and a failed run for an answer made and not returned. */
|
|
268
|
+
function answerOf(returned: unknown, made: Map<Answer, AnswerFields>): AnswerFields {
|
|
269
|
+
if (returned instanceof Answer) {
|
|
270
|
+
return (
|
|
271
|
+
made.get(returned) ?? {
|
|
272
|
+
how: 'retry',
|
|
273
|
+
error: 'the handler returned an answer another run made',
|
|
274
|
+
}
|
|
275
|
+
)
|
|
276
|
+
}
|
|
277
|
+
const [unreturned] = made.keys()
|
|
278
|
+
if (unreturned !== undefined) {
|
|
279
|
+
return {
|
|
280
|
+
how: 'retry',
|
|
281
|
+
error: `the handler made ${unreturned} and returned something else: return the answer that ends the run`,
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
return { how: 'done' }
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** The JSON a handler reports of its job, 4 KiB of it at most. */
|
|
288
|
+
function progressText(progress: unknown): string {
|
|
289
|
+
const text = jsonText(progress, "a job's progress")
|
|
126
290
|
if (new TextEncoder().encode(text).length > 4096) {
|
|
127
|
-
throw new InvalidError("a job's progress
|
|
291
|
+
throw new InvalidError("a job's progress past 4 KiB of JSON")
|
|
128
292
|
}
|
|
129
293
|
return text
|
|
130
294
|
}
|
|
@@ -174,7 +338,7 @@ class Reporter {
|
|
|
174
338
|
})
|
|
175
339
|
}
|
|
176
340
|
|
|
177
|
-
/** Drops what waits and lets what is being sent finish, before the
|
|
341
|
+
/** Drops what waits and lets what is being sent finish, before the run's answer follows it. */
|
|
178
342
|
async stop(): Promise<void> {
|
|
179
343
|
this.#stopped = true
|
|
180
344
|
clearTimeout(this.#timer)
|
|
@@ -185,779 +349,899 @@ class Reporter {
|
|
|
185
349
|
/** Calls a method on the connection a job is held on, answering its body. */
|
|
186
350
|
type Call = (method: number, body: Uint8Array) => Promise<Uint8Array>
|
|
187
351
|
|
|
188
|
-
/**
|
|
189
|
-
* The steps of a held job's run, which the server keeps under the number the
|
|
190
|
-
* job was held by: a step's kept answer, or the step run and its answer kept.
|
|
191
|
-
*/
|
|
352
|
+
/** The steps of a held job's run, which the server keeps under the run's number on its connection. */
|
|
192
353
|
class Steps {
|
|
193
354
|
readonly #call: Call
|
|
194
|
-
readonly #
|
|
355
|
+
readonly #run: number
|
|
356
|
+
readonly #cancelled: AbortSignal
|
|
195
357
|
|
|
196
|
-
constructor(call: Call,
|
|
358
|
+
constructor(call: Call, run: number, cancelled: AbortSignal) {
|
|
197
359
|
this.#call = call
|
|
198
|
-
this.#
|
|
360
|
+
this.#run = run
|
|
361
|
+
this.#cancelled = cancelled
|
|
199
362
|
}
|
|
200
363
|
|
|
201
364
|
async run<R>(name: string, fn: () => R | Promise<R>): Promise<R> {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
)
|
|
365
|
+
this.#cancelled.throwIfAborted()
|
|
366
|
+
const asked = JobsStep.encode({ run: this.#run, name })
|
|
367
|
+
const kept = JobsKept.decode(await this.#call(methods['jobs.step'], asked))
|
|
205
368
|
if (kept.found === true) {
|
|
206
369
|
return JSON.parse(kept.answer ?? 'null') as R
|
|
207
370
|
}
|
|
208
371
|
const answer = await fn()
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
372
|
+
const text = jsonText(answer, `step ${JSON.stringify(name)}'s answer`)
|
|
373
|
+
await this.#call(methods['jobs.keep'], JobsStep.encode({ run: this.#run, name, answer: text }))
|
|
374
|
+
return answer
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** A value's JSON, the text every language reads; undefined writes as null, as a step's answer may be nothing. */
|
|
379
|
+
function jsonText(value: unknown, what: string): string {
|
|
380
|
+
let text: string | undefined
|
|
381
|
+
try {
|
|
382
|
+
text = JSON.stringify(value)
|
|
383
|
+
} catch (err) {
|
|
384
|
+
throw new InvalidError(`${what} JSON cannot write: ${messageOf(err)}`)
|
|
385
|
+
}
|
|
386
|
+
if (text === undefined) {
|
|
387
|
+
if (value === undefined) {
|
|
388
|
+
return 'null'
|
|
216
389
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
390
|
+
throw new InvalidError(`${what} JSON cannot write: a function or a symbol`)
|
|
391
|
+
}
|
|
392
|
+
return text
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** A queue's values: JSON, checked by its schema when it has one. */
|
|
396
|
+
interface Values<T> {
|
|
397
|
+
encode(value: T): string
|
|
398
|
+
decode(text: string): Promise<T>
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function valuesOf<T>(schema: StandardSchemaV1 | undefined): Values<T> {
|
|
402
|
+
return {
|
|
403
|
+
encode: value => {
|
|
404
|
+
if (value === undefined) {
|
|
405
|
+
throw new InvalidError('a value of undefined, which JSON cannot write: give null')
|
|
406
|
+
}
|
|
407
|
+
return jsonText(value, 'a value')
|
|
408
|
+
},
|
|
409
|
+
decode: async text => {
|
|
410
|
+
let value: unknown
|
|
411
|
+
try {
|
|
412
|
+
value = JSON.parse(text)
|
|
413
|
+
} catch (err) {
|
|
414
|
+
throw new CorruptError(`a value that is not JSON any more: ${messageOf(err)}`)
|
|
415
|
+
}
|
|
416
|
+
if (schema === undefined) {
|
|
417
|
+
return value as T
|
|
418
|
+
}
|
|
419
|
+
const checked = await check(schema, value)
|
|
420
|
+
if ('issues' in checked) {
|
|
421
|
+
throw new CorruptError(`a value that no longer meets the queue's schema: ${checked.issues}`)
|
|
422
|
+
}
|
|
423
|
+
return checked.value as T
|
|
424
|
+
},
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/** What a queue's or a schedule's calls and workers need of it. */
|
|
429
|
+
interface Source<T> {
|
|
430
|
+
link: Link
|
|
431
|
+
/** 'jobs queue emails', which an error names */
|
|
432
|
+
describe: string
|
|
433
|
+
openMethod: number
|
|
434
|
+
open: Open
|
|
435
|
+
/** the database whose file keeps the queue; the store's jobs.db when undefined */
|
|
436
|
+
home?: Home | undefined
|
|
437
|
+
/** how its calls reach the server: the link's connection, or a database's transaction */
|
|
438
|
+
via: Via
|
|
439
|
+
values: Values<T>
|
|
440
|
+
/** a run's timeout, in milliseconds */
|
|
441
|
+
timeout: number
|
|
442
|
+
/** the store's running workers, which its close stops */
|
|
443
|
+
workers: Set<Worker>
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function callOf<T>(
|
|
447
|
+
source: Source<T>,
|
|
448
|
+
method: number,
|
|
449
|
+
body: (handle: number) => Uint8Array,
|
|
450
|
+
idempotence: Idempotence,
|
|
451
|
+
): Promise<Uint8Array> {
|
|
452
|
+
return source.via.call(source.openMethod, source.open, method, body, idempotence)
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
async function getJob<T>(source: Source<T>, id: string): Promise<Job<T> | undefined> {
|
|
456
|
+
const body = await callOf(
|
|
457
|
+
source,
|
|
458
|
+
methods['jobs.get'],
|
|
459
|
+
handle => JobsId.encode({ handle, id }),
|
|
460
|
+
'read',
|
|
461
|
+
)
|
|
462
|
+
const found = JobsJob.decode(body)
|
|
463
|
+
return found.found === true ? jobOf(source, found) : undefined
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
async function cancelJob<T>(source: Source<T>, id: string): Promise<boolean> {
|
|
467
|
+
const body = await callOf(
|
|
468
|
+
source,
|
|
469
|
+
methods['jobs.cancel'],
|
|
470
|
+
handle => JobsId.encode({ handle, id }),
|
|
471
|
+
'write',
|
|
472
|
+
)
|
|
473
|
+
return JobsChanged.decode(body).changed === true
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/** A job as the server sent it, its value read as the queue reads one. */
|
|
477
|
+
async function jobOf<T>(source: Source<T>, found: Read<typeof JobsJob.fields>): Promise<Job<T>> {
|
|
478
|
+
let value: T
|
|
479
|
+
try {
|
|
480
|
+
value = await source.values.decode(found.value ?? 'null')
|
|
481
|
+
} catch (err) {
|
|
482
|
+
throw new CorruptError(
|
|
483
|
+
`${source.describe}: id ${JSON.stringify(found.id ?? '')}: ${messageOf(err)}`,
|
|
220
484
|
)
|
|
221
|
-
|
|
485
|
+
}
|
|
486
|
+
return {
|
|
487
|
+
id: found.id,
|
|
488
|
+
value,
|
|
489
|
+
state: (found.state ?? 'waiting') as JobState,
|
|
490
|
+
at: new Date(found.at ?? 0),
|
|
491
|
+
attempt: found.attempt ?? 0,
|
|
492
|
+
ahead: found.ahead ?? 0,
|
|
493
|
+
progress: found.progress === undefined ? undefined : JSON.parse(found.progress),
|
|
494
|
+
error: found.error,
|
|
495
|
+
group: found.group,
|
|
496
|
+
repeat: found.repeat,
|
|
497
|
+
lastRun:
|
|
498
|
+
found.startedAt === undefined
|
|
499
|
+
? undefined
|
|
500
|
+
: {
|
|
501
|
+
startedAt: new Date(found.startedAt),
|
|
502
|
+
endedAt: new Date(found.endedAt ?? found.startedAt),
|
|
503
|
+
},
|
|
222
504
|
}
|
|
223
505
|
}
|
|
224
506
|
|
|
225
507
|
/**
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
508
|
+
* Follows the job under id over the server's watch, until the stream's last
|
|
509
|
+
* DATA. A lost connection or a server going away watches again, and the
|
|
510
|
+
* first report of the new watch is left out when it shows what was seen.
|
|
229
511
|
*/
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
512
|
+
async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T>> {
|
|
513
|
+
let last: Read<typeof JobsJob.fields> | undefined
|
|
514
|
+
for (;;) {
|
|
515
|
+
const stream = await source.link.run('read', connection => openWatch(connection, source, id))
|
|
516
|
+
try {
|
|
517
|
+
for (;;) {
|
|
518
|
+
const event = await stream.next()
|
|
519
|
+
stream.consumed(event.body.length)
|
|
520
|
+
if (event.end) {
|
|
521
|
+
return
|
|
522
|
+
}
|
|
523
|
+
const found = JobsJob.decode(event.body)
|
|
524
|
+
if (last === undefined || !showsTheSame(last, found)) {
|
|
525
|
+
last = found
|
|
526
|
+
yield await jobOf(source, found)
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
} catch (err) {
|
|
530
|
+
if (!(err instanceof ConnectionLostError || err instanceof UnavailableError)) {
|
|
531
|
+
throw err
|
|
532
|
+
}
|
|
533
|
+
} finally {
|
|
534
|
+
stream.cancel()
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Opens a watch of the job under id, once its RESPONSE came. */
|
|
540
|
+
async function openWatch<T>(
|
|
541
|
+
connection: Connection,
|
|
542
|
+
source: Source<T>,
|
|
543
|
+
id: string,
|
|
544
|
+
): Promise<Stream> {
|
|
545
|
+
const handle = await handleOn(connection, source.openMethod, source.open)
|
|
546
|
+
const stream = await connection.session.open(
|
|
547
|
+
methods['jobs.watch'],
|
|
548
|
+
JobsId.encode({ handle, id }),
|
|
549
|
+
true,
|
|
550
|
+
)
|
|
551
|
+
await stream.next()
|
|
552
|
+
return stream
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/** Whether two reports show a watcher the same: the job's state, place, attempt, time, progress and error. */
|
|
556
|
+
function showsTheSame(a: Read<typeof JobsJob.fields>, b: Read<typeof JobsJob.fields>): boolean {
|
|
557
|
+
return (
|
|
558
|
+
a.state === b.state &&
|
|
559
|
+
a.ahead === b.ahead &&
|
|
560
|
+
a.attempt === b.attempt &&
|
|
561
|
+
a.at === b.at &&
|
|
562
|
+
a.progress === b.progress &&
|
|
563
|
+
a.error === b.error
|
|
564
|
+
)
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// a queue's source and name, kept out of its public shape, for a database's transaction
|
|
568
|
+
const queues = new WeakMap<object, QueueParts>()
|
|
569
|
+
|
|
570
|
+
/** What a database's transaction needs of a queue it takes in. */
|
|
571
|
+
export interface QueueParts {
|
|
572
|
+
readonly name: string
|
|
573
|
+
readonly source: Source<unknown>
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/** A queue of jobs of one type, run in the order of their time: open it with `store.queue(name)`. */
|
|
577
|
+
export class Queue<T> {
|
|
578
|
+
readonly name: string
|
|
579
|
+
readonly #source: Source<T>
|
|
580
|
+
|
|
581
|
+
/** Queues come from `store.queue` and `db.queue`. */
|
|
582
|
+
constructor(source: Source<T>, name: string) {
|
|
583
|
+
this.#source = source
|
|
584
|
+
this.name = name
|
|
585
|
+
queues.set(this, { name, source: source as Source<unknown> })
|
|
586
|
+
}
|
|
587
|
+
|
|
237
588
|
/**
|
|
238
|
-
*
|
|
239
|
-
*
|
|
589
|
+
* Adds a job unless its id is taken: by a job scheduled, waiting, running
|
|
590
|
+
* or failed, or done while the queue's dedupe keeps it. It returns once
|
|
591
|
+
* the job is on disk, and says whether it added one; a job without an id
|
|
592
|
+
* is always added.
|
|
593
|
+
*
|
|
594
|
+
* await reminders.add({ userId: 42, text: 'Call mom' }, { delay: '1h' })
|
|
240
595
|
*/
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
596
|
+
async add(value: T, options: AddOptions = {}): Promise<boolean> {
|
|
597
|
+
const fields = {
|
|
598
|
+
...jobFields(options),
|
|
599
|
+
id: options.id,
|
|
600
|
+
value: this.#source.values.encode(value),
|
|
601
|
+
}
|
|
602
|
+
return (await this.#changed(methods['jobs.add'], fields)) === true
|
|
603
|
+
}
|
|
245
604
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
605
|
+
/**
|
|
606
|
+
* Makes the id's job this value at this time, whatever it was: one waiting
|
|
607
|
+
* is made new, a failed one starts again, and a running one runs once more
|
|
608
|
+
* after this run, with this value.
|
|
609
|
+
*
|
|
610
|
+
* await checks.set(`check:${check.id}`, check, { delay: '25h' })
|
|
611
|
+
*/
|
|
612
|
+
async set(id: string, value: T, options: JobOptions = {}): Promise<void> {
|
|
613
|
+
const fields = { ...jobFields(options), id, value: this.#source.values.encode(value) }
|
|
614
|
+
await callOf(
|
|
615
|
+
this.#source,
|
|
616
|
+
methods['jobs.set'],
|
|
617
|
+
handle => JobsCall.encode({ ...fields, handle }),
|
|
618
|
+
'write',
|
|
619
|
+
)
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* Changes the id's job when it has not started: its value, and its time,
|
|
624
|
+
* group or repeat when the options give them. It says whether it did: a
|
|
625
|
+
* job that runs, ran or never was answers false.
|
|
626
|
+
*/
|
|
627
|
+
async update(id: string, value: T, options: JobOptions = {}): Promise<boolean> {
|
|
628
|
+
const fields = { ...jobFields(options), id, value: this.#source.values.encode(value) }
|
|
629
|
+
return (await this.#changed(methods['jobs.update'], fields)) === true
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Takes the job under id, whatever its state, and says whether there was
|
|
634
|
+
* one. A running one's handler sees its signal abort, and what it returns
|
|
635
|
+
* then settles nothing; a repeating one stops.
|
|
636
|
+
*/
|
|
637
|
+
cancel(id: string): Promise<boolean> {
|
|
638
|
+
return cancelJob(this.#source, id)
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Where the job under id is: scheduled, waiting and how many jobs are
|
|
643
|
+
* ahead of it, running and its progress, failed and why, or done while
|
|
644
|
+
* the queue's dedupe keeps its id; undefined for an id with no job.
|
|
645
|
+
*/
|
|
646
|
+
get(id: string): Promise<Job<T> | undefined> {
|
|
647
|
+
return getJob(this.#source, id)
|
|
262
648
|
}
|
|
263
649
|
|
|
264
650
|
/**
|
|
265
|
-
*
|
|
266
|
-
* attempt
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
270
|
-
* 'model:1', 'tool:1', 'model:2'. An answer is at most 1 MiB of JSON.
|
|
651
|
+
* The job under id as it is, then again each time its state, place,
|
|
652
|
+
* attempt, time, progress or error changes; it ends with the job: done,
|
|
653
|
+
* failed, or cancelled, a state only a watcher sees. An id with no job
|
|
654
|
+
* yields nothing. A watch whose connection is lost watches again on the
|
|
655
|
+
* next; a job that ended meanwhile ends it without its last state.
|
|
271
656
|
*
|
|
272
|
-
* const
|
|
657
|
+
* for await (const job of videos.watch(id)) send(job.state, job.ahead, job.progress)
|
|
273
658
|
*/
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
659
|
+
watch(id: string): AsyncGenerator<Job<T>> {
|
|
660
|
+
return watchJob(this.#source, id)
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/** A page of the queue's jobs whose ids start with a prefix, or of its failed ones. */
|
|
664
|
+
async list(options: ListOptions = {}): Promise<JobPage<T>> {
|
|
665
|
+
const body = await callOf(
|
|
666
|
+
this.#source,
|
|
667
|
+
methods['jobs.list'],
|
|
668
|
+
handle => JobsList.encode({ handle, ...options }),
|
|
669
|
+
'read',
|
|
670
|
+
)
|
|
671
|
+
const page = JobsPage.decode(body)
|
|
672
|
+
const jobs: Job<T>[] = []
|
|
673
|
+
for (const found of page.jobs ?? []) {
|
|
674
|
+
jobs.push(await jobOf(this.#source, found))
|
|
675
|
+
}
|
|
676
|
+
return { jobs, next: page.next }
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** Every job the options name, a page at a time, holding nothing between pages. */
|
|
680
|
+
async *all(options: Omit<ListOptions, 'after'> = {}): AsyncGenerator<Job<T>> {
|
|
681
|
+
let after: string | undefined
|
|
682
|
+
for (;;) {
|
|
683
|
+
const page = await this.list({ ...options, after })
|
|
684
|
+
yield* page.jobs
|
|
685
|
+
if (page.next === undefined) {
|
|
686
|
+
return
|
|
687
|
+
}
|
|
688
|
+
after = page.next
|
|
277
689
|
}
|
|
278
|
-
return this.#steps.run(name, fn)
|
|
279
690
|
}
|
|
280
691
|
|
|
281
692
|
/**
|
|
282
|
-
*
|
|
283
|
-
*
|
|
693
|
+
* Runs handler on each job as it falls due, as many at once as the
|
|
694
|
+
* queue's concurrency lets, one when it sets none, and returns the running
|
|
695
|
+
* worker at once: stop it at shutdown, as store.close() does. A worker
|
|
696
|
+
* whose connection is lost reaches the store again, the jobs it held
|
|
697
|
+
* running again as a dead worker's would.
|
|
284
698
|
*
|
|
285
|
-
*
|
|
699
|
+
* const worker = reminders.work(async ({ userId, text }) => push(userId, text))
|
|
286
700
|
*/
|
|
287
|
-
|
|
288
|
-
this.#
|
|
701
|
+
work(handler: Handler<T>, options: WorkOptions = {}): Worker {
|
|
702
|
+
const concurrency = handlersOf(options, this.#source.describe)
|
|
703
|
+
return new Worker(
|
|
704
|
+
working => keepWorking({ source: this.#source, handler, concurrency, working }),
|
|
705
|
+
this.#source.workers,
|
|
706
|
+
)
|
|
289
707
|
}
|
|
290
708
|
|
|
291
|
-
/**
|
|
292
|
-
|
|
293
|
-
|
|
709
|
+
/**
|
|
710
|
+
* Runs what is due when it starts and what falls due meanwhile, then says
|
|
711
|
+
* how many jobs ran; it never waits for a later job. Tests, scripts and
|
|
712
|
+
* commands use it.
|
|
713
|
+
*/
|
|
714
|
+
runDue(handler: Handler<T>, options: WorkOptions = {}): Promise<number> {
|
|
715
|
+
return runDue(this.#source, handler, handlersOf(options, this.#source.describe))
|
|
294
716
|
}
|
|
295
717
|
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
718
|
+
async #changed(
|
|
719
|
+
method: number,
|
|
720
|
+
fields: Omit<Written<typeof JobsCall.fields>, 'handle'>,
|
|
721
|
+
): Promise<boolean | undefined> {
|
|
722
|
+
const body = await callOf(
|
|
723
|
+
this.#source,
|
|
724
|
+
method,
|
|
725
|
+
handle => JobsCall.encode({ ...fields, handle }),
|
|
726
|
+
'write',
|
|
727
|
+
)
|
|
728
|
+
return JobsChanged.decode(body).changed
|
|
299
729
|
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* A queue's calls inside a database's transaction, from `tx.with(queue)`: a
|
|
734
|
+
* job it adds is there only if the transaction commits, and the queue's
|
|
735
|
+
* workers find it once it has. A worker, a watch and a page work outside.
|
|
736
|
+
*/
|
|
737
|
+
export class TxQueue<T> {
|
|
738
|
+
readonly name: string
|
|
739
|
+
readonly #queue: Queue<T>
|
|
300
740
|
|
|
301
|
-
/**
|
|
302
|
-
|
|
303
|
-
this
|
|
741
|
+
/** Comes from `tx.with(queue)`. */
|
|
742
|
+
constructor(queue: Queue<T>) {
|
|
743
|
+
this.#queue = queue
|
|
744
|
+
this.name = queue.name
|
|
304
745
|
}
|
|
305
|
-
}
|
|
306
746
|
|
|
307
|
-
/**
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
readonly at: Date
|
|
312
|
-
readonly attempt: number
|
|
313
|
-
readonly #settle: (outcome: Parameters<typeof JobsOutcome.encode>[0]) => Promise<void>
|
|
314
|
-
readonly #job: number
|
|
315
|
-
readonly #steps: Steps
|
|
747
|
+
/** Adds a job unless its id is taken, as `queue.add` does, with the transaction. */
|
|
748
|
+
add(value: T, options: AddOptions = {}): Promise<boolean> {
|
|
749
|
+
return this.#queue.add(value, options)
|
|
750
|
+
}
|
|
316
751
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
value: T,
|
|
320
|
-
settle: (outcome: Parameters<typeof JobsOutcome.encode>[0]) => Promise<void>,
|
|
321
|
-
call: Call,
|
|
322
|
-
) {
|
|
323
|
-
this.key = held.key ?? ''
|
|
324
|
-
this.value = value
|
|
325
|
-
this.at = new Date(held.at ?? 0)
|
|
326
|
-
this.attempt = held.attempt ?? 0
|
|
327
|
-
this.#job = held.job ?? 0
|
|
328
|
-
this.#settle = settle
|
|
329
|
-
this.#steps = new Steps(call, this.#job)
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
/** Runs fn once in the job's run and keeps its answer, as a work loop's job.step does. */
|
|
333
|
-
step<R>(name: string, fn: () => R | Promise<R>): Promise<R> {
|
|
334
|
-
return this.#steps.run(name, fn)
|
|
752
|
+
set(id: string, value: T, options: JobOptions = {}): Promise<void> {
|
|
753
|
+
return this.#queue.set(id, value, options)
|
|
335
754
|
}
|
|
336
755
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
return this.#settle({ job: this.#job, how: how.ack })
|
|
756
|
+
update(id: string, value: T, options: JobOptions = {}): Promise<boolean> {
|
|
757
|
+
return this.#queue.update(id, value, options)
|
|
340
758
|
}
|
|
341
759
|
|
|
342
|
-
|
|
343
|
-
return this.#
|
|
760
|
+
cancel(id: string): Promise<boolean> {
|
|
761
|
+
return this.#queue.cancel(id)
|
|
344
762
|
}
|
|
345
763
|
|
|
346
|
-
|
|
347
|
-
|
|
764
|
+
/** The job under id as the transaction sees it, its own writes among it. */
|
|
765
|
+
get(id: string): Promise<Job<T> | undefined> {
|
|
766
|
+
return this.#queue.get(id)
|
|
348
767
|
}
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
/** A write the store's transaction keeps for its commit: its method, and its call or the id it cancels. */
|
|
771
|
+
export interface JobOp {
|
|
772
|
+
source: Source<unknown>
|
|
773
|
+
method: number
|
|
774
|
+
call?: Omit<Written<typeof JobsCall.fields>, 'handle'>
|
|
775
|
+
id?: string
|
|
776
|
+
}
|
|
349
777
|
|
|
350
|
-
|
|
351
|
-
|
|
778
|
+
/**
|
|
779
|
+
* A queue's writes inside the store's transaction, from `tx.with(queue)` in
|
|
780
|
+
* `store.tx`: each is kept for the commit, which makes them all or none, so
|
|
781
|
+
* what one did is known once the transaction has committed.
|
|
782
|
+
*
|
|
783
|
+
* await store.tx(async tx => {
|
|
784
|
+
* for (const member of members) await tx.with(pushes).add({ member, message }, { id: `${message}:${member}` })
|
|
785
|
+
* })
|
|
786
|
+
*/
|
|
787
|
+
export class StoreTxQueue<T> {
|
|
788
|
+
readonly name: string
|
|
789
|
+
readonly #source: Source<T>
|
|
790
|
+
readonly #keep: (op: JobOp) => void
|
|
791
|
+
|
|
792
|
+
/** Comes from `tx.with(queue)` in `store.tx`. */
|
|
793
|
+
constructor(parts: QueueParts, keep: (op: JobOp) => void) {
|
|
794
|
+
this.name = parts.name
|
|
795
|
+
this.#source = parts.source as Source<T>
|
|
796
|
+
this.#keep = keep
|
|
352
797
|
}
|
|
353
798
|
|
|
354
|
-
/**
|
|
355
|
-
|
|
356
|
-
|
|
799
|
+
/** Adds a job unless its id is taken, with the commit. */
|
|
800
|
+
async add(value: T, options: AddOptions = {}): Promise<void> {
|
|
801
|
+
this.#write('jobs.add', { ...jobFields(options), id: options.id }, value)
|
|
357
802
|
}
|
|
358
803
|
|
|
359
|
-
/**
|
|
360
|
-
|
|
361
|
-
|
|
804
|
+
/** Makes the id's job this value at this time, whatever it was, with the commit. */
|
|
805
|
+
async set(id: string, value: T, options: JobOptions = {}): Promise<void> {
|
|
806
|
+
this.#write('jobs.set', { ...jobFields(options), id }, value)
|
|
362
807
|
}
|
|
363
|
-
}
|
|
364
808
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
809
|
+
/** Changes the id's job when it has not started, with the commit. */
|
|
810
|
+
async update(id: string, value: T, options: JobOptions = {}): Promise<void> {
|
|
811
|
+
this.#write('jobs.update', { ...jobFields(options), id }, value)
|
|
368
812
|
}
|
|
369
|
-
return error instanceof Error ? error.message : String(error)
|
|
370
|
-
}
|
|
371
813
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
} {
|
|
376
|
-
if (when?.at !== undefined) {
|
|
377
|
-
return { at: unixMs(when.at) }
|
|
814
|
+
/** Takes the job under id, whatever its state, with the commit. */
|
|
815
|
+
async cancel(id: string): Promise<void> {
|
|
816
|
+
this.#keep({ source: this.#source as Source<unknown>, method: methods['jobs.cancel'], id })
|
|
378
817
|
}
|
|
379
|
-
|
|
380
|
-
|
|
818
|
+
|
|
819
|
+
#write(
|
|
820
|
+
method: 'jobs.add' | 'jobs.set' | 'jobs.update',
|
|
821
|
+
fields: Omit<Written<typeof JobsCall.fields>, 'handle' | 'value'>,
|
|
822
|
+
value: T,
|
|
823
|
+
): void {
|
|
824
|
+
const call = { ...fields, value: this.#source.values.encode(value) }
|
|
825
|
+
this.#keep({ source: this.#source as Source<unknown>, method: methods[method], call })
|
|
381
826
|
}
|
|
382
|
-
return {}
|
|
383
827
|
}
|
|
384
828
|
|
|
385
|
-
/**
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
829
|
+
/** Sends a transaction's writes of jobs as one jobs.tx: all of them are made, or none. */
|
|
830
|
+
export async function commitJobs(link: Link, ops: readonly JobOp[]): Promise<void> {
|
|
831
|
+
await link.run('write', async connection => {
|
|
832
|
+
const writes = []
|
|
833
|
+
for (const op of ops) {
|
|
834
|
+
const handle = await handleOn(connection, op.source.openMethod, op.source.open)
|
|
835
|
+
writes.push({
|
|
836
|
+
method: op.method,
|
|
837
|
+
call: op.call === undefined ? undefined : { ...op.call, handle },
|
|
838
|
+
id: op.id === undefined ? undefined : { handle, id: op.id },
|
|
839
|
+
})
|
|
840
|
+
}
|
|
841
|
+
await connection.session.call(methods['jobs.tx'], JobsTx.encode({ writes }))
|
|
842
|
+
})
|
|
389
843
|
}
|
|
390
844
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
try {
|
|
395
|
-
text = JSON.stringify(value)
|
|
396
|
-
} catch (err) {
|
|
397
|
-
throw new InvalidError(`a job's value JSON cannot write: ${(err as Error).message}`)
|
|
398
|
-
}
|
|
399
|
-
if (text === undefined) {
|
|
400
|
-
throw new InvalidError("a job's value JSON cannot write: undefined, a function or a symbol")
|
|
401
|
-
}
|
|
402
|
-
return text
|
|
403
|
-
},
|
|
404
|
-
decode: text => {
|
|
405
|
-
try {
|
|
406
|
-
return JSON.parse(text)
|
|
407
|
-
} catch (err) {
|
|
408
|
-
throw new CorruptError(`a job's value is not JSON any more: ${(err as Error).message}`)
|
|
409
|
-
}
|
|
410
|
-
},
|
|
845
|
+
/** A queue's parts, for a database's transaction that takes it in; undefined for anything else. */
|
|
846
|
+
export function queueParts(handle: object): QueueParts | undefined {
|
|
847
|
+
return queues.get(handle)
|
|
411
848
|
}
|
|
412
849
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
decode: async text => {
|
|
417
|
-
const checked = await check(schema, await jsonValues.decode(text))
|
|
418
|
-
if ('issues' in checked) {
|
|
419
|
-
throw new CorruptError(`a job's value does not meet the queue's schema: ${checked.issues}`)
|
|
420
|
-
}
|
|
421
|
-
return checked.value
|
|
422
|
-
},
|
|
423
|
-
}
|
|
850
|
+
/** The queue's calls inside a transaction, going through its stream. */
|
|
851
|
+
export function queueVia(parts: QueueParts, via: Via): TxQueue<unknown> {
|
|
852
|
+
return new TxQueue(new Queue({ ...parts.source, via }, parts.name))
|
|
424
853
|
}
|
|
425
854
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
855
|
+
/** A repeat the code owns: one repeating job under its name, and the worker that runs it. */
|
|
856
|
+
export class Schedule {
|
|
857
|
+
readonly name: string
|
|
858
|
+
readonly #source: Source<null>
|
|
859
|
+
readonly #handler: Handler<null>
|
|
860
|
+
readonly #worker: Worker
|
|
861
|
+
|
|
862
|
+
/** Schedules come from `store.schedule`, started. */
|
|
863
|
+
constructor(source: Source<null>, name: string, handler: (run: Run) => unknown) {
|
|
864
|
+
this.#source = source
|
|
865
|
+
this.name = name
|
|
866
|
+
this.#handler = (_, run) => handler(run)
|
|
867
|
+
const worked = { source, handler: this.#handler, concurrency: undefined }
|
|
868
|
+
this.#worker = new Worker(working => keepWorking({ ...worked, working }), source.workers)
|
|
429
869
|
}
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
}
|
|
435
|
-
return { cron: `${Number(time[2])} ${Number(time[1])} * * *`, zone: r.zone }
|
|
436
|
-
}
|
|
437
|
-
return { cron: r.cron, zone: r.zone }
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
export class Jobs {
|
|
441
|
-
readonly #link: Link
|
|
442
|
-
|
|
443
|
-
constructor(link: Link) {
|
|
444
|
-
this.#link = link
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
/** A queue of JSON values of type T. */
|
|
448
|
-
queue<T = unknown>(name: string, options?: QueueOptions): Queue<T>
|
|
449
|
-
/** A queue whose values each claim checks, with zod's, valibot's or another Standard Schema. */
|
|
450
|
-
queue<S extends StandardSchemaV1>(
|
|
451
|
-
name: string,
|
|
452
|
-
schema: S,
|
|
453
|
-
options?: QueueOptions,
|
|
454
|
-
): Queue<StandardSchemaV1.InferOutput<S>>
|
|
455
|
-
queue(
|
|
456
|
-
name: string,
|
|
457
|
-
of?: StandardSchemaV1 | QueueOptions,
|
|
458
|
-
options?: QueueOptions,
|
|
459
|
-
): Queue<unknown> {
|
|
460
|
-
checkName(name, 'queue')
|
|
461
|
-
let values = jsonValues
|
|
462
|
-
if (isSchema(of)) {
|
|
463
|
-
values = schemaValues(of)
|
|
464
|
-
} else if (of !== undefined) {
|
|
465
|
-
options = of
|
|
466
|
-
}
|
|
467
|
-
const open = JobsQueue.encode(queueFields(name, options))
|
|
468
|
-
return new Queue(this.#link, name, open, values, options?.in)
|
|
870
|
+
|
|
871
|
+
/** When it runs next and how its last run went: what a page of cron jobs shows. */
|
|
872
|
+
get(): Promise<Job<null> | undefined> {
|
|
873
|
+
return getJob(this.#source, this.name)
|
|
469
874
|
}
|
|
470
875
|
|
|
471
|
-
/**
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
876
|
+
/** Runs the schedule's job when it is due, as a queue's runDue does. */
|
|
877
|
+
runDue(): Promise<number> {
|
|
878
|
+
return runDue(this.#source, this.#handler, undefined)
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
/** Stops the schedule's worker, as a worker's stop does. */
|
|
882
|
+
stop(): Promise<void> {
|
|
883
|
+
return this.#worker.stop()
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/** Removes the schedule's job, until the program opens it again. */
|
|
887
|
+
cancel(): Promise<boolean> {
|
|
888
|
+
return cancelJob(this.#source, this.name)
|
|
480
889
|
}
|
|
481
890
|
}
|
|
482
891
|
|
|
483
|
-
|
|
892
|
+
/** What a queue's open sends, checked before anything leaves. */
|
|
893
|
+
/**
|
|
894
|
+
* A queue, its open checked before anything leaves; `home` is the database
|
|
895
|
+
* whose file keeps it, its open naming the database's handle on each
|
|
896
|
+
* connection.
|
|
897
|
+
*/
|
|
898
|
+
export function openQueue<T>(
|
|
899
|
+
link: Link,
|
|
900
|
+
workers: Set<Worker>,
|
|
484
901
|
name: string,
|
|
485
|
-
options: QueueOptions | undefined,
|
|
486
|
-
|
|
487
|
-
|
|
902
|
+
options: QueueOptions & { schema?: StandardSchemaV1 | undefined },
|
|
903
|
+
home?: Home,
|
|
904
|
+
): Queue<T> {
|
|
905
|
+
checkName(name, 'queue')
|
|
906
|
+
const describe =
|
|
907
|
+
home === undefined ? `jobs queue ${name}` : `${home.describe}: jobs queue ${name}`
|
|
908
|
+
const concurrency =
|
|
909
|
+
typeof options.concurrency === 'number' ? { total: options.concurrency } : options.concurrency
|
|
910
|
+
const fields = {
|
|
488
911
|
name,
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
912
|
+
attempts: whole(options.attempts, describe, 'attempts'),
|
|
913
|
+
backoff: backoffOf(options.backoff),
|
|
914
|
+
timeout: msOf(options.timeout),
|
|
915
|
+
concurrency:
|
|
916
|
+
concurrency === undefined
|
|
917
|
+
? undefined
|
|
918
|
+
: {
|
|
919
|
+
total: whole(concurrency.total, describe, 'a concurrency'),
|
|
920
|
+
group: whole(concurrency.group, describe, "a group's concurrency"),
|
|
921
|
+
},
|
|
922
|
+
rate: options.rate === undefined ? undefined : rateOf(options.rate),
|
|
923
|
+
dedupe: msOf(options.dedupe),
|
|
924
|
+
keep: msOf(options.keep),
|
|
925
|
+
maxWaiting: whole(options.maxWaiting, describe, 'a maxWaiting'),
|
|
498
926
|
}
|
|
927
|
+
const bytes = JobsQueueOpen.encode(fields)
|
|
928
|
+
const open: Open =
|
|
929
|
+
home === undefined
|
|
930
|
+
? bytes
|
|
931
|
+
: home.keep(methods['jobs.queue.open'], bytes, async connection =>
|
|
932
|
+
JobsQueueOpen.encode({ ...fields, database: await home.handle(connection) }),
|
|
933
|
+
)
|
|
934
|
+
const source: Source<T> = {
|
|
935
|
+
link,
|
|
936
|
+
describe,
|
|
937
|
+
openMethod: methods['jobs.queue.open'],
|
|
938
|
+
open,
|
|
939
|
+
home,
|
|
940
|
+
via: home === undefined ? viaLink(link) : home.via(`jobs queue ${name}`),
|
|
941
|
+
values: valuesOf<T>(options.schema),
|
|
942
|
+
timeout: options.timeout === undefined ? 60_000 : ms(options.timeout),
|
|
943
|
+
workers,
|
|
944
|
+
}
|
|
945
|
+
return new Queue<T>(source, name)
|
|
499
946
|
}
|
|
500
947
|
|
|
501
|
-
/** A
|
|
502
|
-
function
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
948
|
+
/** A schedule, its open checked before anything leaves, and its worker started. */
|
|
949
|
+
export function openSchedule(
|
|
950
|
+
link: Link,
|
|
951
|
+
workers: Set<Worker>,
|
|
952
|
+
name: string,
|
|
953
|
+
when: ScheduleWhen,
|
|
954
|
+
handler: (run: Run) => unknown,
|
|
955
|
+
options: ScheduleOptions = {},
|
|
956
|
+
): Schedule {
|
|
957
|
+
checkName(name, 'schedule')
|
|
958
|
+
const describe = `jobs schedule ${name}`
|
|
959
|
+
const repeat =
|
|
960
|
+
'cron' in when
|
|
961
|
+
? { cron: when.cron, timeZone: zoneOf(when.timeZone, describe) }
|
|
962
|
+
: { every: ms(when.every) }
|
|
963
|
+
const open = JobsScheduleOpen.encode({
|
|
964
|
+
name,
|
|
965
|
+
...repeat,
|
|
966
|
+
attempts: whole(options.attempts, describe, 'attempts'),
|
|
967
|
+
backoff: backoffOf(options.backoff),
|
|
968
|
+
timeout: msOf(options.timeout),
|
|
969
|
+
})
|
|
970
|
+
const source: Source<null> = {
|
|
971
|
+
link,
|
|
972
|
+
describe,
|
|
973
|
+
openMethod: methods['jobs.schedule.open'],
|
|
974
|
+
open,
|
|
975
|
+
via: viaLink(link),
|
|
976
|
+
values: valuesOf<null>(undefined),
|
|
977
|
+
timeout: options.timeout === undefined ? 60_000 : ms(options.timeout),
|
|
978
|
+
workers,
|
|
509
979
|
}
|
|
980
|
+
return new Schedule(source, name, handler)
|
|
510
981
|
}
|
|
511
982
|
|
|
512
|
-
/** A
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
readonly #values: Values<T>
|
|
517
|
-
|
|
518
|
-
constructor(tx: SqlBatch, open: Uint8Array, values: Values<T>) {
|
|
519
|
-
this.#tx = tx
|
|
520
|
-
this.#open = open
|
|
521
|
-
this.#values = values
|
|
983
|
+
/** A cron's zone, which an hour means nothing without: undefined, a user's never set, is refused. */
|
|
984
|
+
function zoneOf(zone: string | undefined, describe: string): string {
|
|
985
|
+
if (typeof zone !== 'string' || zone === '') {
|
|
986
|
+
throw new InvalidError(`${describe}: a cron needs a timeZone, 'UTC' among them`)
|
|
522
987
|
}
|
|
988
|
+
return zone
|
|
989
|
+
}
|
|
523
990
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
991
|
+
function jobFields(
|
|
992
|
+
options: JobOptions,
|
|
993
|
+
): Omit<Written<typeof JobsCall.fields>, 'handle' | 'id' | 'value'> {
|
|
994
|
+
return {
|
|
995
|
+
at: options.at === undefined ? undefined : unixMs(options.at),
|
|
996
|
+
delay: msOf(options.delay),
|
|
997
|
+
group: options.group,
|
|
998
|
+
every: msOf(options.every),
|
|
999
|
+
cron: options.cron,
|
|
1000
|
+
timeZone: options.timeZone,
|
|
527
1001
|
}
|
|
528
1002
|
}
|
|
529
1003
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
2: 'running',
|
|
533
|
-
3: 'failed',
|
|
534
|
-
4: 'done',
|
|
535
|
-
5: 'cancelled',
|
|
1004
|
+
function msOf(d: Duration | undefined): number | undefined {
|
|
1005
|
+
return d === undefined ? undefined : ms(d)
|
|
536
1006
|
}
|
|
537
1007
|
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
readonly #open: Uint8Array
|
|
542
|
-
readonly #values: Values<T>
|
|
543
|
-
readonly #in: Database | undefined
|
|
544
|
-
|
|
545
|
-
constructor(
|
|
546
|
-
link: Link,
|
|
547
|
-
name: string,
|
|
548
|
-
open: Uint8Array,
|
|
549
|
-
values: Values<T>,
|
|
550
|
-
inDatabase: Database | undefined,
|
|
551
|
-
) {
|
|
552
|
-
this.#link = link
|
|
553
|
-
this.name = name
|
|
554
|
-
this.#open = open
|
|
555
|
-
this.#values = values as Values<T>
|
|
556
|
-
this.#in = inDatabase
|
|
557
|
-
}
|
|
1008
|
+
function backoffOf(backoff: Backoff | undefined): { initial: number; max: number } | undefined {
|
|
1009
|
+
return backoff === undefined ? undefined : { initial: ms(backoff.initial), max: ms(backoff.max) }
|
|
1010
|
+
}
|
|
558
1011
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
1012
|
+
/** A count the server takes as a whole number; one that is none is refused here, naming what it was. */
|
|
1013
|
+
function whole(n: number | undefined, describe: string, what: string): number | undefined {
|
|
1014
|
+
if (n !== undefined && (!Number.isSafeInteger(n) || n < 0)) {
|
|
1015
|
+
throw new InvalidError(`${describe}: ${what} of ${n}, which is no whole number`)
|
|
563
1016
|
}
|
|
1017
|
+
return n
|
|
1018
|
+
}
|
|
564
1019
|
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
* await db.batch(tx => {
|
|
570
|
-
* tx.exec`update notes set body = ${body} where id = ${id}`
|
|
571
|
-
* index.withTx(tx).enqueue({ id })
|
|
572
|
-
* })
|
|
573
|
-
*/
|
|
574
|
-
withTx(tx: SqlBatch): QueueTx<T> {
|
|
575
|
-
if (this.#in === undefined || this.#in.name !== tx.database) {
|
|
576
|
-
const lives = this.#in === undefined ? 'jobs.db' : `sql ${this.#in.name}`
|
|
577
|
-
throw new InvalidError(
|
|
578
|
-
`the queue ${this.name} lives in ${lives}, not in sql ${tx.database}: ` +
|
|
579
|
-
"open it with { in: db } to enqueue in that database's batches",
|
|
580
|
-
)
|
|
581
|
-
}
|
|
582
|
-
return new QueueTx(tx, this.#open, this.#values)
|
|
1020
|
+
function handlersOf(options: WorkOptions, describe: string): number | undefined {
|
|
1021
|
+
const n = options.concurrency
|
|
1022
|
+
if (n !== undefined && (!Number.isSafeInteger(n) || n < 1 || n > mostHandlers)) {
|
|
1023
|
+
throw new InvalidError(`${describe}: a worker of ${n} handlers, not 1 to ${mostHandlers}`)
|
|
583
1024
|
}
|
|
1025
|
+
return n
|
|
1026
|
+
}
|
|
584
1027
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
1028
|
+
/** A worker's state, which its loop and its stop share. */
|
|
1029
|
+
interface Working {
|
|
1030
|
+
stopping: boolean
|
|
1031
|
+
/** the stream the worker runs on now, which a stop is sent on */
|
|
1032
|
+
stream: Stream | undefined
|
|
1033
|
+
/** wakes a loop pausing before it dials again */
|
|
1034
|
+
wake: (() => void) | undefined
|
|
1035
|
+
}
|
|
589
1036
|
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
})
|
|
1037
|
+
/** A worker running a queue's handlers here while the server's loop claims their jobs. */
|
|
1038
|
+
export class Worker implements AsyncDisposable {
|
|
1039
|
+
readonly #working: Working = { stopping: false, stream: undefined, wake: undefined }
|
|
1040
|
+
readonly #done: Promise<void>
|
|
1041
|
+
|
|
1042
|
+
/** Workers come from a queue's work and from store.schedule, running. */
|
|
1043
|
+
constructor(run: (working: Working) => Promise<void>, workers: Set<Worker>) {
|
|
1044
|
+
workers.add(this)
|
|
1045
|
+
this.#done = run(this.#working).finally(() => workers.delete(this))
|
|
600
1046
|
}
|
|
601
1047
|
|
|
602
1048
|
/**
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
1049
|
+
* Takes no job more and waits for the handlers under way, each bounded by
|
|
1050
|
+
* its timeout: the server gives back the jobs it held for them, and ends
|
|
1051
|
+
* the worker once their answers are written.
|
|
606
1052
|
*/
|
|
607
|
-
|
|
608
|
-
const
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
repeat: options?.repeat === undefined ? undefined : repeatOf(options.repeat),
|
|
1053
|
+
stop(): Promise<void> {
|
|
1054
|
+
const working = this.#working
|
|
1055
|
+
if (!working.stopping) {
|
|
1056
|
+
working.stopping = true
|
|
1057
|
+
working.stream?.send(JobsAnswer.encode({ how: 'stop' }), false).catch(() => {})
|
|
1058
|
+
working.wake?.()
|
|
614
1059
|
}
|
|
615
|
-
|
|
616
|
-
const handle = await this.#handle(connection)
|
|
617
|
-
await connection.session.call(
|
|
618
|
-
methods['jobs.update'],
|
|
619
|
-
JobsChange.encode({ ...change, handle }),
|
|
620
|
-
)
|
|
621
|
-
})
|
|
1060
|
+
return this.#done
|
|
622
1061
|
}
|
|
623
1062
|
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
* whether there was one. A running job's handler sees job.signal abort,
|
|
627
|
-
* and what it returns settles nothing.
|
|
628
|
-
*/
|
|
629
|
-
cancel(key: string): Promise<boolean> {
|
|
630
|
-
return this.#link.run('write', async connection => {
|
|
631
|
-
const handle = await this.#handle(connection)
|
|
632
|
-
const body = await connection.session.call(
|
|
633
|
-
methods['jobs.cancel'],
|
|
634
|
-
JobsKey.encode({ handle, key }),
|
|
635
|
-
)
|
|
636
|
-
return JobsEntry.decode(body).found === true
|
|
637
|
-
})
|
|
1063
|
+
[Symbol.asyncDispose](): Promise<void> {
|
|
1064
|
+
return this.stop()
|
|
638
1065
|
}
|
|
1066
|
+
}
|
|
639
1067
|
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
return JobsEntry.decode(
|
|
648
|
-
await connection.session.call(methods['jobs.get'], JobsKey.encode({ handle, key })),
|
|
649
|
-
)
|
|
650
|
-
})
|
|
651
|
-
return entry.found === true ? this.#entryOf(entry) : undefined
|
|
652
|
-
}
|
|
1068
|
+
/** One worker's parts: what it runs, how many at once, and the state its stop shares. */
|
|
1069
|
+
interface Worked<T> {
|
|
1070
|
+
source: Source<T>
|
|
1071
|
+
handler: Handler<T>
|
|
1072
|
+
concurrency: number | undefined
|
|
1073
|
+
working: Working
|
|
1074
|
+
}
|
|
653
1075
|
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
1076
|
+
/**
|
|
1077
|
+
* Runs a worker until it stops: a stream on the link's connection, and
|
|
1078
|
+
* another once a connection is lost or a server going away has ended one.
|
|
1079
|
+
* What fails between is said in TinyStore's own lines, as background work's
|
|
1080
|
+
* failures are.
|
|
1081
|
+
*/
|
|
1082
|
+
async function keepWorking<T>(worked: Worked<T>): Promise<void> {
|
|
1083
|
+
const { source, working } = worked
|
|
1084
|
+
const failures = new FailureLog(`${source.describe} worker`, sayOwn)
|
|
1085
|
+
let pause = firstPause
|
|
1086
|
+
while (!working.stopping) {
|
|
1087
|
+
try {
|
|
1088
|
+
await source.link.run('read', connection => WorkStream.serve(connection, worked, false))
|
|
1089
|
+
failures.succeeded()
|
|
1090
|
+
pause = firstPause
|
|
1091
|
+
} catch (err) {
|
|
1092
|
+
if (working.stopping) {
|
|
1093
|
+
return
|
|
1094
|
+
}
|
|
1095
|
+
failures.failed(err)
|
|
1096
|
+
pause = Math.min(pause * 2, lastPause)
|
|
671
1097
|
}
|
|
1098
|
+
await pausing(pause, working)
|
|
672
1099
|
}
|
|
1100
|
+
}
|
|
673
1101
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
for (;;) {
|
|
699
|
-
const event = await stream.next()
|
|
700
|
-
stream.consumed(event.body.length)
|
|
701
|
-
if (event.end) {
|
|
702
|
-
return
|
|
703
|
-
}
|
|
704
|
-
const entry = await this.#entryOf(JobsEntry.decode(event.body))
|
|
705
|
-
const seen = JSON.stringify([
|
|
706
|
-
entry.state,
|
|
707
|
-
entry.ahead,
|
|
708
|
-
entry.attempt,
|
|
709
|
-
entry.at,
|
|
710
|
-
entry.progress,
|
|
711
|
-
entry.error,
|
|
712
|
-
])
|
|
713
|
-
if (seen !== last) {
|
|
714
|
-
last = seen
|
|
715
|
-
yield entry
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
} catch (err) {
|
|
719
|
-
if (!(err instanceof LostError)) {
|
|
720
|
-
throw err
|
|
721
|
-
}
|
|
722
|
-
} finally {
|
|
723
|
-
unwatch()
|
|
724
|
-
stream.cancel()
|
|
725
|
-
}
|
|
1102
|
+
/** Runs a stream that ends once nothing is due, and says how many jobs its handlers finished. */
|
|
1103
|
+
async function runDue<T>(
|
|
1104
|
+
source: Source<T>,
|
|
1105
|
+
handler: Handler<T>,
|
|
1106
|
+
concurrency: number | undefined,
|
|
1107
|
+
): Promise<number> {
|
|
1108
|
+
const working: Working = { stopping: false, stream: undefined, wake: undefined }
|
|
1109
|
+
let finished = 0
|
|
1110
|
+
await source.link.run('read', async connection => {
|
|
1111
|
+
finished += await WorkStream.serve(connection, { source, handler, concurrency, working }, true)
|
|
1112
|
+
})
|
|
1113
|
+
return finished
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
/** Waits ms, or until the worker stops. */
|
|
1117
|
+
function pausing(ms: number, working: Working): Promise<void> {
|
|
1118
|
+
if (working.stopping) {
|
|
1119
|
+
return Promise.resolve()
|
|
1120
|
+
}
|
|
1121
|
+
return new Promise(resolve => {
|
|
1122
|
+
const done = () => {
|
|
1123
|
+
clearTimeout(timer)
|
|
1124
|
+
working.wake = undefined
|
|
1125
|
+
resolve()
|
|
726
1126
|
}
|
|
1127
|
+
const timer = setTimeout(done, ms)
|
|
1128
|
+
working.wake = done
|
|
1129
|
+
})
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
/** A worker's stream on one connection: the jobs the server hands over, and the handlers they run here. */
|
|
1133
|
+
class WorkStream<T> {
|
|
1134
|
+
readonly #worked: Worked<T>
|
|
1135
|
+
readonly #connection: Connection
|
|
1136
|
+
readonly #stream: Stream
|
|
1137
|
+
/** the runs under way, by number, and what cancels each */
|
|
1138
|
+
readonly #runs = new Map<number, AbortController>()
|
|
1139
|
+
#finished = 0
|
|
1140
|
+
|
|
1141
|
+
constructor(worked: Worked<T>, connection: Connection, stream: Stream) {
|
|
1142
|
+
this.#worked = worked
|
|
1143
|
+
this.#connection = connection
|
|
1144
|
+
this.#stream = stream
|
|
727
1145
|
}
|
|
728
1146
|
|
|
729
1147
|
/**
|
|
730
|
-
*
|
|
731
|
-
*
|
|
1148
|
+
* Opens a worker's stream on a connection and runs it to its end, which
|
|
1149
|
+
* the server sends: after a stop or a GOAWAY once the jobs in hand are
|
|
1150
|
+
* answered, or once nothing is due when it ends when idle. It says how
|
|
1151
|
+
* many jobs its handlers finished.
|
|
732
1152
|
*/
|
|
733
|
-
async
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
return download(connection, methods['jobs.scan'], body)
|
|
1153
|
+
static async serve<T>(
|
|
1154
|
+
connection: Connection,
|
|
1155
|
+
worked: Worked<T>,
|
|
1156
|
+
untilIdle: boolean,
|
|
1157
|
+
): Promise<number> {
|
|
1158
|
+
const handle = await handleOn(connection, worked.source.openMethod, worked.source.open)
|
|
1159
|
+
const asked = JobsWork.encode({
|
|
1160
|
+
handle,
|
|
1161
|
+
concurrency: worked.concurrency,
|
|
1162
|
+
untilIdle: untilIdle || undefined,
|
|
744
1163
|
})
|
|
745
|
-
const
|
|
746
|
-
|
|
747
|
-
|
|
1164
|
+
const stream = await connection.session.open(methods['jobs.work'], asked, false)
|
|
1165
|
+
const { working } = worked
|
|
1166
|
+
working.stream = stream
|
|
1167
|
+
if (working.stopping) {
|
|
1168
|
+
stream.send(JobsAnswer.encode({ how: 'stop' }), false).catch(() => {})
|
|
748
1169
|
}
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
/** Walks scan's pages, holding no snapshot between them. */
|
|
757
|
-
async *all(query?: {
|
|
758
|
-
prefix?: string
|
|
759
|
-
state?: 'failed'
|
|
760
|
-
limit?: number
|
|
761
|
-
}): AsyncGenerator<JobEntry<T>> {
|
|
762
|
-
let after: string | undefined
|
|
763
|
-
for (;;) {
|
|
764
|
-
const page = await this.scan({ ...query, ...(after === undefined ? {} : { after }) })
|
|
765
|
-
yield* page.items
|
|
766
|
-
if (page.next === undefined) {
|
|
767
|
-
return
|
|
1170
|
+
try {
|
|
1171
|
+
return await new WorkStream(worked, connection, stream).#serve()
|
|
1172
|
+
} finally {
|
|
1173
|
+
if (working.stream === stream) {
|
|
1174
|
+
working.stream = undefined
|
|
768
1175
|
}
|
|
769
|
-
after = page.next
|
|
770
1176
|
}
|
|
771
1177
|
}
|
|
772
1178
|
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
const held = JobsHeld.decode(
|
|
782
|
-
await connection.session.call(methods['jobs.claim'], JobsLease.encode({ handle, lease })),
|
|
783
|
-
)
|
|
784
|
-
if (held.found !== true) {
|
|
785
|
-
return undefined
|
|
786
|
-
}
|
|
787
|
-
const value = await this.#values.decode(held.value ?? 'null')
|
|
788
|
-
return new ClaimedJob(
|
|
789
|
-
held,
|
|
790
|
-
value,
|
|
791
|
-
async outcome => {
|
|
792
|
-
const body = await connection.session.call(
|
|
793
|
-
methods['jobs.settle'],
|
|
794
|
-
JobsOutcomes.encode({ outcomes: [outcome] }),
|
|
795
|
-
)
|
|
796
|
-
const refused = JobsSettled.decode(body).settled?.[0]
|
|
797
|
-
if (refused !== undefined && refused !== null) {
|
|
798
|
-
throw errorOf(refused.code ?? 'internal', refused.message ?? '', refused.what ?? {})
|
|
799
|
-
}
|
|
800
|
-
},
|
|
801
|
-
(method, body) => connection.session.call(method, body),
|
|
802
|
-
)
|
|
803
|
-
})
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
/**
|
|
807
|
-
* Runs the queue's jobs as they come due, workers at once, until the
|
|
808
|
-
* signal aborts: the server's own Work loop claims ahead and extends
|
|
809
|
-
* leases, and nothing polls. A handler's return acknowledges its job and a
|
|
810
|
-
* throw retries it. A connection lost takes the jobs in hand with it, as a
|
|
811
|
-
* process that died would, and the loop connects again.
|
|
812
|
-
*/
|
|
813
|
-
async work(
|
|
814
|
-
handler: (job: Job<T>) => void | Promise<void>,
|
|
815
|
-
options: WorkOptions = {},
|
|
816
|
-
): Promise<void> {
|
|
817
|
-
for (;;) {
|
|
818
|
-
options.signal?.throwIfAborted()
|
|
819
|
-
try {
|
|
820
|
-
await this.#link.run(
|
|
821
|
-
'read',
|
|
822
|
-
connection => this.#workOn(connection, handler, options),
|
|
823
|
-
options.signal,
|
|
824
|
-
)
|
|
825
|
-
if (options.untilIdle === true || options.signal?.aborted === true) {
|
|
826
|
-
return
|
|
827
|
-
}
|
|
828
|
-
} catch (err) {
|
|
829
|
-
if (!(err instanceof LostError) || options.signal?.aborted === true) {
|
|
830
|
-
throw err
|
|
1179
|
+
async #serve(): Promise<number> {
|
|
1180
|
+
try {
|
|
1181
|
+
const started = await this.#stream.next()
|
|
1182
|
+
while (!started.end) {
|
|
1183
|
+
const event = await this.#stream.next()
|
|
1184
|
+
this.#stream.consumed(event.body.length)
|
|
1185
|
+
if (event.end) {
|
|
1186
|
+
break
|
|
831
1187
|
}
|
|
1188
|
+
this.#take(JobsHeld.decode(event.body))
|
|
832
1189
|
}
|
|
1190
|
+
return this.#finished
|
|
1191
|
+
} catch (err) {
|
|
1192
|
+
for (const cancel of this.#runs.values()) {
|
|
1193
|
+
cancel.abort(err)
|
|
1194
|
+
}
|
|
1195
|
+
throw err
|
|
833
1196
|
}
|
|
834
1197
|
}
|
|
835
1198
|
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
const stream = await connection.session.open(
|
|
844
|
-
methods['jobs.work'],
|
|
845
|
-
JobsWorkers.encode({
|
|
846
|
-
handle,
|
|
847
|
-
workers: options.workers,
|
|
848
|
-
timeout,
|
|
849
|
-
untilIdle: options.untilIdle || undefined,
|
|
850
|
-
cancels: true,
|
|
851
|
-
}),
|
|
852
|
-
false,
|
|
853
|
-
)
|
|
854
|
-
const stopping = new AbortController()
|
|
855
|
-
const stop = () => stopping.abort()
|
|
856
|
-
options.signal?.addEventListener('abort', stop, { once: true })
|
|
857
|
-
const inHand = new Set<Promise<void>>()
|
|
858
|
-
const cancels = new Map<number, AbortController>()
|
|
859
|
-
let ended = false
|
|
860
|
-
const endOurSide = async () => {
|
|
861
|
-
if (!ended) {
|
|
862
|
-
ended = true
|
|
863
|
-
await Promise.allSettled(inHand)
|
|
864
|
-
await stream.send(new Uint8Array(0), true).catch(() => {})
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
stopping.signal.addEventListener('abort', () => void endOurSide(), { once: true })
|
|
868
|
-
try {
|
|
869
|
-
await stream.next()
|
|
870
|
-
for (;;) {
|
|
871
|
-
const event = await stream.next()
|
|
872
|
-
stream.consumed(event.body.length)
|
|
873
|
-
if (event.end) {
|
|
874
|
-
return
|
|
875
|
-
}
|
|
876
|
-
const held = JobsHeld.decode(event.body)
|
|
877
|
-
if (held.cancelled === true) {
|
|
878
|
-
cancels.get(held.job ?? 0)?.abort(new CancelledError('cancel took the job while it ran'))
|
|
879
|
-
continue
|
|
880
|
-
}
|
|
881
|
-
const cancel = new AbortController()
|
|
882
|
-
cancels.set(held.job ?? 0, cancel)
|
|
883
|
-
const steps = new Steps(
|
|
884
|
-
(method, body) => connection.session.call(method, body),
|
|
885
|
-
held.job ?? 0,
|
|
886
|
-
)
|
|
887
|
-
const running = this.#run(
|
|
888
|
-
stream,
|
|
889
|
-
held,
|
|
890
|
-
steps,
|
|
891
|
-
handler,
|
|
892
|
-
timeout,
|
|
893
|
-
stopping.signal,
|
|
894
|
-
cancel.signal,
|
|
895
|
-
).finally(() => cancels.delete(held.job ?? 0))
|
|
896
|
-
inHand.add(running)
|
|
897
|
-
running.finally(() => inHand.delete(running))
|
|
898
|
-
}
|
|
899
|
-
} finally {
|
|
900
|
-
options.signal?.removeEventListener('abort', stop)
|
|
901
|
-
stopping.abort()
|
|
1199
|
+
/** A job the server handed over, run at once, since it sends no more than the handlers; or a cancel of one. */
|
|
1200
|
+
#take(held: Read<typeof JobsHeld.fields>): void {
|
|
1201
|
+
const run = held.run ?? 0
|
|
1202
|
+
if (held.cancelled === true) {
|
|
1203
|
+
this.#runs.get(run)?.abort(new CancelledError('a cancel took the job while it ran'))
|
|
1204
|
+
this.#runs.delete(run)
|
|
1205
|
+
return
|
|
902
1206
|
}
|
|
1207
|
+
const cancel = new AbortController()
|
|
1208
|
+
this.#runs.set(run, cancel)
|
|
1209
|
+
void this.#run(held, cancel).finally(() => this.#runs.delete(run))
|
|
903
1210
|
}
|
|
904
1211
|
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
held: ReturnType<typeof JobsHeld.decode>,
|
|
911
|
-
steps: Steps,
|
|
912
|
-
handler: (job: Job<T>) => void | Promise<void>,
|
|
913
|
-
timeout: number,
|
|
914
|
-
stopping: AbortSignal,
|
|
915
|
-
cancelled: AbortSignal,
|
|
916
|
-
): Promise<void> {
|
|
917
|
-
const signal = AbortSignal.any([stopping, AbortSignal.timeout(timeout), cancelled])
|
|
918
|
-
const number = held.job ?? 0
|
|
1212
|
+
/** Runs one job's handler and sends the answer it ended with; a job a cancel took is the server's, and gets none. */
|
|
1213
|
+
async #run(held: Read<typeof JobsHeld.fields>, cancel: AbortController): Promise<void> {
|
|
1214
|
+
const { source, handler } = this.#worked
|
|
1215
|
+
const run = held.run ?? 0
|
|
1216
|
+
const signal = AbortSignal.any([cancel.signal, AbortSignal.timeout(source.timeout)])
|
|
919
1217
|
const reporter = new Reporter(progress =>
|
|
920
|
-
stream.send(
|
|
1218
|
+
this.#stream.send(JobsAnswer.encode({ run, how: 'progress', progress }), false),
|
|
921
1219
|
)
|
|
922
|
-
let
|
|
1220
|
+
let answer: AnswerFields
|
|
923
1221
|
try {
|
|
924
|
-
const
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
signal,
|
|
930
|
-
reporter,
|
|
931
|
-
steps,
|
|
1222
|
+
const value = await source.values.decode(held.value ?? 'null')
|
|
1223
|
+
const made = new Map<Answer, AnswerFields>()
|
|
1224
|
+
const steps = new Steps(
|
|
1225
|
+
(method, body) => this.#connection.session.call(method, body),
|
|
1226
|
+
run,
|
|
1227
|
+
cancel.signal,
|
|
932
1228
|
)
|
|
933
1229
|
try {
|
|
934
|
-
|
|
935
|
-
|
|
1230
|
+
answer = answerOf(
|
|
1231
|
+
await handler(value, new Run({ held, signal, steps, reporter, made })),
|
|
1232
|
+
made,
|
|
1233
|
+
)
|
|
936
1234
|
} catch (err) {
|
|
937
|
-
|
|
938
|
-
stopping.aborted && signal.aborted
|
|
939
|
-
? { job: number, how: how.snooze, at: held.at ?? Date.now() }
|
|
940
|
-
: outcomeOf(number, job.settlement ?? { how: 'retry', error: reasonOf(err) })
|
|
1235
|
+
answer = { how: 'retry', error: messageOf(err) }
|
|
941
1236
|
}
|
|
942
1237
|
} catch (err) {
|
|
943
|
-
|
|
1238
|
+
answer = { how: 'fail', error: `its value no longer reads: ${messageOf(err)}` }
|
|
944
1239
|
}
|
|
945
1240
|
await reporter.stop()
|
|
946
|
-
if (
|
|
947
|
-
|
|
1241
|
+
if (cancel.signal.aborted) {
|
|
1242
|
+
return
|
|
948
1243
|
}
|
|
949
|
-
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
function outcomeOf(job: number, s: Settlement): Parameters<typeof JobsOutcome.encode>[0] {
|
|
953
|
-
switch (s.how) {
|
|
954
|
-
case 'ack':
|
|
955
|
-
return { job, how: how.ack }
|
|
956
|
-
case 'retry':
|
|
957
|
-
return { job, how: how.retry, err: s.error, ...timing(s) }
|
|
958
|
-
case 'fail':
|
|
959
|
-
return { job, how: how.fail, err: s.error }
|
|
960
|
-
case 'snooze':
|
|
961
|
-
return { job, how: how.snooze, ...timing(s) }
|
|
1244
|
+
this.#finished++
|
|
1245
|
+
await this.#stream.send(JobsAnswer.encode({ run, ...answer }), false).catch(() => {})
|
|
962
1246
|
}
|
|
963
1247
|
}
|