@shaferllc/keel 0.59.0 → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +167 -0
- package/README.md +30 -1
- package/bin/keel-mcp.mjs +9 -0
- package/dist/core/application.d.ts +5 -5
- package/dist/core/application.js +14 -2
- package/dist/core/auth.d.ts +47 -0
- package/dist/core/auth.js +77 -0
- package/dist/core/authorization.d.ts +9 -1
- package/dist/core/authorization.js +22 -2
- package/dist/core/cache.d.ts +82 -5
- package/dist/core/cache.js +181 -23
- package/dist/core/cli/stubs.d.ts +12 -0
- package/dist/core/cli/stubs.js +120 -0
- package/dist/core/container.d.ts +20 -0
- package/dist/core/container.js +52 -0
- package/dist/core/cors.d.ts +29 -0
- package/dist/core/cors.js +72 -0
- package/dist/core/crypto.d.ts +40 -4
- package/dist/core/crypto.js +66 -6
- package/dist/core/csrf.d.ts +25 -0
- package/dist/core/csrf.js +78 -0
- package/dist/core/database.d.ts +49 -4
- package/dist/core/database.js +89 -21
- package/dist/core/events.d.ts +129 -5
- package/dist/core/events.js +165 -7
- package/dist/core/health.d.ts +141 -0
- package/dist/core/health.js +226 -0
- package/dist/core/helpers.d.ts +15 -3
- package/dist/core/helpers.js +23 -3
- package/dist/core/index.d.ts +33 -18
- package/dist/core/index.js +16 -8
- package/dist/core/lock.d.ts +139 -0
- package/dist/core/lock.js +215 -0
- package/dist/core/logger.d.ts +82 -4
- package/dist/core/logger.js +141 -23
- package/dist/core/mail.d.ts +128 -7
- package/dist/core/mail.js +264 -16
- package/dist/core/model.d.ts +2 -0
- package/dist/core/model.js +16 -14
- package/dist/core/provider.d.ts +7 -0
- package/dist/core/provider.js +7 -0
- package/dist/core/queue.d.ts +134 -9
- package/dist/core/queue.js +304 -14
- package/dist/core/rate-limit.js +3 -0
- package/dist/core/relations.js +13 -13
- package/dist/core/request.d.ts +26 -0
- package/dist/core/request.js +77 -0
- package/dist/core/shield.d.ts +39 -0
- package/dist/core/shield.js +60 -0
- package/dist/core/social.d.ts +173 -0
- package/dist/core/social.js +337 -0
- package/dist/core/storage.d.ts +159 -6
- package/dist/core/storage.js +287 -7
- package/dist/core/tokens.d.ts +74 -0
- package/dist/core/tokens.js +155 -0
- package/dist/db/d1.d.ts +32 -0
- package/dist/db/d1.js +26 -0
- package/dist/db/libsql.d.ts +29 -0
- package/dist/db/libsql.js +32 -0
- package/dist/db/pg.d.ts +29 -0
- package/dist/db/pg.js +33 -0
- package/dist/mcp/server.d.ts +19 -0
- package/dist/mcp/server.js +355 -0
- package/docs/ai-manifest.json +2472 -0
- package/docs/ai.md +128 -0
- package/docs/architecture.md +331 -0
- package/docs/authentication.md +453 -0
- package/docs/authorization.md +167 -0
- package/docs/broadcasting.md +137 -0
- package/docs/broker.md +500 -0
- package/docs/cache.md +558 -0
- package/docs/configuration.md +311 -0
- package/docs/console.md +356 -0
- package/docs/container.md +467 -0
- package/docs/controllers.md +265 -0
- package/docs/cors.md +51 -0
- package/docs/database.md +530 -0
- package/docs/debugging.md +129 -0
- package/docs/decorators.md +127 -0
- package/docs/errors.md +395 -0
- package/docs/events.md +496 -0
- package/docs/examples/architecture-app.ts +27 -0
- package/docs/examples/authentication.ts +61 -0
- package/docs/examples/authorization.ts +79 -0
- package/docs/examples/broadcasting.ts +60 -0
- package/docs/examples/broker-cache-validate.ts +34 -0
- package/docs/examples/broker-fault-tolerance.ts +29 -0
- package/docs/examples/broker-middleware.ts +27 -0
- package/docs/examples/broker.ts +203 -0
- package/docs/examples/cache.ts +222 -0
- package/docs/examples/configuration.ts +81 -0
- package/docs/examples/container.ts +134 -0
- package/docs/examples/controllers.ts +86 -0
- package/docs/examples/database.ts +118 -0
- package/docs/examples/debugging.ts +41 -0
- package/docs/examples/decorators.ts +40 -0
- package/docs/examples/errors.ts +121 -0
- package/docs/examples/events.ts +204 -0
- package/docs/examples/factories.ts +84 -0
- package/docs/examples/hashing.ts +71 -0
- package/docs/examples/health.ts +94 -0
- package/docs/examples/helpers.ts +171 -0
- package/docs/examples/hooks.ts +54 -0
- package/docs/examples/inertia.ts +81 -0
- package/docs/examples/locks.ts +120 -0
- package/docs/examples/logger.ts +92 -0
- package/docs/examples/mail.ts +160 -0
- package/docs/examples/middleware.ts +119 -0
- package/docs/examples/migrations.ts +126 -0
- package/docs/examples/models.ts +239 -0
- package/docs/examples/notification.ts +124 -0
- package/docs/examples/providers.ts +123 -0
- package/docs/examples/queues.ts +254 -0
- package/docs/examples/rate-limiting.ts +42 -0
- package/docs/examples/redis.ts +99 -0
- package/docs/examples/request-response.ts +197 -0
- package/docs/examples/routing.ts +186 -0
- package/docs/examples/scheduling.ts +62 -0
- package/docs/examples/sessions.ts +102 -0
- package/docs/examples/static-files.ts +63 -0
- package/docs/examples/storage.ts +132 -0
- package/docs/examples/templates.ts +58 -0
- package/docs/examples/testing.ts +66 -0
- package/docs/examples/transformer.ts +141 -0
- package/docs/examples/transformers.ts +49 -0
- package/docs/examples/url-builder.ts +86 -0
- package/docs/examples/validation.ts +102 -0
- package/docs/examples/views.tsx +62 -0
- package/docs/examples/vite.ts +106 -0
- package/docs/factories.md +166 -0
- package/docs/getting-started.md +290 -0
- package/docs/hashing.md +259 -0
- package/docs/health.md +225 -0
- package/docs/helpers.md +347 -0
- package/docs/hono.md +186 -0
- package/docs/hooks.md +118 -0
- package/docs/inertia.md +241 -0
- package/docs/locks.md +323 -0
- package/docs/logger.md +290 -0
- package/docs/mail.md +678 -0
- package/docs/middleware.md +425 -0
- package/docs/migrations.md +476 -0
- package/docs/models.md +810 -0
- package/docs/notifications.md +474 -0
- package/docs/providers.md +363 -0
- package/docs/queues.md +679 -0
- package/docs/rate-limiting.md +155 -0
- package/docs/redis.md +178 -0
- package/docs/request-response.md +953 -0
- package/docs/routing.md +804 -0
- package/docs/scheduling.md +110 -0
- package/docs/security.md +85 -0
- package/docs/sessions.md +354 -0
- package/docs/social-auth.md +174 -0
- package/docs/static-files.md +211 -0
- package/docs/storage.md +450 -0
- package/docs/templates.md +315 -0
- package/docs/testing.md +125 -0
- package/docs/transformers.md +381 -0
- package/docs/url-builder.md +295 -0
- package/docs/validation.md +288 -0
- package/docs/views.md +267 -0
- package/docs/vite.md +434 -0
- package/llms-full.txt +17694 -0
- package/llms.txt +116 -0
- package/package.json +38 -7
package/dist/core/queue.d.ts
CHANGED
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* so it stays edge-safe.
|
|
8
8
|
*
|
|
9
9
|
* class SendWelcome extends Job {
|
|
10
|
+
* static maxRetries = 3;
|
|
10
11
|
* constructor(private userId: number) { super(); }
|
|
11
12
|
* async handle() { await mail().to(...).send(); }
|
|
13
|
+
* async failed(error: unknown) { logger().error("welcome failed", { error }); }
|
|
12
14
|
* }
|
|
13
15
|
*
|
|
14
16
|
* await dispatch(new SendWelcome(user.id)); // runs now with the sync driver
|
|
@@ -17,18 +19,91 @@
|
|
|
17
19
|
* setQueue(new MemoryDriver());
|
|
18
20
|
* await dispatch(new SendWelcome(user.id)); // queued
|
|
19
21
|
* await work(); // runs everything pending
|
|
22
|
+
*
|
|
23
|
+
* A job that throws is **retried** with backoff up to `maxRetries`, and only then
|
|
24
|
+
* declared failed — at which point `failed()` runs and it lands in the driver's
|
|
25
|
+
* dead-letter list rather than vanishing.
|
|
26
|
+
*/
|
|
27
|
+
/** What a job knows about its own execution while `handle()` runs. */
|
|
28
|
+
export interface JobContext {
|
|
29
|
+
/** This dispatch's id — stable across retries. */
|
|
30
|
+
jobId: string;
|
|
31
|
+
/** 1 on the first run, 2 on the first retry, and so on. */
|
|
32
|
+
attempt: number;
|
|
33
|
+
/** The lane it was placed on. */
|
|
34
|
+
queue: string;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* A unit of background work. Subclass and implement `handle`.
|
|
38
|
+
*
|
|
39
|
+
* Retry policy is per-class, via statics:
|
|
40
|
+
*
|
|
41
|
+
* class ChargeCard extends Job {
|
|
42
|
+
* static maxRetries = 5;
|
|
43
|
+
* static backoff = exponentialBackoff(1_000);
|
|
44
|
+
* async handle() { … }
|
|
45
|
+
* }
|
|
20
46
|
*/
|
|
21
|
-
/** A unit of background work. Subclass and implement `handle`. */
|
|
22
47
|
export declare abstract class Job {
|
|
48
|
+
/** How many times to retry after the first failure. Default: 0 (no retries). */
|
|
49
|
+
static maxRetries: number;
|
|
50
|
+
/** How long to wait before each retry. Default: exponential from 1s. */
|
|
51
|
+
static backoff: Backoff;
|
|
52
|
+
/** Default lane for this job class. */
|
|
53
|
+
static queue?: string;
|
|
54
|
+
/** Default priority for this job class (lower runs first). */
|
|
55
|
+
static priority?: number;
|
|
56
|
+
/** Set by the driver before `handle()` runs. */
|
|
57
|
+
context?: JobContext;
|
|
23
58
|
abstract handle(): void | Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* Called once the job has exhausted its retries — the last chance to record
|
|
61
|
+
* the failure, alert, or compensate. A throw in here is logged and swallowed:
|
|
62
|
+
* failing to handle a failure must not itself crash the worker.
|
|
63
|
+
*/
|
|
64
|
+
failed(_error: unknown): void | Promise<void>;
|
|
24
65
|
}
|
|
25
66
|
/** Anything dispatchable: a `Job` instance or a plain function. */
|
|
26
67
|
export type Dispatchable = Job | (() => void | Promise<void>);
|
|
68
|
+
/** Milliseconds to wait before attempt `attempt` (1 = the first retry). */
|
|
69
|
+
export type Backoff = (attempt: number) => number;
|
|
70
|
+
/** Doubles each time: 1s, 2s, 4s, 8s… Capped at `maxMs`. */
|
|
71
|
+
export declare function exponentialBackoff(baseMs?: number, maxMs?: number): Backoff;
|
|
72
|
+
/** Adds a fixed step each time: 5s, 10s, 15s… Capped at `maxMs`. */
|
|
73
|
+
export declare function linearBackoff(stepMs?: number, maxMs?: number): Backoff;
|
|
74
|
+
/** The same delay every time. */
|
|
75
|
+
export declare function fixedBackoff(delayMs?: number): Backoff;
|
|
76
|
+
/** No delay at all — retry straight away. */
|
|
77
|
+
export declare const noBackoff: Backoff;
|
|
27
78
|
export interface JobOptions {
|
|
28
79
|
/** Seconds to wait before the job becomes available (drivers may honor it). */
|
|
29
80
|
delay?: number;
|
|
30
|
-
/** Named queue/lane to place the job on. */
|
|
81
|
+
/** Named queue/lane to place the job on. Default: `"default"`. */
|
|
31
82
|
queue?: string;
|
|
83
|
+
/** Lower runs first. Default: 0. */
|
|
84
|
+
priority?: number;
|
|
85
|
+
/** Override the job class's `maxRetries`. */
|
|
86
|
+
maxRetries?: number;
|
|
87
|
+
/** Override the job class's `backoff`. */
|
|
88
|
+
backoff?: Backoff;
|
|
89
|
+
}
|
|
90
|
+
/** A job sitting on a queue, with everything needed to run and retry it. */
|
|
91
|
+
export interface QueuedJob {
|
|
92
|
+
id: string;
|
|
93
|
+
job: Dispatchable;
|
|
94
|
+
options: JobOptions;
|
|
95
|
+
/** How many times it has been attempted so far. */
|
|
96
|
+
attempts: number;
|
|
97
|
+
/** Epoch ms before which it must not run (delay / backoff). */
|
|
98
|
+
availableAt: number;
|
|
99
|
+
}
|
|
100
|
+
/** A job that exhausted its retries. */
|
|
101
|
+
export interface FailedJob {
|
|
102
|
+
id: string;
|
|
103
|
+
job: Dispatchable;
|
|
104
|
+
options: JobOptions;
|
|
105
|
+
attempts: number;
|
|
106
|
+
error: unknown;
|
|
32
107
|
}
|
|
33
108
|
/** The bridge to your queue backend. */
|
|
34
109
|
export interface QueueDriver {
|
|
@@ -39,20 +114,36 @@ export interface Drainable {
|
|
|
39
114
|
readonly size: number;
|
|
40
115
|
work(): Promise<number>;
|
|
41
116
|
}
|
|
42
|
-
/**
|
|
117
|
+
/**
|
|
118
|
+
* Runs jobs the moment they're dispatched — the default; ideal for dev/tests.
|
|
119
|
+
*
|
|
120
|
+
* Retries still apply, but the backoff delay is **not** slept through: an
|
|
121
|
+
* inline driver blocking the request for a 30-second backoff would be worse than
|
|
122
|
+
* useless. Use `MemoryDriver` or a real broker to exercise real delays.
|
|
123
|
+
*/
|
|
43
124
|
export declare class SyncDriver implements QueueDriver {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
job: Dispatchable;
|
|
48
|
-
options: JobOptions;
|
|
125
|
+
/** Jobs that exhausted their retries. */
|
|
126
|
+
readonly failed: FailedJob[];
|
|
127
|
+
push(job: Dispatchable, options: JobOptions): Promise<void>;
|
|
49
128
|
}
|
|
50
129
|
/** Holds jobs in memory; `work()` drains them. Assert on `.jobs` in tests. */
|
|
51
130
|
export declare class MemoryDriver implements QueueDriver, Drainable {
|
|
52
131
|
readonly jobs: QueuedJob[];
|
|
132
|
+
/** Jobs that exhausted their retries. */
|
|
133
|
+
readonly failed: FailedJob[];
|
|
53
134
|
push(job: Dispatchable, options: JobOptions): Promise<void>;
|
|
54
135
|
get size(): number;
|
|
55
|
-
/**
|
|
136
|
+
/** The next job that's due, by priority then insertion order. */
|
|
137
|
+
private takeNext;
|
|
138
|
+
/**
|
|
139
|
+
* Run every job that's currently due, highest priority first; returns how many
|
|
140
|
+
* ran. A job that throws is re-queued with its backoff delay until it runs out
|
|
141
|
+
* of retries, then moves to `failed`.
|
|
142
|
+
*
|
|
143
|
+
* Jobs whose delay or backoff hasn't elapsed are left in place — `work()`
|
|
144
|
+
* drains what's *due*, it doesn't sleep. Call it again later, or advance your
|
|
145
|
+
* test's clock.
|
|
146
|
+
*/
|
|
56
147
|
work(): Promise<number>;
|
|
57
148
|
}
|
|
58
149
|
export declare class Queue {
|
|
@@ -62,7 +153,29 @@ export declare class Queue {
|
|
|
62
153
|
dispatch(job: Dispatchable, options?: JobOptions): Promise<void>;
|
|
63
154
|
/** Drain the driver if it holds jobs locally; returns how many ran. */
|
|
64
155
|
work(): Promise<number>;
|
|
156
|
+
/** Jobs that exhausted their retries, if the driver tracks them. */
|
|
157
|
+
get failed(): FailedJob[];
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* A driver that records dispatches without running anything, plus assertions —
|
|
161
|
+
* what `fakeQueue()` installs so a test can check a job was *queued* without
|
|
162
|
+
* paying for it to run.
|
|
163
|
+
*/
|
|
164
|
+
export declare class FakeQueue extends Queue {
|
|
165
|
+
readonly pushed: QueuedJob[];
|
|
166
|
+
constructor();
|
|
167
|
+
/** Dispatches of a job class (or all of them, with no argument). */
|
|
168
|
+
private matching;
|
|
169
|
+
assertPushed<T extends Job>(type: JobClass<T>, where?: (job: T) => boolean): void;
|
|
170
|
+
assertNotPushed<T extends Job>(type: JobClass<T>, where?: (job: T) => boolean): void;
|
|
171
|
+
assertPushedCount(expected: number, type?: JobClass): void;
|
|
172
|
+
assertNothingPushed(): void;
|
|
173
|
+
/** The queued entry for a job class — to assert on its delay, lane, or priority. */
|
|
174
|
+
pushedJobs<T extends Job>(type: JobClass<T>): QueuedJob[];
|
|
175
|
+
private summary;
|
|
65
176
|
}
|
|
177
|
+
/** A `Job` subclass, as passed to the fake's assertions. */
|
|
178
|
+
export type JobClass<T extends Job = Job> = abstract new (...args: never[]) => T;
|
|
66
179
|
/** Register the default queue driver used by `dispatch()`. */
|
|
67
180
|
export declare function setQueue(driver: QueueDriver): Queue;
|
|
68
181
|
/** The default queue instance. */
|
|
@@ -71,3 +184,15 @@ export declare function getQueue(): Queue;
|
|
|
71
184
|
export declare function dispatch(job: Dispatchable, options?: JobOptions): Promise<void>;
|
|
72
185
|
/** Drain the default queue's pending jobs (no-op for immediate drivers). */
|
|
73
186
|
export declare function work(): Promise<number>;
|
|
187
|
+
/**
|
|
188
|
+
* Swap the queue for one that records dispatches without running them, so a test
|
|
189
|
+
* can assert a job was queued without paying for it to run. Undo with
|
|
190
|
+
* `restoreQueue()`.
|
|
191
|
+
*
|
|
192
|
+
* const q = fakeQueue();
|
|
193
|
+
* await register(user);
|
|
194
|
+
* q.assertPushed(SendWelcome, (job) => job.userId === user.id);
|
|
195
|
+
*/
|
|
196
|
+
export declare function fakeQueue(): FakeQueue;
|
|
197
|
+
/** Restore the real queue after `fakeQueue()`. */
|
|
198
|
+
export declare function restoreQueue(): void;
|
package/dist/core/queue.js
CHANGED
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* so it stays edge-safe.
|
|
8
8
|
*
|
|
9
9
|
* class SendWelcome extends Job {
|
|
10
|
+
* static maxRetries = 3;
|
|
10
11
|
* constructor(private userId: number) { super(); }
|
|
11
12
|
* async handle() { await mail().to(...).send(); }
|
|
13
|
+
* async failed(error: unknown) { logger().error("welcome failed", { error }); }
|
|
12
14
|
* }
|
|
13
15
|
*
|
|
14
16
|
* await dispatch(new SendWelcome(user.id)); // runs now with the sync driver
|
|
@@ -17,38 +19,227 @@
|
|
|
17
19
|
* setQueue(new MemoryDriver());
|
|
18
20
|
* await dispatch(new SendWelcome(user.id)); // queued
|
|
19
21
|
* await work(); // runs everything pending
|
|
22
|
+
*
|
|
23
|
+
* A job that throws is **retried** with backoff up to `maxRetries`, and only then
|
|
24
|
+
* declared failed — at which point `failed()` runs and it lands in the driver's
|
|
25
|
+
* dead-letter list rather than vanishing.
|
|
26
|
+
*/
|
|
27
|
+
import { logger, hasApplication } from "./helpers.js";
|
|
28
|
+
/**
|
|
29
|
+
* A unit of background work. Subclass and implement `handle`.
|
|
30
|
+
*
|
|
31
|
+
* Retry policy is per-class, via statics:
|
|
32
|
+
*
|
|
33
|
+
* class ChargeCard extends Job {
|
|
34
|
+
* static maxRetries = 5;
|
|
35
|
+
* static backoff = exponentialBackoff(1_000);
|
|
36
|
+
* async handle() { … }
|
|
37
|
+
* }
|
|
20
38
|
*/
|
|
21
|
-
/** A unit of background work. Subclass and implement `handle`. */
|
|
22
39
|
export class Job {
|
|
40
|
+
/** How many times to retry after the first failure. Default: 0 (no retries). */
|
|
41
|
+
static maxRetries = 0;
|
|
42
|
+
/** How long to wait before each retry. Default: exponential from 1s. */
|
|
43
|
+
static backoff = exponentialBackoff(1_000);
|
|
44
|
+
/** Default lane for this job class. */
|
|
45
|
+
static queue;
|
|
46
|
+
/** Default priority for this job class (lower runs first). */
|
|
47
|
+
static priority;
|
|
48
|
+
/** Set by the driver before `handle()` runs. */
|
|
49
|
+
context;
|
|
50
|
+
/**
|
|
51
|
+
* Called once the job has exhausted its retries — the last chance to record
|
|
52
|
+
* the failure, alert, or compensate. A throw in here is logged and swallowed:
|
|
53
|
+
* failing to handle a failure must not itself crash the worker.
|
|
54
|
+
*/
|
|
55
|
+
failed(_error) { }
|
|
23
56
|
}
|
|
24
|
-
|
|
25
|
-
|
|
57
|
+
/** Doubles each time: 1s, 2s, 4s, 8s… Capped at `maxMs`. */
|
|
58
|
+
export function exponentialBackoff(baseMs = 1_000, maxMs = 60_000) {
|
|
59
|
+
return (attempt) => Math.min(baseMs * 2 ** (attempt - 1), maxMs);
|
|
60
|
+
}
|
|
61
|
+
/** Adds a fixed step each time: 5s, 10s, 15s… Capped at `maxMs`. */
|
|
62
|
+
export function linearBackoff(stepMs = 1_000, maxMs = 60_000) {
|
|
63
|
+
return (attempt) => Math.min(stepMs * attempt, maxMs);
|
|
64
|
+
}
|
|
65
|
+
/** The same delay every time. */
|
|
66
|
+
export function fixedBackoff(delayMs = 1_000) {
|
|
67
|
+
return () => delayMs;
|
|
68
|
+
}
|
|
69
|
+
/** No delay at all — retry straight away. */
|
|
70
|
+
export const noBackoff = () => 0;
|
|
71
|
+
/* ------------------------------- execution -------------------------------- */
|
|
72
|
+
/** The retry policy for a dispatchable, from its class statics and any overrides. */
|
|
73
|
+
function policy(job, options) {
|
|
74
|
+
const cls = job instanceof Job ? job.constructor : undefined;
|
|
75
|
+
return {
|
|
76
|
+
maxRetries: options.maxRetries ?? cls?.maxRetries ?? 0,
|
|
77
|
+
backoff: options.backoff ?? cls?.backoff ?? exponentialBackoff(1_000),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
function invoke(job, context) {
|
|
81
|
+
if (typeof job === "function")
|
|
82
|
+
return Promise.resolve(job());
|
|
83
|
+
job.context = context;
|
|
84
|
+
return Promise.resolve(job.handle());
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Log through the application logger, falling back to the console. The queue is
|
|
88
|
+
* a primitive — it has to work in a worker that never bootstrapped an
|
|
89
|
+
* Application — and reporting a failure must never itself be the thing that
|
|
90
|
+
* throws.
|
|
91
|
+
*/
|
|
92
|
+
function log(message, context) {
|
|
93
|
+
if (hasApplication())
|
|
94
|
+
logger().error(message, context);
|
|
95
|
+
else
|
|
96
|
+
console.error(message, context);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* A job has exhausted its retries. Log it — loudly, because a worker that keeps
|
|
100
|
+
* running past a failure must not do so silently — then run the `failed()` hook.
|
|
101
|
+
* A throw in the hook is logged too, never rethrown: failing to handle a failure
|
|
102
|
+
* must not itself crash the worker.
|
|
103
|
+
*/
|
|
104
|
+
async function reportFailure(job, error, context) {
|
|
105
|
+
log("queue: job failed", {
|
|
106
|
+
job: job instanceof Job ? job.constructor.name : "fn",
|
|
107
|
+
jobId: context.jobId,
|
|
108
|
+
attempts: context.attempt,
|
|
109
|
+
queue: context.queue,
|
|
110
|
+
error,
|
|
111
|
+
});
|
|
112
|
+
if (!(job instanceof Job))
|
|
113
|
+
return;
|
|
114
|
+
try {
|
|
115
|
+
await job.failed(error);
|
|
116
|
+
}
|
|
117
|
+
catch (hookError) {
|
|
118
|
+
log("queue: a job's failed() hook threw", { error: hookError });
|
|
119
|
+
}
|
|
26
120
|
}
|
|
27
121
|
/* -------------------------------- drivers --------------------------------- */
|
|
28
|
-
|
|
122
|
+
let counter = 0;
|
|
123
|
+
function nextId() {
|
|
124
|
+
return `job_${++counter}`;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Runs jobs the moment they're dispatched — the default; ideal for dev/tests.
|
|
128
|
+
*
|
|
129
|
+
* Retries still apply, but the backoff delay is **not** slept through: an
|
|
130
|
+
* inline driver blocking the request for a 30-second backoff would be worse than
|
|
131
|
+
* useless. Use `MemoryDriver` or a real broker to exercise real delays.
|
|
132
|
+
*/
|
|
29
133
|
export class SyncDriver {
|
|
30
|
-
|
|
31
|
-
|
|
134
|
+
/** Jobs that exhausted their retries. */
|
|
135
|
+
failed = [];
|
|
136
|
+
async push(job, options) {
|
|
137
|
+
const { maxRetries } = policy(job, options);
|
|
138
|
+
const id = nextId();
|
|
139
|
+
const queue = options.queue ?? "default";
|
|
140
|
+
for (let attempt = 1;; attempt++) {
|
|
141
|
+
const context = { jobId: id, attempt, queue };
|
|
142
|
+
try {
|
|
143
|
+
await invoke(job, context);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (attempt > maxRetries) {
|
|
148
|
+
await reportFailure(job, error, context);
|
|
149
|
+
this.failed.push({ id, job, options, attempts: attempt, error });
|
|
150
|
+
// The sync driver *is* the work — it ran inline, so the caller gets the
|
|
151
|
+
// error. A background driver can't do that; it records instead.
|
|
152
|
+
throw error;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
32
156
|
}
|
|
33
157
|
}
|
|
34
158
|
/** Holds jobs in memory; `work()` drains them. Assert on `.jobs` in tests. */
|
|
35
159
|
export class MemoryDriver {
|
|
36
160
|
jobs = [];
|
|
161
|
+
/** Jobs that exhausted their retries. */
|
|
162
|
+
failed = [];
|
|
37
163
|
async push(job, options) {
|
|
38
|
-
this.jobs.push({
|
|
164
|
+
this.jobs.push({
|
|
165
|
+
id: nextId(),
|
|
166
|
+
job,
|
|
167
|
+
options,
|
|
168
|
+
attempts: 0,
|
|
169
|
+
availableAt: Date.now() + (options.delay ?? 0) * 1000,
|
|
170
|
+
});
|
|
39
171
|
}
|
|
40
172
|
get size() {
|
|
41
173
|
return this.jobs.length;
|
|
42
174
|
}
|
|
43
|
-
/**
|
|
175
|
+
/** The next job that's due, by priority then insertion order. */
|
|
176
|
+
takeNext(now) {
|
|
177
|
+
let best = -1;
|
|
178
|
+
for (let i = 0; i < this.jobs.length; i++) {
|
|
179
|
+
const candidate = this.jobs[i];
|
|
180
|
+
if (candidate.availableAt > now)
|
|
181
|
+
continue; // not due yet
|
|
182
|
+
if (best === -1) {
|
|
183
|
+
best = i;
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
const incumbent = this.jobs[best];
|
|
187
|
+
if ((candidate.options.priority ?? 0) < (incumbent.options.priority ?? 0))
|
|
188
|
+
best = i;
|
|
189
|
+
}
|
|
190
|
+
return best === -1 ? undefined : this.jobs.splice(best, 1)[0];
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Run every job that's currently due, highest priority first; returns how many
|
|
194
|
+
* ran. A job that throws is re-queued with its backoff delay until it runs out
|
|
195
|
+
* of retries, then moves to `failed`.
|
|
196
|
+
*
|
|
197
|
+
* Jobs whose delay or backoff hasn't elapsed are left in place — `work()`
|
|
198
|
+
* drains what's *due*, it doesn't sleep. Call it again later, or advance your
|
|
199
|
+
* test's clock.
|
|
200
|
+
*/
|
|
44
201
|
async work() {
|
|
45
202
|
let count = 0;
|
|
46
|
-
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
203
|
+
for (;;) {
|
|
204
|
+
const now = Date.now();
|
|
205
|
+
const next = this.takeNext(now);
|
|
206
|
+
if (!next)
|
|
207
|
+
return count;
|
|
208
|
+
const { maxRetries, backoff } = policy(next.job, next.options);
|
|
209
|
+
const attempt = next.attempts + 1;
|
|
210
|
+
const context = {
|
|
211
|
+
jobId: next.id,
|
|
212
|
+
attempt,
|
|
213
|
+
queue: next.options.queue ?? "default",
|
|
214
|
+
};
|
|
215
|
+
try {
|
|
216
|
+
await invoke(next.job, context);
|
|
217
|
+
count++;
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
if (attempt > maxRetries) {
|
|
221
|
+
// A failed job must not take down the worker — record it, log it, and
|
|
222
|
+
// carry on with the rest of the queue.
|
|
223
|
+
await reportFailure(next.job, error, context);
|
|
224
|
+
this.failed.push({
|
|
225
|
+
id: next.id,
|
|
226
|
+
job: next.job,
|
|
227
|
+
options: next.options,
|
|
228
|
+
attempts: attempt,
|
|
229
|
+
error,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
// Back on the queue, not runnable until the backoff has elapsed.
|
|
234
|
+
this.jobs.push({
|
|
235
|
+
...next,
|
|
236
|
+
attempts: attempt,
|
|
237
|
+
availableAt: Date.now() + backoff(attempt),
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
count++;
|
|
241
|
+
}
|
|
50
242
|
}
|
|
51
|
-
return count;
|
|
52
243
|
}
|
|
53
244
|
}
|
|
54
245
|
/* --------------------------------- queue ---------------------------------- */
|
|
@@ -59,16 +250,93 @@ export class Queue {
|
|
|
59
250
|
}
|
|
60
251
|
/** Place a job on the queue (the driver decides when it runs). */
|
|
61
252
|
async dispatch(job, options = {}) {
|
|
62
|
-
|
|
253
|
+
const cls = job instanceof Job ? job.constructor : undefined;
|
|
254
|
+
await this.driver.push(job, {
|
|
255
|
+
...options,
|
|
256
|
+
queue: options.queue ?? cls?.queue ?? "default",
|
|
257
|
+
priority: options.priority ?? cls?.priority ?? 0,
|
|
258
|
+
});
|
|
63
259
|
}
|
|
64
260
|
/** Drain the driver if it holds jobs locally; returns how many ran. */
|
|
65
261
|
async work() {
|
|
66
262
|
const drainable = this.driver;
|
|
67
263
|
return typeof drainable.work === "function" ? drainable.work() : 0;
|
|
68
264
|
}
|
|
265
|
+
/** Jobs that exhausted their retries, if the driver tracks them. */
|
|
266
|
+
get failed() {
|
|
267
|
+
return this.driver.failed ?? [];
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/* --------------------------------- faking --------------------------------- */
|
|
271
|
+
/**
|
|
272
|
+
* A driver that records dispatches without running anything, plus assertions —
|
|
273
|
+
* what `fakeQueue()` installs so a test can check a job was *queued* without
|
|
274
|
+
* paying for it to run.
|
|
275
|
+
*/
|
|
276
|
+
export class FakeQueue extends Queue {
|
|
277
|
+
pushed = [];
|
|
278
|
+
constructor() {
|
|
279
|
+
const pushed = [];
|
|
280
|
+
super({
|
|
281
|
+
async push(job, options) {
|
|
282
|
+
pushed.push({
|
|
283
|
+
id: nextId(),
|
|
284
|
+
job,
|
|
285
|
+
options,
|
|
286
|
+
attempts: 0,
|
|
287
|
+
availableAt: Date.now() + (options.delay ?? 0) * 1000,
|
|
288
|
+
});
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
this.pushed = pushed;
|
|
292
|
+
}
|
|
293
|
+
/** Dispatches of a job class (or all of them, with no argument). */
|
|
294
|
+
matching(type, where) {
|
|
295
|
+
return this.pushed.filter((entry) => {
|
|
296
|
+
if (type && !(entry.job instanceof type))
|
|
297
|
+
return false;
|
|
298
|
+
return where ? where(entry.job) : true;
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
assertPushed(type, where) {
|
|
302
|
+
if (this.matching(type, where).length)
|
|
303
|
+
return;
|
|
304
|
+
const total = this.pushed.filter((e) => e.job instanceof type).length;
|
|
305
|
+
throw new Error(where && total
|
|
306
|
+
? `Expected a ${type.name} matching the predicate. ${total} were pushed, but none matched.`
|
|
307
|
+
: `Expected ${type.name} to be pushed, but it was not. ${this.summary()}`);
|
|
308
|
+
}
|
|
309
|
+
assertNotPushed(type, where) {
|
|
310
|
+
const found = this.matching(type, where).length;
|
|
311
|
+
if (found)
|
|
312
|
+
throw new Error(`Expected no ${type.name}, but ${found} were pushed.`);
|
|
313
|
+
}
|
|
314
|
+
assertPushedCount(expected, type) {
|
|
315
|
+
const found = this.matching(type).length;
|
|
316
|
+
if (found !== expected) {
|
|
317
|
+
const what = type ? `${type.name} job(s)` : "job(s)";
|
|
318
|
+
throw new Error(`Expected ${expected} ${what} to be pushed, but ${found} were.`);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
assertNothingPushed() {
|
|
322
|
+
if (this.pushed.length) {
|
|
323
|
+
throw new Error(`Expected nothing to be pushed, but ${this.pushed.length} were. ${this.summary()}`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
/** The queued entry for a job class — to assert on its delay, lane, or priority. */
|
|
327
|
+
pushedJobs(type) {
|
|
328
|
+
return this.matching(type);
|
|
329
|
+
}
|
|
330
|
+
summary() {
|
|
331
|
+
if (!this.pushed.length)
|
|
332
|
+
return "Nothing was pushed.";
|
|
333
|
+
const names = [...new Set(this.pushed.map((e) => (e.job instanceof Job ? e.job.constructor.name : "fn")))];
|
|
334
|
+
return `Pushed: ${names.join(", ")}.`;
|
|
335
|
+
}
|
|
69
336
|
}
|
|
70
337
|
/* --------------------------------- global --------------------------------- */
|
|
71
338
|
let queue = new Queue(new SyncDriver());
|
|
339
|
+
let real;
|
|
72
340
|
/** Register the default queue driver used by `dispatch()`. */
|
|
73
341
|
export function setQueue(driver) {
|
|
74
342
|
queue = new Queue(driver);
|
|
@@ -86,3 +354,25 @@ export function dispatch(job, options) {
|
|
|
86
354
|
export function work() {
|
|
87
355
|
return queue.work();
|
|
88
356
|
}
|
|
357
|
+
/**
|
|
358
|
+
* Swap the queue for one that records dispatches without running them, so a test
|
|
359
|
+
* can assert a job was queued without paying for it to run. Undo with
|
|
360
|
+
* `restoreQueue()`.
|
|
361
|
+
*
|
|
362
|
+
* const q = fakeQueue();
|
|
363
|
+
* await register(user);
|
|
364
|
+
* q.assertPushed(SendWelcome, (job) => job.userId === user.id);
|
|
365
|
+
*/
|
|
366
|
+
export function fakeQueue() {
|
|
367
|
+
if (!real)
|
|
368
|
+
real = queue; // only remember the *real* queue — faking twice must not stash a fake
|
|
369
|
+
const fake = new FakeQueue();
|
|
370
|
+
queue = fake;
|
|
371
|
+
return fake;
|
|
372
|
+
}
|
|
373
|
+
/** Restore the real queue after `fakeQueue()`. */
|
|
374
|
+
export function restoreQueue() {
|
|
375
|
+
if (real)
|
|
376
|
+
queue = real;
|
|
377
|
+
real = undefined;
|
|
378
|
+
}
|
package/dist/core/rate-limit.js
CHANGED
|
@@ -35,10 +35,12 @@ export function rateLimiter(options = {}) {
|
|
|
35
35
|
}
|
|
36
36
|
bucket.count++;
|
|
37
37
|
const remaining = Math.max(0, max - bucket.count);
|
|
38
|
+
const resetSeconds = Math.ceil(bucket.reset / 1000);
|
|
38
39
|
if (bucket.count > max) {
|
|
39
40
|
return c.json({ error: options.message ?? "Too Many Requests", status: 429 }, 429, {
|
|
40
41
|
"X-RateLimit-Limit": String(max),
|
|
41
42
|
"X-RateLimit-Remaining": "0",
|
|
43
|
+
"X-RateLimit-Reset": String(resetSeconds),
|
|
42
44
|
"Retry-After": String(Math.ceil((bucket.reset - now) / 1000)),
|
|
43
45
|
});
|
|
44
46
|
}
|
|
@@ -46,5 +48,6 @@ export function rateLimiter(options = {}) {
|
|
|
46
48
|
// Set headers on the final response (survives any handler type).
|
|
47
49
|
c.header("X-RateLimit-Limit", String(max));
|
|
48
50
|
c.header("X-RateLimit-Remaining", String(remaining));
|
|
51
|
+
c.header("X-RateLimit-Reset", String(resetSeconds));
|
|
49
52
|
};
|
|
50
53
|
}
|
package/dist/core/relations.js
CHANGED
|
@@ -56,7 +56,7 @@ export class HasMany extends Relation {
|
|
|
56
56
|
return this.parent[this.localKey];
|
|
57
57
|
}
|
|
58
58
|
query() {
|
|
59
|
-
return db(this.related.table).where(this.foreignKey, this.localValue());
|
|
59
|
+
return db(this.related.table, this.related.connection).where(this.foreignKey, this.localValue());
|
|
60
60
|
}
|
|
61
61
|
async get() {
|
|
62
62
|
return this.hydrate(await this.query().get());
|
|
@@ -64,7 +64,7 @@ export class HasMany extends Relation {
|
|
|
64
64
|
async eager(models, name) {
|
|
65
65
|
const keys = unique(models.map((m) => m[this.localKey]).filter((v) => v != null));
|
|
66
66
|
const rows = keys.length
|
|
67
|
-
? await db(this.related.table).whereIn(this.foreignKey, keys).get()
|
|
67
|
+
? await db(this.related.table, this.related.connection).whereIn(this.foreignKey, keys).get()
|
|
68
68
|
: [];
|
|
69
69
|
const grouped = new Map();
|
|
70
70
|
for (const row of rows) {
|
|
@@ -90,7 +90,7 @@ export class HasOne extends Relation {
|
|
|
90
90
|
return this.parent[this.localKey];
|
|
91
91
|
}
|
|
92
92
|
query() {
|
|
93
|
-
return db(this.related.table).where(this.foreignKey, this.localValue());
|
|
93
|
+
return db(this.related.table, this.related.connection).where(this.foreignKey, this.localValue());
|
|
94
94
|
}
|
|
95
95
|
async get() {
|
|
96
96
|
const row = await this.query().first();
|
|
@@ -99,7 +99,7 @@ export class HasOne extends Relation {
|
|
|
99
99
|
async eager(models, name) {
|
|
100
100
|
const keys = unique(models.map((m) => m[this.localKey]).filter((v) => v != null));
|
|
101
101
|
const rows = keys.length
|
|
102
|
-
? await db(this.related.table).whereIn(this.foreignKey, keys).get()
|
|
102
|
+
? await db(this.related.table, this.related.connection).whereIn(this.foreignKey, keys).get()
|
|
103
103
|
: [];
|
|
104
104
|
const byKey = new Map();
|
|
105
105
|
for (const row of rows) {
|
|
@@ -124,7 +124,7 @@ export class BelongsTo extends Relation {
|
|
|
124
124
|
return this.parent[this.foreignKey];
|
|
125
125
|
}
|
|
126
126
|
query() {
|
|
127
|
-
return db(this.related.table).where(this.ownerKey, this.foreignValue());
|
|
127
|
+
return db(this.related.table, this.related.connection).where(this.ownerKey, this.foreignValue());
|
|
128
128
|
}
|
|
129
129
|
async get() {
|
|
130
130
|
if (this.foreignValue() == null)
|
|
@@ -135,7 +135,7 @@ export class BelongsTo extends Relation {
|
|
|
135
135
|
async eager(models, name) {
|
|
136
136
|
const keys = unique(models.map((m) => m[this.foreignKey]).filter((v) => v != null));
|
|
137
137
|
const rows = keys.length
|
|
138
|
-
? await db(this.related.table).whereIn(this.ownerKey, keys).get()
|
|
138
|
+
? await db(this.related.table, this.related.connection).whereIn(this.ownerKey, keys).get()
|
|
139
139
|
: [];
|
|
140
140
|
const byKey = new Map();
|
|
141
141
|
for (const row of rows)
|
|
@@ -165,18 +165,18 @@ export class BelongsToMany extends Relation {
|
|
|
165
165
|
}
|
|
166
166
|
/** The query against the related table, once pivot rows are known. */
|
|
167
167
|
query() {
|
|
168
|
-
return db(this.related.table);
|
|
168
|
+
return db(this.related.table, this.related.connection);
|
|
169
169
|
}
|
|
170
170
|
async get() {
|
|
171
171
|
if (this.parentValue() == null)
|
|
172
172
|
return [];
|
|
173
|
-
const pivots = await db(this.pivotTable)
|
|
173
|
+
const pivots = await db(this.pivotTable, this.related.connection)
|
|
174
174
|
.where(this.foreignPivotKey, this.parentValue())
|
|
175
175
|
.get();
|
|
176
176
|
const relatedIds = unique(pivots.map((p) => p[this.relatedPivotKey]));
|
|
177
177
|
if (!relatedIds.length)
|
|
178
178
|
return [];
|
|
179
|
-
const rows = await db(this.related.table).whereIn(this.relatedKey, relatedIds).get();
|
|
179
|
+
const rows = await db(this.related.table, this.related.connection).whereIn(this.relatedKey, relatedIds).get();
|
|
180
180
|
return this.hydrate(rows);
|
|
181
181
|
}
|
|
182
182
|
async eager(models, name) {
|
|
@@ -186,10 +186,10 @@ export class BelongsToMany extends Relation {
|
|
|
186
186
|
m.setRelation(name, []);
|
|
187
187
|
return;
|
|
188
188
|
}
|
|
189
|
-
const pivots = await db(this.pivotTable).whereIn(this.foreignPivotKey, parentIds).get();
|
|
189
|
+
const pivots = await db(this.pivotTable, this.related.connection).whereIn(this.foreignPivotKey, parentIds).get();
|
|
190
190
|
const relatedIds = unique(pivots.map((p) => p[this.relatedPivotKey]));
|
|
191
191
|
const rows = relatedIds.length
|
|
192
|
-
? await db(this.related.table).whereIn(this.relatedKey, relatedIds).get()
|
|
192
|
+
? await db(this.related.table, this.related.connection).whereIn(this.relatedKey, relatedIds).get()
|
|
193
193
|
: [];
|
|
194
194
|
const relatedById = new Map(rows.map((row) => [row[this.relatedKey], row]));
|
|
195
195
|
const grouped = new Map();
|
|
@@ -207,7 +207,7 @@ export class BelongsToMany extends Relation {
|
|
|
207
207
|
}
|
|
208
208
|
/** Attach a related row by linking it through the pivot table. */
|
|
209
209
|
async attach(id, extra = {}) {
|
|
210
|
-
await db(this.pivotTable).insert({
|
|
210
|
+
await db(this.pivotTable, this.related.connection).insert({
|
|
211
211
|
[this.foreignPivotKey]: this.parentValue(),
|
|
212
212
|
[this.relatedPivotKey]: id,
|
|
213
213
|
...extra,
|
|
@@ -215,7 +215,7 @@ export class BelongsToMany extends Relation {
|
|
|
215
215
|
}
|
|
216
216
|
/** Detach one related row (or all, when no id is given). */
|
|
217
217
|
async detach(id) {
|
|
218
|
-
let q = db(this.pivotTable).where(this.foreignPivotKey, this.parentValue());
|
|
218
|
+
let q = db(this.pivotTable, this.related.connection).where(this.foreignPivotKey, this.parentValue());
|
|
219
219
|
if (id !== undefined)
|
|
220
220
|
q = q.where(this.relatedPivotKey, id);
|
|
221
221
|
await q.delete();
|