@workglow/job-queue 0.2.27 → 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +457 -43
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +457 -43
- package/dist/bun.js.map +15 -10
- package/dist/common.d.ts +5 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/job/Job.d.ts +1 -1
- package/dist/job/Job.d.ts.map +1 -1
- package/dist/job/JobQueueClient.d.ts +2 -1
- package/dist/job/JobQueueClient.d.ts.map +1 -1
- package/dist/job/JobQueueServer.d.ts +1 -1
- package/dist/job/JobQueueServer.d.ts.map +1 -1
- package/dist/job/JobQueueWorker.d.ts +1 -1
- package/dist/job/JobQueueWorker.d.ts.map +1 -1
- package/dist/job/JobStorageConverters.d.ts +1 -1
- package/dist/job/JobStorageConverters.d.ts.map +1 -1
- package/dist/limiter/CompositeLimiter.d.ts.map +1 -1
- package/dist/limiter/RateLimiter.d.ts +1 -1
- package/dist/limiter/RateLimiter.d.ts.map +1 -1
- package/dist/node.js +457 -43
- package/dist/node.js.map +15 -10
- package/dist/queue-storage/IQueueStorage.d.ts +229 -0
- package/dist/queue-storage/IQueueStorage.d.ts.map +1 -0
- package/dist/queue-storage/InMemoryQueueStorage.d.ts +149 -0
- package/dist/queue-storage/InMemoryQueueStorage.d.ts.map +1 -0
- package/dist/queue-storage/TelemetryQueueStorage.d.ts +33 -0
- package/dist/queue-storage/TelemetryQueueStorage.d.ts.map +1 -0
- package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts +127 -0
- package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts +43 -0
- package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/package.json +3 -8
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
export declare const QUEUE_STORAGE: import("@workglow/util").ServiceToken<IQueueStorage<any, any>>;
|
|
7
|
+
/**
|
|
8
|
+
* The type of a prefix column.
|
|
9
|
+
* - "uuid" maps to UUID in PostgreSQL/Supabase, TEXT in SQLite/IndexedDB/InMemory
|
|
10
|
+
* - "number" maps to INTEGER in PostgreSQL/Supabase/SQLite, number in IndexedDB/InMemory
|
|
11
|
+
*/
|
|
12
|
+
export type PrefixColumnType = "uuid" | "number";
|
|
13
|
+
/**
|
|
14
|
+
* Defines a prefix column for queue storage filtering.
|
|
15
|
+
*/
|
|
16
|
+
export interface PrefixColumn {
|
|
17
|
+
readonly name: string;
|
|
18
|
+
readonly type: PrefixColumnType;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Options for configuring queue storage with prefix filters.
|
|
22
|
+
*/
|
|
23
|
+
export interface QueueStorageOptions {
|
|
24
|
+
/** The prefix column definitions for this storage */
|
|
25
|
+
readonly prefixes?: readonly PrefixColumn[];
|
|
26
|
+
/** The values for each prefix column */
|
|
27
|
+
readonly prefixValues?: Readonly<Record<string, string | number>>;
|
|
28
|
+
}
|
|
29
|
+
export type JobStatus = "PENDING" | "PROCESSING" | "COMPLETED" | "ABORTING" | "FAILED" | "DISABLED";
|
|
30
|
+
export declare const JobStatus: {
|
|
31
|
+
readonly PENDING: "PENDING";
|
|
32
|
+
readonly PROCESSING: "PROCESSING";
|
|
33
|
+
readonly COMPLETED: "COMPLETED";
|
|
34
|
+
readonly ABORTING: "ABORTING";
|
|
35
|
+
readonly FAILED: "FAILED";
|
|
36
|
+
readonly DISABLED: "DISABLED";
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Type of change that occurred in the queue.
|
|
40
|
+
*
|
|
41
|
+
* `RESYNC` is a synthetic event emitted by some backends (e.g. Postgres
|
|
42
|
+
* LISTEN/NOTIFY) after a (re)connect to indicate that arbitrary changes may
|
|
43
|
+
* have happened during a disconnect window. Subscribers should treat it as a
|
|
44
|
+
* "kick the workers, re-poll state" signal — `old` and `new` are both
|
|
45
|
+
* undefined.
|
|
46
|
+
*/
|
|
47
|
+
export type QueueChangeType = "INSERT" | "UPDATE" | "DELETE" | "RESYNC";
|
|
48
|
+
/**
|
|
49
|
+
* Payload describing a change to a job
|
|
50
|
+
*/
|
|
51
|
+
export interface QueueChangePayload<Input, Output> {
|
|
52
|
+
readonly type: QueueChangeType;
|
|
53
|
+
readonly old?: JobStorageFormat<Input, Output>;
|
|
54
|
+
readonly new?: JobStorageFormat<Input, Output>;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Options for subscribing to queue changes
|
|
58
|
+
*/
|
|
59
|
+
export interface QueueSubscribeOptions {
|
|
60
|
+
/** Polling interval in milliseconds (used by implementations that rely on polling) */
|
|
61
|
+
readonly pollingIntervalMs?: number;
|
|
62
|
+
/**
|
|
63
|
+
* Custom prefix filter for this subscription.
|
|
64
|
+
*
|
|
65
|
+
* - If not provided (undefined): Uses the storage instance's configured prefixValues
|
|
66
|
+
* - If empty object ({}): Receives ALL changes across all prefix combinations
|
|
67
|
+
* - If partial object: Receives changes matching the specified subset of prefixes
|
|
68
|
+
*
|
|
69
|
+
* @example
|
|
70
|
+
* // Storage configured with prefixes: [{name: "user_id"}, {name: "project_id"}]
|
|
71
|
+
* // and prefixValues: {user_id: "abc", project_id: "123"}
|
|
72
|
+
*
|
|
73
|
+
* // Subscribe to only this user+project (default behavior)
|
|
74
|
+
* storage.subscribeToChanges(callback);
|
|
75
|
+
*
|
|
76
|
+
* // Subscribe to all projects for this user
|
|
77
|
+
* storage.subscribeToChanges(callback, { prefixFilter: { user_id: "abc" } });
|
|
78
|
+
*
|
|
79
|
+
* // Subscribe to ALL jobs in this queue (admin/supervisor view)
|
|
80
|
+
* storage.subscribeToChanges(callback, { prefixFilter: {} });
|
|
81
|
+
*/
|
|
82
|
+
readonly prefixFilter?: Readonly<Record<string, string | number>>;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Details about a job that reflect the structure in the database.
|
|
86
|
+
*/
|
|
87
|
+
export type JobStorageFormat<Input, Output> = {
|
|
88
|
+
id?: unknown;
|
|
89
|
+
job_run_id?: string;
|
|
90
|
+
queue?: string;
|
|
91
|
+
input: Input;
|
|
92
|
+
output?: Output | null;
|
|
93
|
+
error?: string | null;
|
|
94
|
+
error_code?: string | null;
|
|
95
|
+
fingerprint?: string;
|
|
96
|
+
max_retries?: number;
|
|
97
|
+
status?: JobStatus;
|
|
98
|
+
created_at?: string;
|
|
99
|
+
deadline_at?: string | null;
|
|
100
|
+
last_ran_at?: string | null;
|
|
101
|
+
run_after: string | null;
|
|
102
|
+
completed_at: string | null;
|
|
103
|
+
run_attempts?: number;
|
|
104
|
+
progress?: number;
|
|
105
|
+
progress_message?: string;
|
|
106
|
+
progress_details?: Record<string, any> | null;
|
|
107
|
+
worker_id?: string | null;
|
|
108
|
+
};
|
|
109
|
+
/**
|
|
110
|
+
* Whether a queue storage's state is shared across processes.
|
|
111
|
+
*
|
|
112
|
+
* - `"process"` — in-memory / per-process state. Workers in the same process
|
|
113
|
+
* share it, but separate processes do not.
|
|
114
|
+
* - `"cluster"` — state lives in shared external storage (Postgres, Supabase,
|
|
115
|
+
* etc.) visible to every process. Pairing a `"process"`-scoped limiter with
|
|
116
|
+
* a `"cluster"`-scoped queue almost always indicates a misconfiguration.
|
|
117
|
+
*/
|
|
118
|
+
export type QueueStorageScope = "process" | "cluster";
|
|
119
|
+
/**
|
|
120
|
+
* Interface defining the storage operations for a job queue
|
|
121
|
+
*/
|
|
122
|
+
export interface IQueueStorage<Input, Output> {
|
|
123
|
+
/**
|
|
124
|
+
* Whether this storage is shared across processes. In-memory / browser
|
|
125
|
+
* backends MUST report `"process"`. Shared databases (Postgres, Supabase)
|
|
126
|
+
* report `"cluster"`. Used by JobQueueServer to detect process-scoped
|
|
127
|
+
* limiters paired with cluster-scoped queues.
|
|
128
|
+
*/
|
|
129
|
+
readonly scope: QueueStorageScope;
|
|
130
|
+
/**
|
|
131
|
+
* Adds a job to the queue storage
|
|
132
|
+
* @param job - The job to add to the queue storage
|
|
133
|
+
* @returns The ID of the job
|
|
134
|
+
*/
|
|
135
|
+
add(job: JobStorageFormat<Input, Output>): Promise<unknown>;
|
|
136
|
+
/**
|
|
137
|
+
* Gets a job from the queue storage by ID
|
|
138
|
+
* @param id - The ID of the job to get
|
|
139
|
+
* @returns The job with the given ID
|
|
140
|
+
*/
|
|
141
|
+
get(id: unknown): Promise<JobStorageFormat<Input, Output> | undefined>;
|
|
142
|
+
/**
|
|
143
|
+
* Gets the next job from the queue storage
|
|
144
|
+
* @param workerId - Worker ID to associate with the job (required)
|
|
145
|
+
* @returns The next job from the queue storage
|
|
146
|
+
*/
|
|
147
|
+
next(workerId: string): Promise<JobStorageFormat<Input, Output> | undefined>;
|
|
148
|
+
/**
|
|
149
|
+
* Releases a job that was just claimed by {@link next} but won't be
|
|
150
|
+
* processed (e.g. the worker was stopped mid-claim). Resets status to
|
|
151
|
+
* PENDING and clears worker_id WITHOUT incrementing run_attempts —
|
|
152
|
+
* the worker never actually attempted execution, so the retry budget
|
|
153
|
+
* must be preserved.
|
|
154
|
+
* @param id - The id of the claimed job to release.
|
|
155
|
+
*/
|
|
156
|
+
release(id: unknown): Promise<void>;
|
|
157
|
+
/**
|
|
158
|
+
* Peeks at the next job(s) from the queue storage without removing them
|
|
159
|
+
* @param status - The status of the jobs to peek at
|
|
160
|
+
* @param num - The number of jobs to peek at
|
|
161
|
+
* @returns The jobs with the given status
|
|
162
|
+
*/
|
|
163
|
+
peek(status?: JobStatus, num?: number): Promise<Array<JobStorageFormat<Input, Output>>>;
|
|
164
|
+
/**
|
|
165
|
+
* Gets the size of the queue storage
|
|
166
|
+
* @param status - The status of the jobs to get the size for
|
|
167
|
+
* @returns The size of the queue storage
|
|
168
|
+
*/
|
|
169
|
+
size(status?: JobStatus): Promise<number>;
|
|
170
|
+
/**
|
|
171
|
+
* Completes a job in the queue storage
|
|
172
|
+
* @param job - The job to complete
|
|
173
|
+
*/
|
|
174
|
+
complete(job: JobStorageFormat<Input, Output>): Promise<void>;
|
|
175
|
+
/**
|
|
176
|
+
* Deletes all jobs from the queue storage
|
|
177
|
+
*/
|
|
178
|
+
deleteAll(): Promise<void>;
|
|
179
|
+
/**
|
|
180
|
+
* Gets the output for a given input from the queue storage
|
|
181
|
+
* @param input - The input to get the output for
|
|
182
|
+
* @returns The output of the job
|
|
183
|
+
*/
|
|
184
|
+
outputForInput(input: Input): Promise<Output | null>;
|
|
185
|
+
/**
|
|
186
|
+
* Aborts a job in the queue storage
|
|
187
|
+
* @param id - The ID of the job to abort
|
|
188
|
+
*/
|
|
189
|
+
abort(id: unknown): Promise<void>;
|
|
190
|
+
/**
|
|
191
|
+
* Gets the jobs by job run ID from the queue storage
|
|
192
|
+
* @param runId - The job run ID of the jobs to get
|
|
193
|
+
* @returns The jobs with the given job run ID
|
|
194
|
+
*/
|
|
195
|
+
getByRunId(runId: string): Promise<Array<JobStorageFormat<Input, Output>>>;
|
|
196
|
+
/**
|
|
197
|
+
* Saves progress updates for a job in the queue storage
|
|
198
|
+
* @param id - The ID of the job to save the progress for
|
|
199
|
+
* @param progress - The progress of the job
|
|
200
|
+
* @param message - The message of the job
|
|
201
|
+
* @param details - The details of the job
|
|
202
|
+
*/
|
|
203
|
+
saveProgress(id: unknown, progress: number, message: string, details: Record<string, any> | null): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* Deletes a job by its ID from the queue storage
|
|
206
|
+
* @param id - The ID of the job to delete
|
|
207
|
+
*/
|
|
208
|
+
delete(id: unknown): Promise<void>;
|
|
209
|
+
/**
|
|
210
|
+
* Deletes jobs from the queue storage that are of a specific status and older than the specified time
|
|
211
|
+
* @param status - The status of the jobs to delete
|
|
212
|
+
* @param olderThanMs - The time in milliseconds that the jobs must be older than to be deleted
|
|
213
|
+
*/
|
|
214
|
+
deleteJobsByStatusAndAge(status: JobStatus, olderThanMs: number): Promise<void>;
|
|
215
|
+
/**
|
|
216
|
+
* Sets up the database schema and tables.
|
|
217
|
+
* This method should be called before using the storage in tests.
|
|
218
|
+
* For production use, database setup should be done via migrations.
|
|
219
|
+
*/
|
|
220
|
+
setupDatabase(): Promise<void>;
|
|
221
|
+
/**
|
|
222
|
+
* Subscribes to changes in the queue (including remote changes).
|
|
223
|
+
* @param callback - Function called when a change occurs
|
|
224
|
+
* @param options - Optional subscription options (e.g., polling interval)
|
|
225
|
+
* @returns Unsubscribe function
|
|
226
|
+
*/
|
|
227
|
+
subscribeToChanges(callback: (change: QueueChangePayload<Input, Output>) => void, options?: QueueSubscribeOptions): () => void;
|
|
228
|
+
}
|
|
229
|
+
//# sourceMappingURL=IQueueStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IQueueStorage.d.ts","sourceRoot":"","sources":["../../src/queue-storage/IQueueStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,eAAO,MAAM,aAAa,gEAAkE,CAAC;AAE7F;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,qDAAqD;IACrD,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC5C,wCAAwC;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;CACnE;AAED,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,CAAC;AACpG,eAAO,MAAM,SAAS;sBACX,SAAS;yBACN,YAAY;wBACb,WAAW;uBACZ,UAAU;qBACZ,QAAQ;uBACN,UAAU;CAC2B,CAAC;AAElD;;;;;;;;GAQG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,kBAAkB,CAAC,KAAK,EAAE,MAAM;IAC/C,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC/C,QAAQ,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,sFAAsF;IACtF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;CACnE;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,KAAK,EAAE,MAAM,IAAI;IAC5C,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC;IAC9C,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,EAAE,MAAM;IAC1C;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;IAElC;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5D;;;;OAIG;IACH,GAAG,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC;IAEvE;;;;OAIG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC;IAE7E;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEpC;;;;;OAKG;IACH,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAExF;;;;OAIG;IACH,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1C;;;OAGG;IACH,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;OAEG;IACH,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3B;;;;OAIG;IACH,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAErD;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElC;;;;OAIG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAE3E;;;;;;OAMG;IACH,YAAY,CACV,EAAE,EAAE,OAAO,EACX,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,GAClC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjB;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC;;;;OAIG;IACH,wBAAwB,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhF;;;;OAIG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;;;OAKG;IACH,kBAAkB,CAChB,QAAQ,EAAE,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAC7D,OAAO,CAAC,EAAE,qBAAqB,GAC9B,MAAM,IAAI,CAAC;CACf"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { EventEmitter } from "@workglow/util";
|
|
7
|
+
import { IQueueStorage, JobStatus, JobStorageFormat, QueueChangePayload, QueueStorageOptions, QueueSubscribeOptions } from "./IQueueStorage";
|
|
8
|
+
/**
|
|
9
|
+
* Event listeners for queue storage events
|
|
10
|
+
*/
|
|
11
|
+
type QueueEventListeners<Input, Output> = {
|
|
12
|
+
change: (payload: QueueChangePayload<Input, Output>) => void;
|
|
13
|
+
};
|
|
14
|
+
export declare const IN_MEMORY_QUEUE_STORAGE: import("@workglow/util").ServiceToken<IQueueStorage<any, any>>;
|
|
15
|
+
/**
|
|
16
|
+
* In-memory implementation of a job queue that manages asynchronous tasks.
|
|
17
|
+
* Supports job scheduling, status tracking, result caching, and prefix-based filtering.
|
|
18
|
+
*/
|
|
19
|
+
export declare class InMemoryQueueStorage<Input, Output> implements IQueueStorage<Input, Output> {
|
|
20
|
+
readonly queueName: string;
|
|
21
|
+
readonly scope: "process";
|
|
22
|
+
/** The prefix values for filtering jobs */
|
|
23
|
+
protected readonly prefixValues: Readonly<Record<string, string | number>>;
|
|
24
|
+
/** Event emitter for change notifications */
|
|
25
|
+
protected readonly events: EventEmitter<QueueEventListeners<Input, Output>>;
|
|
26
|
+
/**
|
|
27
|
+
* Creates a new in-memory job queue
|
|
28
|
+
* @param queueName - Name of the queue
|
|
29
|
+
* @param options - Optional configuration including prefix filters
|
|
30
|
+
*/
|
|
31
|
+
constructor(queueName: string, options?: QueueStorageOptions);
|
|
32
|
+
/** Internal array storing all jobs */
|
|
33
|
+
jobQueue: Array<JobStorageFormat<Input, Output> & Record<string, unknown>>;
|
|
34
|
+
/**
|
|
35
|
+
* Checks if a job matches the current prefix values
|
|
36
|
+
*/
|
|
37
|
+
private matchesPrefixes;
|
|
38
|
+
/**
|
|
39
|
+
* Returns a filtered and sorted list of pending jobs that are ready to run
|
|
40
|
+
* Sorts by creation time to maintain FIFO order
|
|
41
|
+
*/
|
|
42
|
+
private pendingQueue;
|
|
43
|
+
/**
|
|
44
|
+
* Adds a new job to the queue
|
|
45
|
+
* Generates an ID and fingerprint if not provided
|
|
46
|
+
*/
|
|
47
|
+
add(job: JobStorageFormat<Input, Output>): Promise<unknown>;
|
|
48
|
+
/**
|
|
49
|
+
* Retrieves a job from the queue by its id.
|
|
50
|
+
* @param id - The id of the job to retrieve.
|
|
51
|
+
* @returns A promise that resolves to the job or undefined if the job is not found.
|
|
52
|
+
*/
|
|
53
|
+
get(id: unknown): Promise<JobStorageFormat<Input, Output> | undefined>;
|
|
54
|
+
/**
|
|
55
|
+
* Retrieves a slice of jobs from the queue.
|
|
56
|
+
* @param status - The status of the jobs to retrieve.
|
|
57
|
+
* @param num - The number of jobs to retrieve.
|
|
58
|
+
* @returns A promise that resolves to an array of jobs.
|
|
59
|
+
*/
|
|
60
|
+
peek(status?: JobStatus, num?: number): Promise<Array<JobStorageFormat<Input, Output>>>;
|
|
61
|
+
/**
|
|
62
|
+
* Retrieves the next available job that is ready to be processed
|
|
63
|
+
* Updates the job status to PROCESSING before returning
|
|
64
|
+
* @param workerId - Worker ID to associate with the job
|
|
65
|
+
* @returns The next job or undefined if no job is available
|
|
66
|
+
*/
|
|
67
|
+
next(workerId: string): Promise<JobStorageFormat<Input, Output> | undefined>;
|
|
68
|
+
/**
|
|
69
|
+
* Retrieves the size of the queue for a given status
|
|
70
|
+
* @param status - The status of the jobs to retrieve.
|
|
71
|
+
* @returns A promise that resolves to the number of jobs.
|
|
72
|
+
*/
|
|
73
|
+
size(status?: "PENDING"): Promise<number>;
|
|
74
|
+
/**
|
|
75
|
+
* Saves the progress of a job
|
|
76
|
+
* @param jobId - The id of the job to save the progress for.
|
|
77
|
+
* @param progress - The progress of the job.
|
|
78
|
+
* @param message - The message of the job.
|
|
79
|
+
* @param details - The details of the job.
|
|
80
|
+
*/
|
|
81
|
+
saveProgress(id: unknown, progress: number, message: string, details: Record<string, any> | null): Promise<void>;
|
|
82
|
+
/**
|
|
83
|
+
* Marks a job as complete with its output or error
|
|
84
|
+
* Handles run_attempts for failed jobs and triggers completion callbacks
|
|
85
|
+
* @param id - ID of the job to complete
|
|
86
|
+
* @param output - Result of the job execution
|
|
87
|
+
* @param error - Optional error message if job failed
|
|
88
|
+
*/
|
|
89
|
+
complete(job: JobStorageFormat<Input, Output>): Promise<void>;
|
|
90
|
+
/**
|
|
91
|
+
* Releases a claimed job back to PENDING without incrementing run_attempts.
|
|
92
|
+
* @param id - The id of the claimed job to release.
|
|
93
|
+
*/
|
|
94
|
+
release(id: unknown): Promise<void>;
|
|
95
|
+
/**
|
|
96
|
+
* Aborts a job
|
|
97
|
+
* @param id - The id of the job to abort.
|
|
98
|
+
*/
|
|
99
|
+
abort(id: unknown): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Retrieves all jobs by their job_run_id.
|
|
102
|
+
* @param job_run_id - The job_run_id of the jobs to retrieve.
|
|
103
|
+
* @returns A promise that resolves to an array of jobs.
|
|
104
|
+
*/
|
|
105
|
+
getByRunId(runId: string): Promise<Array<JobStorageFormat<Input, Output>>>;
|
|
106
|
+
/**
|
|
107
|
+
* Deletes all jobs from the queue that match the current prefix values.
|
|
108
|
+
*/
|
|
109
|
+
deleteAll(): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Looks up cached output for a given input
|
|
112
|
+
* Uses input fingerprinting for efficient matching
|
|
113
|
+
* @param input - The input to look up the cached output for.
|
|
114
|
+
* @returns The cached output or null if not found
|
|
115
|
+
*/
|
|
116
|
+
outputForInput(input: Input): Promise<Output | null>;
|
|
117
|
+
/**
|
|
118
|
+
* Deletes a job by its ID
|
|
119
|
+
*/
|
|
120
|
+
delete(id: unknown): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Delete jobs with a specific status older than a cutoff date
|
|
123
|
+
* @param status - Status of jobs to delete
|
|
124
|
+
* @param olderThanMs - Delete jobs completed more than this many milliseconds ago
|
|
125
|
+
*/
|
|
126
|
+
deleteJobsByStatusAndAge(status: JobStatus, olderThanMs: number): Promise<void>;
|
|
127
|
+
/**
|
|
128
|
+
* Sets up the database schema and tables.
|
|
129
|
+
* No-op for in-memory storage as it doesn't require database setup.
|
|
130
|
+
*/
|
|
131
|
+
setupDatabase(): Promise<void>;
|
|
132
|
+
/**
|
|
133
|
+
* Checks if a job matches the specified prefix filter
|
|
134
|
+
* @param job - The job to check
|
|
135
|
+
* @param prefixFilter - The prefix filter (undefined = use instance prefixes, {} = no filter)
|
|
136
|
+
*/
|
|
137
|
+
private matchesPrefixFilter;
|
|
138
|
+
/**
|
|
139
|
+
* Subscribes to changes in the queue.
|
|
140
|
+
* Since InMemory is both client and server, changes are detected via local events.
|
|
141
|
+
*
|
|
142
|
+
* @param callback - Function called when a change occurs
|
|
143
|
+
* @param options - Subscription options including prefix filter
|
|
144
|
+
* @returns Unsubscribe function
|
|
145
|
+
*/
|
|
146
|
+
subscribeToChanges(callback: (change: QueueChangePayload<Input, Output>) => void, options?: QueueSubscribeOptions): () => void;
|
|
147
|
+
}
|
|
148
|
+
export {};
|
|
149
|
+
//# sourceMappingURL=InMemoryQueueStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"InMemoryQueueStorage.d.ts","sourceRoot":"","sources":["../../src/queue-storage/InMemoryQueueStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEL,YAAY,EAKb,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,aAAa,EACb,SAAS,EACT,gBAAgB,EAChB,kBAAkB,EAClB,mBAAmB,EACnB,qBAAqB,EACtB,MAAM,iBAAiB,CAAC;AAEzB;;GAEG;AACH,KAAK,mBAAmB,CAAC,KAAK,EAAE,MAAM,IAAI;IACxC,MAAM,EAAE,CAAC,OAAO,EAAE,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;CAC9D,CAAC;AAEF,eAAO,MAAM,uBAAuB,gEAEnC,CAAC;AAEF;;;GAGG;AACH,qBAAa,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAE,YAAW,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;aAapE,SAAS,EAAE,MAAM;IAZnC,SAAgB,KAAK,EAAG,SAAS,CAAU;IAC3C,2CAA2C;IAC3C,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IAC3E,6CAA6C;IAC7C,SAAS,CAAC,QAAQ,CAAC,MAAM,mDAA0D;IAEnF;;;;OAIG;IACH,YACkB,SAAS,EAAE,MAAM,EACjC,OAAO,CAAC,EAAE,mBAAmB,EAI9B;IAED,sCAAsC;IAC/B,QAAQ,EAAE,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAElF;;OAEG;IACH,OAAO,CAAC,eAAe;IASvB;;;OAGG;IACH,OAAO,CAAC,YAAY;IASpB;;;OAGG;IACU,GAAG,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAuBvE;IAED;;;;OAIG;IACU,GAAG,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,CAOlF;IAED;;;;;OAKG;IACU,IAAI,CACf,MAAM,GAAE,SAA6B,EACrC,GAAG,GAAE,MAAY,GAChB,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAQjD;IAED;;;;;OAKG;IACU,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,CAaxF;IAED;;;;OAIG;IACU,IAAI,CAAC,MAAM,YAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAG7D;IAED;;;;;;OAMG;IACU,YAAY,CACvB,EAAE,EAAE,OAAO,EACX,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,GAClC,OAAO,CAAC,IAAI,CAAC,CAkCf;IAED;;;;;;OAMG;IACU,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,iBAezD;IAED;;;OAGG;IACU,OAAO,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAY/C;IAED;;;OAGG;IACU,KAAK,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ7C;IAED;;;;OAIG;IACU,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAGtF;IAED;;OAEG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAOtC;IAED;;;;;OAKG;IACU,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAWhE;IAED;;OAEG;IACU,MAAM,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAO9C;IAED;;;;OAIG;IACU,wBAAwB,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAoB3F;IAED;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1C;IAED;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IA2B3B;;;;;;;OAOG;IACI,kBAAkB,CACvB,QAAQ,EAAE,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAC7D,OAAO,CAAC,EAAE,qBAAqB,GAC9B,MAAM,IAAI,CAiBZ;CACF"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { IQueueStorage, JobStatus, JobStorageFormat, QueueChangePayload, QueueStorageScope, QueueSubscribeOptions } from "./IQueueStorage";
|
|
7
|
+
/**
|
|
8
|
+
* Telemetry wrapper for any IQueueStorage implementation.
|
|
9
|
+
* Creates spans for all queue storage operations.
|
|
10
|
+
*/
|
|
11
|
+
export declare class TelemetryQueueStorage<Input, Output> implements IQueueStorage<Input, Output> {
|
|
12
|
+
private readonly storageName;
|
|
13
|
+
private readonly inner;
|
|
14
|
+
constructor(storageName: string, inner: IQueueStorage<Input, Output>);
|
|
15
|
+
get scope(): QueueStorageScope;
|
|
16
|
+
add(job: JobStorageFormat<Input, Output>): Promise<unknown>;
|
|
17
|
+
get(id: unknown): Promise<JobStorageFormat<Input, Output> | undefined>;
|
|
18
|
+
next(workerId: string): Promise<JobStorageFormat<Input, Output> | undefined>;
|
|
19
|
+
peek(status?: JobStatus, num?: number): Promise<Array<JobStorageFormat<Input, Output>>>;
|
|
20
|
+
size(status?: JobStatus): Promise<number>;
|
|
21
|
+
complete(job: JobStorageFormat<Input, Output>): Promise<void>;
|
|
22
|
+
release(id: unknown): Promise<void>;
|
|
23
|
+
deleteAll(): Promise<void>;
|
|
24
|
+
outputForInput(input: Input): Promise<Output | null>;
|
|
25
|
+
abort(id: unknown): Promise<void>;
|
|
26
|
+
getByRunId(runId: string): Promise<Array<JobStorageFormat<Input, Output>>>;
|
|
27
|
+
saveProgress(id: unknown, progress: number, message: string, details: Record<string, any> | null): Promise<void>;
|
|
28
|
+
delete(id: unknown): Promise<void>;
|
|
29
|
+
deleteJobsByStatusAndAge(status: JobStatus, olderThanMs: number): Promise<void>;
|
|
30
|
+
setupDatabase(): Promise<void>;
|
|
31
|
+
subscribeToChanges(callback: (change: QueueChangePayload<Input, Output>) => void, options?: QueueSubscribeOptions): () => void;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=TelemetryQueueStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TelemetryQueueStorage.d.ts","sourceRoot":"","sources":["../../src/queue-storage/TelemetryQueueStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EACV,aAAa,EACb,SAAS,EACT,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACtB,MAAM,iBAAiB,CAAC;AAEzB;;;GAGG;AACH,qBAAa,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAE,YAAW,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;IAErF,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,KAAK;IAFxB,YACmB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,EAClD;IAEJ,IAAW,KAAK,IAAI,iBAAiB,CAEpC;IAED,GAAG,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAE1D;IACD,GAAG,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,CAErE;IACD,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,CAE3E;IACD,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAItF;IACD,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAExC;IACD,QAAQ,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAI5D;IACD,OAAO,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElC;IACD,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAIzB;IACD,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAInD;IACD,KAAK,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhC;IACD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAIzE;IACD,YAAY,CACV,EAAE,EAAE,OAAO,EACX,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,GAClC,OAAO,CAAC,IAAI,CAAC,CAIf;IACD,MAAM,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjC;IACD,wBAAwB,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI9E;IACD,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;IACD,kBAAkB,CAChB,QAAQ,EAAE,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAC7D,OAAO,CAAC,EAAE,qBAAqB,GAC9B,MAAM,IAAI,CAEZ;CACF"}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { PrefixColumn } from "../queue-storage/IQueueStorage";
|
|
7
|
+
export declare const RATE_LIMITER_STORAGE: import("@workglow/util").ServiceToken<IRateLimiterStorage>;
|
|
8
|
+
/**
|
|
9
|
+
* Whether a rate-limiter storage's state is shared across processes.
|
|
10
|
+
*
|
|
11
|
+
* - `"process"` — in-memory / per-process state. Multiple workers in the same
|
|
12
|
+
* process share it, but separate processes do not.
|
|
13
|
+
* - `"cluster"` — state lives in shared external storage (Postgres, Supabase,
|
|
14
|
+
* etc.) visible to every process.
|
|
15
|
+
*/
|
|
16
|
+
export type RateLimiterStorageScope = "process" | "cluster";
|
|
17
|
+
/**
|
|
18
|
+
* Options for configuring rate limiter storage with prefix filters.
|
|
19
|
+
*/
|
|
20
|
+
export interface RateLimiterStorageOptions {
|
|
21
|
+
/** The prefix column definitions for this storage */
|
|
22
|
+
readonly prefixes?: readonly PrefixColumn[];
|
|
23
|
+
/** The values for each prefix column */
|
|
24
|
+
readonly prefixValues?: Readonly<Record<string, string | number>>;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Record of a job execution for rate limiting tracking.
|
|
28
|
+
*/
|
|
29
|
+
export interface ExecutionRecord {
|
|
30
|
+
readonly id?: unknown;
|
|
31
|
+
readonly queue_name: string;
|
|
32
|
+
readonly executed_at: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Record of the next available time for a queue.
|
|
36
|
+
*/
|
|
37
|
+
export interface NextAvailableRecord {
|
|
38
|
+
readonly queue_name: string;
|
|
39
|
+
readonly next_available_at: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Interface defining the storage operations for a rate limiter.
|
|
43
|
+
* This separates the storage concerns from the rate limiting logic.
|
|
44
|
+
*/
|
|
45
|
+
export interface IRateLimiterStorage {
|
|
46
|
+
/**
|
|
47
|
+
* Whether this storage is shared across processes. In-memory backends MUST
|
|
48
|
+
* report `"process"`. Shared databases (Postgres, Supabase) report
|
|
49
|
+
* `"cluster"`.
|
|
50
|
+
*/
|
|
51
|
+
readonly scope: RateLimiterStorageScope;
|
|
52
|
+
/**
|
|
53
|
+
* Sets up the database schema and tables.
|
|
54
|
+
* This method should be called before using the storage.
|
|
55
|
+
* For production use, database setup should be done via migrations.
|
|
56
|
+
*/
|
|
57
|
+
setupDatabase(): Promise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Atomic check-and-record. Inserts an execution row and returns the
|
|
60
|
+
* inserted row's id iff BOTH (a) fewer than `maxExecutions` rows have
|
|
61
|
+
* `executed_at > (now - windowMs)` AND (b) any persisted `nextAvailableAt`
|
|
62
|
+
* is in the past or absent. Returns `null` without writing anything
|
|
63
|
+
* otherwise.
|
|
64
|
+
*
|
|
65
|
+
* The returned id MUST be passed to {@link releaseExecution} to free the
|
|
66
|
+
* slot — otherwise concurrent acquirers would race to delete the wrong
|
|
67
|
+
* worker's row when one of them rolls back.
|
|
68
|
+
*
|
|
69
|
+
* Implementations MUST serialize concurrent callers (advisory locks,
|
|
70
|
+
* `BEGIN IMMEDIATE`, per-key mutex, etc.) so the count-then-insert window
|
|
71
|
+
* is uninterruptible.
|
|
72
|
+
*
|
|
73
|
+
* @param queueName - The name of the queue
|
|
74
|
+
* @param maxExecutions - Max allowed executions in the window
|
|
75
|
+
* @param windowMs - Window size in milliseconds
|
|
76
|
+
* @returns the inserted row's id on success, or `null` on failure
|
|
77
|
+
*/
|
|
78
|
+
tryReserveExecution(queueName: string, maxExecutions: number, windowMs: number): Promise<unknown | null>;
|
|
79
|
+
/**
|
|
80
|
+
* Release the execution row identified by `token` (the value previously
|
|
81
|
+
* returned from {@link tryReserveExecution}). No-op if the row no longer
|
|
82
|
+
* exists.
|
|
83
|
+
*
|
|
84
|
+
* Critical: implementations MUST delete by id, NOT by recency or position.
|
|
85
|
+
* Two concurrent workers can hold tokens for different rows; deleting the
|
|
86
|
+
* "most recent" row would release another worker's reservation.
|
|
87
|
+
*/
|
|
88
|
+
releaseExecution(queueName: string, token: unknown): Promise<void>;
|
|
89
|
+
/**
|
|
90
|
+
* Records a job execution for rate limiting tracking.
|
|
91
|
+
* @param queueName - The name of the queue
|
|
92
|
+
*/
|
|
93
|
+
recordExecution(queueName: string): Promise<void>;
|
|
94
|
+
/**
|
|
95
|
+
* Gets the count of executions within a time window.
|
|
96
|
+
* @param queueName - The name of the queue
|
|
97
|
+
* @param windowStartTime - The start of the time window (ISO string)
|
|
98
|
+
* @returns The count of executions within the window
|
|
99
|
+
*/
|
|
100
|
+
getExecutionCount(queueName: string, windowStartTime: string): Promise<number>;
|
|
101
|
+
/**
|
|
102
|
+
* Gets the oldest execution time within the window, offset by a count.
|
|
103
|
+
* Used to calculate when the rate limit will allow the next execution.
|
|
104
|
+
* @param queueName - The name of the queue
|
|
105
|
+
* @param offset - The offset (typically maxExecutions - 1)
|
|
106
|
+
* @returns The execution time or undefined if not enough executions
|
|
107
|
+
*/
|
|
108
|
+
getOldestExecutionAtOffset(queueName: string, offset: number): Promise<string | undefined>;
|
|
109
|
+
/**
|
|
110
|
+
* Gets the next available time for a queue.
|
|
111
|
+
* @param queueName - The name of the queue
|
|
112
|
+
* @returns The next available time or undefined if not set
|
|
113
|
+
*/
|
|
114
|
+
getNextAvailableTime(queueName: string): Promise<string | undefined>;
|
|
115
|
+
/**
|
|
116
|
+
* Sets the next available time for a queue.
|
|
117
|
+
* @param queueName - The name of the queue
|
|
118
|
+
* @param nextAvailableAt - The next available time (ISO string)
|
|
119
|
+
*/
|
|
120
|
+
setNextAvailableTime(queueName: string, nextAvailableAt: string): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Clears all rate limit entries for a queue.
|
|
123
|
+
* @param queueName - The name of the queue
|
|
124
|
+
*/
|
|
125
|
+
clear(queueName: string): Promise<void>;
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=IRateLimiterStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IRateLimiterStorage.d.ts","sourceRoot":"","sources":["../../src/rate-limiter-storage/IRateLimiterStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAEnE,eAAO,MAAM,oBAAoB,4DAAiE,CAAC;AAEnG;;;;;;;GAOG;AACH,MAAM,MAAM,uBAAuB,GAAG,SAAS,GAAG,SAAS,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,qDAAqD;IACrD,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC5C,wCAAwC;IACxC,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;CACnE;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,uBAAuB,CAAC;IAExC;;;;OAIG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;;;;;;;;;;;;;;;;;OAmBG;IACH,mBAAmB,CACjB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IAE3B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnE;;;OAGG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElD;;;;;OAKG;IACH,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE/E;;;;;;OAMG;IACH,0BAA0B,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IAE3F;;;;OAIG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IAErE;;;;OAIG;IACH,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhF;;;OAGG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { IRateLimiterStorage, RateLimiterStorageOptions, RateLimiterStorageScope } from "./IRateLimiterStorage";
|
|
7
|
+
export declare const IN_MEMORY_RATE_LIMITER_STORAGE: import("@workglow/util").ServiceToken<IRateLimiterStorage>;
|
|
8
|
+
/**
|
|
9
|
+
* In-memory implementation of rate limiter storage.
|
|
10
|
+
* Manages execution records and next available times for rate limiting.
|
|
11
|
+
*/
|
|
12
|
+
export declare class InMemoryRateLimiterStorage implements IRateLimiterStorage {
|
|
13
|
+
readonly scope: RateLimiterStorageScope;
|
|
14
|
+
/** The prefix values for filtering */
|
|
15
|
+
protected readonly prefixValues: Readonly<Record<string, string | number>>;
|
|
16
|
+
/** Execution records keyed by a composite of prefix values and queue name */
|
|
17
|
+
private readonly executions;
|
|
18
|
+
/** Next available times keyed by a composite of prefix values and queue name */
|
|
19
|
+
private readonly nextAvailableTimes;
|
|
20
|
+
/**
|
|
21
|
+
* Per-key promise chain used to serialize {@link tryReserveExecution} so
|
|
22
|
+
* concurrent callers cannot both observe `count < max` before either
|
|
23
|
+
* inserts. Each key's chain is replaced with the next pending operation
|
|
24
|
+
* before the current one returns, giving FIFO mutex semantics.
|
|
25
|
+
*/
|
|
26
|
+
private readonly reserveChains;
|
|
27
|
+
constructor(options?: RateLimiterStorageOptions);
|
|
28
|
+
/**
|
|
29
|
+
* Creates a storage key from the queue name and prefix values.
|
|
30
|
+
*/
|
|
31
|
+
private makeKey;
|
|
32
|
+
setupDatabase(): Promise<void>;
|
|
33
|
+
private withKeyLock;
|
|
34
|
+
tryReserveExecution(queueName: string, maxExecutions: number, windowMs: number): Promise<unknown | null>;
|
|
35
|
+
releaseExecution(queueName: string, token: unknown): Promise<void>;
|
|
36
|
+
recordExecution(queueName: string): Promise<void>;
|
|
37
|
+
getExecutionCount(queueName: string, windowStartTime: string): Promise<number>;
|
|
38
|
+
getOldestExecutionAtOffset(queueName: string, offset: number): Promise<string | undefined>;
|
|
39
|
+
getNextAvailableTime(queueName: string): Promise<string | undefined>;
|
|
40
|
+
setNextAvailableTime(queueName: string, nextAvailableAt: string): Promise<void>;
|
|
41
|
+
clear(queueName: string): Promise<void>;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=InMemoryRateLimiterStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"InMemoryRateLimiterStorage.d.ts","sourceRoot":"","sources":["../../src/rate-limiter-storage/InMemoryRateLimiterStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EACL,mBAAmB,EACnB,yBAAyB,EACzB,uBAAuB,EACxB,MAAM,uBAAuB,CAAC;AAE/B,eAAO,MAAM,8BAA8B,4DAE1C,CAAC;AAYF;;;GAGG;AACH,qBAAa,0BAA2B,YAAW,mBAAmB;IACpE,SAAgB,KAAK,EAAE,uBAAuB,CAAa;IAE3D,sCAAsC;IACtC,SAAS,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IAE3E,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA4C;IAEvE,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAgC;IAEnE;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA4C;IAE1E,YAAY,OAAO,CAAC,EAAE,yBAAyB,EAE9C;IAED;;OAEG;IACH,OAAO,CAAC,OAAO;IAQF,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1C;YAOa,WAAW;IAkBZ,mBAAmB,CAC9B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAuBzB;IAEY,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAc9E;IAEY,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAU7D;IAEY,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAM1F;IAEY,0BAA0B,CACrC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAO7B;IAEY,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAKhF;IAEY,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI3F;IAEY,KAAK,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKnD;CACF"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@workglow/job-queue",
|
|
3
3
|
"type": "module",
|
|
4
4
|
"sideEffects": false,
|
|
5
|
-
"version": "0.2.
|
|
5
|
+
"version": "0.2.28",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/workglow-dev/workglow.git",
|
|
@@ -45,20 +45,15 @@
|
|
|
45
45
|
}
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
|
-
"@workglow/
|
|
49
|
-
"@workglow/util": "0.2.27"
|
|
48
|
+
"@workglow/util": "0.2.28"
|
|
50
49
|
},
|
|
51
50
|
"peerDependenciesMeta": {
|
|
52
|
-
"@workglow/storage": {
|
|
53
|
-
"optional": false
|
|
54
|
-
},
|
|
55
51
|
"@workglow/util": {
|
|
56
52
|
"optional": false
|
|
57
53
|
}
|
|
58
54
|
},
|
|
59
55
|
"devDependencies": {
|
|
60
|
-
"@workglow/
|
|
61
|
-
"@workglow/util": "0.2.27"
|
|
56
|
+
"@workglow/util": "0.2.28"
|
|
62
57
|
},
|
|
63
58
|
"files": [
|
|
64
59
|
"dist"
|