@workglow/supabase 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/job-queue/SupabaseQueueStorage.d.ts +196 -0
- package/dist/job-queue/SupabaseQueueStorage.d.ts.map +1 -0
- package/dist/job-queue/SupabaseRateLimiterStorage.d.ts +54 -0
- package/dist/job-queue/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/job-queue/browser.d.ts +7 -0
- package/dist/job-queue/browser.d.ts.map +1 -0
- package/dist/job-queue/browser.js +765 -0
- package/dist/job-queue/browser.js.map +11 -0
- package/dist/job-queue/bun.d.ts +7 -0
- package/dist/job-queue/bun.d.ts.map +1 -0
- package/dist/job-queue/common.d.ts +8 -0
- package/dist/job-queue/common.d.ts.map +1 -0
- package/dist/job-queue/node.d.ts +7 -0
- package/dist/job-queue/node.d.ts.map +1 -0
- package/dist/job-queue/node.js +765 -0
- package/dist/job-queue/node.js.map +11 -0
- package/dist/storage/SupabaseKvStorage.d.ts +32 -0
- package/dist/storage/SupabaseKvStorage.d.ts.map +1 -0
- package/dist/storage/SupabaseTabularStorage.d.ts +172 -0
- package/dist/storage/SupabaseTabularStorage.d.ts.map +1 -0
- package/dist/storage/browser.d.ts +7 -0
- package/dist/storage/browser.d.ts.map +1 -0
- package/dist/storage/browser.js +590 -0
- package/dist/storage/browser.js.map +11 -0
- package/dist/storage/bun.d.ts +7 -0
- package/dist/storage/bun.d.ts.map +1 -0
- package/dist/storage/common.d.ts +8 -0
- package/dist/storage/common.d.ts.map +1 -0
- package/dist/storage/node.d.ts +7 -0
- package/dist/storage/node.d.ts.map +1 -0
- package/dist/storage/node.js +590 -0
- package/dist/storage/node.js.map +11 -0
- package/package.json +76 -0
|
@@ -0,0 +1,765 @@
|
|
|
1
|
+
// src/job-queue/SupabaseQueueStorage.ts
|
|
2
|
+
import { createServiceToken, deepEqual, makeFingerprint, uuid4 } from "@workglow/util";
|
|
3
|
+
import { PollingSubscriptionManager } from "@workglow/storage";
|
|
4
|
+
import { JobStatus } from "@workglow/job-queue";
|
|
5
|
+
var SUPABASE_QUEUE_STORAGE = createServiceToken("jobqueue.storage.supabase");
|
|
6
|
+
|
|
7
|
+
class SupabaseQueueStorage {
|
|
8
|
+
queueName;
|
|
9
|
+
scope = "cluster";
|
|
10
|
+
client;
|
|
11
|
+
prefixes;
|
|
12
|
+
prefixValues;
|
|
13
|
+
tableName;
|
|
14
|
+
realtimeChannel = null;
|
|
15
|
+
pollingManager = null;
|
|
16
|
+
constructor(client, queueName, options) {
|
|
17
|
+
this.queueName = queueName;
|
|
18
|
+
this.client = client;
|
|
19
|
+
this.prefixes = options?.prefixes ?? [];
|
|
20
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
21
|
+
if (this.prefixes.length > 0) {
|
|
22
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
23
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
24
|
+
} else {
|
|
25
|
+
this.tableName = "job_queue";
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
getPrefixColumnType(type) {
|
|
29
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
30
|
+
}
|
|
31
|
+
buildPrefixColumnsSql() {
|
|
32
|
+
if (this.prefixes.length === 0)
|
|
33
|
+
return "";
|
|
34
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
35
|
+
`) + `,
|
|
36
|
+
`;
|
|
37
|
+
}
|
|
38
|
+
getPrefixColumnNames() {
|
|
39
|
+
return this.prefixes.map((p) => p.name);
|
|
40
|
+
}
|
|
41
|
+
applyPrefixFilters(query) {
|
|
42
|
+
let result = query;
|
|
43
|
+
for (const prefix of this.prefixes) {
|
|
44
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
45
|
+
}
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
getPrefixInsertValues() {
|
|
49
|
+
const values = {};
|
|
50
|
+
for (const prefix of this.prefixes) {
|
|
51
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
52
|
+
}
|
|
53
|
+
return values;
|
|
54
|
+
}
|
|
55
|
+
buildPrefixWhereSql() {
|
|
56
|
+
if (this.prefixes.length === 0) {
|
|
57
|
+
return "";
|
|
58
|
+
}
|
|
59
|
+
const conditions = this.prefixes.map((p) => {
|
|
60
|
+
const value = this.prefixValues[p.name];
|
|
61
|
+
if (p.type === "uuid") {
|
|
62
|
+
const validated = this.validateSqlValue(String(value), `prefix "${p.name}"`);
|
|
63
|
+
return `${p.name} = '${this.escapeSqlString(validated)}'`;
|
|
64
|
+
}
|
|
65
|
+
const numValue = Number(value ?? 0);
|
|
66
|
+
if (!Number.isFinite(numValue)) {
|
|
67
|
+
throw new Error(`Invalid numeric prefix value for "${p.name}": ${value}`);
|
|
68
|
+
}
|
|
69
|
+
return `${p.name} = ${numValue}`;
|
|
70
|
+
}).join(" AND ");
|
|
71
|
+
return " AND " + conditions;
|
|
72
|
+
}
|
|
73
|
+
static SAFE_SQL_VALUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
|
|
74
|
+
validateSqlValue(value, context) {
|
|
75
|
+
if (!SupabaseQueueStorage.SAFE_SQL_VALUE_RE.test(value)) {
|
|
76
|
+
throw new Error(`Unsafe value for ${context}: "${value}". Values must match /^[a-zA-Z0-9_\\-.:]+$/.`);
|
|
77
|
+
}
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
escapeSqlString(value) {
|
|
81
|
+
return value.replace(/'/g, "''");
|
|
82
|
+
}
|
|
83
|
+
async setupDatabase() {
|
|
84
|
+
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
85
|
+
const { error: typeError } = await this.client.rpc("exec_sql", { query: createTypeSql });
|
|
86
|
+
if (typeError && typeError.code !== "42710") {
|
|
87
|
+
throw typeError;
|
|
88
|
+
}
|
|
89
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
90
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
91
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
92
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
93
|
+
const createTableSql = `
|
|
94
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
95
|
+
id SERIAL NOT NULL,
|
|
96
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
97
|
+
queue text NOT NULL,
|
|
98
|
+
job_run_id text NOT NULL,
|
|
99
|
+
status job_status NOT NULL default 'PENDING',
|
|
100
|
+
input jsonb NOT NULL,
|
|
101
|
+
output jsonb,
|
|
102
|
+
run_attempts integer default 0,
|
|
103
|
+
max_retries integer default 20,
|
|
104
|
+
run_after timestamp with time zone DEFAULT now(),
|
|
105
|
+
last_ran_at timestamp with time zone,
|
|
106
|
+
created_at timestamp with time zone DEFAULT now(),
|
|
107
|
+
deadline_at timestamp with time zone,
|
|
108
|
+
completed_at timestamp with time zone,
|
|
109
|
+
error text,
|
|
110
|
+
error_code text,
|
|
111
|
+
progress real DEFAULT 0,
|
|
112
|
+
progress_message text DEFAULT '',
|
|
113
|
+
progress_details jsonb,
|
|
114
|
+
worker_id text
|
|
115
|
+
)`;
|
|
116
|
+
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
117
|
+
if (tableError) {
|
|
118
|
+
if (tableError.code !== "42P07") {
|
|
119
|
+
throw tableError;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const indexes = [
|
|
123
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
124
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
125
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
126
|
+
];
|
|
127
|
+
for (const indexSql of indexes) {
|
|
128
|
+
await this.client.rpc("exec_sql", { query: indexSql });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
async add(job) {
|
|
132
|
+
const now = new Date().toISOString();
|
|
133
|
+
job.queue = this.queueName;
|
|
134
|
+
job.job_run_id = job.job_run_id ?? uuid4();
|
|
135
|
+
job.fingerprint = await makeFingerprint(job.input);
|
|
136
|
+
job.status = JobStatus.PENDING;
|
|
137
|
+
job.progress = 0;
|
|
138
|
+
job.progress_message = "";
|
|
139
|
+
job.progress_details = null;
|
|
140
|
+
job.created_at = now;
|
|
141
|
+
job.run_after = now;
|
|
142
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
143
|
+
const { data, error } = await this.client.from(this.tableName).insert({
|
|
144
|
+
...prefixInsertValues,
|
|
145
|
+
queue: job.queue,
|
|
146
|
+
fingerprint: job.fingerprint,
|
|
147
|
+
input: job.input,
|
|
148
|
+
run_after: job.run_after,
|
|
149
|
+
created_at: job.created_at,
|
|
150
|
+
deadline_at: job.deadline_at,
|
|
151
|
+
max_retries: job.max_retries,
|
|
152
|
+
job_run_id: job.job_run_id,
|
|
153
|
+
progress: job.progress,
|
|
154
|
+
progress_message: job.progress_message,
|
|
155
|
+
progress_details: job.progress_details
|
|
156
|
+
}).select("id").single();
|
|
157
|
+
if (error)
|
|
158
|
+
throw error;
|
|
159
|
+
if (!data)
|
|
160
|
+
throw new Error("Failed to add to queue");
|
|
161
|
+
job.id = data.id;
|
|
162
|
+
return job.id;
|
|
163
|
+
}
|
|
164
|
+
async get(id) {
|
|
165
|
+
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
166
|
+
query = this.applyPrefixFilters(query);
|
|
167
|
+
const { data, error } = await query.single();
|
|
168
|
+
if (error) {
|
|
169
|
+
if (error.code === "PGRST116")
|
|
170
|
+
return;
|
|
171
|
+
throw error;
|
|
172
|
+
}
|
|
173
|
+
return data;
|
|
174
|
+
}
|
|
175
|
+
async peek(status = JobStatus.PENDING, num = 100) {
|
|
176
|
+
num = Number(num) || 100;
|
|
177
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
178
|
+
query = this.applyPrefixFilters(query);
|
|
179
|
+
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
180
|
+
if (error)
|
|
181
|
+
throw error;
|
|
182
|
+
return data ?? [];
|
|
183
|
+
}
|
|
184
|
+
async next(workerId) {
|
|
185
|
+
const prefixConditions = this.buildPrefixWhereSql();
|
|
186
|
+
const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
|
|
187
|
+
const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
|
|
188
|
+
const escapedQueueName = this.escapeSqlString(validatedQueueName);
|
|
189
|
+
const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
|
|
190
|
+
const sql = `
|
|
191
|
+
UPDATE ${this.tableName}
|
|
192
|
+
SET status = '${JobStatus.PROCESSING}', last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = '${escapedWorkerId}'
|
|
193
|
+
WHERE id = (
|
|
194
|
+
SELECT id
|
|
195
|
+
FROM ${this.tableName}
|
|
196
|
+
WHERE queue = '${escapedQueueName}'
|
|
197
|
+
AND status = '${JobStatus.PENDING}'
|
|
198
|
+
${prefixConditions}
|
|
199
|
+
AND run_after <= NOW() AT TIME ZONE 'UTC'
|
|
200
|
+
ORDER BY run_after ASC
|
|
201
|
+
FOR UPDATE SKIP LOCKED
|
|
202
|
+
LIMIT 1
|
|
203
|
+
)
|
|
204
|
+
RETURNING *`;
|
|
205
|
+
const { data, error } = await this.client.rpc("exec_sql", { query: sql });
|
|
206
|
+
if (error)
|
|
207
|
+
throw error;
|
|
208
|
+
if (!data || !Array.isArray(data) || data.length === 0) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
return data[0];
|
|
212
|
+
}
|
|
213
|
+
async size(status = JobStatus.PENDING) {
|
|
214
|
+
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
215
|
+
query = this.applyPrefixFilters(query);
|
|
216
|
+
const { count, error } = await query;
|
|
217
|
+
if (error)
|
|
218
|
+
throw error;
|
|
219
|
+
return count ?? 0;
|
|
220
|
+
}
|
|
221
|
+
async getAllJobs() {
|
|
222
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
223
|
+
query = this.applyPrefixFilters(query);
|
|
224
|
+
const { data, error } = await query;
|
|
225
|
+
if (error)
|
|
226
|
+
throw error;
|
|
227
|
+
return data ?? [];
|
|
228
|
+
}
|
|
229
|
+
async complete(jobDetails) {
|
|
230
|
+
const now = new Date().toISOString();
|
|
231
|
+
if (jobDetails.status === JobStatus.DISABLED) {
|
|
232
|
+
let query2 = this.client.from(this.tableName).update({
|
|
233
|
+
status: jobDetails.status,
|
|
234
|
+
progress: 100,
|
|
235
|
+
progress_message: "",
|
|
236
|
+
progress_details: null,
|
|
237
|
+
completed_at: now,
|
|
238
|
+
last_ran_at: now
|
|
239
|
+
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
240
|
+
query2 = this.applyPrefixFilters(query2);
|
|
241
|
+
const { error: error2 } = await query2;
|
|
242
|
+
if (error2)
|
|
243
|
+
throw error2;
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
let getQuery = this.client.from(this.tableName).select("run_attempts, max_retries").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
247
|
+
getQuery = this.applyPrefixFilters(getQuery);
|
|
248
|
+
const { data: current, error: getError } = await getQuery.single();
|
|
249
|
+
if (getError)
|
|
250
|
+
throw getError;
|
|
251
|
+
const currentAttempts = current?.run_attempts ?? 0;
|
|
252
|
+
const maxRetries = current?.max_retries ?? jobDetails.max_retries ?? 10;
|
|
253
|
+
const nextAttempts = currentAttempts + 1;
|
|
254
|
+
if (jobDetails.status === JobStatus.PENDING) {
|
|
255
|
+
if (nextAttempts > maxRetries) {
|
|
256
|
+
let failQuery = this.client.from(this.tableName).update({
|
|
257
|
+
status: JobStatus.FAILED,
|
|
258
|
+
error: "Max retries reached",
|
|
259
|
+
error_code: "MAX_RETRIES_REACHED",
|
|
260
|
+
progress: 100,
|
|
261
|
+
progress_message: "",
|
|
262
|
+
progress_details: null,
|
|
263
|
+
completed_at: now,
|
|
264
|
+
last_ran_at: now
|
|
265
|
+
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
266
|
+
failQuery = this.applyPrefixFilters(failQuery);
|
|
267
|
+
const { error: failError } = await failQuery;
|
|
268
|
+
if (failError)
|
|
269
|
+
throw failError;
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
let query2 = this.client.from(this.tableName).update({
|
|
273
|
+
error: jobDetails.error ?? null,
|
|
274
|
+
error_code: jobDetails.error_code ?? null,
|
|
275
|
+
status: jobDetails.status,
|
|
276
|
+
run_after: jobDetails.run_after,
|
|
277
|
+
progress: 0,
|
|
278
|
+
progress_message: "",
|
|
279
|
+
progress_details: null,
|
|
280
|
+
run_attempts: nextAttempts,
|
|
281
|
+
last_ran_at: now
|
|
282
|
+
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
283
|
+
query2 = this.applyPrefixFilters(query2);
|
|
284
|
+
const { error: error2 } = await query2;
|
|
285
|
+
if (error2)
|
|
286
|
+
throw error2;
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
if (jobDetails.status === JobStatus.COMPLETED || jobDetails.status === JobStatus.FAILED) {
|
|
290
|
+
let query2 = this.client.from(this.tableName).update({
|
|
291
|
+
output: jobDetails.output ?? null,
|
|
292
|
+
error: jobDetails.error ?? null,
|
|
293
|
+
error_code: jobDetails.error_code ?? null,
|
|
294
|
+
status: jobDetails.status,
|
|
295
|
+
progress: 100,
|
|
296
|
+
progress_message: "",
|
|
297
|
+
progress_details: null,
|
|
298
|
+
run_attempts: nextAttempts,
|
|
299
|
+
completed_at: now,
|
|
300
|
+
last_ran_at: now
|
|
301
|
+
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
302
|
+
query2 = this.applyPrefixFilters(query2);
|
|
303
|
+
const { error: error2 } = await query2;
|
|
304
|
+
if (error2)
|
|
305
|
+
throw error2;
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
let query = this.client.from(this.tableName).update({
|
|
309
|
+
status: jobDetails.status,
|
|
310
|
+
output: jobDetails.output ?? null,
|
|
311
|
+
error: jobDetails.error ?? null,
|
|
312
|
+
error_code: jobDetails.error_code ?? null,
|
|
313
|
+
run_after: jobDetails.run_after ?? null,
|
|
314
|
+
run_attempts: nextAttempts,
|
|
315
|
+
last_ran_at: now
|
|
316
|
+
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
317
|
+
query = this.applyPrefixFilters(query);
|
|
318
|
+
const { error } = await query;
|
|
319
|
+
if (error)
|
|
320
|
+
throw error;
|
|
321
|
+
}
|
|
322
|
+
async release(jobId) {
|
|
323
|
+
let query = this.client.from(this.tableName).update({
|
|
324
|
+
status: JobStatus.PENDING,
|
|
325
|
+
worker_id: null,
|
|
326
|
+
progress: 0,
|
|
327
|
+
progress_message: "",
|
|
328
|
+
progress_details: null
|
|
329
|
+
}).eq("id", jobId).eq("queue", this.queueName);
|
|
330
|
+
query = this.applyPrefixFilters(query);
|
|
331
|
+
const { error } = await query;
|
|
332
|
+
if (error)
|
|
333
|
+
throw error;
|
|
334
|
+
}
|
|
335
|
+
async deleteAll() {
|
|
336
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
337
|
+
query = this.applyPrefixFilters(query);
|
|
338
|
+
const { error } = await query;
|
|
339
|
+
if (error)
|
|
340
|
+
throw error;
|
|
341
|
+
}
|
|
342
|
+
async outputForInput(input) {
|
|
343
|
+
const fingerprint = await makeFingerprint(input);
|
|
344
|
+
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", JobStatus.COMPLETED);
|
|
345
|
+
query = this.applyPrefixFilters(query);
|
|
346
|
+
const { data, error } = await query.single();
|
|
347
|
+
if (error) {
|
|
348
|
+
if (error.code === "PGRST116")
|
|
349
|
+
return null;
|
|
350
|
+
throw error;
|
|
351
|
+
}
|
|
352
|
+
return data?.output ?? null;
|
|
353
|
+
}
|
|
354
|
+
async abort(jobId) {
|
|
355
|
+
let query = this.client.from(this.tableName).update({ status: JobStatus.ABORTING }).eq("id", jobId).eq("queue", this.queueName);
|
|
356
|
+
query = this.applyPrefixFilters(query);
|
|
357
|
+
const { error } = await query;
|
|
358
|
+
if (error)
|
|
359
|
+
throw error;
|
|
360
|
+
}
|
|
361
|
+
async getByRunId(job_run_id) {
|
|
362
|
+
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
363
|
+
query = this.applyPrefixFilters(query);
|
|
364
|
+
const { data, error } = await query;
|
|
365
|
+
if (error)
|
|
366
|
+
throw error;
|
|
367
|
+
return data ?? [];
|
|
368
|
+
}
|
|
369
|
+
async saveProgress(jobId, progress, message, details) {
|
|
370
|
+
let query = this.client.from(this.tableName).update({
|
|
371
|
+
progress,
|
|
372
|
+
progress_message: message,
|
|
373
|
+
progress_details: details
|
|
374
|
+
}).eq("id", jobId).eq("queue", this.queueName);
|
|
375
|
+
query = this.applyPrefixFilters(query);
|
|
376
|
+
const { error } = await query;
|
|
377
|
+
if (error)
|
|
378
|
+
throw error;
|
|
379
|
+
}
|
|
380
|
+
async delete(jobId) {
|
|
381
|
+
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
382
|
+
query = this.applyPrefixFilters(query);
|
|
383
|
+
const { error } = await query;
|
|
384
|
+
if (error)
|
|
385
|
+
throw error;
|
|
386
|
+
}
|
|
387
|
+
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
388
|
+
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
389
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
390
|
+
query = this.applyPrefixFilters(query);
|
|
391
|
+
const { error } = await query;
|
|
392
|
+
if (error)
|
|
393
|
+
throw error;
|
|
394
|
+
}
|
|
395
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
396
|
+
if (!job)
|
|
397
|
+
return false;
|
|
398
|
+
if (job.queue !== this.queueName) {
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
402
|
+
return true;
|
|
403
|
+
}
|
|
404
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
405
|
+
if (Object.keys(filterValues).length === 0) {
|
|
406
|
+
return true;
|
|
407
|
+
}
|
|
408
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
409
|
+
if (job[key] !== value) {
|
|
410
|
+
return false;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return true;
|
|
414
|
+
}
|
|
415
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
416
|
+
if (prefixFilter === undefined) {
|
|
417
|
+
return false;
|
|
418
|
+
}
|
|
419
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
420
|
+
return true;
|
|
421
|
+
}
|
|
422
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
423
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
424
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
425
|
+
return true;
|
|
426
|
+
}
|
|
427
|
+
for (const key of instanceKeys) {
|
|
428
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
429
|
+
return true;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return false;
|
|
433
|
+
}
|
|
434
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
435
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
436
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
437
|
+
query = query.eq(key, value);
|
|
438
|
+
}
|
|
439
|
+
const { data, error } = await query;
|
|
440
|
+
if (error)
|
|
441
|
+
throw error;
|
|
442
|
+
return data ?? [];
|
|
443
|
+
}
|
|
444
|
+
subscribeToChanges(callback, options) {
|
|
445
|
+
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
446
|
+
}
|
|
447
|
+
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
448
|
+
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
449
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
450
|
+
event: "*",
|
|
451
|
+
schema: "public",
|
|
452
|
+
table: this.tableName,
|
|
453
|
+
filter: `queue=eq.${this.queueName}`
|
|
454
|
+
}, (payload) => {
|
|
455
|
+
const newJob = payload.new;
|
|
456
|
+
const oldJob = payload.old;
|
|
457
|
+
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
458
|
+
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
459
|
+
if (!newMatches && !oldMatches) {
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
callback({
|
|
463
|
+
type: payload.eventType.toUpperCase(),
|
|
464
|
+
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
465
|
+
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
466
|
+
});
|
|
467
|
+
}).subscribe();
|
|
468
|
+
return () => {
|
|
469
|
+
if (this.realtimeChannel) {
|
|
470
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
471
|
+
this.realtimeChannel = null;
|
|
472
|
+
}
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
getPollingManager() {
|
|
476
|
+
if (!this.pollingManager) {
|
|
477
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
478
|
+
const jobs = await this.getAllJobs();
|
|
479
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
480
|
+
}, (a, b) => deepEqual(a, b), {
|
|
481
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
482
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
483
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
return this.pollingManager;
|
|
487
|
+
}
|
|
488
|
+
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
489
|
+
let lastKnownJobs = new Map;
|
|
490
|
+
let cancelled = false;
|
|
491
|
+
const poll = async () => {
|
|
492
|
+
if (cancelled)
|
|
493
|
+
return;
|
|
494
|
+
try {
|
|
495
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
496
|
+
if (cancelled)
|
|
497
|
+
return;
|
|
498
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
499
|
+
for (const [id, job] of currentMap) {
|
|
500
|
+
const old = lastKnownJobs.get(id);
|
|
501
|
+
if (!old) {
|
|
502
|
+
callback({ type: "INSERT", new: job });
|
|
503
|
+
} else if (!deepEqual(old, job)) {
|
|
504
|
+
callback({ type: "UPDATE", old, new: job });
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
for (const [id, job] of lastKnownJobs) {
|
|
508
|
+
if (!currentMap.has(id)) {
|
|
509
|
+
callback({ type: "DELETE", old: job });
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
lastKnownJobs = currentMap;
|
|
513
|
+
} catch {}
|
|
514
|
+
};
|
|
515
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
516
|
+
poll();
|
|
517
|
+
return () => {
|
|
518
|
+
cancelled = true;
|
|
519
|
+
clearInterval(intervalId);
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
subscribeToChangesWithPolling(callback, options) {
|
|
523
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
524
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
525
|
+
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
526
|
+
}
|
|
527
|
+
const manager = this.getPollingManager();
|
|
528
|
+
return manager.subscribe(callback, { intervalMs });
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
// src/job-queue/SupabaseRateLimiterStorage.ts
|
|
532
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
533
|
+
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken2("ratelimiter.storage.supabase");
|
|
534
|
+
|
|
535
|
+
class SupabaseRateLimiterStorage {
|
|
536
|
+
scope = "cluster";
|
|
537
|
+
client;
|
|
538
|
+
prefixes;
|
|
539
|
+
prefixValues;
|
|
540
|
+
executionTableName;
|
|
541
|
+
nextAvailableTableName;
|
|
542
|
+
constructor(client, options) {
|
|
543
|
+
this.client = client;
|
|
544
|
+
this.prefixes = options?.prefixes ?? [];
|
|
545
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
546
|
+
if (this.prefixes.length > 0) {
|
|
547
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
548
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
549
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
550
|
+
} else {
|
|
551
|
+
this.executionTableName = "rate_limit_executions";
|
|
552
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
getPrefixColumnType(type) {
|
|
556
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
557
|
+
}
|
|
558
|
+
buildPrefixColumnsSql() {
|
|
559
|
+
if (this.prefixes.length === 0)
|
|
560
|
+
return "";
|
|
561
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
562
|
+
`) + `,
|
|
563
|
+
`;
|
|
564
|
+
}
|
|
565
|
+
getPrefixColumnNames() {
|
|
566
|
+
return this.prefixes.map((p) => p.name);
|
|
567
|
+
}
|
|
568
|
+
applyPrefixFilters(query) {
|
|
569
|
+
let result = query;
|
|
570
|
+
for (const prefix of this.prefixes) {
|
|
571
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
572
|
+
}
|
|
573
|
+
return result;
|
|
574
|
+
}
|
|
575
|
+
getPrefixInsertValues() {
|
|
576
|
+
const values = {};
|
|
577
|
+
for (const prefix of this.prefixes) {
|
|
578
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
579
|
+
}
|
|
580
|
+
return values;
|
|
581
|
+
}
|
|
582
|
+
async setupDatabase() {
|
|
583
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
584
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
585
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
586
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
587
|
+
const createExecTableSql = `
|
|
588
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
589
|
+
id SERIAL PRIMARY KEY,
|
|
590
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
591
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
592
|
+
)
|
|
593
|
+
`;
|
|
594
|
+
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
595
|
+
query: createExecTableSql
|
|
596
|
+
});
|
|
597
|
+
if (execTableError && execTableError.code !== "42P07") {
|
|
598
|
+
throw execTableError;
|
|
599
|
+
}
|
|
600
|
+
const createExecIndexSql = `
|
|
601
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
602
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
603
|
+
`;
|
|
604
|
+
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
605
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
606
|
+
const createNextTableSql = `
|
|
607
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
608
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
609
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
610
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
611
|
+
)
|
|
612
|
+
`;
|
|
613
|
+
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
614
|
+
query: createNextTableSql
|
|
615
|
+
});
|
|
616
|
+
if (nextTableError && nextTableError.code !== "42P07") {
|
|
617
|
+
throw nextTableError;
|
|
618
|
+
}
|
|
619
|
+
const fnName = this.atomicReserveFunctionName();
|
|
620
|
+
const prefixSig = this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)}`).join(", ");
|
|
621
|
+
const prefixSigPrefix = prefixSig ? prefixSig + ", " : "";
|
|
622
|
+
const prefixWhere = this.prefixes.length > 0 ? " AND " + this.prefixes.map((p) => `${p.name} = _${p.name}`).join(" AND ") : "";
|
|
623
|
+
const prefixInsertCols = this.prefixes.length > 0 ? this.prefixes.map((p) => p.name).join(", ") + ", " : "";
|
|
624
|
+
const prefixInsertVals = this.prefixes.length > 0 ? this.prefixes.map((p) => `_${p.name}`).join(", ") + ", " : "";
|
|
625
|
+
const lockKeyParts = [
|
|
626
|
+
`'${this.executionTableName}'`,
|
|
627
|
+
...this.prefixes.map((p) => `_${p.name}::text`),
|
|
628
|
+
`_queue_name::text`
|
|
629
|
+
];
|
|
630
|
+
const lockKeyExpr = `hashtextextended(${lockKeyParts.join(" || '|' || ")}, 0)`;
|
|
631
|
+
const createFnSql = `
|
|
632
|
+
CREATE OR REPLACE FUNCTION ${fnName}(
|
|
633
|
+
${prefixSigPrefix}_queue_name TEXT, _window_start TIMESTAMPTZ, _max_exec INT
|
|
634
|
+
) RETURNS BIGINT AS $fn$
|
|
635
|
+
DECLARE
|
|
636
|
+
_count INT;
|
|
637
|
+
_next TIMESTAMPTZ;
|
|
638
|
+
_new_id BIGINT;
|
|
639
|
+
BEGIN
|
|
640
|
+
PERFORM pg_advisory_xact_lock(${lockKeyExpr});
|
|
641
|
+
SELECT COUNT(*) INTO _count FROM ${this.executionTableName}
|
|
642
|
+
WHERE queue_name = _queue_name AND executed_at > _window_start${prefixWhere};
|
|
643
|
+
IF _count >= _max_exec THEN RETURN NULL; END IF;
|
|
644
|
+
SELECT next_available_at INTO _next FROM ${this.nextAvailableTableName}
|
|
645
|
+
WHERE queue_name = _queue_name${prefixWhere};
|
|
646
|
+
IF _next IS NOT NULL AND _next > NOW() THEN RETURN NULL; END IF;
|
|
647
|
+
INSERT INTO ${this.executionTableName} (${prefixInsertCols}queue_name)
|
|
648
|
+
VALUES (${prefixInsertVals}_queue_name)
|
|
649
|
+
RETURNING id INTO _new_id;
|
|
650
|
+
RETURN _new_id;
|
|
651
|
+
END;
|
|
652
|
+
$fn$ LANGUAGE plpgsql;
|
|
653
|
+
`;
|
|
654
|
+
const { error: fnError } = await this.client.rpc("exec_sql", { query: createFnSql });
|
|
655
|
+
if (fnError) {
|
|
656
|
+
throw fnError;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
atomicReserveFunctionName() {
|
|
660
|
+
return `${this.executionTableName}_try_reserve`.slice(0, 63);
|
|
661
|
+
}
|
|
662
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
663
|
+
const args = {
|
|
664
|
+
_queue_name: queueName,
|
|
665
|
+
_window_start: new Date(Date.now() - windowMs).toISOString(),
|
|
666
|
+
_max_exec: maxExecutions
|
|
667
|
+
};
|
|
668
|
+
for (const p of this.prefixes) {
|
|
669
|
+
args[`_${p.name}`] = this.prefixValues[p.name];
|
|
670
|
+
}
|
|
671
|
+
const { data, error } = await this.client.rpc(this.atomicReserveFunctionName(), args);
|
|
672
|
+
if (error)
|
|
673
|
+
throw error;
|
|
674
|
+
if (data === null || data === undefined)
|
|
675
|
+
return null;
|
|
676
|
+
if (Array.isArray(data)) {
|
|
677
|
+
if (data.length === 0)
|
|
678
|
+
return null;
|
|
679
|
+
const first = Object.values(data[0])[0];
|
|
680
|
+
return first ?? null;
|
|
681
|
+
}
|
|
682
|
+
return data;
|
|
683
|
+
}
|
|
684
|
+
async releaseExecution(queueName, token) {
|
|
685
|
+
if (token === null || token === undefined)
|
|
686
|
+
return;
|
|
687
|
+
let del = this.client.from(this.executionTableName).delete().eq("id", token).eq("queue_name", queueName);
|
|
688
|
+
del = this.applyPrefixFilters(del);
|
|
689
|
+
const { error: delError } = await del;
|
|
690
|
+
if (delError)
|
|
691
|
+
throw delError;
|
|
692
|
+
}
|
|
693
|
+
async recordExecution(queueName) {
|
|
694
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
695
|
+
const { error } = await this.client.from(this.executionTableName).insert({
|
|
696
|
+
...prefixInsertValues,
|
|
697
|
+
queue_name: queueName
|
|
698
|
+
});
|
|
699
|
+
if (error)
|
|
700
|
+
throw error;
|
|
701
|
+
}
|
|
702
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
703
|
+
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
704
|
+
query = this.applyPrefixFilters(query);
|
|
705
|
+
const { count, error } = await query;
|
|
706
|
+
if (error)
|
|
707
|
+
throw error;
|
|
708
|
+
return count ?? 0;
|
|
709
|
+
}
|
|
710
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
711
|
+
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
712
|
+
query = this.applyPrefixFilters(query);
|
|
713
|
+
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
714
|
+
if (error)
|
|
715
|
+
throw error;
|
|
716
|
+
if (!data || data.length === 0)
|
|
717
|
+
return;
|
|
718
|
+
return new Date(data[0].executed_at).toISOString();
|
|
719
|
+
}
|
|
720
|
+
async getNextAvailableTime(queueName) {
|
|
721
|
+
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
722
|
+
query = this.applyPrefixFilters(query);
|
|
723
|
+
const { data, error } = await query.single();
|
|
724
|
+
if (error) {
|
|
725
|
+
if (error.code === "PGRST116")
|
|
726
|
+
return;
|
|
727
|
+
throw error;
|
|
728
|
+
}
|
|
729
|
+
if (!data?.next_available_at)
|
|
730
|
+
return;
|
|
731
|
+
return new Date(data.next_available_at).toISOString();
|
|
732
|
+
}
|
|
733
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
734
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
735
|
+
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
736
|
+
...prefixInsertValues,
|
|
737
|
+
queue_name: queueName,
|
|
738
|
+
next_available_at: nextAvailableAt
|
|
739
|
+
}, {
|
|
740
|
+
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
741
|
+
});
|
|
742
|
+
if (error)
|
|
743
|
+
throw error;
|
|
744
|
+
}
|
|
745
|
+
async clear(queueName) {
|
|
746
|
+
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
747
|
+
execQuery = this.applyPrefixFilters(execQuery);
|
|
748
|
+
const { error: execError } = await execQuery;
|
|
749
|
+
if (execError)
|
|
750
|
+
throw execError;
|
|
751
|
+
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
752
|
+
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
753
|
+
const { error: nextError } = await nextQuery;
|
|
754
|
+
if (nextError)
|
|
755
|
+
throw nextError;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
export {
|
|
759
|
+
SupabaseRateLimiterStorage,
|
|
760
|
+
SupabaseQueueStorage,
|
|
761
|
+
SUPABASE_RATE_LIMITER_STORAGE,
|
|
762
|
+
SUPABASE_QUEUE_STORAGE
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
//# debugId=FFFCF982F7410DEF64756E2164756E21
|