@workglow/supabase 0.2.36 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/job-queue/SupabaseJobStore.d.ts +42 -0
- package/dist/job-queue/SupabaseJobStore.d.ts.map +1 -0
- package/dist/job-queue/SupabaseMessageQueue.d.ts +38 -0
- package/dist/job-queue/SupabaseMessageQueue.d.ts.map +1 -0
- package/dist/job-queue/SupabaseQueueStorage.d.ts +76 -9
- package/dist/job-queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/job-queue/SupabaseRateLimiterStorage.d.ts +1 -2
- package/dist/job-queue/SupabaseRateLimiterStorage.d.ts.map +1 -1
- package/dist/job-queue/browser.js +477 -43
- package/dist/job-queue/browser.js.map +8 -5
- package/dist/job-queue/common.d.ts +3 -0
- package/dist/job-queue/common.d.ts.map +1 -1
- package/dist/job-queue/createSupabaseQueue.d.ts +22 -0
- package/dist/job-queue/createSupabaseQueue.d.ts.map +1 -0
- package/dist/job-queue/node.js +477 -43
- package/dist/job-queue/node.js.map +8 -5
- package/dist/storage/SupabaseKvStorage.d.ts +1 -1
- package/dist/storage/SupabaseKvStorage.d.ts.map +1 -1
- package/dist/storage/SupabaseTabularStorage.d.ts +1 -1
- package/dist/storage/SupabaseTabularStorage.d.ts.map +1 -1
- package/dist/storage/browser.js +10 -10
- package/dist/storage/browser.js.map +4 -4
- package/dist/storage/node.js +10 -10
- package/dist/storage/node.js.map +4 -4
- package/package.json +8 -8
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// src/job-queue/SupabaseQueueStorage.ts
|
|
2
|
-
import {
|
|
2
|
+
import { JobStatus, validateLeaseMs } from "@workglow/job-queue";
|
|
3
3
|
import { PollingSubscriptionManager } from "@workglow/storage";
|
|
4
|
-
import {
|
|
4
|
+
import { createServiceToken, deepEqual, makeFingerprint, uuid4 } from "@workglow/util";
|
|
5
5
|
var SUPABASE_QUEUE_STORAGE = createServiceToken("jobqueue.storage.supabase");
|
|
6
6
|
|
|
7
7
|
class SupabaseQueueStorage {
|
|
@@ -81,7 +81,8 @@ class SupabaseQueueStorage {
|
|
|
81
81
|
return value.replace(/'/g, "''");
|
|
82
82
|
}
|
|
83
83
|
async migrate() {
|
|
84
|
-
const
|
|
84
|
+
const enumValues = [...Object.values(JobStatus), "ABORTING"].filter((v, i, a) => a.indexOf(v) === i).map((v) => `'${v}'`).join(",");
|
|
85
|
+
const createTypeSql = `CREATE TYPE job_status AS ENUM (${enumValues})`;
|
|
85
86
|
const { error: typeError } = await this.client.rpc("exec_sql", { query: createTypeSql });
|
|
86
87
|
if (typeError && typeError.code !== "42710") {
|
|
87
88
|
throw typeError;
|
|
@@ -99,10 +100,10 @@ class SupabaseQueueStorage {
|
|
|
99
100
|
status job_status NOT NULL default 'PENDING',
|
|
100
101
|
input jsonb NOT NULL,
|
|
101
102
|
output jsonb,
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
103
|
+
attempts integer default 0,
|
|
104
|
+
max_attempts integer default 10,
|
|
105
|
+
visible_at timestamp with time zone DEFAULT now(),
|
|
106
|
+
last_attempted_at timestamp with time zone,
|
|
106
107
|
created_at timestamp with time zone DEFAULT now(),
|
|
107
108
|
deadline_at timestamp with time zone,
|
|
108
109
|
completed_at timestamp with time zone,
|
|
@@ -111,7 +112,9 @@ class SupabaseQueueStorage {
|
|
|
111
112
|
progress real DEFAULT 0,
|
|
112
113
|
progress_message text DEFAULT '',
|
|
113
114
|
progress_details jsonb,
|
|
114
|
-
|
|
115
|
+
lease_owner text,
|
|
116
|
+
abort_requested_at timestamp with time zone,
|
|
117
|
+
lease_expires_at timestamp with time zone
|
|
115
118
|
)`;
|
|
116
119
|
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
117
120
|
if (tableError) {
|
|
@@ -119,10 +122,26 @@ class SupabaseQueueStorage {
|
|
|
119
122
|
throw tableError;
|
|
120
123
|
}
|
|
121
124
|
}
|
|
125
|
+
const alterSqls = [
|
|
126
|
+
`ALTER TABLE ${this.tableName} ADD COLUMN IF NOT EXISTS abort_requested_at timestamp with time zone`,
|
|
127
|
+
`ALTER TABLE ${this.tableName} ADD COLUMN IF NOT EXISTS lease_expires_at timestamp with time zone`,
|
|
128
|
+
`ALTER TABLE ${this.tableName} RENAME COLUMN run_after TO visible_at`,
|
|
129
|
+
`ALTER TABLE ${this.tableName} RENAME COLUMN last_ran_at TO last_attempted_at`,
|
|
130
|
+
`ALTER TABLE ${this.tableName} RENAME COLUMN run_attempts TO attempts`,
|
|
131
|
+
`ALTER TABLE ${this.tableName} RENAME COLUMN max_retries TO max_attempts`,
|
|
132
|
+
`ALTER TABLE ${this.tableName} RENAME COLUMN worker_id TO lease_owner`
|
|
133
|
+
];
|
|
134
|
+
for (const sql of alterSqls) {
|
|
135
|
+
const { error } = await this.client.rpc("exec_sql", { query: sql });
|
|
136
|
+
if (error && error.code !== "42703") {
|
|
137
|
+
throw new Error(`Failed to rename column: ${error.message}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
122
140
|
const indexes = [
|
|
123
|
-
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status,
|
|
124
|
-
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status,
|
|
125
|
-
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)
|
|
141
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, visible_at)`,
|
|
142
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, visible_at)`,
|
|
143
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`,
|
|
144
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_${this.tableName}_fingerprint_active ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint) WHERE status IN ('PENDING','PROCESSING')`
|
|
126
145
|
];
|
|
127
146
|
for (const indexSql of indexes) {
|
|
128
147
|
await this.client.rpc("exec_sql", { query: indexSql });
|
|
@@ -135,30 +154,41 @@ class SupabaseQueueStorage {
|
|
|
135
154
|
const now = new Date().toISOString();
|
|
136
155
|
job.queue = this.queueName;
|
|
137
156
|
job.job_run_id = job.job_run_id ?? uuid4();
|
|
138
|
-
job.fingerprint = await makeFingerprint(job.input);
|
|
157
|
+
job.fingerprint = job.fingerprint ?? await makeFingerprint(job.input);
|
|
139
158
|
job.status = JobStatus.PENDING;
|
|
140
159
|
job.progress = 0;
|
|
141
160
|
job.progress_message = "";
|
|
142
161
|
job.progress_details = null;
|
|
143
162
|
job.created_at = now;
|
|
144
|
-
job.
|
|
163
|
+
job.visible_at = now;
|
|
145
164
|
const prefixInsertValues = this.getPrefixInsertValues();
|
|
146
165
|
const { data, error } = await this.client.from(this.tableName).insert({
|
|
147
166
|
...prefixInsertValues,
|
|
148
167
|
queue: job.queue,
|
|
149
168
|
fingerprint: job.fingerprint,
|
|
150
169
|
input: job.input,
|
|
151
|
-
|
|
170
|
+
visible_at: job.visible_at,
|
|
152
171
|
created_at: job.created_at,
|
|
153
172
|
deadline_at: job.deadline_at,
|
|
154
|
-
|
|
173
|
+
max_attempts: job.max_attempts,
|
|
155
174
|
job_run_id: job.job_run_id,
|
|
156
175
|
progress: job.progress,
|
|
157
176
|
progress_message: job.progress_message,
|
|
158
177
|
progress_details: job.progress_details
|
|
159
178
|
}).select("id").single();
|
|
160
|
-
if (error)
|
|
179
|
+
if (error) {
|
|
180
|
+
const e = error;
|
|
181
|
+
const isUniqueViolation = e?.code === "23505";
|
|
182
|
+
const involvesFingerprint = typeof e?.details === "string" && /fingerprint/i.test(e.details) || typeof e?.message === "string" && /fingerprint/i.test(e.message);
|
|
183
|
+
if (isUniqueViolation && involvesFingerprint && job.fingerprint) {
|
|
184
|
+
const winner = await this.findActiveByFingerprint(job.fingerprint, this.queueName);
|
|
185
|
+
if (winner?.id != null) {
|
|
186
|
+
job.id = winner.id;
|
|
187
|
+
return winner.id;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
161
190
|
throw error;
|
|
191
|
+
}
|
|
162
192
|
if (!data)
|
|
163
193
|
throw new Error("Failed to add to queue");
|
|
164
194
|
job.id = data.id;
|
|
@@ -179,12 +209,14 @@ class SupabaseQueueStorage {
|
|
|
179
209
|
num = Number(num) || 100;
|
|
180
210
|
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
181
211
|
query = this.applyPrefixFilters(query);
|
|
182
|
-
const { data, error } = await query.order("
|
|
212
|
+
const { data, error } = await query.order("visible_at", { ascending: true }).limit(num);
|
|
183
213
|
if (error)
|
|
184
214
|
throw error;
|
|
185
215
|
return data ?? [];
|
|
186
216
|
}
|
|
187
|
-
async next(workerId) {
|
|
217
|
+
async next(workerId, opts) {
|
|
218
|
+
const leaseMs = opts?.leaseMs ?? 30000;
|
|
219
|
+
validateLeaseMs(leaseMs, "leaseMs");
|
|
188
220
|
const prefixConditions = this.buildPrefixWhereSql();
|
|
189
221
|
const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
|
|
190
222
|
const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
|
|
@@ -192,15 +224,27 @@ class SupabaseQueueStorage {
|
|
|
192
224
|
const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
|
|
193
225
|
const sql = `
|
|
194
226
|
UPDATE ${this.tableName}
|
|
195
|
-
SET status = '${JobStatus.PROCESSING}',
|
|
227
|
+
SET status = '${JobStatus.PROCESSING}',
|
|
228
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC',
|
|
229
|
+
lease_owner = '${escapedWorkerId}',
|
|
230
|
+
lease_expires_at = NOW() AT TIME ZONE 'UTC' + (${Number(leaseMs)} * INTERVAL '1 millisecond'),
|
|
231
|
+
-- Lease-expiry reclaim consumes one attempt against max_attempts;
|
|
232
|
+
-- PENDING claims do not (the worker's validateJobState will FAIL
|
|
233
|
+
-- the job when attempts >= max_attempts at next-step time).
|
|
234
|
+
attempts = CASE WHEN status = '${JobStatus.PROCESSING}' THEN attempts + 1 ELSE attempts END,
|
|
235
|
+
-- Always clear stale abort_requested_at on (re)claim so a flag set
|
|
236
|
+
-- by an earlier worker doesn't immediately abort the new lease.
|
|
237
|
+
abort_requested_at = NULL
|
|
196
238
|
WHERE id = (
|
|
197
239
|
SELECT id
|
|
198
240
|
FROM ${this.tableName}
|
|
199
241
|
WHERE queue = '${escapedQueueName}'
|
|
200
|
-
AND
|
|
242
|
+
AND (
|
|
243
|
+
(status = '${JobStatus.PENDING}' AND visible_at <= NOW() AT TIME ZONE 'UTC')
|
|
244
|
+
OR (status = '${JobStatus.PROCESSING}' AND (lease_expires_at IS NULL OR lease_expires_at < NOW() AT TIME ZONE 'UTC'))
|
|
245
|
+
)
|
|
201
246
|
${prefixConditions}
|
|
202
|
-
|
|
203
|
-
ORDER BY run_after ASC
|
|
247
|
+
ORDER BY visible_at ASC
|
|
204
248
|
FOR UPDATE SKIP LOCKED
|
|
205
249
|
LIMIT 1
|
|
206
250
|
)
|
|
@@ -213,6 +257,31 @@ class SupabaseQueueStorage {
|
|
|
213
257
|
}
|
|
214
258
|
return data[0];
|
|
215
259
|
}
|
|
260
|
+
async extendLease(id, workerId, ms) {
|
|
261
|
+
validateLeaseMs(ms, "ms");
|
|
262
|
+
const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
|
|
263
|
+
const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
|
|
264
|
+
const numericId = Number(id);
|
|
265
|
+
if (!Number.isFinite(numericId)) {
|
|
266
|
+
throw new Error(`Invalid job id: ${id}`);
|
|
267
|
+
}
|
|
268
|
+
const prefixConditions = this.buildPrefixWhereSql();
|
|
269
|
+
const sql = `
|
|
270
|
+
UPDATE ${this.tableName}
|
|
271
|
+
SET lease_expires_at = NOW() AT TIME ZONE 'UTC' + (${Number(ms)} * INTERVAL '1 millisecond')
|
|
272
|
+
WHERE id = ${numericId}
|
|
273
|
+
AND queue = '${this.escapeSqlString(this.validateSqlValue(this.queueName, "queueName"))}'
|
|
274
|
+
AND lease_owner = '${escapedWorkerId}'
|
|
275
|
+
AND status = '${JobStatus.PROCESSING}'
|
|
276
|
+
${prefixConditions}
|
|
277
|
+
RETURNING id`;
|
|
278
|
+
const { data, error } = await this.client.rpc("exec_sql", { query: sql });
|
|
279
|
+
if (error)
|
|
280
|
+
throw error;
|
|
281
|
+
if (!data || !Array.isArray(data) || data.length === 0) {
|
|
282
|
+
throw new Error(`extendLease failed: job ${String(id)} is not PROCESSING or lease is not owned by worker ${workerId}`);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
216
285
|
async size(status = JobStatus.PENDING) {
|
|
217
286
|
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
218
287
|
query = this.applyPrefixFilters(query);
|
|
@@ -238,7 +307,7 @@ class SupabaseQueueStorage {
|
|
|
238
307
|
progress_message: "",
|
|
239
308
|
progress_details: null,
|
|
240
309
|
completed_at: now,
|
|
241
|
-
|
|
310
|
+
last_attempted_at: now
|
|
242
311
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
243
312
|
query2 = this.applyPrefixFilters(query2);
|
|
244
313
|
const { error: error2 } = await query2;
|
|
@@ -246,25 +315,25 @@ class SupabaseQueueStorage {
|
|
|
246
315
|
throw error2;
|
|
247
316
|
return;
|
|
248
317
|
}
|
|
249
|
-
let getQuery = this.client.from(this.tableName).select("
|
|
318
|
+
let getQuery = this.client.from(this.tableName).select("attempts, max_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
250
319
|
getQuery = this.applyPrefixFilters(getQuery);
|
|
251
320
|
const { data: current, error: getError } = await getQuery.single();
|
|
252
321
|
if (getError)
|
|
253
322
|
throw getError;
|
|
254
|
-
const currentAttempts = current?.
|
|
255
|
-
const
|
|
323
|
+
const currentAttempts = current?.attempts ?? 0;
|
|
324
|
+
const maxAttempts = current?.max_attempts ?? jobDetails.max_attempts ?? 10;
|
|
256
325
|
const nextAttempts = currentAttempts + 1;
|
|
257
326
|
if (jobDetails.status === JobStatus.PENDING) {
|
|
258
|
-
if (nextAttempts
|
|
327
|
+
if (nextAttempts >= maxAttempts) {
|
|
259
328
|
let failQuery = this.client.from(this.tableName).update({
|
|
260
329
|
status: JobStatus.FAILED,
|
|
261
|
-
error: "Max
|
|
262
|
-
error_code: "
|
|
330
|
+
error: "Max attempts reached",
|
|
331
|
+
error_code: "MAX_ATTEMPTS_REACHED",
|
|
263
332
|
progress: 100,
|
|
264
333
|
progress_message: "",
|
|
265
334
|
progress_details: null,
|
|
266
335
|
completed_at: now,
|
|
267
|
-
|
|
336
|
+
last_attempted_at: now
|
|
268
337
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
269
338
|
failQuery = this.applyPrefixFilters(failQuery);
|
|
270
339
|
const { error: failError } = await failQuery;
|
|
@@ -276,12 +345,13 @@ class SupabaseQueueStorage {
|
|
|
276
345
|
error: jobDetails.error ?? null,
|
|
277
346
|
error_code: jobDetails.error_code ?? null,
|
|
278
347
|
status: jobDetails.status,
|
|
279
|
-
|
|
348
|
+
visible_at: jobDetails.visible_at,
|
|
280
349
|
progress: 0,
|
|
281
350
|
progress_message: "",
|
|
282
351
|
progress_details: null,
|
|
283
|
-
|
|
284
|
-
|
|
352
|
+
attempts: nextAttempts,
|
|
353
|
+
last_attempted_at: now,
|
|
354
|
+
abort_requested_at: null
|
|
285
355
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
286
356
|
query2 = this.applyPrefixFilters(query2);
|
|
287
357
|
const { error: error2 } = await query2;
|
|
@@ -298,9 +368,9 @@ class SupabaseQueueStorage {
|
|
|
298
368
|
progress: 100,
|
|
299
369
|
progress_message: "",
|
|
300
370
|
progress_details: null,
|
|
301
|
-
|
|
371
|
+
attempts: nextAttempts,
|
|
302
372
|
completed_at: now,
|
|
303
|
-
|
|
373
|
+
last_attempted_at: now
|
|
304
374
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
305
375
|
query2 = this.applyPrefixFilters(query2);
|
|
306
376
|
const { error: error2 } = await query2;
|
|
@@ -313,28 +383,62 @@ class SupabaseQueueStorage {
|
|
|
313
383
|
output: jobDetails.output ?? null,
|
|
314
384
|
error: jobDetails.error ?? null,
|
|
315
385
|
error_code: jobDetails.error_code ?? null,
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
386
|
+
visible_at: jobDetails.visible_at ?? null,
|
|
387
|
+
attempts: nextAttempts,
|
|
388
|
+
last_attempted_at: now
|
|
319
389
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
320
390
|
query = this.applyPrefixFilters(query);
|
|
321
391
|
const { error } = await query;
|
|
322
392
|
if (error)
|
|
323
393
|
throw error;
|
|
324
394
|
}
|
|
325
|
-
async
|
|
395
|
+
async releaseClaim(jobId) {
|
|
326
396
|
let query = this.client.from(this.tableName).update({
|
|
327
397
|
status: JobStatus.PENDING,
|
|
328
|
-
|
|
398
|
+
lease_owner: null,
|
|
329
399
|
progress: 0,
|
|
330
400
|
progress_message: "",
|
|
331
|
-
progress_details: null
|
|
401
|
+
progress_details: null,
|
|
402
|
+
abort_requested_at: null
|
|
332
403
|
}).eq("id", jobId).eq("queue", this.queueName);
|
|
333
404
|
query = this.applyPrefixFilters(query);
|
|
334
405
|
const { error } = await query;
|
|
335
406
|
if (error)
|
|
336
407
|
throw error;
|
|
337
408
|
}
|
|
409
|
+
async finalize(id, fields) {
|
|
410
|
+
const patch = {};
|
|
411
|
+
if ("output" in fields)
|
|
412
|
+
patch.output = fields.output ?? null;
|
|
413
|
+
if ("error" in fields)
|
|
414
|
+
patch.error = fields.error ?? null;
|
|
415
|
+
if ("error_code" in fields)
|
|
416
|
+
patch.error_code = fields.error_code ?? null;
|
|
417
|
+
if ("status" in fields)
|
|
418
|
+
patch.status = fields.status;
|
|
419
|
+
if ("completed_at" in fields)
|
|
420
|
+
patch.completed_at = fields.completed_at ?? null;
|
|
421
|
+
if ("abort_requested_at" in fields) {
|
|
422
|
+
patch.abort_requested_at = fields.abort_requested_at ?? null;
|
|
423
|
+
}
|
|
424
|
+
if ("lease_owner" in fields)
|
|
425
|
+
patch.lease_owner = fields.lease_owner ?? null;
|
|
426
|
+
if ("progress" in fields)
|
|
427
|
+
patch.progress = fields.progress ?? 0;
|
|
428
|
+
if ("progress_message" in fields)
|
|
429
|
+
patch.progress_message = fields.progress_message ?? "";
|
|
430
|
+
if ("progress_details" in fields)
|
|
431
|
+
patch.progress_details = fields.progress_details ?? null;
|
|
432
|
+
if ("visible_at" in fields)
|
|
433
|
+
patch.visible_at = fields.visible_at ?? null;
|
|
434
|
+
if (Object.keys(patch).length === 0)
|
|
435
|
+
return;
|
|
436
|
+
let query = this.client.from(this.tableName).update(patch).eq("id", id).eq("queue", this.queueName);
|
|
437
|
+
query = this.applyPrefixFilters(query);
|
|
438
|
+
const { error } = await query;
|
|
439
|
+
if (error)
|
|
440
|
+
throw error;
|
|
441
|
+
}
|
|
338
442
|
async deleteAll() {
|
|
339
443
|
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
340
444
|
query = this.applyPrefixFilters(query);
|
|
@@ -355,7 +459,28 @@ class SupabaseQueueStorage {
|
|
|
355
459
|
return data?.output ?? null;
|
|
356
460
|
}
|
|
357
461
|
async abort(jobId) {
|
|
358
|
-
|
|
462
|
+
const now = new Date().toISOString();
|
|
463
|
+
{
|
|
464
|
+
let query = this.client.from(this.tableName).update({
|
|
465
|
+
status: JobStatus.FAILED,
|
|
466
|
+
abort_requested_at: now,
|
|
467
|
+
completed_at: now
|
|
468
|
+
}).eq("id", jobId).eq("queue", this.queueName).eq("status", JobStatus.PENDING);
|
|
469
|
+
query = this.applyPrefixFilters(query);
|
|
470
|
+
const { error } = await query;
|
|
471
|
+
if (error)
|
|
472
|
+
throw error;
|
|
473
|
+
}
|
|
474
|
+
{
|
|
475
|
+
let query = this.client.from(this.tableName).update({ abort_requested_at: now }).eq("id", jobId).eq("queue", this.queueName).eq("status", JobStatus.PROCESSING);
|
|
476
|
+
query = this.applyPrefixFilters(query);
|
|
477
|
+
const { error } = await query;
|
|
478
|
+
if (error)
|
|
479
|
+
throw error;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
async saveStatus(jobId, status) {
|
|
483
|
+
let query = this.client.from(this.tableName).update({ status }).eq("id", jobId).eq("queue", this.queueName);
|
|
359
484
|
query = this.applyPrefixFilters(query);
|
|
360
485
|
const { error } = await query;
|
|
361
486
|
if (error)
|
|
@@ -387,6 +512,76 @@ class SupabaseQueueStorage {
|
|
|
387
512
|
if (error)
|
|
388
513
|
throw error;
|
|
389
514
|
}
|
|
515
|
+
async findActiveByFingerprint(fingerprint, queueName) {
|
|
516
|
+
const prefixConditions = this.buildPrefixWhereSql();
|
|
517
|
+
const validatedQueueName = this.validateSqlValue(queueName, "queueName");
|
|
518
|
+
const escapedQueueName = this.escapeSqlString(validatedQueueName);
|
|
519
|
+
const escapedFingerprint = this.escapeSqlString(fingerprint);
|
|
520
|
+
const sql = `
|
|
521
|
+
SELECT * FROM ${this.tableName}
|
|
522
|
+
WHERE fingerprint = '${escapedFingerprint}' AND queue = '${escapedQueueName}'
|
|
523
|
+
AND status IN ('PENDING','PROCESSING')${prefixConditions}
|
|
524
|
+
ORDER BY created_at DESC
|
|
525
|
+
LIMIT 1`;
|
|
526
|
+
const { data, error } = await this.client.rpc("exec_sql", { query: sql });
|
|
527
|
+
if (error)
|
|
528
|
+
throw error;
|
|
529
|
+
if (!data || !Array.isArray(data) || data.length === 0)
|
|
530
|
+
return;
|
|
531
|
+
return data[0];
|
|
532
|
+
}
|
|
533
|
+
async getMany(ids) {
|
|
534
|
+
if (ids.length === 0)
|
|
535
|
+
return [];
|
|
536
|
+
let query = this.client.from(this.tableName).select("*").in("id", ids).eq("queue", this.queueName);
|
|
537
|
+
query = this.applyPrefixFilters(query);
|
|
538
|
+
const { data, error } = await query;
|
|
539
|
+
if (error)
|
|
540
|
+
throw error;
|
|
541
|
+
const map = new Map;
|
|
542
|
+
for (const row of data ?? []) {
|
|
543
|
+
map.set(row.id, row);
|
|
544
|
+
}
|
|
545
|
+
return ids.map((id) => map.get(id));
|
|
546
|
+
}
|
|
547
|
+
async completeWithResult(id, result) {
|
|
548
|
+
const now = new Date().toISOString();
|
|
549
|
+
let query = this.client.from(this.tableName).update({
|
|
550
|
+
output: result ?? null,
|
|
551
|
+
status: "COMPLETED",
|
|
552
|
+
progress: 100,
|
|
553
|
+
progress_message: "",
|
|
554
|
+
progress_details: null,
|
|
555
|
+
completed_at: now
|
|
556
|
+
}).eq("id", id).eq("queue", this.queueName);
|
|
557
|
+
query = this.applyPrefixFilters(query);
|
|
558
|
+
const { error } = await query;
|
|
559
|
+
if (error)
|
|
560
|
+
throw error;
|
|
561
|
+
}
|
|
562
|
+
async failWithError(id, opts) {
|
|
563
|
+
const numericId = Number(id);
|
|
564
|
+
if (!Number.isFinite(numericId)) {
|
|
565
|
+
throw new Error(`Invalid job id: ${id}`);
|
|
566
|
+
}
|
|
567
|
+
const prefixConditions = this.buildPrefixWhereSql();
|
|
568
|
+
const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
|
|
569
|
+
const escapedQueueName = this.escapeSqlString(validatedQueueName);
|
|
570
|
+
const errorLiteral = "error" in opts ? opts.error != null ? `'${this.escapeSqlString(opts.error)}'` : "NULL" : "NULL";
|
|
571
|
+
const errorCodeLiteral = "errorCode" in opts ? opts.errorCode != null ? `'${this.escapeSqlString(opts.errorCode)}'` : "NULL" : "NULL";
|
|
572
|
+
const abortClause = opts.abortRequested === true ? `CASE WHEN abort_requested_at IS NOT NULL THEN abort_requested_at ELSE NOW() AT TIME ZONE 'UTC' END` : `abort_requested_at`;
|
|
573
|
+
const sql = `
|
|
574
|
+
UPDATE ${this.tableName}
|
|
575
|
+
SET error = COALESCE(${errorLiteral}, error),
|
|
576
|
+
error_code = COALESCE(${errorCodeLiteral}, error_code),
|
|
577
|
+
abort_requested_at = ${abortClause},
|
|
578
|
+
status = 'FAILED',
|
|
579
|
+
completed_at = COALESCE(completed_at, NOW() AT TIME ZONE 'UTC')
|
|
580
|
+
WHERE id = ${numericId} AND queue = '${escapedQueueName}'${prefixConditions}`;
|
|
581
|
+
const { error } = await this.client.rpc("exec_sql", { query: sql });
|
|
582
|
+
if (error)
|
|
583
|
+
throw error;
|
|
584
|
+
}
|
|
390
585
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
391
586
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
392
587
|
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
@@ -761,11 +956,250 @@ class SupabaseRateLimiterStorage {
|
|
|
761
956
|
throw nextError;
|
|
762
957
|
}
|
|
763
958
|
}
|
|
959
|
+
// src/job-queue/SupabaseMessageQueue.ts
|
|
960
|
+
class SupabaseClaim {
|
|
961
|
+
core;
|
|
962
|
+
pending;
|
|
963
|
+
id;
|
|
964
|
+
body;
|
|
965
|
+
attempts;
|
|
966
|
+
workerId;
|
|
967
|
+
constructor(core, pending, id, body, attempts, workerId) {
|
|
968
|
+
this.core = core;
|
|
969
|
+
this.pending = pending;
|
|
970
|
+
this.id = id;
|
|
971
|
+
this.body = body;
|
|
972
|
+
this.attempts = attempts;
|
|
973
|
+
this.workerId = workerId;
|
|
974
|
+
}
|
|
975
|
+
async ack(result) {
|
|
976
|
+
const buf = this.pending.get(this.id);
|
|
977
|
+
this.pending.delete(this.id);
|
|
978
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
979
|
+
const output = result !== undefined ? result : buf?.output !== undefined ? buf.output : current.output ?? null;
|
|
980
|
+
await this.core.finalize(this.id, {
|
|
981
|
+
output,
|
|
982
|
+
error: null,
|
|
983
|
+
error_code: null,
|
|
984
|
+
status: "COMPLETED",
|
|
985
|
+
completed_at: current.completed_at ?? new Date().toISOString()
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
async retry(opts) {
|
|
989
|
+
this.pending.delete(this.id);
|
|
990
|
+
const delay = opts?.delaySeconds ?? 0;
|
|
991
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
992
|
+
await this.core.complete({
|
|
993
|
+
...current,
|
|
994
|
+
status: "PENDING",
|
|
995
|
+
lease_owner: null,
|
|
996
|
+
lease_expires_at: null,
|
|
997
|
+
visible_at: new Date(Date.now() + delay * 1000).toISOString(),
|
|
998
|
+
progress: 0,
|
|
999
|
+
progress_message: "",
|
|
1000
|
+
progress_details: null
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
async fail(opts) {
|
|
1004
|
+
opts?.permanent;
|
|
1005
|
+
const buf = this.pending.get(this.id);
|
|
1006
|
+
this.pending.delete(this.id);
|
|
1007
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1008
|
+
const error = opts?.error !== undefined ? opts.error : buf?.error !== undefined ? buf.error : current.error ?? null;
|
|
1009
|
+
const errorCode = opts?.errorCode !== undefined ? opts.errorCode : buf?.errorCode !== undefined ? buf.errorCode : current.error_code ?? null;
|
|
1010
|
+
const abortRequested = opts?.abortRequested !== undefined ? opts.abortRequested : buf?.abortRequested ?? false;
|
|
1011
|
+
await this.core.finalize(this.id, {
|
|
1012
|
+
error,
|
|
1013
|
+
error_code: errorCode,
|
|
1014
|
+
abort_requested_at: abortRequested ? current.abort_requested_at ?? new Date().toISOString() : current.abort_requested_at ?? null,
|
|
1015
|
+
status: "FAILED",
|
|
1016
|
+
completed_at: current.completed_at ?? new Date().toISOString()
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
async extendLease(ms) {
|
|
1020
|
+
await this.core.extendLease(this.id, this.workerId, ms);
|
|
1021
|
+
}
|
|
1022
|
+
async disable() {
|
|
1023
|
+
this.pending.delete(this.id);
|
|
1024
|
+
const current = await this.core.get(this.id);
|
|
1025
|
+
const completedAt = current?.completed_at ?? new Date().toISOString();
|
|
1026
|
+
await this.core.finalize(this.id, {
|
|
1027
|
+
status: "DISABLED",
|
|
1028
|
+
completed_at: completedAt,
|
|
1029
|
+
lease_owner: null,
|
|
1030
|
+
progress: 0,
|
|
1031
|
+
progress_message: "",
|
|
1032
|
+
progress_details: null
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
class SupabaseMessageQueue {
|
|
1038
|
+
scope;
|
|
1039
|
+
core;
|
|
1040
|
+
pending;
|
|
1041
|
+
constructor(core, pending) {
|
|
1042
|
+
this.core = core;
|
|
1043
|
+
this.pending = pending;
|
|
1044
|
+
this.scope = core.scope;
|
|
1045
|
+
}
|
|
1046
|
+
async send(body, opts) {
|
|
1047
|
+
return this.core.add(applySendOptions(body, opts));
|
|
1048
|
+
}
|
|
1049
|
+
async sendBatch(bodies, opts) {
|
|
1050
|
+
const ids = [];
|
|
1051
|
+
for (const body of bodies) {
|
|
1052
|
+
ids.push(await this.send(body, opts));
|
|
1053
|
+
}
|
|
1054
|
+
return ids;
|
|
1055
|
+
}
|
|
1056
|
+
async receive(opts) {
|
|
1057
|
+
const max = Math.max(1, opts.max ?? 1);
|
|
1058
|
+
const claims = [];
|
|
1059
|
+
while (claims.length < max) {
|
|
1060
|
+
const job = await this.core.next(opts.workerId, { leaseMs: opts.leaseMs });
|
|
1061
|
+
if (!job)
|
|
1062
|
+
break;
|
|
1063
|
+
claims.push(new SupabaseClaim(this.core, this.pending, job.id, job, job.attempts ?? 0, opts.workerId));
|
|
1064
|
+
}
|
|
1065
|
+
return claims;
|
|
1066
|
+
}
|
|
1067
|
+
async releaseClaim(id) {
|
|
1068
|
+
this.pending.delete(id);
|
|
1069
|
+
await this.core.releaseClaim(id);
|
|
1070
|
+
}
|
|
1071
|
+
async migrate() {
|
|
1072
|
+
await this.core.migrate();
|
|
1073
|
+
}
|
|
1074
|
+
getMigrations() {
|
|
1075
|
+
return this.core.getMigrations();
|
|
1076
|
+
}
|
|
1077
|
+
subscribeToChanges(callback, options) {
|
|
1078
|
+
return this.core.subscribeToChanges(callback, options);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function applySendOptions(body, opts) {
|
|
1082
|
+
if (!opts)
|
|
1083
|
+
return body;
|
|
1084
|
+
const out = { ...body };
|
|
1085
|
+
if (opts.delaySeconds != null) {
|
|
1086
|
+
out.visible_at = new Date(Date.now() + opts.delaySeconds * 1000).toISOString();
|
|
1087
|
+
}
|
|
1088
|
+
if (opts.timeoutSeconds != null) {
|
|
1089
|
+
out.deadline_at = new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString();
|
|
1090
|
+
}
|
|
1091
|
+
if (opts.fingerprint != null)
|
|
1092
|
+
out.fingerprint = opts.fingerprint;
|
|
1093
|
+
if (opts.jobRunId != null)
|
|
1094
|
+
out.job_run_id = opts.jobRunId;
|
|
1095
|
+
if (opts.maxAttempts != null)
|
|
1096
|
+
out.max_attempts = opts.maxAttempts;
|
|
1097
|
+
return out;
|
|
1098
|
+
}
|
|
1099
|
+
// src/job-queue/SupabaseJobStore.ts
|
|
1100
|
+
class SupabaseJobStore {
|
|
1101
|
+
core;
|
|
1102
|
+
pending;
|
|
1103
|
+
constructor(core, pending) {
|
|
1104
|
+
this.core = core;
|
|
1105
|
+
this.pending = pending;
|
|
1106
|
+
}
|
|
1107
|
+
get(id) {
|
|
1108
|
+
return this.core.get(id);
|
|
1109
|
+
}
|
|
1110
|
+
async peek(status, num) {
|
|
1111
|
+
return this.core.peek(status, num);
|
|
1112
|
+
}
|
|
1113
|
+
size(status) {
|
|
1114
|
+
return this.core.size(status);
|
|
1115
|
+
}
|
|
1116
|
+
async getByRunId(runId) {
|
|
1117
|
+
return this.core.getByRunId(runId);
|
|
1118
|
+
}
|
|
1119
|
+
outputForInput(input) {
|
|
1120
|
+
return this.core.outputForInput(input);
|
|
1121
|
+
}
|
|
1122
|
+
async saveProgress(id, progress, message, details) {
|
|
1123
|
+
await this.core.saveProgress(id, progress, message, details);
|
|
1124
|
+
}
|
|
1125
|
+
async saveResult(id, output) {
|
|
1126
|
+
const buf = this.pending.get(id) ?? {};
|
|
1127
|
+
buf.output = output ?? null;
|
|
1128
|
+
this.pending.set(id, buf);
|
|
1129
|
+
}
|
|
1130
|
+
async saveError(id, error, errorCode, abortRequested) {
|
|
1131
|
+
const buf = this.pending.get(id) ?? {};
|
|
1132
|
+
buf.error = error;
|
|
1133
|
+
buf.errorCode = errorCode;
|
|
1134
|
+
buf.abortRequested = abortRequested;
|
|
1135
|
+
this.pending.set(id, buf);
|
|
1136
|
+
}
|
|
1137
|
+
async deleteByStatusAndAge(status, olderThanMs) {
|
|
1138
|
+
await this.core.deleteJobsByStatusAndAge(status, olderThanMs);
|
|
1139
|
+
}
|
|
1140
|
+
async delete(id) {
|
|
1141
|
+
this.pending.delete(id);
|
|
1142
|
+
await this.core.delete(id);
|
|
1143
|
+
}
|
|
1144
|
+
async deleteAll() {
|
|
1145
|
+
this.pending.clear();
|
|
1146
|
+
await this.core.deleteAll();
|
|
1147
|
+
}
|
|
1148
|
+
async abort(id) {
|
|
1149
|
+
await this.core.abort(id);
|
|
1150
|
+
}
|
|
1151
|
+
async saveStatus(id, status) {
|
|
1152
|
+
await this.core.saveStatus(id, status);
|
|
1153
|
+
}
|
|
1154
|
+
async create(body, opts) {
|
|
1155
|
+
const enriched = {
|
|
1156
|
+
...body,
|
|
1157
|
+
fingerprint: opts.fingerprint ?? body.fingerprint,
|
|
1158
|
+
job_run_id: opts.jobRunId ?? body.job_run_id,
|
|
1159
|
+
max_attempts: opts.maxAttempts ?? body.max_attempts,
|
|
1160
|
+
deadline_at: opts.timeoutSeconds != null ? new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString() : body.deadline_at
|
|
1161
|
+
};
|
|
1162
|
+
return this.core.add(enriched);
|
|
1163
|
+
}
|
|
1164
|
+
async findActiveByFingerprint(fingerprint, queueName) {
|
|
1165
|
+
return this.core.findActiveByFingerprint(fingerprint, queueName);
|
|
1166
|
+
}
|
|
1167
|
+
async getMany(ids) {
|
|
1168
|
+
return this.core.getMany(ids);
|
|
1169
|
+
}
|
|
1170
|
+
async completeWithResult(id, result) {
|
|
1171
|
+
this.pending.delete(id);
|
|
1172
|
+
await this.core.completeWithResult(id, result);
|
|
1173
|
+
}
|
|
1174
|
+
async failWithError(id, opts) {
|
|
1175
|
+
this.pending.delete(id);
|
|
1176
|
+
await this.core.failWithError(id, opts);
|
|
1177
|
+
}
|
|
1178
|
+
async markEnqueueDeferred(id, opts) {
|
|
1179
|
+
await this.core.finalize(id, {
|
|
1180
|
+
visible_at: opts.visible_at.toISOString(),
|
|
1181
|
+
error_code: opts.errorCode
|
|
1182
|
+
});
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
// src/job-queue/createSupabaseQueue.ts
|
|
1186
|
+
function createSupabaseQueue(queueName, client, opts) {
|
|
1187
|
+
const core = new SupabaseQueueStorage(client, queueName, opts);
|
|
1188
|
+
const pending = new Map;
|
|
1189
|
+
return {
|
|
1190
|
+
messageQueue: new SupabaseMessageQueue(core, pending),
|
|
1191
|
+
jobStore: new SupabaseJobStore(core, pending),
|
|
1192
|
+
core
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
764
1195
|
export {
|
|
1196
|
+
createSupabaseQueue,
|
|
765
1197
|
SupabaseRateLimiterStorage,
|
|
766
1198
|
SupabaseQueueStorage,
|
|
1199
|
+
SupabaseMessageQueue,
|
|
1200
|
+
SupabaseJobStore,
|
|
767
1201
|
SUPABASE_RATE_LIMITER_STORAGE,
|
|
768
1202
|
SUPABASE_QUEUE_STORAGE
|
|
769
1203
|
};
|
|
770
1204
|
|
|
771
|
-
//# debugId=
|
|
1205
|
+
//# debugId=9D95D30C4B098D6464756E2164756E21
|