@workglow/postgres 0.2.36 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/job-queue/PostgresJobStore.d.ts +42 -0
- package/dist/job-queue/PostgresJobStore.d.ts.map +1 -0
- package/dist/job-queue/PostgresMessageQueue.d.ts +38 -0
- package/dist/job-queue/PostgresMessageQueue.d.ts.map +1 -0
- package/dist/job-queue/PostgresQueueStorage.d.ts +65 -10
- package/dist/job-queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts +1 -2
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts.map +1 -1
- package/dist/job-queue/browser.js +546 -55
- package/dist/job-queue/browser.js.map +11 -8
- package/dist/job-queue/common.d.ts +3 -0
- package/dist/job-queue/common.d.ts.map +1 -1
- package/dist/job-queue/createPostgresQueue.d.ts +22 -0
- package/dist/job-queue/createPostgresQueue.d.ts.map +1 -0
- package/dist/job-queue/node.js +546 -55
- package/dist/job-queue/node.js.map +11 -8
- package/dist/migrations/PostgresMigrationRunner.d.ts +6 -1
- package/dist/migrations/PostgresMigrationRunner.d.ts.map +1 -1
- package/dist/migrations/postgresQueueMigrations.d.ts +9 -1
- package/dist/migrations/postgresQueueMigrations.d.ts.map +1 -1
- package/dist/migrations/postgresRateLimiterMigrations.d.ts +1 -1
- package/dist/migrations/postgresRateLimiterMigrations.d.ts.map +1 -1
- package/dist/storage/PostgresKvStorage.d.ts +1 -1
- package/dist/storage/PostgresKvStorage.d.ts.map +1 -1
- package/dist/storage/PostgresTabularStorage.d.ts +30 -1
- package/dist/storage/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/storage/PostgresVectorStorage.d.ts +5 -1
- package/dist/storage/PostgresVectorStorage.d.ts.map +1 -1
- package/dist/storage/browser.js +28 -12
- package/dist/storage/browser.js.map +5 -5
- package/dist/storage/node.js +28 -12
- package/dist/storage/node.js.map +6 -6
- package/dist/text/PostgresFtsTextIndex.d.ts +10 -0
- package/dist/text/PostgresFtsTextIndex.d.ts.map +1 -1
- package/dist/text/browser.js.map +2 -2
- package/dist/text/node.js.map +2 -2
- package/package.json +8 -8
package/dist/job-queue/node.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
// src/job-queue/PostgresQueueStorage.ts
|
|
2
|
-
import {
|
|
3
|
-
import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
|
|
4
|
-
import { JobStatus as JobStatus2 } from "@workglow/job-queue";
|
|
2
|
+
import { JobStatus as JobStatus2, validateLeaseMs } from "@workglow/job-queue";
|
|
5
3
|
import {
|
|
6
4
|
assertPrefixesSafe,
|
|
7
5
|
buildPrefixInsertFragments,
|
|
@@ -10,6 +8,8 @@ import {
|
|
|
10
8
|
getPrefixParamValues,
|
|
11
9
|
PostgresDialect as PostgresDialect2
|
|
12
10
|
} from "@workglow/storage";
|
|
11
|
+
import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
13
|
|
|
14
14
|
// src/migrations/PostgresMigrationRunner.ts
|
|
15
15
|
import {
|
|
@@ -149,11 +149,6 @@ var JOB_STATUS_V1 = [
|
|
|
149
149
|
];
|
|
150
150
|
function assertJobStatusMatchesV1() {
|
|
151
151
|
const current = new Set(Object.values(JobStatus));
|
|
152
|
-
for (const v of JOB_STATUS_V1) {
|
|
153
|
-
if (!current.has(v)) {
|
|
154
|
-
throw new Error(`JobStatus const is missing v1 enum value "${v}"; v1 migration values are frozen.`);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
152
|
for (const v of current) {
|
|
158
153
|
if (!JOB_STATUS_V1.includes(v)) {
|
|
159
154
|
throw new Error(`JobStatus contains "${v}" which is not in JOB_STATUS_V1. ` + `Add a new migration that runs "ALTER TYPE job_status ADD VALUE IF NOT EXISTS '${v}'" ` + `instead of mutating the v1 enum literal.`);
|
|
@@ -254,6 +249,92 @@ function postgresQueueMigrations(tableName, prefixes) {
|
|
|
254
249
|
});
|
|
255
250
|
}
|
|
256
251
|
}
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
component,
|
|
255
|
+
version: 2,
|
|
256
|
+
description: "Add abort_requested_at and lease_expires_at columns",
|
|
257
|
+
async up(db) {
|
|
258
|
+
await db.query(`
|
|
259
|
+
ALTER TABLE ${tableName}
|
|
260
|
+
ADD COLUMN IF NOT EXISTS abort_requested_at timestamp with time zone,
|
|
261
|
+
ADD COLUMN IF NOT EXISTS lease_expires_at timestamp with time zone
|
|
262
|
+
`);
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
component,
|
|
267
|
+
version: 3,
|
|
268
|
+
description: "Rename run_after→visible_at, last_ran_at→last_attempted_at, run_attempts→attempts, max_retries→max_attempts, worker_id→lease_owner; drop run_after-keyed indexes and recreate visible_at-keyed",
|
|
269
|
+
async up(db) {
|
|
270
|
+
await db.query(`
|
|
271
|
+
DO $$
|
|
272
|
+
BEGIN
|
|
273
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='run_after' AND table_schema=current_schema()) THEN
|
|
274
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN run_after TO visible_at';
|
|
275
|
+
END IF;
|
|
276
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='last_ran_at' AND table_schema=current_schema()) THEN
|
|
277
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN last_ran_at TO last_attempted_at';
|
|
278
|
+
END IF;
|
|
279
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='run_attempts' AND table_schema=current_schema()) THEN
|
|
280
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN run_attempts TO attempts';
|
|
281
|
+
END IF;
|
|
282
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='max_retries' AND table_schema=current_schema()) THEN
|
|
283
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN max_retries TO max_attempts';
|
|
284
|
+
EXECUTE 'ALTER TABLE ${tableName} ALTER COLUMN max_attempts SET DEFAULT 10';
|
|
285
|
+
END IF;
|
|
286
|
+
IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='${tableName}' AND column_name='worker_id' AND table_schema=current_schema()) THEN
|
|
287
|
+
EXECUTE 'ALTER TABLE ${tableName} RENAME COLUMN worker_id TO lease_owner';
|
|
288
|
+
END IF;
|
|
289
|
+
END $$
|
|
290
|
+
`);
|
|
291
|
+
await db.query(`DROP INDEX IF EXISTS job_fetcher${indexSuffix}_idx`);
|
|
292
|
+
await db.query(`DROP INDEX IF EXISTS job_queue_fetcher${indexSuffix}_idx`);
|
|
293
|
+
await db.query(`
|
|
294
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
295
|
+
ON ${tableName} (${prefixIndexPrefix}id, status, visible_at)
|
|
296
|
+
`);
|
|
297
|
+
await db.query(`
|
|
298
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
299
|
+
ON ${tableName} (${prefixIndexPrefix}queue, status, visible_at)
|
|
300
|
+
`);
|
|
301
|
+
}
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
component,
|
|
305
|
+
version: 4,
|
|
306
|
+
description: "Add UNIQUE partial index for findActiveByFingerprint O(1) lookup + fingerprint dedup at the DB layer (H2)",
|
|
307
|
+
async up(db) {
|
|
308
|
+
await db.query(`
|
|
309
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_${tableName}_fingerprint_active
|
|
310
|
+
ON ${tableName}(${prefixIndexPrefix}queue, fingerprint)
|
|
311
|
+
WHERE status IN ('PENDING','PROCESSING')
|
|
312
|
+
`);
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
component,
|
|
317
|
+
version: 5,
|
|
318
|
+
description: "Converge idx_<table>_fingerprint_active to UNIQUE for DBs that applied the pre-edit v4 (non-unique) variant",
|
|
319
|
+
async up(db) {
|
|
320
|
+
const indexName = `idx_${tableName}_fingerprint_active`;
|
|
321
|
+
const result = await db.query(`SELECT i.indisunique
|
|
322
|
+
FROM pg_class c
|
|
323
|
+
JOIN pg_index i ON i.indexrelid = c.oid
|
|
324
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
325
|
+
WHERE c.relname = $1
|
|
326
|
+
AND n.nspname = current_schema()`, [indexName]);
|
|
327
|
+
const existing = result.rows[0];
|
|
328
|
+
if (existing && existing.indisunique) {
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
await db.query(`DROP INDEX IF EXISTS ${indexName}`);
|
|
332
|
+
await db.query(`
|
|
333
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${indexName}
|
|
334
|
+
ON ${tableName}(${prefixIndexPrefix}queue, fingerprint)
|
|
335
|
+
WHERE status IN ('PENDING','PROCESSING')
|
|
336
|
+
`);
|
|
337
|
+
}
|
|
257
338
|
}
|
|
258
339
|
];
|
|
259
340
|
}
|
|
@@ -302,32 +383,32 @@ class PostgresQueueStorage {
|
|
|
302
383
|
const now = new Date().toISOString();
|
|
303
384
|
job.queue = this.queueName;
|
|
304
385
|
job.job_run_id = job.job_run_id ?? uuid4();
|
|
305
|
-
job.fingerprint = await makeFingerprint(job.input);
|
|
386
|
+
job.fingerprint = job.fingerprint ?? await makeFingerprint(job.input);
|
|
306
387
|
job.status = JobStatus2.PENDING;
|
|
307
388
|
job.progress = 0;
|
|
308
389
|
job.progress_message = "";
|
|
309
390
|
job.progress_details = null;
|
|
310
391
|
job.created_at = now;
|
|
311
|
-
job.
|
|
392
|
+
job.visible_at = now;
|
|
312
393
|
const prefixColumnNames = getPrefixColumnNames(this.prefixes);
|
|
313
394
|
const { columns: prefixColumnsInsert, placeholders: prefixParamPlaceholders } = buildPrefixInsertFragments(PostgresDialect2, this.prefixes, 1);
|
|
314
395
|
const prefixParamValues = this.getPrefixParamValues();
|
|
315
396
|
const baseParamStart = prefixColumnNames.length + 1;
|
|
316
397
|
const sql = `
|
|
317
398
|
INSERT INTO ${this.tableName}(
|
|
318
|
-
${prefixColumnsInsert}queue,
|
|
319
|
-
fingerprint,
|
|
320
|
-
input,
|
|
321
|
-
|
|
399
|
+
${prefixColumnsInsert}queue,
|
|
400
|
+
fingerprint,
|
|
401
|
+
input,
|
|
402
|
+
visible_at,
|
|
322
403
|
created_at,
|
|
323
404
|
deadline_at,
|
|
324
|
-
|
|
325
|
-
job_run_id,
|
|
326
|
-
progress,
|
|
327
|
-
progress_message,
|
|
405
|
+
max_attempts,
|
|
406
|
+
job_run_id,
|
|
407
|
+
progress,
|
|
408
|
+
progress_message,
|
|
328
409
|
progress_details
|
|
329
410
|
)
|
|
330
|
-
VALUES
|
|
411
|
+
VALUES
|
|
331
412
|
(${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
|
|
332
413
|
RETURNING id`;
|
|
333
414
|
const params = [
|
|
@@ -335,16 +416,31 @@ class PostgresQueueStorage {
|
|
|
335
416
|
job.queue,
|
|
336
417
|
job.fingerprint,
|
|
337
418
|
JSON.stringify(job.input),
|
|
338
|
-
job.
|
|
419
|
+
job.visible_at,
|
|
339
420
|
job.created_at,
|
|
340
421
|
job.deadline_at,
|
|
341
|
-
job.
|
|
422
|
+
job.max_attempts,
|
|
342
423
|
job.job_run_id,
|
|
343
424
|
job.progress,
|
|
344
425
|
job.progress_message,
|
|
345
426
|
job.progress_details ? JSON.stringify(job.progress_details) : null
|
|
346
427
|
];
|
|
347
|
-
|
|
428
|
+
let result;
|
|
429
|
+
try {
|
|
430
|
+
result = await this.db.query(sql, params);
|
|
431
|
+
} catch (err) {
|
|
432
|
+
const e = err;
|
|
433
|
+
const isUniqueViolation = e?.code === "23505";
|
|
434
|
+
const involvesFingerprint = typeof e?.constraint === "string" ? e.constraint.includes("fingerprint") : typeof e?.message === "string" ? e.message.includes("fingerprint") : false;
|
|
435
|
+
if (isUniqueViolation && involvesFingerprint && job.fingerprint) {
|
|
436
|
+
const winner = await this.findActiveByFingerprint(job.fingerprint, this.queueName);
|
|
437
|
+
if (winner?.id != null) {
|
|
438
|
+
job.id = winner.id;
|
|
439
|
+
return winner.id;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
throw err;
|
|
443
|
+
}
|
|
348
444
|
if (!result)
|
|
349
445
|
throw new Error("Failed to add to queue");
|
|
350
446
|
job.id = result.rows[0].id;
|
|
@@ -369,32 +465,68 @@ class PostgresQueueStorage {
|
|
|
369
465
|
FROM ${this.tableName}
|
|
370
466
|
WHERE queue = $1
|
|
371
467
|
AND status = $2${prefixConditions}
|
|
372
|
-
ORDER BY
|
|
468
|
+
ORDER BY visible_at ASC
|
|
373
469
|
LIMIT $3
|
|
374
470
|
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
|
|
375
471
|
if (!result)
|
|
376
472
|
return [];
|
|
377
473
|
return result.rows;
|
|
378
474
|
}
|
|
379
|
-
async next(workerId) {
|
|
380
|
-
const
|
|
475
|
+
async next(workerId, opts) {
|
|
476
|
+
const leaseMs = opts?.leaseMs ?? 30000;
|
|
477
|
+
validateLeaseMs(leaseMs, "leaseMs");
|
|
478
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(7);
|
|
381
479
|
const result = await this.db.query(`
|
|
382
|
-
UPDATE ${this.tableName}
|
|
383
|
-
SET status = $1,
|
|
480
|
+
UPDATE ${this.tableName}
|
|
481
|
+
SET status = $1,
|
|
482
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC',
|
|
483
|
+
lease_owner = $4,
|
|
484
|
+
lease_expires_at = NOW() AT TIME ZONE 'UTC' + ($2 * INTERVAL '1 millisecond'),
|
|
485
|
+
-- A reclaimed PROCESSING row was claimed by a now-crashed worker;
|
|
486
|
+
-- that constitutes one used-up attempt against max_attempts.
|
|
487
|
+
-- PENDING claims must not be charged here — JobQueueWorker's
|
|
488
|
+
-- existing validateJobState() will FAIL the job in the next-step
|
|
489
|
+
-- branch when attempts >= max_attempts.
|
|
490
|
+
attempts = CASE WHEN status = $6 THEN attempts + 1 ELSE attempts END,
|
|
491
|
+
-- Always clear any stale abort_requested_at on (re)claim. A PROCESSING
|
|
492
|
+
-- row may have had abort_requested_at set before the worker crashed;
|
|
493
|
+
-- the new owner must start with a clean slate or the worker will see
|
|
494
|
+
-- the abort flag immediately and never run user code.
|
|
495
|
+
abort_requested_at = NULL
|
|
384
496
|
WHERE id = (
|
|
385
|
-
SELECT id
|
|
386
|
-
FROM ${this.tableName}
|
|
387
|
-
WHERE queue = $
|
|
388
|
-
AND
|
|
497
|
+
SELECT id
|
|
498
|
+
FROM ${this.tableName}
|
|
499
|
+
WHERE queue = $3
|
|
500
|
+
AND (
|
|
501
|
+
(status = $5 AND visible_at <= NOW() AT TIME ZONE 'UTC')
|
|
502
|
+
OR (status = $6 AND (lease_expires_at IS NULL OR lease_expires_at < NOW() AT TIME ZONE 'UTC'))
|
|
503
|
+
)
|
|
389
504
|
${prefixConditions}
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
FOR UPDATE SKIP LOCKED
|
|
505
|
+
ORDER BY visible_at ASC
|
|
506
|
+
FOR UPDATE SKIP LOCKED
|
|
393
507
|
LIMIT 1
|
|
394
508
|
)
|
|
395
|
-
RETURNING *`, [
|
|
509
|
+
RETURNING *`, [
|
|
510
|
+
JobStatus2.PROCESSING,
|
|
511
|
+
leaseMs,
|
|
512
|
+
this.queueName,
|
|
513
|
+
workerId,
|
|
514
|
+
JobStatus2.PENDING,
|
|
515
|
+
JobStatus2.PROCESSING,
|
|
516
|
+
...prefixParams
|
|
517
|
+
]);
|
|
396
518
|
return result?.rows?.[0] ?? undefined;
|
|
397
519
|
}
|
|
520
|
+
async extendLease(id, workerId, ms) {
|
|
521
|
+
validateLeaseMs(ms, "ms");
|
|
522
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
523
|
+
const result = await this.db.query(`UPDATE ${this.tableName}
|
|
524
|
+
SET lease_expires_at = NOW() AT TIME ZONE 'UTC' + ($1 * INTERVAL '1 millisecond')
|
|
525
|
+
WHERE id = $2 AND queue = $3 AND lease_owner = $4 AND status = 'PROCESSING'${prefixConditions}`, [ms, id, this.queueName, workerId, ...prefixParams]);
|
|
526
|
+
if (!result || result.rowCount === 0) {
|
|
527
|
+
throw new Error(`extendLease failed: job ${String(id)} is not PROCESSING or lease is not owned by worker ${workerId}`);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
398
530
|
async size(status = JobStatus2.PENDING) {
|
|
399
531
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
400
532
|
const result = await this.db.query(`
|
|
@@ -420,22 +552,23 @@ class PostgresQueueStorage {
|
|
|
420
552
|
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
421
553
|
} else if (jobDetails.status === JobStatus2.PENDING) {
|
|
422
554
|
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
423
|
-
await this.db.query(`UPDATE ${this.tableName}
|
|
424
|
-
SET
|
|
425
|
-
error = $1,
|
|
555
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
556
|
+
SET
|
|
557
|
+
error = $1,
|
|
426
558
|
error_code = $2,
|
|
427
|
-
status = $3,
|
|
428
|
-
|
|
559
|
+
status = $3,
|
|
560
|
+
visible_at = $4,
|
|
429
561
|
progress = 0,
|
|
430
562
|
progress_message = '',
|
|
431
563
|
progress_details = NULL,
|
|
432
|
-
|
|
433
|
-
|
|
564
|
+
attempts = attempts + 1,
|
|
565
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC',
|
|
566
|
+
abort_requested_at = NULL
|
|
434
567
|
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
435
568
|
jobDetails.error,
|
|
436
569
|
jobDetails.error_code,
|
|
437
570
|
jobDetails.status,
|
|
438
|
-
jobDetails.
|
|
571
|
+
jobDetails.visible_at,
|
|
439
572
|
jobDetails.id,
|
|
440
573
|
this.queueName,
|
|
441
574
|
...prefixParams
|
|
@@ -452,9 +585,9 @@ class PostgresQueueStorage {
|
|
|
452
585
|
progress = 100,
|
|
453
586
|
progress_message = '',
|
|
454
587
|
progress_details = NULL,
|
|
455
|
-
|
|
588
|
+
attempts = attempts + 1,
|
|
456
589
|
completed_at = NOW() AT TIME ZONE 'UTC',
|
|
457
|
-
|
|
590
|
+
last_attempted_at = NOW() AT TIME ZONE 'UTC'
|
|
458
591
|
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
459
592
|
jobDetails.output ? JSON.stringify(jobDetails.output) : null,
|
|
460
593
|
jobDetails.error ?? null,
|
|
@@ -466,6 +599,51 @@ class PostgresQueueStorage {
|
|
|
466
599
|
]);
|
|
467
600
|
}
|
|
468
601
|
}
|
|
602
|
+
async finalize(id, fields) {
|
|
603
|
+
const sets = [];
|
|
604
|
+
const params = [];
|
|
605
|
+
let nextParam = 1;
|
|
606
|
+
const push = (col, value) => {
|
|
607
|
+
sets.push(`${col} = $${nextParam}`);
|
|
608
|
+
params.push(value);
|
|
609
|
+
nextParam += 1;
|
|
610
|
+
};
|
|
611
|
+
if ("output" in fields) {
|
|
612
|
+
push("output", fields.output != null ? JSON.stringify(fields.output) : null);
|
|
613
|
+
}
|
|
614
|
+
if ("error" in fields)
|
|
615
|
+
push("error", fields.error ?? null);
|
|
616
|
+
if ("error_code" in fields)
|
|
617
|
+
push("error_code", fields.error_code ?? null);
|
|
618
|
+
if ("status" in fields)
|
|
619
|
+
push("status", fields.status);
|
|
620
|
+
if ("completed_at" in fields)
|
|
621
|
+
push("completed_at", fields.completed_at ?? null);
|
|
622
|
+
if ("abort_requested_at" in fields) {
|
|
623
|
+
push("abort_requested_at", fields.abort_requested_at ?? null);
|
|
624
|
+
}
|
|
625
|
+
if ("lease_owner" in fields)
|
|
626
|
+
push("lease_owner", fields.lease_owner ?? null);
|
|
627
|
+
if ("progress" in fields)
|
|
628
|
+
push("progress", fields.progress ?? 0);
|
|
629
|
+
if ("progress_message" in fields)
|
|
630
|
+
push("progress_message", fields.progress_message ?? "");
|
|
631
|
+
if ("progress_details" in fields) {
|
|
632
|
+
push("progress_details", fields.progress_details != null ? JSON.stringify(fields.progress_details) : null);
|
|
633
|
+
}
|
|
634
|
+
if ("visible_at" in fields)
|
|
635
|
+
push("visible_at", fields.visible_at ?? null);
|
|
636
|
+
if (sets.length === 0)
|
|
637
|
+
return;
|
|
638
|
+
const idParam = nextParam;
|
|
639
|
+
nextParam += 1;
|
|
640
|
+
const queueParam = nextParam;
|
|
641
|
+
nextParam += 1;
|
|
642
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(nextParam);
|
|
643
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
644
|
+
SET ${sets.join(", ")}
|
|
645
|
+
WHERE id = $${idParam} AND queue = $${queueParam}${prefixConditions}`, [...params, id, this.queueName, ...prefixParams]);
|
|
646
|
+
}
|
|
469
647
|
async deleteAll() {
|
|
470
648
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
471
649
|
await this.db.query(`
|
|
@@ -484,23 +662,37 @@ class PostgresQueueStorage {
|
|
|
484
662
|
return result.rows[0].output;
|
|
485
663
|
}
|
|
486
664
|
async abort(jobId) {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
UPDATE ${this.tableName}
|
|
490
|
-
|
|
491
|
-
|
|
665
|
+
{
|
|
666
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
667
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
668
|
+
SET status = 'FAILED',
|
|
669
|
+
abort_requested_at = NOW() AT TIME ZONE 'UTC',
|
|
670
|
+
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
671
|
+
WHERE id = $1 AND queue = $2 AND status = $3${prefixConditions}`, [jobId, this.queueName, JobStatus2.PENDING, ...prefixParams]);
|
|
672
|
+
}
|
|
673
|
+
{
|
|
674
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
675
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
676
|
+
SET abort_requested_at = NOW() AT TIME ZONE 'UTC'
|
|
677
|
+
WHERE id = $1 AND queue = $2 AND status = $3${prefixConditions}`, [jobId, this.queueName, JobStatus2.PROCESSING, ...prefixParams]);
|
|
678
|
+
}
|
|
492
679
|
}
|
|
493
|
-
async
|
|
680
|
+
async releaseClaim(jobId) {
|
|
494
681
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
495
682
|
await this.db.query(`
|
|
496
683
|
UPDATE ${this.tableName}
|
|
497
684
|
SET status = 'PENDING',
|
|
498
|
-
|
|
685
|
+
lease_owner = NULL,
|
|
499
686
|
progress = 0,
|
|
500
687
|
progress_message = '',
|
|
501
|
-
progress_details = NULL
|
|
688
|
+
progress_details = NULL,
|
|
689
|
+
abort_requested_at = NULL
|
|
502
690
|
WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
503
691
|
}
|
|
692
|
+
async saveStatus(jobId, status) {
|
|
693
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
694
|
+
await this.db.query(`UPDATE ${this.tableName} SET status = $1 WHERE id = $2 AND queue = $3${prefixConditions}`, [status, jobId, this.queueName, ...prefixParams]);
|
|
695
|
+
}
|
|
504
696
|
async getByRunId(job_run_id) {
|
|
505
697
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
506
698
|
const result = await this.db.query(`
|
|
@@ -529,6 +721,66 @@ class PostgresQueueStorage {
|
|
|
529
721
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
530
722
|
await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
531
723
|
}
|
|
724
|
+
async findActiveByFingerprint(fingerprint, queueName) {
|
|
725
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
726
|
+
const result = await this.db.query(`SELECT * FROM ${this.tableName}
|
|
727
|
+
WHERE fingerprint = $1 AND queue = $2
|
|
728
|
+
AND status IN ('PENDING','PROCESSING')${prefixConditions}
|
|
729
|
+
ORDER BY created_at DESC
|
|
730
|
+
LIMIT 1`, [fingerprint, queueName, ...prefixParams]);
|
|
731
|
+
if (!result || result.rows.length === 0)
|
|
732
|
+
return;
|
|
733
|
+
return result.rows[0];
|
|
734
|
+
}
|
|
735
|
+
async getMany(ids) {
|
|
736
|
+
if (ids.length === 0)
|
|
737
|
+
return [];
|
|
738
|
+
const numericIds = ids.map((id) => Number(id)).filter((n) => Number.isInteger(n) && n > 0);
|
|
739
|
+
if (numericIds.length === 0)
|
|
740
|
+
return ids.map(() => {
|
|
741
|
+
return;
|
|
742
|
+
});
|
|
743
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
744
|
+
const result = await this.db.query(`SELECT * FROM ${this.tableName}
|
|
745
|
+
WHERE id = ANY($1::int[]) AND queue = $2${prefixConditions}`, [numericIds, this.queueName, ...prefixParams]);
|
|
746
|
+
if (!result)
|
|
747
|
+
return ids.map(() => {
|
|
748
|
+
return;
|
|
749
|
+
});
|
|
750
|
+
const map = new Map;
|
|
751
|
+
for (const row of result.rows) {
|
|
752
|
+
map.set(Number(row.id), row);
|
|
753
|
+
}
|
|
754
|
+
return ids.map((id) => map.get(Number(id)));
|
|
755
|
+
}
|
|
756
|
+
async completeWithResult(id, result) {
|
|
757
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
758
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
759
|
+
SET output = $1,
|
|
760
|
+
status = 'COMPLETED',
|
|
761
|
+
progress = 100,
|
|
762
|
+
progress_message = '',
|
|
763
|
+
progress_details = NULL,
|
|
764
|
+
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
765
|
+
WHERE id = $2 AND queue = $3${prefixConditions}`, [result != null ? JSON.stringify(result) : null, id, this.queueName, ...prefixParams]);
|
|
766
|
+
}
|
|
767
|
+
async failWithError(id, opts) {
|
|
768
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
769
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
770
|
+
SET error = COALESCE($1, error),
|
|
771
|
+
error_code = COALESCE($2, error_code),
|
|
772
|
+
abort_requested_at = CASE WHEN $3 THEN NOW() AT TIME ZONE 'UTC' ELSE abort_requested_at END,
|
|
773
|
+
status = 'FAILED',
|
|
774
|
+
completed_at = COALESCE(completed_at, NOW() AT TIME ZONE 'UTC')
|
|
775
|
+
WHERE id = $4 AND queue = $5${prefixConditions}`, [
|
|
776
|
+
"error" in opts ? opts.error ?? null : null,
|
|
777
|
+
"errorCode" in opts ? opts.errorCode ?? null : null,
|
|
778
|
+
opts.abortRequested === true,
|
|
779
|
+
id,
|
|
780
|
+
this.queueName,
|
|
781
|
+
...prefixParams
|
|
782
|
+
]);
|
|
783
|
+
}
|
|
532
784
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
533
785
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
534
786
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
@@ -663,7 +915,6 @@ class PostgresQueueStorage {
|
|
|
663
915
|
}
|
|
664
916
|
}
|
|
665
917
|
// src/job-queue/PostgresRateLimiterStorage.ts
|
|
666
|
-
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
667
918
|
import {
|
|
668
919
|
assertPrefixesSafe as assertPrefixesSafe2,
|
|
669
920
|
buildPrefixWhereClause as buildPrefixWhereClause2,
|
|
@@ -671,6 +922,7 @@ import {
|
|
|
671
922
|
getPrefixParamValues as getPrefixParamValues2,
|
|
672
923
|
PostgresDialect as PostgresDialect4
|
|
673
924
|
} from "@workglow/storage";
|
|
925
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
674
926
|
|
|
675
927
|
// src/migrations/postgresRateLimiterMigrations.ts
|
|
676
928
|
import {
|
|
@@ -895,14 +1147,253 @@ class PostgresRateLimiterStorage {
|
|
|
895
1147
|
await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
896
1148
|
}
|
|
897
1149
|
}
|
|
1150
|
+
// src/job-queue/PostgresMessageQueue.ts
|
|
1151
|
+
class PostgresClaim {
|
|
1152
|
+
core;
|
|
1153
|
+
pending;
|
|
1154
|
+
id;
|
|
1155
|
+
body;
|
|
1156
|
+
attempts;
|
|
1157
|
+
workerId;
|
|
1158
|
+
constructor(core, pending, id, body, attempts, workerId) {
|
|
1159
|
+
this.core = core;
|
|
1160
|
+
this.pending = pending;
|
|
1161
|
+
this.id = id;
|
|
1162
|
+
this.body = body;
|
|
1163
|
+
this.attempts = attempts;
|
|
1164
|
+
this.workerId = workerId;
|
|
1165
|
+
}
|
|
1166
|
+
async ack(result) {
|
|
1167
|
+
const buf = this.pending.get(this.id);
|
|
1168
|
+
this.pending.delete(this.id);
|
|
1169
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1170
|
+
const output = result !== undefined ? result : buf?.output !== undefined ? buf.output : current.output ?? null;
|
|
1171
|
+
await this.core.finalize(this.id, {
|
|
1172
|
+
output,
|
|
1173
|
+
error: null,
|
|
1174
|
+
error_code: null,
|
|
1175
|
+
status: "COMPLETED",
|
|
1176
|
+
completed_at: current.completed_at ?? new Date().toISOString()
|
|
1177
|
+
});
|
|
1178
|
+
}
|
|
1179
|
+
async retry(opts) {
|
|
1180
|
+
this.pending.delete(this.id);
|
|
1181
|
+
const delay = opts?.delaySeconds ?? 0;
|
|
1182
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1183
|
+
await this.core.complete({
|
|
1184
|
+
...current,
|
|
1185
|
+
status: "PENDING",
|
|
1186
|
+
lease_owner: null,
|
|
1187
|
+
lease_expires_at: null,
|
|
1188
|
+
visible_at: new Date(Date.now() + delay * 1000).toISOString(),
|
|
1189
|
+
progress: 0,
|
|
1190
|
+
progress_message: "",
|
|
1191
|
+
progress_details: null
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
async fail(opts) {
|
|
1195
|
+
opts?.permanent;
|
|
1196
|
+
const buf = this.pending.get(this.id);
|
|
1197
|
+
this.pending.delete(this.id);
|
|
1198
|
+
const current = await this.core.get(this.id) ?? this.body;
|
|
1199
|
+
const error = opts?.error !== undefined ? opts.error : buf?.error !== undefined ? buf.error : current.error ?? null;
|
|
1200
|
+
const errorCode = opts?.errorCode !== undefined ? opts.errorCode : buf?.errorCode !== undefined ? buf.errorCode : current.error_code ?? null;
|
|
1201
|
+
const abortRequested = opts?.abortRequested !== undefined ? opts.abortRequested : buf?.abortRequested ?? false;
|
|
1202
|
+
await this.core.finalize(this.id, {
|
|
1203
|
+
error,
|
|
1204
|
+
error_code: errorCode,
|
|
1205
|
+
abort_requested_at: abortRequested ? current.abort_requested_at ?? new Date().toISOString() : current.abort_requested_at ?? null,
|
|
1206
|
+
status: "FAILED",
|
|
1207
|
+
completed_at: current.completed_at ?? new Date().toISOString()
|
|
1208
|
+
});
|
|
1209
|
+
}
|
|
1210
|
+
async extendLease(ms) {
|
|
1211
|
+
await this.core.extendLease(this.id, this.workerId, ms);
|
|
1212
|
+
}
|
|
1213
|
+
async disable() {
|
|
1214
|
+
this.pending.delete(this.id);
|
|
1215
|
+
const current = await this.core.get(this.id);
|
|
1216
|
+
const completedAt = current?.completed_at ?? new Date().toISOString();
|
|
1217
|
+
await this.core.finalize(this.id, {
|
|
1218
|
+
status: "DISABLED",
|
|
1219
|
+
completed_at: completedAt,
|
|
1220
|
+
lease_owner: null,
|
|
1221
|
+
progress: 0,
|
|
1222
|
+
progress_message: "",
|
|
1223
|
+
progress_details: null
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
class PostgresMessageQueue {
|
|
1229
|
+
scope;
|
|
1230
|
+
core;
|
|
1231
|
+
pending;
|
|
1232
|
+
constructor(core, pending) {
|
|
1233
|
+
this.core = core;
|
|
1234
|
+
this.pending = pending;
|
|
1235
|
+
this.scope = core.scope;
|
|
1236
|
+
}
|
|
1237
|
+
async send(body, opts) {
|
|
1238
|
+
return this.core.add(applySendOptions(body, opts));
|
|
1239
|
+
}
|
|
1240
|
+
async sendBatch(bodies, opts) {
|
|
1241
|
+
const ids = [];
|
|
1242
|
+
for (const body of bodies) {
|
|
1243
|
+
ids.push(await this.send(body, opts));
|
|
1244
|
+
}
|
|
1245
|
+
return ids;
|
|
1246
|
+
}
|
|
1247
|
+
async receive(opts) {
|
|
1248
|
+
const max = Math.max(1, opts.max ?? 1);
|
|
1249
|
+
const claims = [];
|
|
1250
|
+
while (claims.length < max) {
|
|
1251
|
+
const job = await this.core.next(opts.workerId, { leaseMs: opts.leaseMs });
|
|
1252
|
+
if (!job)
|
|
1253
|
+
break;
|
|
1254
|
+
claims.push(new PostgresClaim(this.core, this.pending, job.id, job, job.attempts ?? 0, opts.workerId));
|
|
1255
|
+
}
|
|
1256
|
+
return claims;
|
|
1257
|
+
}
|
|
1258
|
+
async releaseClaim(id) {
|
|
1259
|
+
this.pending.delete(id);
|
|
1260
|
+
await this.core.releaseClaim(id);
|
|
1261
|
+
}
|
|
1262
|
+
async migrate() {
|
|
1263
|
+
await this.core.migrate();
|
|
1264
|
+
}
|
|
1265
|
+
getMigrations() {
|
|
1266
|
+
return this.core.getMigrations();
|
|
1267
|
+
}
|
|
1268
|
+
subscribeToChanges(callback, options) {
|
|
1269
|
+
return this.core.subscribeToChanges(callback, options);
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
function applySendOptions(body, opts) {
|
|
1273
|
+
if (!opts)
|
|
1274
|
+
return body;
|
|
1275
|
+
const out = { ...body };
|
|
1276
|
+
if (opts.delaySeconds != null) {
|
|
1277
|
+
out.visible_at = new Date(Date.now() + opts.delaySeconds * 1000).toISOString();
|
|
1278
|
+
}
|
|
1279
|
+
if (opts.timeoutSeconds != null) {
|
|
1280
|
+
out.deadline_at = new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString();
|
|
1281
|
+
}
|
|
1282
|
+
if (opts.fingerprint != null)
|
|
1283
|
+
out.fingerprint = opts.fingerprint;
|
|
1284
|
+
if (opts.jobRunId != null)
|
|
1285
|
+
out.job_run_id = opts.jobRunId;
|
|
1286
|
+
if (opts.maxAttempts != null)
|
|
1287
|
+
out.max_attempts = opts.maxAttempts;
|
|
1288
|
+
return out;
|
|
1289
|
+
}
|
|
1290
|
+
// src/job-queue/PostgresJobStore.ts
|
|
1291
|
+
class PostgresJobStore {
|
|
1292
|
+
core;
|
|
1293
|
+
pending;
|
|
1294
|
+
constructor(core, pending) {
|
|
1295
|
+
this.core = core;
|
|
1296
|
+
this.pending = pending;
|
|
1297
|
+
}
|
|
1298
|
+
get(id) {
|
|
1299
|
+
return this.core.get(id);
|
|
1300
|
+
}
|
|
1301
|
+
async peek(status, num) {
|
|
1302
|
+
return this.core.peek(status, num);
|
|
1303
|
+
}
|
|
1304
|
+
size(status) {
|
|
1305
|
+
return this.core.size(status);
|
|
1306
|
+
}
|
|
1307
|
+
async getByRunId(runId) {
|
|
1308
|
+
return this.core.getByRunId(runId);
|
|
1309
|
+
}
|
|
1310
|
+
outputForInput(input) {
|
|
1311
|
+
return this.core.outputForInput(input);
|
|
1312
|
+
}
|
|
1313
|
+
async saveProgress(id, progress, message, details) {
|
|
1314
|
+
await this.core.saveProgress(id, progress, message, details);
|
|
1315
|
+
}
|
|
1316
|
+
async saveResult(id, output) {
|
|
1317
|
+
const buf = this.pending.get(id) ?? {};
|
|
1318
|
+
buf.output = output ?? null;
|
|
1319
|
+
this.pending.set(id, buf);
|
|
1320
|
+
}
|
|
1321
|
+
async saveError(id, error, errorCode, abortRequested) {
|
|
1322
|
+
const buf = this.pending.get(id) ?? {};
|
|
1323
|
+
buf.error = error;
|
|
1324
|
+
buf.errorCode = errorCode;
|
|
1325
|
+
buf.abortRequested = abortRequested;
|
|
1326
|
+
this.pending.set(id, buf);
|
|
1327
|
+
}
|
|
1328
|
+
async deleteByStatusAndAge(status, olderThanMs) {
|
|
1329
|
+
await this.core.deleteJobsByStatusAndAge(status, olderThanMs);
|
|
1330
|
+
}
|
|
1331
|
+
async delete(id) {
|
|
1332
|
+
this.pending.delete(id);
|
|
1333
|
+
await this.core.delete(id);
|
|
1334
|
+
}
|
|
1335
|
+
async deleteAll() {
|
|
1336
|
+
this.pending.clear();
|
|
1337
|
+
await this.core.deleteAll();
|
|
1338
|
+
}
|
|
1339
|
+
async abort(id) {
|
|
1340
|
+
await this.core.abort(id);
|
|
1341
|
+
}
|
|
1342
|
+
async saveStatus(id, status) {
|
|
1343
|
+
await this.core.saveStatus(id, status);
|
|
1344
|
+
}
|
|
1345
|
+
async create(body, opts) {
|
|
1346
|
+
const enriched = {
|
|
1347
|
+
...body,
|
|
1348
|
+
fingerprint: opts.fingerprint ?? body.fingerprint,
|
|
1349
|
+
job_run_id: opts.jobRunId ?? body.job_run_id,
|
|
1350
|
+
max_attempts: opts.maxAttempts ?? body.max_attempts,
|
|
1351
|
+
deadline_at: opts.timeoutSeconds != null ? new Date(Date.now() + opts.timeoutSeconds * 1000).toISOString() : body.deadline_at
|
|
1352
|
+
};
|
|
1353
|
+
return this.core.add(enriched);
|
|
1354
|
+
}
|
|
1355
|
+
async findActiveByFingerprint(fingerprint, queueName) {
|
|
1356
|
+
return this.core.findActiveByFingerprint(fingerprint, queueName);
|
|
1357
|
+
}
|
|
1358
|
+
async getMany(ids) {
|
|
1359
|
+
return this.core.getMany(ids);
|
|
1360
|
+
}
|
|
1361
|
+
async completeWithResult(id, result) {
|
|
1362
|
+
this.pending.delete(id);
|
|
1363
|
+
await this.core.completeWithResult(id, result);
|
|
1364
|
+
}
|
|
1365
|
+
async failWithError(id, opts) {
|
|
1366
|
+
this.pending.delete(id);
|
|
1367
|
+
await this.core.failWithError(id, opts);
|
|
1368
|
+
}
|
|
1369
|
+
async markEnqueueDeferred(id, opts) {
|
|
1370
|
+
await this.core.finalize(id, {
|
|
1371
|
+
visible_at: opts.visible_at.toISOString(),
|
|
1372
|
+
error_code: opts.errorCode
|
|
1373
|
+
});
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
// src/job-queue/createPostgresQueue.ts
|
|
1377
|
+
function createPostgresQueue(queueName, pool, opts) {
|
|
1378
|
+
const core = new PostgresQueueStorage(pool, queueName, opts);
|
|
1379
|
+
const pending = new Map;
|
|
1380
|
+
return {
|
|
1381
|
+
messageQueue: new PostgresMessageQueue(core, pending),
|
|
1382
|
+
jobStore: new PostgresJobStore(core, pending),
|
|
1383
|
+
core
|
|
1384
|
+
};
|
|
1385
|
+
}
|
|
898
1386
|
export {
|
|
899
1387
|
postgresRateLimiterMigrations,
|
|
900
1388
|
postgresQueueMigrations,
|
|
1389
|
+
createPostgresQueue,
|
|
901
1390
|
PostgresRateLimiterStorage,
|
|
902
1391
|
PostgresQueueStorage,
|
|
903
1392
|
PostgresMigrationRunner,
|
|
1393
|
+
PostgresMessageQueue,
|
|
1394
|
+
PostgresJobStore,
|
|
904
1395
|
POSTGRES_RATE_LIMITER_STORAGE,
|
|
905
1396
|
POSTGRES_QUEUE_STORAGE
|
|
906
1397
|
};
|
|
907
1398
|
|
|
908
|
-
//# debugId=
|
|
1399
|
+
//# debugId=D9E27F175B26F25364756E2164756E21
|