@workglow/postgres 0.2.30 → 0.2.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/job-queue/PostgresQueueStorage.d.ts +12 -19
- package/dist/job-queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts +9 -19
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts.map +1 -1
- package/dist/job-queue/browser.js +331 -165
- package/dist/job-queue/browser.js.map +8 -5
- package/dist/job-queue/common.d.ts +3 -0
- package/dist/job-queue/common.d.ts.map +1 -1
- package/dist/job-queue/node.js +331 -165
- package/dist/job-queue/node.js.map +8 -5
- package/dist/migrations/PostgresMigrationRunner.d.ts +31 -0
- package/dist/migrations/PostgresMigrationRunner.d.ts.map +1 -0
- package/dist/migrations/common.d.ts +9 -0
- package/dist/migrations/common.d.ts.map +1 -0
- package/dist/migrations/postgresQueueMigrations.d.ts +18 -0
- package/dist/migrations/postgresQueueMigrations.d.ts.map +1 -0
- package/dist/migrations/postgresRateLimiterMigrations.d.ts +11 -0
- package/dist/migrations/postgresRateLimiterMigrations.d.ts.map +1 -0
- package/dist/storage/PostgresTabularStorage.d.ts +175 -10
- package/dist/storage/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/storage/PostgresVectorStorage.d.ts +52 -2
- package/dist/storage/PostgresVectorStorage.d.ts.map +1 -1
- package/dist/storage/browser.js +454 -75
- package/dist/storage/browser.js.map +4 -4
- package/dist/storage/common.d.ts +1 -0
- package/dist/storage/common.d.ts.map +1 -1
- package/dist/storage/node.js +564 -75
- package/dist/storage/node.js.map +6 -5
- package/package.json +7 -7
|
@@ -1,9 +1,255 @@
|
|
|
1
1
|
// src/job-queue/PostgresQueueStorage.ts
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { createServiceToken, getLogger, makeFingerprint, uuid4 } from "@workglow/util";
|
|
4
|
+
import { JobStatus as JobStatus2 } from "@workglow/job-queue";
|
|
5
|
+
import {
|
|
6
|
+
assertPrefixesSafe,
|
|
7
|
+
buildPrefixInsertFragments,
|
|
8
|
+
buildPrefixWhereClause,
|
|
9
|
+
getPrefixColumnNames,
|
|
10
|
+
getPrefixParamValues,
|
|
11
|
+
PostgresDialect as PostgresDialect2
|
|
12
|
+
} from "@workglow/storage";
|
|
13
|
+
|
|
14
|
+
// src/migrations/PostgresMigrationRunner.ts
|
|
15
|
+
import {
|
|
16
|
+
MIGRATIONS_TABLE,
|
|
17
|
+
sortMigrations
|
|
18
|
+
} from "@workglow/storage";
|
|
19
|
+
|
|
20
|
+
class PostgresMigrationRunner {
|
|
21
|
+
db;
|
|
22
|
+
constructor(db) {
|
|
23
|
+
this.db = db;
|
|
24
|
+
}
|
|
25
|
+
async ensureBookkeepingTable() {
|
|
26
|
+
await this.db.query(`
|
|
27
|
+
CREATE TABLE IF NOT EXISTS ${MIGRATIONS_TABLE} (
|
|
28
|
+
component TEXT NOT NULL,
|
|
29
|
+
version INTEGER NOT NULL,
|
|
30
|
+
description TEXT,
|
|
31
|
+
applied_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
|
32
|
+
PRIMARY KEY (component, version)
|
|
33
|
+
)
|
|
34
|
+
`);
|
|
35
|
+
}
|
|
36
|
+
async appliedVersions(component) {
|
|
37
|
+
const result = await this.db.query(`SELECT version FROM ${MIGRATIONS_TABLE} WHERE component = $1`, [component]);
|
|
38
|
+
return new Set(result.rows.map((r) => Number(r.version)));
|
|
39
|
+
}
|
|
40
|
+
async acquireClient() {
|
|
41
|
+
const pool = this.db;
|
|
42
|
+
if (typeof pool.connect === "function") {
|
|
43
|
+
const client = await pool.connect();
|
|
44
|
+
return { client, release: () => client.release() };
|
|
45
|
+
}
|
|
46
|
+
return { client: this.db, release: () => {
|
|
47
|
+
return;
|
|
48
|
+
} };
|
|
49
|
+
}
|
|
50
|
+
async run(migrations, options = {}) {
|
|
51
|
+
await this.ensureBookkeepingTable();
|
|
52
|
+
const sorted = sortMigrations(migrations);
|
|
53
|
+
const applied = [];
|
|
54
|
+
const cache = new Map;
|
|
55
|
+
const onProgress = options.onProgress;
|
|
56
|
+
for (const m of sorted) {
|
|
57
|
+
let seen = cache.get(m.component);
|
|
58
|
+
if (!seen) {
|
|
59
|
+
seen = await this.appliedVersions(m.component);
|
|
60
|
+
cache.set(m.component, seen);
|
|
61
|
+
}
|
|
62
|
+
if (seen.has(m.version))
|
|
63
|
+
continue;
|
|
64
|
+
onProgress?.({
|
|
65
|
+
component: m.component,
|
|
66
|
+
version: m.version,
|
|
67
|
+
phase: "starting",
|
|
68
|
+
description: m.description
|
|
69
|
+
});
|
|
70
|
+
const { client, release } = await this.acquireClient();
|
|
71
|
+
try {
|
|
72
|
+
await client.query("BEGIN");
|
|
73
|
+
await m.up(client, (fraction) => {
|
|
74
|
+
onProgress?.({
|
|
75
|
+
component: m.component,
|
|
76
|
+
version: m.version,
|
|
77
|
+
phase: "running",
|
|
78
|
+
description: m.description,
|
|
79
|
+
fraction
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
await client.query(`INSERT INTO ${MIGRATIONS_TABLE}(component, version, description) VALUES ($1, $2, $3)`, [m.component, m.version, m.description ?? null]);
|
|
83
|
+
await client.query("COMMIT");
|
|
84
|
+
seen.add(m.version);
|
|
85
|
+
applied.push(m);
|
|
86
|
+
onProgress?.({
|
|
87
|
+
component: m.component,
|
|
88
|
+
version: m.version,
|
|
89
|
+
phase: "completed",
|
|
90
|
+
description: m.description,
|
|
91
|
+
fraction: 1
|
|
92
|
+
});
|
|
93
|
+
} catch (err) {
|
|
94
|
+
await client.query("ROLLBACK").catch(() => {
|
|
95
|
+
return;
|
|
96
|
+
});
|
|
97
|
+
if (err?.code === "23505") {
|
|
98
|
+
seen.add(m.version);
|
|
99
|
+
onProgress?.({
|
|
100
|
+
component: m.component,
|
|
101
|
+
version: m.version,
|
|
102
|
+
phase: "completed",
|
|
103
|
+
description: m.description,
|
|
104
|
+
fraction: 1
|
|
105
|
+
});
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
onProgress?.({
|
|
109
|
+
component: m.component,
|
|
110
|
+
version: m.version,
|
|
111
|
+
phase: "failed",
|
|
112
|
+
description: m.description,
|
|
113
|
+
error: err
|
|
114
|
+
});
|
|
115
|
+
throw err;
|
|
116
|
+
} finally {
|
|
117
|
+
release();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return applied;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// src/migrations/postgresQueueMigrations.ts
|
|
4
125
|
import { JobStatus } from "@workglow/job-queue";
|
|
126
|
+
import {
|
|
127
|
+
buildPrefixColumnsSql,
|
|
128
|
+
getPrefixIndexPrefix,
|
|
129
|
+
getPrefixIndexSuffix,
|
|
130
|
+
PostgresDialect
|
|
131
|
+
} from "@workglow/storage";
|
|
132
|
+
var JOB_STATUS_V1 = [
|
|
133
|
+
"PENDING",
|
|
134
|
+
"PROCESSING",
|
|
135
|
+
"COMPLETED",
|
|
136
|
+
"ABORTING",
|
|
137
|
+
"FAILED",
|
|
138
|
+
"DISABLED"
|
|
139
|
+
];
|
|
140
|
+
function assertJobStatusMatchesV1() {
|
|
141
|
+
const current = new Set(Object.values(JobStatus));
|
|
142
|
+
for (const v of JOB_STATUS_V1) {
|
|
143
|
+
if (!current.has(v)) {
|
|
144
|
+
throw new Error(`JobStatus const is missing v1 enum value "${v}"; v1 migration values are frozen.`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const v of current) {
|
|
148
|
+
if (!JOB_STATUS_V1.includes(v)) {
|
|
149
|
+
throw new Error(`JobStatus contains "${v}" which is not in JOB_STATUS_V1. ` + `Add a new migration that runs "ALTER TYPE job_status ADD VALUE IF NOT EXISTS '${v}'" ` + `instead of mutating the v1 enum literal.`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
function postgresQueueMigrations(tableName, prefixes) {
|
|
154
|
+
assertJobStatusMatchesV1();
|
|
155
|
+
const component = `queue:postgres:${tableName}`;
|
|
156
|
+
const prefixColumnsSql = buildPrefixColumnsSql(PostgresDialect, prefixes);
|
|
157
|
+
const prefixIndexPrefix = getPrefixIndexPrefix(prefixes);
|
|
158
|
+
const indexSuffix = getPrefixIndexSuffix(prefixes);
|
|
159
|
+
return [
|
|
160
|
+
{
|
|
161
|
+
component,
|
|
162
|
+
version: 1,
|
|
163
|
+
description: "Create job_status enum + queue table + indexes + notify trigger",
|
|
164
|
+
async up(db) {
|
|
165
|
+
const enumLiteral = JOB_STATUS_V1.map((v) => `'${v}'`).join(",");
|
|
166
|
+
await db.query(`
|
|
167
|
+
DO $$
|
|
168
|
+
BEGIN
|
|
169
|
+
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'job_status') THEN
|
|
170
|
+
CREATE TYPE job_status AS ENUM (${enumLiteral});
|
|
171
|
+
END IF;
|
|
172
|
+
END $$;
|
|
173
|
+
`);
|
|
174
|
+
await db.query(`
|
|
175
|
+
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
176
|
+
id SERIAL NOT NULL,
|
|
177
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
178
|
+
queue text NOT NULL,
|
|
179
|
+
job_run_id text NOT NULL,
|
|
180
|
+
status job_status NOT NULL default 'PENDING',
|
|
181
|
+
input jsonb NOT NULL,
|
|
182
|
+
output jsonb,
|
|
183
|
+
run_attempts integer default 0,
|
|
184
|
+
max_retries integer default 20,
|
|
185
|
+
run_after timestamp with time zone DEFAULT now(),
|
|
186
|
+
last_ran_at timestamp with time zone,
|
|
187
|
+
created_at timestamp with time zone DEFAULT now(),
|
|
188
|
+
deadline_at timestamp with time zone,
|
|
189
|
+
completed_at timestamp with time zone,
|
|
190
|
+
error text,
|
|
191
|
+
error_code text,
|
|
192
|
+
progress real DEFAULT 0,
|
|
193
|
+
progress_message text DEFAULT '',
|
|
194
|
+
progress_details jsonb,
|
|
195
|
+
worker_id text
|
|
196
|
+
)
|
|
197
|
+
`);
|
|
198
|
+
await db.query(`
|
|
199
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
200
|
+
ON ${tableName} (${prefixIndexPrefix}id, status, run_after)
|
|
201
|
+
`);
|
|
202
|
+
await db.query(`
|
|
203
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
204
|
+
ON ${tableName} (${prefixIndexPrefix}queue, status, run_after)
|
|
205
|
+
`);
|
|
206
|
+
await db.query(`
|
|
207
|
+
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
208
|
+
ON ${tableName} (${prefixIndexPrefix}queue, fingerprint, status)
|
|
209
|
+
`);
|
|
210
|
+
const fnName = `${tableName}_notify`;
|
|
211
|
+
const trgName = `${tableName}_notify_trg`;
|
|
212
|
+
await db.query("SAVEPOINT install_notify_trigger");
|
|
213
|
+
try {
|
|
214
|
+
await db.query(`
|
|
215
|
+
CREATE OR REPLACE FUNCTION ${fnName}() RETURNS trigger AS $fn$
|
|
216
|
+
DECLARE
|
|
217
|
+
channel TEXT := 'wglw_q_' || md5('${tableName}' || COALESCE(NEW.queue, OLD.queue));
|
|
218
|
+
payload TEXT;
|
|
219
|
+
BEGIN
|
|
220
|
+
payload := json_build_object(
|
|
221
|
+
'op', TG_OP,
|
|
222
|
+
'id', COALESCE(NEW.id, OLD.id),
|
|
223
|
+
'queue', COALESCE(NEW.queue, OLD.queue),
|
|
224
|
+
'status', COALESCE(NEW.status::text, OLD.status::text)
|
|
225
|
+
)::text;
|
|
226
|
+
PERFORM pg_notify(channel, payload);
|
|
227
|
+
RETURN NULL;
|
|
228
|
+
END;
|
|
229
|
+
$fn$ LANGUAGE plpgsql;
|
|
230
|
+
`);
|
|
231
|
+
await db.query(`DROP TRIGGER IF EXISTS ${trgName} ON ${tableName}`);
|
|
232
|
+
await db.query(`
|
|
233
|
+
CREATE TRIGGER ${trgName}
|
|
234
|
+
AFTER INSERT OR UPDATE ON ${tableName}
|
|
235
|
+
FOR EACH ROW EXECUTE FUNCTION ${fnName}();
|
|
236
|
+
`);
|
|
237
|
+
await db.query("RELEASE SAVEPOINT install_notify_trigger");
|
|
238
|
+
} catch {
|
|
239
|
+
await db.query("ROLLBACK TO SAVEPOINT install_notify_trigger").catch(() => {
|
|
240
|
+
return;
|
|
241
|
+
});
|
|
242
|
+
await db.query("RELEASE SAVEPOINT install_notify_trigger").catch(() => {
|
|
243
|
+
return;
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
];
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// src/job-queue/PostgresQueueStorage.ts
|
|
5
252
|
var POSTGRES_QUEUE_STORAGE = createServiceToken("jobqueue.storage.postgres");
|
|
6
|
-
var SAFE_IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9_]*$/;
|
|
7
253
|
|
|
8
254
|
class PostgresQueueStorage {
|
|
9
255
|
db;
|
|
@@ -17,11 +263,7 @@ class PostgresQueueStorage {
|
|
|
17
263
|
this.queueName = queueName;
|
|
18
264
|
this.prefixes = options?.prefixes ?? [];
|
|
19
265
|
this.prefixValues = options?.prefixValues ?? {};
|
|
20
|
-
|
|
21
|
-
if (!SAFE_IDENTIFIER.test(prefix.name)) {
|
|
22
|
-
throw new Error(`Prefix column name must start with a letter and contain only letters, digits, and underscores, got: ${prefix.name}`);
|
|
23
|
-
}
|
|
24
|
-
}
|
|
266
|
+
assertPrefixesSafe(this.prefixes);
|
|
25
267
|
if (this.prefixes.length > 0) {
|
|
26
268
|
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
27
269
|
this.tableName = `job_queue_${prefixNames}`;
|
|
@@ -29,112 +271,17 @@ class PostgresQueueStorage {
|
|
|
29
271
|
this.tableName = "job_queue";
|
|
30
272
|
}
|
|
31
273
|
}
|
|
32
|
-
getPrefixColumnType(type) {
|
|
33
|
-
return type === "uuid" ? "UUID" : "INTEGER";
|
|
34
|
-
}
|
|
35
|
-
buildPrefixColumnsSql() {
|
|
36
|
-
if (this.prefixes.length === 0)
|
|
37
|
-
return "";
|
|
38
|
-
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
39
|
-
`) + `,
|
|
40
|
-
`;
|
|
41
|
-
}
|
|
42
|
-
getPrefixColumnNames() {
|
|
43
|
-
return this.prefixes.map((p) => p.name);
|
|
44
|
-
}
|
|
45
274
|
buildPrefixWhereClause(startParam) {
|
|
46
|
-
|
|
47
|
-
return { conditions: "", params: [] };
|
|
48
|
-
}
|
|
49
|
-
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
50
|
-
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
51
|
-
return { conditions: " AND " + conditions, params };
|
|
275
|
+
return buildPrefixWhereClause(PostgresDialect2, this.prefixes, this.prefixValues, startParam);
|
|
52
276
|
}
|
|
53
277
|
getPrefixParamValues() {
|
|
54
|
-
return this.prefixes
|
|
278
|
+
return getPrefixParamValues(this.prefixes, this.prefixValues);
|
|
55
279
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if (!SAFE_IDENTIFIER.test(v)) {
|
|
62
|
-
throw new Error(`Invalid JobStatus enum value: ${v}`);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
sql = `CREATE TYPE job_status AS ENUM (${enumValues.map((v) => `'${v}'`).join(",")})`;
|
|
66
|
-
await this.db.query(sql);
|
|
67
|
-
} catch (e) {
|
|
68
|
-
if (e.code !== "42710")
|
|
69
|
-
throw e;
|
|
70
|
-
}
|
|
71
|
-
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
72
|
-
const prefixColumnNames = this.getPrefixColumnNames();
|
|
73
|
-
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
74
|
-
sql = `
|
|
75
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
76
|
-
id SERIAL NOT NULL,
|
|
77
|
-
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
78
|
-
queue text NOT NULL,
|
|
79
|
-
job_run_id text NOT NULL,
|
|
80
|
-
status job_status NOT NULL default 'PENDING',
|
|
81
|
-
input jsonb NOT NULL,
|
|
82
|
-
output jsonb,
|
|
83
|
-
run_attempts integer default 0,
|
|
84
|
-
max_retries integer default 20,
|
|
85
|
-
run_after timestamp with time zone DEFAULT now(),
|
|
86
|
-
last_ran_at timestamp with time zone,
|
|
87
|
-
created_at timestamp with time zone DEFAULT now(),
|
|
88
|
-
deadline_at timestamp with time zone,
|
|
89
|
-
completed_at timestamp with time zone,
|
|
90
|
-
error text,
|
|
91
|
-
error_code text,
|
|
92
|
-
progress real DEFAULT 0,
|
|
93
|
-
progress_message text DEFAULT '',
|
|
94
|
-
progress_details jsonb,
|
|
95
|
-
worker_id text
|
|
96
|
-
)`;
|
|
97
|
-
await this.db.query(sql);
|
|
98
|
-
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
99
|
-
sql = `
|
|
100
|
-
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
101
|
-
ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
|
|
102
|
-
await this.db.query(sql);
|
|
103
|
-
sql = `
|
|
104
|
-
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
105
|
-
ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
|
|
106
|
-
await this.db.query(sql);
|
|
107
|
-
sql = `
|
|
108
|
-
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
109
|
-
ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
|
|
110
|
-
await this.db.query(sql);
|
|
111
|
-
const fnName = `${this.tableName}_notify`;
|
|
112
|
-
const trgName = `${this.tableName}_notify_trg`;
|
|
113
|
-
try {
|
|
114
|
-
await this.db.query(`
|
|
115
|
-
CREATE OR REPLACE FUNCTION ${fnName}() RETURNS trigger AS $fn$
|
|
116
|
-
DECLARE
|
|
117
|
-
channel TEXT := 'wglw_q_' || md5('${this.tableName}' || COALESCE(NEW.queue, OLD.queue));
|
|
118
|
-
payload TEXT;
|
|
119
|
-
BEGIN
|
|
120
|
-
payload := json_build_object(
|
|
121
|
-
'op', TG_OP,
|
|
122
|
-
'id', COALESCE(NEW.id, OLD.id),
|
|
123
|
-
'queue', COALESCE(NEW.queue, OLD.queue),
|
|
124
|
-
'status', COALESCE(NEW.status::text, OLD.status::text)
|
|
125
|
-
)::text;
|
|
126
|
-
PERFORM pg_notify(channel, payload);
|
|
127
|
-
RETURN NULL;
|
|
128
|
-
END;
|
|
129
|
-
$fn$ LANGUAGE plpgsql;
|
|
130
|
-
`);
|
|
131
|
-
await this.db.query(`DROP TRIGGER IF EXISTS ${trgName} ON ${this.tableName}`);
|
|
132
|
-
await this.db.query(`
|
|
133
|
-
CREATE TRIGGER ${trgName}
|
|
134
|
-
AFTER INSERT OR UPDATE ON ${this.tableName}
|
|
135
|
-
FOR EACH ROW EXECUTE FUNCTION ${fnName}();
|
|
136
|
-
`);
|
|
137
|
-
} catch {}
|
|
280
|
+
getMigrations() {
|
|
281
|
+
return postgresQueueMigrations(this.tableName, this.prefixes);
|
|
282
|
+
}
|
|
283
|
+
async migrate() {
|
|
284
|
+
await new PostgresMigrationRunner(this.db).run(this.getMigrations());
|
|
138
285
|
}
|
|
139
286
|
notifyChannelName() {
|
|
140
287
|
const tableAndQueue = `${this.tableName}${this.queueName}`;
|
|
@@ -146,16 +293,15 @@ class PostgresQueueStorage {
|
|
|
146
293
|
job.queue = this.queueName;
|
|
147
294
|
job.job_run_id = job.job_run_id ?? uuid4();
|
|
148
295
|
job.fingerprint = await makeFingerprint(job.input);
|
|
149
|
-
job.status =
|
|
296
|
+
job.status = JobStatus2.PENDING;
|
|
150
297
|
job.progress = 0;
|
|
151
298
|
job.progress_message = "";
|
|
152
299
|
job.progress_details = null;
|
|
153
300
|
job.created_at = now;
|
|
154
301
|
job.run_after = now;
|
|
155
|
-
const prefixColumnNames = this.
|
|
156
|
-
const prefixColumnsInsert
|
|
302
|
+
const prefixColumnNames = getPrefixColumnNames(this.prefixes);
|
|
303
|
+
const { columns: prefixColumnsInsert, placeholders: prefixParamPlaceholders } = buildPrefixInsertFragments(PostgresDialect2, this.prefixes, 1);
|
|
157
304
|
const prefixParamValues = this.getPrefixParamValues();
|
|
158
|
-
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
|
|
159
305
|
const baseParamStart = prefixColumnNames.length + 1;
|
|
160
306
|
const sql = `
|
|
161
307
|
INSERT INTO ${this.tableName}(
|
|
@@ -205,7 +351,7 @@ class PostgresQueueStorage {
|
|
|
205
351
|
return;
|
|
206
352
|
return result.rows[0];
|
|
207
353
|
}
|
|
208
|
-
async peek(status =
|
|
354
|
+
async peek(status = JobStatus2.PENDING, num = 100) {
|
|
209
355
|
num = Number(num) || 100;
|
|
210
356
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
211
357
|
const result = await this.db.query(`
|
|
@@ -236,10 +382,10 @@ class PostgresQueueStorage {
|
|
|
236
382
|
FOR UPDATE SKIP LOCKED
|
|
237
383
|
LIMIT 1
|
|
238
384
|
)
|
|
239
|
-
RETURNING *`, [
|
|
385
|
+
RETURNING *`, [JobStatus2.PROCESSING, this.queueName, JobStatus2.PENDING, workerId, ...prefixParams]);
|
|
240
386
|
return result?.rows?.[0] ?? undefined;
|
|
241
387
|
}
|
|
242
|
-
async size(status =
|
|
388
|
+
async size(status = JobStatus2.PENDING) {
|
|
243
389
|
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
244
390
|
const result = await this.db.query(`
|
|
245
391
|
SELECT COUNT(*) as count
|
|
@@ -252,7 +398,7 @@ class PostgresQueueStorage {
|
|
|
252
398
|
}
|
|
253
399
|
async complete(jobDetails) {
|
|
254
400
|
const prefixParams = this.getPrefixParamValues();
|
|
255
|
-
if (jobDetails.status ===
|
|
401
|
+
if (jobDetails.status === JobStatus2.DISABLED) {
|
|
256
402
|
const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
|
|
257
403
|
await this.db.query(`UPDATE ${this.tableName}
|
|
258
404
|
SET
|
|
@@ -262,7 +408,7 @@ class PostgresQueueStorage {
|
|
|
262
408
|
progress_details = NULL,
|
|
263
409
|
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
264
410
|
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
265
|
-
} else if (jobDetails.status ===
|
|
411
|
+
} else if (jobDetails.status === JobStatus2.PENDING) {
|
|
266
412
|
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
267
413
|
await this.db.query(`UPDATE ${this.tableName}
|
|
268
414
|
SET
|
|
@@ -508,6 +654,59 @@ class PostgresQueueStorage {
|
|
|
508
654
|
}
|
|
509
655
|
// src/job-queue/PostgresRateLimiterStorage.ts
|
|
510
656
|
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
657
|
+
import {
|
|
658
|
+
assertPrefixesSafe as assertPrefixesSafe2,
|
|
659
|
+
buildPrefixWhereClause as buildPrefixWhereClause2,
|
|
660
|
+
getPrefixColumnNames as getPrefixColumnNames3,
|
|
661
|
+
getPrefixParamValues as getPrefixParamValues2,
|
|
662
|
+
PostgresDialect as PostgresDialect4
|
|
663
|
+
} from "@workglow/storage";
|
|
664
|
+
|
|
665
|
+
// src/migrations/postgresRateLimiterMigrations.ts
|
|
666
|
+
import {
|
|
667
|
+
buildPrefixColumnsSql as buildPrefixColumnsSql2,
|
|
668
|
+
getPrefixColumnNames as getPrefixColumnNames2,
|
|
669
|
+
getPrefixIndexPrefix as getPrefixIndexPrefix2,
|
|
670
|
+
getPrefixIndexSuffix as getPrefixIndexSuffix2,
|
|
671
|
+
PostgresDialect as PostgresDialect3
|
|
672
|
+
} from "@workglow/storage";
|
|
673
|
+
function postgresRateLimiterMigrations(executionTableName, nextAvailableTableName, prefixes) {
|
|
674
|
+
const component = `rate-limiter:postgres:${executionTableName}`;
|
|
675
|
+
const prefixColumnsSql = buildPrefixColumnsSql2(PostgresDialect3, prefixes);
|
|
676
|
+
const prefixColumnNames = getPrefixColumnNames2(prefixes);
|
|
677
|
+
const prefixIndexPrefix = getPrefixIndexPrefix2(prefixes);
|
|
678
|
+
const indexSuffix = getPrefixIndexSuffix2(prefixes);
|
|
679
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
680
|
+
return [
|
|
681
|
+
{
|
|
682
|
+
component,
|
|
683
|
+
version: 1,
|
|
684
|
+
description: "Create rate-limiter execution + next_available tables",
|
|
685
|
+
async up(db) {
|
|
686
|
+
await db.query(`
|
|
687
|
+
CREATE TABLE IF NOT EXISTS ${executionTableName} (
|
|
688
|
+
id SERIAL PRIMARY KEY,
|
|
689
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
690
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
691
|
+
)
|
|
692
|
+
`);
|
|
693
|
+
await db.query(`
|
|
694
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
695
|
+
ON ${executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
696
|
+
`);
|
|
697
|
+
await db.query(`
|
|
698
|
+
CREATE TABLE IF NOT EXISTS ${nextAvailableTableName} (
|
|
699
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
700
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
701
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
702
|
+
)
|
|
703
|
+
`);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
];
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// src/job-queue/PostgresRateLimiterStorage.ts
|
|
511
710
|
var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken2("ratelimiter.storage.postgres");
|
|
512
711
|
|
|
513
712
|
class PostgresRateLimiterStorage {
|
|
@@ -521,6 +720,7 @@ class PostgresRateLimiterStorage {
|
|
|
521
720
|
this.db = db;
|
|
522
721
|
this.prefixes = options?.prefixes ?? [];
|
|
523
722
|
this.prefixValues = options?.prefixValues ?? {};
|
|
723
|
+
assertPrefixesSafe2(this.prefixes);
|
|
524
724
|
if (this.prefixes.length > 0) {
|
|
525
725
|
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
526
726
|
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
@@ -530,57 +730,20 @@ class PostgresRateLimiterStorage {
|
|
|
530
730
|
this.nextAvailableTableName = "rate_limit_next_available";
|
|
531
731
|
}
|
|
532
732
|
}
|
|
533
|
-
getPrefixColumnType(type) {
|
|
534
|
-
return type === "uuid" ? "UUID" : "INTEGER";
|
|
535
|
-
}
|
|
536
|
-
buildPrefixColumnsSql() {
|
|
537
|
-
if (this.prefixes.length === 0)
|
|
538
|
-
return "";
|
|
539
|
-
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
540
|
-
`) + `,
|
|
541
|
-
`;
|
|
542
|
-
}
|
|
543
|
-
getPrefixColumnNames() {
|
|
544
|
-
return this.prefixes.map((p) => p.name);
|
|
545
|
-
}
|
|
546
733
|
buildPrefixWhereClause(startParam) {
|
|
547
|
-
|
|
548
|
-
return { conditions: "", params: [] };
|
|
549
|
-
}
|
|
550
|
-
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
551
|
-
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
552
|
-
return { conditions: " AND " + conditions, params };
|
|
734
|
+
return buildPrefixWhereClause2(PostgresDialect4, this.prefixes, this.prefixValues, startParam);
|
|
553
735
|
}
|
|
554
736
|
getPrefixParamValues() {
|
|
555
|
-
return this.prefixes
|
|
737
|
+
return getPrefixParamValues2(this.prefixes, this.prefixValues);
|
|
556
738
|
}
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
await this.db.query(`
|
|
563
|
-
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
564
|
-
id SERIAL PRIMARY KEY,
|
|
565
|
-
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
566
|
-
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
567
|
-
)
|
|
568
|
-
`);
|
|
569
|
-
await this.db.query(`
|
|
570
|
-
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
571
|
-
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
572
|
-
`);
|
|
573
|
-
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
574
|
-
await this.db.query(`
|
|
575
|
-
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
576
|
-
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
577
|
-
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
578
|
-
PRIMARY KEY (${primaryKeyColumns})
|
|
579
|
-
)
|
|
580
|
-
`);
|
|
739
|
+
getMigrations() {
|
|
740
|
+
return postgresRateLimiterMigrations(this.executionTableName, this.nextAvailableTableName, this.prefixes);
|
|
741
|
+
}
|
|
742
|
+
async migrate() {
|
|
743
|
+
await new PostgresMigrationRunner(this.db).run(this.getMigrations());
|
|
581
744
|
}
|
|
582
745
|
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
583
|
-
const prefixColumnNames = this.
|
|
746
|
+
const prefixColumnNames = getPrefixColumnNames3(this.prefixes);
|
|
584
747
|
const prefixParamValues = this.getPrefixParamValues();
|
|
585
748
|
const prefixCount = prefixColumnNames.length;
|
|
586
749
|
const queueParam = `$${prefixCount + 1}`;
|
|
@@ -657,7 +820,7 @@ class PostgresRateLimiterStorage {
|
|
|
657
820
|
await this.db.query(`DELETE FROM ${this.executionTableName} WHERE id = $1 AND queue_name = $2${prefixConditions}`, [token, queueName, ...prefixParams]);
|
|
658
821
|
}
|
|
659
822
|
async recordExecution(queueName) {
|
|
660
|
-
const prefixColumnNames = this.
|
|
823
|
+
const prefixColumnNames = getPrefixColumnNames3(this.prefixes);
|
|
661
824
|
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
662
825
|
const prefixParamValues = this.getPrefixParamValues();
|
|
663
826
|
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
@@ -703,7 +866,7 @@ class PostgresRateLimiterStorage {
|
|
|
703
866
|
return new Date(nextAvailableAt).toISOString();
|
|
704
867
|
}
|
|
705
868
|
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
706
|
-
const prefixColumnNames = this.
|
|
869
|
+
const prefixColumnNames = getPrefixColumnNames3(this.prefixes);
|
|
707
870
|
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
708
871
|
const prefixParamValues = this.getPrefixParamValues();
|
|
709
872
|
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
@@ -723,10 +886,13 @@ class PostgresRateLimiterStorage {
|
|
|
723
886
|
}
|
|
724
887
|
}
|
|
725
888
|
export {
|
|
889
|
+
postgresRateLimiterMigrations,
|
|
890
|
+
postgresQueueMigrations,
|
|
726
891
|
PostgresRateLimiterStorage,
|
|
727
892
|
PostgresQueueStorage,
|
|
893
|
+
PostgresMigrationRunner,
|
|
728
894
|
POSTGRES_RATE_LIMITER_STORAGE,
|
|
729
895
|
POSTGRES_QUEUE_STORAGE
|
|
730
896
|
};
|
|
731
897
|
|
|
732
|
-
//# debugId=
|
|
898
|
+
//# debugId=9559FE4247FC32D964756E2164756E21
|