@workflow/world-postgres 4.1.0-beta.36 → 4.1.0-beta.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -9
- package/dist/index.js.map +1 -1
- package/dist/{boss.d.ts → message.d.ts} +5 -5
- package/dist/message.d.ts.map +1 -0
- package/dist/{boss.js → message.js} +5 -5
- package/dist/message.js.map +1 -0
- package/dist/queue.d.ts +4 -4
- package/dist/queue.d.ts.map +1 -1
- package/dist/queue.js +121 -42
- package/dist/queue.js.map +1 -1
- package/dist/streamer.d.ts.map +1 -1
- package/dist/streamer.js +4 -6
- package/dist/streamer.js.map +1 -1
- package/package.json +7 -7
- package/src/drizzle/migrations/0008_migrate_pgboss_to_graphile.sql +20 -0
- package/src/drizzle/migrations/meta/_journal.json +7 -0
- package/dist/boss.d.ts.map +0 -1
- package/dist/boss.js.map +0 -1
package/README.md
CHANGED
|
@@ -73,7 +73,7 @@ const world = createWorld({
|
|
|
73
73
|
|
|
74
74
|
This package uses PostgreSQL with the following components:
|
|
75
75
|
|
|
76
|
-
- **
|
|
76
|
+
- **graphile-worker**: For queue processing and job management
|
|
77
77
|
- **Drizzle ORM**: For database operations and schema management
|
|
78
78
|
- **postgres**: For PostgreSQL client connections
|
|
79
79
|
|
|
@@ -115,7 +115,7 @@ Make sure your PostgreSQL database is accessible and the user has sufficient per
|
|
|
115
115
|
## Features
|
|
116
116
|
|
|
117
117
|
- **Durable Storage**: Stores workflow runs, events, steps, hooks, and webhooks in PostgreSQL
|
|
118
|
-
- **Queue Processing**: Uses
|
|
118
|
+
- **Queue Processing**: Uses graphile-worker for reliable job queue processing
|
|
119
119
|
- **Streaming**: Real-time event streaming capabilities
|
|
120
120
|
- **Health Checks**: Built-in connection health monitoring
|
|
121
121
|
- **Configurable Concurrency**: Adjustable worker concurrency for queue processing
|
|
@@ -137,6 +137,20 @@ export WORKFLOW_POSTGRES_URL="postgres://world:world@localhost:5432/world"
|
|
|
137
137
|
export WORKFLOW_TARGET_WORLD="@workflow/world-postgres"
|
|
138
138
|
```
|
|
139
139
|
|
|
140
|
+
## Testing
|
|
141
|
+
|
|
142
|
+
Integration tests use [Testcontainers](https://testcontainers.com/) to start a PostgreSQL container. **Docker must be installed and running** before you run tests.
|
|
143
|
+
|
|
144
|
+
- **Linux/macOS**: Start the Docker daemon (e.g. `sudo systemctl start docker` or Docker Desktop).
|
|
145
|
+
- **WSL2**: Use Docker Desktop with WSL2 integration, or run the Docker engine inside WSL and ensure the daemon is started. Verify with `docker info`.
|
|
146
|
+
|
|
147
|
+
Then from the package directory:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
pnpm build
|
|
151
|
+
pnpm test
|
|
152
|
+
```
|
|
153
|
+
|
|
140
154
|
## World Selection
|
|
141
155
|
|
|
142
156
|
To use the PostgreSQL world, set the `WORKFLOW_TARGET_WORLD` environment variable to the package name:
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAoBvD,wBAAgB,WAAW,CACzB,MAAM,GAAE,mBAQP,GACA,KAAK,GAAG;IAAE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CA+BpC;AAGD,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,cAAc,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import PgBoss from 'pg-boss';
|
|
2
1
|
import createPostgres from 'postgres';
|
|
3
2
|
import { createClient } from './drizzle/index.js';
|
|
4
3
|
import { createQueue } from './queue.js';
|
|
@@ -19,12 +18,9 @@ export function createWorld(config = {
|
|
|
19
18
|
queueConcurrency: parseInt(process.env.WORKFLOW_POSTGRES_WORKER_CONCURRENCY || '10', 10) ||
|
|
20
19
|
10,
|
|
21
20
|
}) {
|
|
22
|
-
const boss = new PgBoss({
|
|
23
|
-
connectionString: config.connectionString,
|
|
24
|
-
});
|
|
25
21
|
const postgres = createPostgres(config.connectionString);
|
|
26
22
|
const drizzle = createClient(postgres);
|
|
27
|
-
const queue = createQueue(
|
|
23
|
+
const queue = createQueue(config, postgres);
|
|
28
24
|
const storage = createStorage(drizzle);
|
|
29
25
|
const streamer = createStreamer(postgres, drizzle);
|
|
30
26
|
return {
|
|
@@ -35,10 +31,6 @@ export function createWorld(config = {
|
|
|
35
31
|
await queue.start();
|
|
36
32
|
},
|
|
37
33
|
async close() {
|
|
38
|
-
await boss.stop();
|
|
39
|
-
const bossDb = boss.getDb();
|
|
40
|
-
if (bossDb.opened)
|
|
41
|
-
await bossDb.close?.();
|
|
42
34
|
await streamer.close();
|
|
43
35
|
await queue.close();
|
|
44
36
|
await postgres.end();
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,cAAc,MAAM,UAAU,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAgB,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,SAAS,aAAa,CAAC,OAAgB;IACrC,OAAO;QACL,IAAI,EAAE,iBAAiB,CAAC,OAAO,CAAC;QAChC,MAAM,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACpC,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC;QAClC,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC;KACnC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,SAA8B;IAC5B,gBAAgB,EACd,OAAO,CAAC,GAAG,CAAC,qBAAqB;QACjC,6CAA6C;IAC/C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,4BAA4B;IACnD,gBAAgB,EACd,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,IAAI,EAAE,EAAE,CAAC;QACtE,EAAE;CACL;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEnD,OAAO;QACL,GAAG,OAAO;QACV,GAAG,QAAQ;QACX,GAAG,KAAK;QACR,KAAK,CAAC,KAAK;YACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,QAAQ,CAAC,GAAG,EAAE,CAAC;YACrB,8DAA8D;YAC9D,kEAAkE;YAClE,kEAAkE;YAClE,iEAAiE;YACjE,yBAAyB;YACzB,wDAAwD;YACxD,KAAK,MAAM,CAAC,IAAK,OAAe,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC7D,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;oBACjE,CAAY,CAAC,OAAO,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAID,cAAc,qBAAqB,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import * as z from 'zod';
|
|
2
2
|
/**
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
* graphile-worker is using JSON under the hood, so we need to base64 encode
|
|
4
|
+
* the body to ensure binary safety
|
|
5
|
+
* maybe later we can have a `blobs` table for larger payloads
|
|
6
|
+
*/
|
|
7
7
|
export declare const MessageData: z.ZodObject<{
|
|
8
8
|
attempt: z.ZodNumber;
|
|
9
9
|
messageId: z.core.$ZodBranded<z.ZodString, "MessageId">;
|
|
@@ -12,4 +12,4 @@ export declare const MessageData: z.ZodObject<{
|
|
|
12
12
|
data: z.ZodCodec<z.ZodBase64, z.ZodCustom<Buffer<ArrayBufferLike>, Buffer<ArrayBufferLike>>>;
|
|
13
13
|
}, z.core.$strip>;
|
|
14
14
|
export type MessageData = z.infer<typeof MessageData>;
|
|
15
|
-
//# sourceMappingURL=
|
|
15
|
+
//# sourceMappingURL=message.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"message.d.ts","sourceRoot":"","sources":["../src/message.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;GAIG;AACH,eAAO,MAAM,WAAW;;;;;;iBAUtB,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC"}
|
|
@@ -2,10 +2,10 @@ import { MessageId } from '@workflow/world';
|
|
|
2
2
|
import * as z from 'zod';
|
|
3
3
|
import { Base64Buffer } from './zod.js';
|
|
4
4
|
/**
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
* graphile-worker is using JSON under the hood, so we need to base64 encode
|
|
6
|
+
* the body to ensure binary safety
|
|
7
|
+
* maybe later we can have a `blobs` table for larger payloads
|
|
8
|
+
*/
|
|
9
9
|
export const MessageData = z.object({
|
|
10
10
|
attempt: z.number().describe('The attempt number of the message'),
|
|
11
11
|
messageId: MessageId.describe('The unique ID of the message'),
|
|
@@ -15,4 +15,4 @@ export const MessageData = z.object({
|
|
|
15
15
|
.describe("The ID of the sub-queue. For workflows, it's the workflow name. For steps, it's the step name."),
|
|
16
16
|
data: Base64Buffer.describe('The message that was sent'),
|
|
17
17
|
});
|
|
18
|
-
//# sourceMappingURL=
|
|
18
|
+
//# sourceMappingURL=message.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"message.js","sourceRoot":"","sources":["../src/message.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC;;;;GAIG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IACjE,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAC7D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,QAAQ,CACP,gGAAgG,CACjG;IACH,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CACzD,CAAC,CAAC"}
|
package/dist/queue.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { type Queue } from '@workflow/world';
|
|
2
|
-
import type
|
|
2
|
+
import type Postgres from 'postgres';
|
|
3
3
|
import type { PostgresWorldConfig } from './config.js';
|
|
4
4
|
/**
|
|
5
|
-
* The Postgres queue works by creating two job types in
|
|
5
|
+
* The Postgres queue works by creating two job types in graphile-worker:
|
|
6
6
|
* - `workflow` for workflow jobs
|
|
7
7
|
* - `step` for step jobs
|
|
8
8
|
*
|
|
9
|
-
* When a message is queued, it is sent to
|
|
9
|
+
* When a message is queued, it is sent to graphile-worker with the appropriate job type.
|
|
10
10
|
* When a job is processed, it is deserialized and then re-queued into the _local world_, showing that
|
|
11
11
|
* we can reuse the local world, mix and match worlds to build
|
|
12
12
|
* hybrid architectures, and even migrate between worlds.
|
|
@@ -15,5 +15,5 @@ export type PostgresQueue = Queue & {
|
|
|
15
15
|
start(): Promise<void>;
|
|
16
16
|
close(): Promise<void>;
|
|
17
17
|
};
|
|
18
|
-
export declare function createQueue(
|
|
18
|
+
export declare function createQueue(config: PostgresWorldConfig, postgres: Postgres.Sql): PostgresQueue;
|
|
19
19
|
//# sourceMappingURL=queue.d.ts.map
|
package/dist/queue.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,KAAK,EAIX,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,KAAK,EAIX,MAAM,iBAAiB,CAAC;AASzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAqBvD;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG;IAClC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAgB,WAAW,CACzB,MAAM,EAAE,mBAAmB,EAC3B,QAAQ,EAAE,QAAQ,CAAC,GAAG,GACrB,aAAa,CAyKf"}
|
package/dist/queue.js
CHANGED
|
@@ -2,9 +2,28 @@ import * as Stream from 'node:stream';
|
|
|
2
2
|
import { JsonTransport } from '@vercel/queue';
|
|
3
3
|
import { MessageId, QueuePayloadSchema, } from '@workflow/world';
|
|
4
4
|
import { createLocalWorld } from '@workflow/world-local';
|
|
5
|
+
import { Logger, makeWorkerUtils, run, } from 'graphile-worker';
|
|
5
6
|
import { monotonicFactory } from 'ulid';
|
|
6
|
-
import { MessageData } from './
|
|
7
|
-
|
|
7
|
+
import { MessageData } from './message.js';
|
|
8
|
+
function createGraphileLogger() {
|
|
9
|
+
const isJsonMode = () => process.env.WORKFLOW_JSON_MODE === '1';
|
|
10
|
+
const isVerbose = () => Boolean(process.env.DEBUG);
|
|
11
|
+
return new Logger(() => (level, message, meta) => {
|
|
12
|
+
if (isJsonMode())
|
|
13
|
+
return;
|
|
14
|
+
if ((level === 'debug' || level === 'info') && !isVerbose())
|
|
15
|
+
return;
|
|
16
|
+
const pipe = level === 'error' ? process.stderr : process.stdout;
|
|
17
|
+
if (meta) {
|
|
18
|
+
pipe.write(`[Graphile Worker] ${message} ${JSON.stringify(meta, null, 2)}\n`);
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
pipe.write(`[Graphile Worker] ${message}\n`);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
const graphileLogger = createGraphileLogger();
|
|
26
|
+
export function createQueue(config, postgres) {
|
|
8
27
|
const port = process.env.PORT ? Number(process.env.PORT) : undefined;
|
|
9
28
|
const localWorld = createLocalWorld({ dataDir: undefined, port });
|
|
10
29
|
const transport = new JsonTransport();
|
|
@@ -18,75 +37,135 @@ export function createQueue(boss, config) {
|
|
|
18
37
|
const getDeploymentId = async () => {
|
|
19
38
|
return 'postgres';
|
|
20
39
|
};
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
40
|
+
let workerUtils = null;
|
|
41
|
+
let runner = null;
|
|
42
|
+
let startPromise = null;
|
|
43
|
+
async function migratePgBossJobs(utils) {
|
|
44
|
+
// Scenario A: Drizzle migration already ran — staging table exists
|
|
45
|
+
const hasStaging = await postgres `
|
|
46
|
+
SELECT EXISTS (
|
|
47
|
+
SELECT 1 FROM information_schema.tables
|
|
48
|
+
WHERE table_schema = 'workflow'
|
|
49
|
+
AND table_name = '_pgboss_pending_jobs'
|
|
50
|
+
) AS exists
|
|
51
|
+
`;
|
|
52
|
+
if (hasStaging[0].exists) {
|
|
53
|
+
const jobs = await postgres `
|
|
54
|
+
SELECT name, data, singleton_key, retry_limit
|
|
55
|
+
FROM "workflow"."_pgboss_pending_jobs"
|
|
56
|
+
`;
|
|
57
|
+
for (const job of jobs) {
|
|
58
|
+
await utils.addJob(job.name, job.data, {
|
|
59
|
+
jobKey: job.singleton_key ?? undefined,
|
|
60
|
+
maxAttempts: job.retry_limit ?? 3,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
await postgres `DROP TABLE "workflow"."_pgboss_pending_jobs"`;
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
// Scenario B: Drizzle migration didn't run — pgboss schema still exists
|
|
67
|
+
const hasPgBoss = await postgres `
|
|
68
|
+
SELECT EXISTS (
|
|
69
|
+
SELECT 1 FROM information_schema.schemata
|
|
70
|
+
WHERE schema_name = 'pgboss'
|
|
71
|
+
) AS exists
|
|
72
|
+
`;
|
|
73
|
+
if (hasPgBoss[0].exists) {
|
|
74
|
+
const jobs = await postgres `
|
|
75
|
+
SELECT name, data, singleton_key, retry_limit
|
|
76
|
+
FROM pgboss.job
|
|
77
|
+
WHERE state IN ('created', 'retry')
|
|
78
|
+
`;
|
|
79
|
+
for (const job of jobs) {
|
|
80
|
+
await utils.addJob(job.name, job.data, {
|
|
81
|
+
jobKey: job.singleton_key ?? undefined,
|
|
82
|
+
maxAttempts: job.retry_limit ?? 3,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
await postgres `DROP SCHEMA pgboss CASCADE`;
|
|
27
86
|
}
|
|
28
|
-
|
|
87
|
+
}
|
|
88
|
+
async function start() {
|
|
89
|
+
if (!startPromise) {
|
|
90
|
+
startPromise = (async () => {
|
|
91
|
+
try {
|
|
92
|
+
workerUtils = await makeWorkerUtils({
|
|
93
|
+
connectionString: config.connectionString,
|
|
94
|
+
logger: graphileLogger,
|
|
95
|
+
});
|
|
96
|
+
await workerUtils.migrate();
|
|
97
|
+
await migratePgBossJobs(workerUtils);
|
|
98
|
+
await setupListeners();
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
startPromise = null;
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
})();
|
|
105
|
+
}
|
|
106
|
+
await startPromise;
|
|
29
107
|
}
|
|
30
108
|
const queue = async (queue, message, opts) => {
|
|
31
|
-
await
|
|
109
|
+
await start();
|
|
32
110
|
const [prefix, queueId] = parseQueueName(queue);
|
|
33
111
|
const jobName = Queues[prefix];
|
|
34
|
-
await createQueue(jobName);
|
|
35
112
|
const body = transport.serialize(message);
|
|
36
113
|
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
37
|
-
await
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
attempt: 1,
|
|
47
|
-
messageId,
|
|
48
|
-
idempotencyKey: opts?.idempotencyKey,
|
|
49
|
-
}),
|
|
114
|
+
await workerUtils.addJob(jobName, MessageData.encode({
|
|
115
|
+
id: queueId,
|
|
116
|
+
data: body,
|
|
117
|
+
attempt: 1,
|
|
118
|
+
messageId,
|
|
119
|
+
idempotencyKey: opts?.idempotencyKey,
|
|
120
|
+
}), {
|
|
121
|
+
jobKey: opts?.idempotencyKey ?? messageId,
|
|
122
|
+
maxAttempts: 3,
|
|
50
123
|
});
|
|
51
124
|
return { messageId };
|
|
52
125
|
};
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
await boss.work(jobName, {
|
|
57
|
-
// The default is 2s, which is far too slow for running steps in quick succession.
|
|
58
|
-
// The min is 0.5s, which is still too slow. We should move to a pg NOTIFY/LISTEN-based job system.
|
|
59
|
-
pollingIntervalSeconds: 0.5,
|
|
60
|
-
}, work);
|
|
61
|
-
}));
|
|
62
|
-
async function work([job]) {
|
|
63
|
-
const messageData = MessageData.parse(job.data);
|
|
126
|
+
function createTaskHandler(queue) {
|
|
127
|
+
return async (payload) => {
|
|
128
|
+
const messageData = MessageData.parse(payload);
|
|
64
129
|
const bodyStream = Stream.Readable.toWeb(Stream.Readable.from([messageData.data]));
|
|
65
130
|
const body = await transport.deserialize(bodyStream);
|
|
66
131
|
const message = QueuePayloadSchema.parse(body);
|
|
67
132
|
const queueName = `${queue}${messageData.id}`;
|
|
68
133
|
// TODO: Custom headers from opts.headers are not propagated into MessageData.
|
|
69
134
|
// To support this, MessageData schema would need to include a headers field
|
|
70
|
-
// and the headers would need to be stored/retrieved from
|
|
135
|
+
// and the headers would need to be stored/retrieved from graphile-worker job data.
|
|
71
136
|
await localWorld.queue(queueName, message, {
|
|
72
137
|
idempotencyKey: messageData.idempotencyKey,
|
|
73
138
|
});
|
|
74
|
-
}
|
|
139
|
+
};
|
|
75
140
|
}
|
|
76
141
|
async function setupListeners() {
|
|
142
|
+
const taskList = {};
|
|
77
143
|
for (const [prefix, jobName] of Object.entries(Queues)) {
|
|
78
|
-
|
|
144
|
+
taskList[jobName] = createTaskHandler(prefix);
|
|
79
145
|
}
|
|
146
|
+
runner = await run({
|
|
147
|
+
connectionString: config.connectionString,
|
|
148
|
+
concurrency: config.queueConcurrency || 10,
|
|
149
|
+
logger: graphileLogger,
|
|
150
|
+
pollInterval: 500, // 500ms = 0.5s (graphile-worker uses LISTEN/NOTIFY when available)
|
|
151
|
+
taskList,
|
|
152
|
+
});
|
|
80
153
|
}
|
|
81
154
|
return {
|
|
82
155
|
createQueueHandler,
|
|
83
156
|
getDeploymentId,
|
|
84
157
|
queue,
|
|
85
|
-
|
|
86
|
-
boss = await boss.start();
|
|
87
|
-
await setupListeners();
|
|
88
|
-
},
|
|
158
|
+
start,
|
|
89
159
|
async close() {
|
|
160
|
+
if (runner) {
|
|
161
|
+
await runner.stop();
|
|
162
|
+
runner = null;
|
|
163
|
+
}
|
|
164
|
+
if (workerUtils) {
|
|
165
|
+
await workerUtils.release();
|
|
166
|
+
workerUtils = null;
|
|
167
|
+
}
|
|
168
|
+
startPromise = null;
|
|
90
169
|
await localWorld.close?.();
|
|
91
170
|
},
|
|
92
171
|
};
|
package/dist/queue.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,SAAS,EAET,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,SAAS,EAET,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EACL,MAAM,EACN,eAAe,EAEf,GAAG,GAEJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,SAAS,oBAAoB;IAC3B,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC;IAChE,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAEnD,OAAO,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,KAAa,EAAE,OAAe,EAAE,IAAc,EAAE,EAAE;QACzE,IAAI,UAAU,EAAE;YAAE,OAAO;QACzB,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACpE,MAAM,IAAI,GAAG,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACjE,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,CAAC,qBAAqB,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAChF,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,qBAAqB,OAAO,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,cAAc,GAAG,oBAAoB,EAAE,CAAC;AAiB9C,MAAM,UAAU,WAAW,CACzB,MAA2B,EAC3B,QAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,UAAU,GAAG,gBAAgB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElE,MAAM,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;IACtC,MAAM,iBAAiB,GAAG,gBAAgB,EAAE,CAAC;IAE7C,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,IAAI,WAAW,CAAC;IAC/C,MAAM,MAAM,GAAG;QACb,eAAe,EAAE,GAAG,MAAM,OAAO;QACjC,WAAW,EAAE,GAAG,MAAM,OAAO;KACiB,CAAC;IAEjD,MAAM,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;IAEzD,MAAM,eAAe,GAA6B,KAAK,IAAI,EAAE;QAC3D,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;IAEF,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,YAAY,GAAyB,IAAI,CAAC;IAE9C,KAAK,UAAU,iBAAiB,CAAC,KAAkB;QACjD,mEAAmE;QACnE,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAA;;;;;;KAMhC,CAAC;QACF,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAA;;;OAG1B,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAA+B,EAAE;oBAChE,MAAM,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;oBACtC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC;iBAClC,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,CAAA,8CAA8C,CAAC;YAC7D,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAA;;;;;KAK/B,CAAC;QACF,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAA;;;;OAI1B,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAA+B,EAAE;oBAChE,MAAM,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;oBACtC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC;iBAClC,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,CAAA,4BAA4B,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,KAAK,UAAU,KAAK;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;gBACzB,IAAI,CAAC;oBACH,WAAW,GAAG,MAAM,eAAe,CAAC;wBAClC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;wBACzC,MAAM,EAAE,cAAc;qBACvB,CAAC,CAAC;oBACH,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;oBAC5B,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;oBACrC,MAAM,cAAc,EAAE,CAAC;gBACzB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QACD,MAAM,YAAY,CAAC;IACrB,CAAC;IAED,MAAM,KAAK,GAAmB,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC3D,MAAM,KAAK,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAChE,MAAM,WAAY,CAAC,MAAM,CACvB,OAAO,EACP,WAAW,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,IAAI;YACV,OAAO,EAAE,CAAC;YACV,SAAS;YACT,cAAc,EAAE,IAAI,EAAE,cAAc;SACrC,CAAC,EACF;YACE,MAAM,EAAE,IAAI,EAAE,cAAc,IAAI,SAAS;YACzC,WAAW,EAAE,CAAC;SACf,CACF,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,CAAC;IACvB,CAAC,CAAC;IAEF,SAAS,iBAAiB,CAAC,KAAkB;QAC3C,OAAO,KAAK,EAAE,OAAgB,EAAE,EAAE;YAChC,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CACtC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CACzC,CAAC;YACF,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,WAAW,CACtC,UAAwC,CACzC,CAAC;YACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,SAAS,GAAG,GAAG,KAAK,GAAG,WAAW,CAAC,EAAE,EAAW,CAAC;YACvD,8EAA8E;YAC9E,4EAA4E;YAC5E,mFAAmF;YACnF,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE;gBACzC,cAAc,EAAE,WAAW,CAAC,cAAc;aAC3C,CAAC,CAAC;QACL,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,UAAU,cAAc;QAC3B,MAAM,QAAQ,GAAwD,EAAE,CAAC;QACzE,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAGlD,EAAE,CAAC;YACJ,QAAQ,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,GAAG,MAAM,GAAG,CAAC;YACjB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,WAAW,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;YAC1C,MAAM,EAAE,cAAc;YACtB,YAAY,EAAE,GAAG,EAAE,mEAAmE;YACtF,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,kBAAkB;QAClB,eAAe;QACf,KAAK;QACL,KAAK;QACL,KAAK,CAAC,KAAK;YACT,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBACpB,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC5B,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;QAC7B,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,cAAc,GAAG,CAAC,IAAoB,EAAyB,EAAE;IACrE,MAAM,QAAQ,GAAkB,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACnE,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC"}
|
package/dist/streamer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamer.d.ts","sourceRoot":"","sources":["../src/streamer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,OAAO,EAAE,KAAK,OAAO,EAAU,MAAM,oBAAoB,CAAC;AAkC1D,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG;IACxC,mEAAmE;IACnE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,GAAG,EACb,OAAO,EAAE,OAAO,GACf,gBAAgB,
|
|
1
|
+
{"version":3,"file":"streamer.d.ts","sourceRoot":"","sources":["../src/streamer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,OAAO,EAAE,KAAK,OAAO,EAAU,MAAM,oBAAoB,CAAC;AAkC1D,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG;IACxC,mEAAmE;IACnE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,GAAG,EACb,OAAO,EAAE,OAAO,GACf,gBAAgB,CAmOlB"}
|
package/dist/streamer.js
CHANGED
|
@@ -49,9 +49,7 @@ export function createStreamer(postgres, drizzle) {
|
|
|
49
49
|
};
|
|
50
50
|
const STREAM_TOPIC = 'workflow_event_chunk';
|
|
51
51
|
const listenSubscription = postgres.listen(STREAM_TOPIC, async (msg) => {
|
|
52
|
-
const parsed =
|
|
53
|
-
.then(JSON.parse)
|
|
54
|
-
.then(StreamPublishMessage.parse);
|
|
52
|
+
const parsed = StreamPublishMessage.parse(JSON.parse(msg));
|
|
55
53
|
const key = `strm:${parsed.streamId}`;
|
|
56
54
|
if (!events.listenerCount(key)) {
|
|
57
55
|
return;
|
|
@@ -83,7 +81,7 @@ export function createStreamer(postgres, drizzle) {
|
|
|
83
81
|
chunkData: toBuffer(chunk),
|
|
84
82
|
eof: false,
|
|
85
83
|
});
|
|
86
|
-
postgres.notify(STREAM_TOPIC, JSON.stringify(StreamPublishMessage.encode({
|
|
84
|
+
await postgres.notify(STREAM_TOPIC, JSON.stringify(StreamPublishMessage.encode({
|
|
87
85
|
chunkId,
|
|
88
86
|
streamId: name,
|
|
89
87
|
})));
|
|
@@ -105,7 +103,7 @@ export function createStreamer(postgres, drizzle) {
|
|
|
105
103
|
})));
|
|
106
104
|
// Notify for each chunk (could be batched in future if needed)
|
|
107
105
|
for (const chunkId of chunkIds) {
|
|
108
|
-
postgres.notify(STREAM_TOPIC, JSON.stringify(StreamPublishMessage.encode({
|
|
106
|
+
await postgres.notify(STREAM_TOPIC, JSON.stringify(StreamPublishMessage.encode({
|
|
109
107
|
chunkId,
|
|
110
108
|
streamId: name,
|
|
111
109
|
})));
|
|
@@ -122,7 +120,7 @@ export function createStreamer(postgres, drizzle) {
|
|
|
122
120
|
chunkData: Buffer.from([]),
|
|
123
121
|
eof: true,
|
|
124
122
|
});
|
|
125
|
-
postgres.notify('workflow_event_chunk', JSON.stringify(StreamPublishMessage.encode({
|
|
123
|
+
await postgres.notify('workflow_event_chunk', JSON.stringify(StreamPublishMessage.encode({
|
|
126
124
|
streamId: name,
|
|
127
125
|
chunkId,
|
|
128
126
|
})));
|
package/dist/streamer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamer.js","sourceRoot":"","sources":["../src/streamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;CAClD,CAAC,CAAC;AAQH,MAAM,EAAE;IAEc;IADZ,QAAQ,GAAG,CAAC,CAAC;IACrB,YAAoB,QAAW;QAAX,aAAQ,GAAR,QAAQ,CAAG;IAAG,CAAC;IACnC,OAAO;QACL,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO;YACL,GAAG,IAAI,CAAC,QAAQ;YAChB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;gBACrB,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,CAAC;SACF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;CACF;AAOD,MAAM,UAAU,cAAc,CAC5B,QAAa,EACb,OAAgB;IAEhB,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,YAAY,EAE3B,CAAC;IACL,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,EAAW,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8C,CAAC;IACtE,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;QAC/B,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,IAAI,KAAK,EAAE;gBAClB,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;aAChC,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,sBAAsB,CAAC;IAC5C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrE,MAAM,MAAM,GAAG,
|
|
1
|
+
{"version":3,"file":"streamer.js","sourceRoot":"","sources":["../src/streamer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAgB,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;CAClD,CAAC,CAAC;AAQH,MAAM,EAAE;IAEc;IADZ,QAAQ,GAAG,CAAC,CAAC;IACrB,YAAoB,QAAW;QAAX,aAAQ,GAAR,QAAQ,CAAG;IAAG,CAAC;IACnC,OAAO;QACL,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,OAAO;YACL,GAAG,IAAI,CAAC,QAAQ;YAChB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;gBACrB,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,CAAC;SACF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;CACF;AAOD,MAAM,UAAU,cAAc,CAC5B,QAAa,EACb,OAAgB;IAEhB,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,YAAY,EAE3B,CAAC;IACL,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,EAAW,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8C,CAAC;IACtE,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;QAC/B,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,IAAI,KAAK,EAAE;gBAClB,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;aAChC,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,sBAAsB,CAAC;IAC5C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrE,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAG,QAAQ,MAAM,CAAC,QAAQ,EAAW,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YACtC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO;iBAC1B,MAAM,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;iBACrD,IAAI,CAAC,OAAO,CAAC;iBACb,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,EACrC,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CACpC,CACF;iBACA,KAAK,CAAC,CAAC,CAAC,CAAC;YACZ,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,oCAAoC;IACpC,MAAM,QAAQ,GAAG,CAAC,KAA0B,EAAU,EAAE,CACtD,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEvD,OAAO;QACL,KAAK,CAAC,aAAa,CACjB,IAAY,EACZ,MAAgC,EAChC,KAA0B;YAE1B,0DAA0D;YAC1D,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;YAE3B,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;gBACnC,OAAO;gBACP,QAAQ,EAAE,IAAI;gBACd,KAAK;gBACL,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC;gBAC1B,GAAG,EAAE,KAAK;aACX,CAAC,CAAC;YACH,MAAM,QAAQ,CAAC,MAAM,CACnB,YAAY,EACZ,IAAI,CAAC,SAAS,CACZ,oBAAoB,CAAC,MAAM,CAAC;gBAC1B,OAAO;gBACP,QAAQ,EAAE,IAAI;aACf,CAAC,CACH,CACF,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,kBAAkB,CACtB,IAAY,EACZ,MAAgC,EAChC,MAA+B;YAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEhC,uDAAuD;YACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YAEhD,0DAA0D;YAC1D,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;YAE3B,4CAA4C;YAC5C,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAClC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxB,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACpB,QAAQ,EAAE,IAAI;gBACd,KAAK;gBACL,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC;gBAC1B,GAAG,EAAE,KAAK;aACX,CAAC,CAAC,CACJ,CAAC;YAEF,+DAA+D;YAC/D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,QAAQ,CAAC,MAAM,CACnB,YAAY,EACZ,IAAI,CAAC,SAAS,CACZ,oBAAoB,CAAC,MAAM,CAAC;oBAC1B,OAAO;oBACP,QAAQ,EAAE,IAAI;iBACf,CAAC,CACH,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,KAAK,CAAC,WAAW,CACf,IAAY,EACZ,MAAgC;YAEhC,0DAA0D;YAC1D,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;YAE3B,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;YAC7B,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;gBACnC,OAAO;gBACP,QAAQ,EAAE,IAAI;gBACd,KAAK;gBACL,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1B,GAAG,EAAE,IAAI;aACV,CAAC,CAAC;YACH,MAAM,QAAQ,CAAC,MAAM,CACnB,sBAAsB,EACtB,IAAI,CAAC,SAAS,CACZ,oBAAoB,CAAC,MAAM,CAAC;gBAC1B,QAAQ,EAAE,IAAI;gBACd,OAAO;aACR,CAAC,CACH,CACF,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,cAAc,CAClB,IAAY,EACZ,UAAmB;YAEnB,MAAM,QAAQ,GAAmB,EAAE,CAAC;YAEpC,OAAO,IAAI,cAAc,CAAa;gBACpC,KAAK,CAAC,KAAK,CAAC,UAAU;oBACpB,+CAA+C;oBAC/C,6DAA6D;oBAC7D,IAAI,WAAW,GAAG,EAAE,CAAC;oBACrB,IAAI,MAAM,GAAG,UAAU,IAAI,CAAC,CAAC;oBAC7B,IAAI,MAAM,GAAG,EAA+B,CAAC;oBAE7C,SAAS,OAAO,CAAC,GAIhB;wBACC,IAAI,WAAW,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;4BAC1B,+BAA+B;4BAC/B,OAAO;wBACT,CAAC;wBAED,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;4BACf,MAAM,EAAE,CAAC;4BACT,OAAO;wBACT,CAAC;wBAED,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;4BACxB,UAAU,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;wBAC/C,CAAC;wBACD,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;4BACZ,UAAU,CAAC,KAAK,EAAE,CAAC;wBACrB,CAAC;wBACD,WAAW,GAAG,GAAG,CAAC,EAAE,CAAC;oBACvB,CAAC;oBAED,SAAS,MAAM,CAAC,IAAsB;wBACpC,IAAI,MAAM,EAAE,CAAC;4BACX,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAClB,OAAO;wBACT,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC;oBACD,MAAM,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE;wBACjB,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;oBACrC,CAAC,CAAC,CAAC;oBAEH,MAAM,MAAM,GAAG,MAAM,OAAO;yBACzB,MAAM,CAAC;wBACN,EAAE,EAAE,OAAO,CAAC,OAAO;wBACnB,GAAG,EAAE,OAAO,CAAC,GAAG;wBAChB,IAAI,EAAE,OAAO,CAAC,SAAS;qBACxB,CAAC;yBACD,IAAI,CAAC,OAAO,CAAC;yBACb,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;yBACtC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oBAE5B,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;wBACnD,OAAO,CAAC,KAAK,CAAC,CAAC;oBACjB,CAAC;oBACD,MAAM,GAAG,IAAI,CAAC;gBAChB,CAAC;gBACD,MAAM;oBACJ,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACjC,CAAC;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,CAAC,kBAAkB,CAAC,KAAa;YACpC,sDAAsD;YACtD,MAAM,OAAO,GAAG,MAAM,OAAO;iBAC1B,cAAc,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;iBAC9C,IAAI,CAAC,OAAO,CAAC;iBACb,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YAEnC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,CAAC,KAAK;YACT,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;YACrC,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@workflow/world-postgres",
|
|
3
|
-
"version": "4.1.0-beta.
|
|
3
|
+
"version": "4.1.0-beta.37",
|
|
4
4
|
"description": "A reference World implementation based on PostgreSQL",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -37,17 +37,17 @@
|
|
|
37
37
|
"./migrations/*.sql": "./src/drizzle/migrations/*.sql"
|
|
38
38
|
},
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@vercel/queue": "0.0.
|
|
40
|
+
"@vercel/queue": "0.0.2",
|
|
41
41
|
"cbor-x": "1.6.0",
|
|
42
42
|
"dotenv": "16.4.5",
|
|
43
43
|
"drizzle-orm": "0.44.7",
|
|
44
|
-
"
|
|
44
|
+
"graphile-worker": "0.16.6",
|
|
45
45
|
"postgres": "3.4.7",
|
|
46
46
|
"ulid": "3.0.1",
|
|
47
47
|
"zod": "4.1.11",
|
|
48
|
-
"@workflow/
|
|
49
|
-
"@workflow/world
|
|
50
|
-
"@workflow/
|
|
48
|
+
"@workflow/errors": "4.1.0-beta.16",
|
|
49
|
+
"@workflow/world": "4.1.0-beta.7",
|
|
50
|
+
"@workflow/world-local": "4.1.0-beta.35"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
53
|
"@testcontainers/postgresql": "11.7.1",
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
"vitest": "^3.2.4",
|
|
57
57
|
"@workflow/errors": "4.1.0-beta.16",
|
|
58
58
|
"@workflow/tsconfig": "4.0.1-beta.0",
|
|
59
|
-
"@workflow/world-testing": "4.1.0-beta.
|
|
59
|
+
"@workflow/world-testing": "4.1.0-beta.62"
|
|
60
60
|
},
|
|
61
61
|
"keywords": [],
|
|
62
62
|
"author": "",
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
-- Migrate pending pg-boss jobs to a staging table before dropping the pgboss schema.
|
|
2
|
+
-- The application code will re-enqueue these jobs into graphile-worker on first start.
|
|
3
|
+
DO $$
|
|
4
|
+
BEGIN
|
|
5
|
+
IF EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'pgboss') THEN
|
|
6
|
+
CREATE TABLE IF NOT EXISTS "workflow"."_pgboss_pending_jobs" (
|
|
7
|
+
name text NOT NULL,
|
|
8
|
+
data jsonb,
|
|
9
|
+
singleton_key text,
|
|
10
|
+
retry_limit integer
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
INSERT INTO "workflow"."_pgboss_pending_jobs" (name, data, singleton_key, retry_limit)
|
|
14
|
+
SELECT name, data, singleton_key, retry_limit
|
|
15
|
+
FROM pgboss.job
|
|
16
|
+
WHERE state IN ('created', 'retry');
|
|
17
|
+
|
|
18
|
+
DROP SCHEMA pgboss CASCADE;
|
|
19
|
+
END IF;
|
|
20
|
+
END $$;
|
package/dist/boss.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"boss.d.ts","sourceRoot":"","sources":["../src/boss.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;IAII;AACJ,eAAO,MAAM,WAAW;;;;;;iBAUtB,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC"}
|
package/dist/boss.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"boss.js","sourceRoot":"","sources":["../src/boss.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC;;;;IAII;AACJ,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IACjE,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAC7D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,QAAQ,CACP,gGAAgG,CACjG;IACH,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CACzD,CAAC,CAAC"}
|