@workflow/world-postgres 4.1.0-beta.4 → 4.1.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +201 -21
- package/README.md +17 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +12 -12
- package/dist/cli.js.map +1 -1
- package/dist/drizzle/cbor.d.ts +38 -0
- package/dist/drizzle/cbor.d.ts.map +1 -0
- package/dist/drizzle/cbor.js +10 -0
- package/dist/drizzle/cbor.js.map +1 -0
- package/dist/drizzle/index.d.ts +3 -1
- package/dist/drizzle/index.d.ts.map +1 -1
- package/dist/drizzle/index.js.map +1 -1
- package/dist/drizzle/schema.d.ts +756 -59
- package/dist/drizzle/schema.d.ts.map +1 -1
- package/dist/drizzle/schema.js +68 -37
- package/dist/drizzle/schema.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -5
- package/dist/index.js.map +1 -1
- package/dist/{boss.d.ts → message.d.ts} +6 -6
- package/dist/message.d.ts.map +1 -0
- package/dist/{boss.js → message.js} +5 -5
- package/dist/message.js.map +1 -0
- package/dist/queue.d.ts +8 -6
- package/dist/queue.d.ts.map +1 -1
- package/dist/queue.js +129 -50
- package/dist/queue.js.map +1 -1
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +854 -212
- package/dist/storage.js.map +1 -1
- package/dist/streamer.d.ts +5 -1
- package/dist/streamer.d.ts.map +1 -1
- package/dist/streamer.js +50 -9
- package/dist/streamer.js.map +1 -1
- package/package.json +21 -20
- package/src/drizzle/migrations/{0000_redundant_smasher.sql → 0000_cultured_the_anarchist.sql} +17 -13
- package/src/drizzle/migrations/0001_tricky_sersi.sql +11 -0
- package/src/drizzle/migrations/0002_add_expired_at.sql +2 -0
- package/src/drizzle/migrations/0003_add_stream_run_id.sql +3 -0
- package/src/drizzle/migrations/0004_remove_run_pause_status.sql +14 -0
- package/src/drizzle/migrations/0005_add_spec_version.sql +5 -0
- package/src/drizzle/migrations/0006_add_error_cbor.sql +5 -0
- package/src/drizzle/migrations/0007_add_waits_table.sql +13 -0
- package/src/drizzle/migrations/0008_migrate_pgboss_to_graphile.sql +20 -0
- package/src/drizzle/migrations/0009_add_is_webhook.sql +1 -0
- package/src/drizzle/migrations/meta/0000_snapshot.json +499 -0
- package/src/drizzle/migrations/meta/0001_snapshot.json +548 -0
- package/src/drizzle/migrations/meta/0002_snapshot.json +554 -0
- package/src/drizzle/migrations/meta/0003_snapshot.json +576 -0
- package/src/drizzle/migrations/meta/0005_snapshot.json +575 -0
- package/src/drizzle/migrations/meta/_journal.json +76 -0
- package/dist/boss.d.ts.map +0 -1
- package/dist/boss.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/drizzle/schema.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/drizzle/schema.ts"],"names":[],"mappings":"AA+BA,eAAO,MAAM,iBAAiB,2KAG7B,CAAC;AAEF,eAAO,MAAM,UAAU,2KAGtB,CAAC;AAEF,eAAO,MAAM,UAAU,iGAGtB,CAAC;AAYF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC;AAEtC,eAAO,MAAM,MAAM,oDAAuB,CAAC;AAE3C,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoChB,CAAC;AAEF,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgBlB,CAAC;AAEF,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoCjB,CAAC;AAEF,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiBjB,CAAC;AAEF,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgBjB,CAAC;AAQF,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAcnB,CAAC"}
|
package/dist/drizzle/schema.js
CHANGED
|
@@ -1,20 +1,33 @@
|
|
|
1
|
-
import { StepStatusSchema, WorkflowRunStatusSchema, } from '@workflow/world';
|
|
2
|
-
import { boolean, customType, index, integer,
|
|
1
|
+
import { StepStatusSchema, WaitStatusSchema, WorkflowRunStatusSchema, } from '@workflow/world';
|
|
2
|
+
import { boolean, customType, index, integer,
|
|
3
|
+
/** @deprecated: use Cbor instead */
|
|
4
|
+
jsonb, pgEnum, pgSchema, primaryKey, text, timestamp, varchar, } from 'drizzle-orm/pg-core';
|
|
5
|
+
import { Cbor } from './cbor.js';
|
|
3
6
|
function mustBeMoreThanOne(t) {
|
|
4
7
|
return t;
|
|
5
8
|
}
|
|
6
9
|
export const workflowRunStatus = pgEnum('status', mustBeMoreThanOne(WorkflowRunStatusSchema.options));
|
|
7
10
|
export const stepStatus = pgEnum('step_status', mustBeMoreThanOne(StepStatusSchema.options));
|
|
8
|
-
export const
|
|
11
|
+
export const waitStatus = pgEnum('wait_status', mustBeMoreThanOne(WaitStatusSchema.options));
|
|
12
|
+
export const schema = pgSchema('workflow');
|
|
13
|
+
export const runs = schema.table('workflow_runs', {
|
|
9
14
|
runId: varchar('id').primaryKey(),
|
|
10
|
-
|
|
15
|
+
/** @deprecated */
|
|
16
|
+
outputJson: jsonb('output').$type(),
|
|
17
|
+
output: Cbor()('output_cbor'),
|
|
11
18
|
deploymentId: varchar('deployment_id').notNull(),
|
|
12
19
|
status: workflowRunStatus('status').notNull(),
|
|
13
20
|
workflowName: varchar('name').notNull(),
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
21
|
+
specVersion: integer('spec_version'),
|
|
22
|
+
/** @deprecated */
|
|
23
|
+
executionContextJson: jsonb('execution_context').$type(),
|
|
24
|
+
executionContext: Cbor()('execution_context_cbor'),
|
|
25
|
+
/** @deprecated */
|
|
26
|
+
inputJson: jsonb('input').$type(),
|
|
27
|
+
input: Cbor()('input_cbor'),
|
|
28
|
+
/** @deprecated - use error instead */
|
|
29
|
+
errorJson: text('error'),
|
|
30
|
+
error: Cbor()('error_cbor'),
|
|
18
31
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
19
32
|
updatedAt: timestamp('updated_at')
|
|
20
33
|
.defaultNow()
|
|
@@ -22,31 +35,35 @@ export const runs = pgTable('workflow_runs', {
|
|
|
22
35
|
.notNull(),
|
|
23
36
|
completedAt: timestamp('completed_at'),
|
|
24
37
|
startedAt: timestamp('started_at'),
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
}));
|
|
29
|
-
export const events = pgTable('workflow_events', {
|
|
38
|
+
expiredAt: timestamp('expired_at'),
|
|
39
|
+
}, (tb) => [index().on(tb.workflowName), index().on(tb.status)]);
|
|
40
|
+
export const events = schema.table('workflow_events', {
|
|
30
41
|
eventId: varchar('id').primaryKey(),
|
|
31
42
|
eventType: varchar('type').$type().notNull(),
|
|
32
43
|
correlationId: varchar('correlation_id'),
|
|
33
44
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
34
45
|
runId: varchar('run_id').notNull(),
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
}));
|
|
40
|
-
export const steps =
|
|
46
|
+
/** @deprecated */
|
|
47
|
+
eventDataJson: jsonb('payload'),
|
|
48
|
+
eventData: Cbor()('payload_cbor'),
|
|
49
|
+
specVersion: integer('spec_version'),
|
|
50
|
+
}, (tb) => [index().on(tb.runId), index().on(tb.correlationId)]);
|
|
51
|
+
export const steps = schema.table('workflow_steps', {
|
|
41
52
|
runId: varchar('run_id').notNull(),
|
|
42
53
|
stepId: varchar('step_id').primaryKey(),
|
|
43
54
|
stepName: varchar('step_name').notNull(),
|
|
44
55
|
status: stepStatus('status').notNull(),
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
56
|
+
/** @deprecated */
|
|
57
|
+
inputJson: jsonb('input').$type(),
|
|
58
|
+
input: Cbor()('input_cbor'),
|
|
59
|
+
/** @deprecated we stream binary data */
|
|
60
|
+
outputJson: jsonb('output').$type(),
|
|
61
|
+
output: Cbor()('output_cbor'),
|
|
62
|
+
/** @deprecated - use error instead */
|
|
63
|
+
errorJson: text('error'),
|
|
64
|
+
error: Cbor()('error_cbor'),
|
|
49
65
|
attempt: integer('attempt').notNull(),
|
|
66
|
+
/** Maps to startedAt in Step interface */
|
|
50
67
|
startedAt: timestamp('started_at'),
|
|
51
68
|
completedAt: timestamp('completed_at'),
|
|
52
69
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
@@ -55,11 +72,9 @@ export const steps = pgTable('workflow_steps', {
|
|
|
55
72
|
.$onUpdateFn(() => new Date())
|
|
56
73
|
.notNull(),
|
|
57
74
|
retryAfter: timestamp('retry_after'),
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}));
|
|
62
|
-
export const hooks = pgTable('workflow_hooks', {
|
|
75
|
+
specVersion: integer('spec_version'),
|
|
76
|
+
}, (tb) => [index().on(tb.runId), index().on(tb.status)]);
|
|
77
|
+
export const hooks = schema.table('workflow_hooks', {
|
|
63
78
|
runId: varchar('run_id').notNull(),
|
|
64
79
|
hookId: varchar('hook_id').primaryKey(),
|
|
65
80
|
token: varchar('token').notNull(),
|
|
@@ -67,23 +82,39 @@ export const hooks = pgTable('workflow_hooks', {
|
|
|
67
82
|
projectId: varchar('project_id').notNull(),
|
|
68
83
|
environment: varchar('environment').notNull(),
|
|
69
84
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
85
|
+
/** @deprecated */
|
|
86
|
+
metadataJson: jsonb('metadata').$type(),
|
|
87
|
+
metadata: Cbor()('metadata_cbor'),
|
|
88
|
+
specVersion: integer('spec_version'),
|
|
89
|
+
isWebhook: boolean('is_webhook').default(true),
|
|
90
|
+
}, (tb) => [index().on(tb.runId), index().on(tb.token)]);
|
|
91
|
+
export const waits = schema.table('workflow_waits', {
|
|
92
|
+
waitId: varchar('wait_id').primaryKey(),
|
|
93
|
+
runId: varchar('run_id').notNull(),
|
|
94
|
+
status: waitStatus('status').notNull(),
|
|
95
|
+
resumeAt: timestamp('resume_at'),
|
|
96
|
+
completedAt: timestamp('completed_at'),
|
|
97
|
+
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
98
|
+
updatedAt: timestamp('updated_at')
|
|
99
|
+
.defaultNow()
|
|
100
|
+
.$onUpdateFn(() => new Date())
|
|
101
|
+
.notNull(),
|
|
102
|
+
specVersion: integer('spec_version'),
|
|
103
|
+
}, (tb) => [index().on(tb.runId)]);
|
|
75
104
|
const bytea = customType({
|
|
76
105
|
dataType() {
|
|
77
106
|
return 'bytea';
|
|
78
107
|
},
|
|
79
108
|
});
|
|
80
|
-
export const streams =
|
|
109
|
+
export const streams = schema.table('workflow_stream_chunks', {
|
|
81
110
|
chunkId: varchar('id').$type().notNull(),
|
|
82
111
|
streamId: varchar('stream_id').notNull(),
|
|
112
|
+
runId: varchar('run_id'),
|
|
83
113
|
chunkData: bytea('data').notNull(),
|
|
84
114
|
createdAt: timestamp('created_at').defaultNow().notNull(),
|
|
85
115
|
eof: boolean('eof').notNull(),
|
|
86
|
-
}, (tb) =>
|
|
87
|
-
primaryKey
|
|
88
|
-
|
|
116
|
+
}, (tb) => [
|
|
117
|
+
primaryKey({ columns: [tb.streamId, tb.chunkId] }),
|
|
118
|
+
index().on(tb.runId),
|
|
119
|
+
]);
|
|
89
120
|
//# sourceMappingURL=schema.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/drizzle/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,gBAAgB,EAEhB,uBAAuB,GACxB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,OAAO,EACP,UAAU,EACV,KAAK,EACL,OAAO,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/drizzle/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,gBAAgB,EAGhB,gBAAgB,EAEhB,uBAAuB,GACxB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,OAAO,EACP,UAAU,EACV,KAAK,EACL,OAAO;AACP,oCAAoC;AACpC,KAAK,EACL,MAAM,EACN,QAAQ,EACR,UAAU,EACV,IAAI,EACJ,SAAS,EACT,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAiB,MAAM,WAAW,CAAC;AAEhD,SAAS,iBAAiB,CAAI,CAAM;IAClC,OAAO,CAAgB,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAM,CACrC,QAAQ,EACR,iBAAiB,CAAC,uBAAuB,CAAC,OAAO,CAAC,CACnD,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAC9B,aAAa,EACb,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAC5C,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAC9B,aAAa,EACb,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAC5C,CAAC;AAiBF,MAAM,CAAC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,CAAC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAC9B,eAAe,EACf;IACE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IACjC,kBAAkB;IAClB,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAqB;IACtD,MAAM,EAAE,IAAI,EAAqB,CAAC,aAAa,CAAC;IAChD,YAAY,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;IAChD,MAAM,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IAC7C,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IACvC,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC;IACpC,kBAAkB;IAClB,oBAAoB,EAClB,KAAK,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAuB;IACzD,gBAAgB,EAAE,IAAI,EAAuB,CAAC,wBAAwB,CAAC;IACvE,kBAAkB;IAClB,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAqB;IACpD,KAAK,EAAE,IAAI,EAAqB,CAAC,YAAY,CAAC;IAC9C,sCAAsC;IACtC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC;IACxB,KAAK,EAAE,IAAI,EAAmB,CAAC,YAAY,CAAC;IAC5C,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE;IACzD,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;SAC/B,UAAU,EAAE;SACZ,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;SAC7B,OAAO,EAAE;IACZ,WAAW,EAAE,SAAS,CAAC,cAAc,CAAC;IACtC,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;IAClC,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;CAMnC,EACD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAC7D,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAChC,iBAAiB,EACjB;IACE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IACnC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAsB,CAAC,OAAO,EAAE;IAChE,aAAa,EAAE,OAAO,CAAC,gBAAgB,CAAC;IACxC,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE;IACzD,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IAClC,kBAAkB;IAClB,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC;IAC/B,SAAS,EAAE,IAAI,EAAW,CAAC,cAAc,CAAC;IAC1C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC;CAGrC,EACD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAC7D,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAC/B,gBAAgB,EAChB;IACE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IAClC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE;IACvC,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;IACxC,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IACtC,kBAAkB;IAClB,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAqB;IACpD,KAAK,EAAE,IAAI,EAAqB,CAAC,YAAY,CAAC;IAC9C,wCAAwC;IACxC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAqB;IACtD,MAAM,EAAE,IAAI,EAAqB,CAAC,aAAa,CAAC;IAChD,sCAAsC;IACtC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC;IACxB,KAAK,EAAE,IAAI,EAAmB,CAAC,YAAY,CAAC;IAC5C,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE;IACrC,0CAA0C;IAC1C,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;IAClC,WAAW,EAAE,SAAS,CAAC,cAAc,CAAC;IACtC,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE;IACzD,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;SAC/B,UAAU,EAAE;SACZ,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;SAC7B,OAAO,EAAE;IACZ,UAAU,EAAE,SAAS,CAAC,aAAa,CAAC;IACpC,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC;CAQrC,EACD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CACtD,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAC/B,gBAAgB,EAChB;IACE,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IAClC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE;IACvC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IACjC,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE;IACtC,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC1C,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE;IAC7C,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE;IACzD,kBAAkB;IAClB,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,EAAqB;IAC1D,QAAQ,EAAE,IAAI,EAAqB,CAAC,eAAe,CAAC;IACpD,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC;IACpC,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACO,EACvD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CACrD,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAC/B,gBAAgB,EAChB;IACE,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE;IACvC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IAClC,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;IACtC,QAAQ,EAAE,SAAS,CAAC,WAAW,CAAC;IAChC,WAAW,EAAE,SAAS,CAAC,cAAc,CAAC;IACtC,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE;IACzD,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;SAC/B,UAAU,EAAE;SACZ,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;SAC7B,OAAO,EAAE;IACZ,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC;CACL,EACjC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAC/B,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAmD;IACzE,QAAQ;QACN,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CACjC,wBAAwB,EACxB;IACE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAoB,CAAC,OAAO,EAAE;IAC1D,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;IACxC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC;IACxB,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAClC,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE;IACzD,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE;CAC9B,EACD,CAAC,EAAE,EAAE,EAAE,CAAC;IACN,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAClD,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC;CACrB,CACF,CAAC"}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAW,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAoBvD,wBAAgB,WAAW,CACzB,MAAM,GAAE,mBAQP,GACA,KAAK,GAAG;IAAE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CA+BpC;AAGD,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,cAAc,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import PgBoss from 'pg-boss';
|
|
2
1
|
import createPostgres from 'postgres';
|
|
3
2
|
import { createClient } from './drizzle/index.js';
|
|
4
3
|
import { createQueue } from './queue.js';
|
|
@@ -19,12 +18,9 @@ export function createWorld(config = {
|
|
|
19
18
|
queueConcurrency: parseInt(process.env.WORKFLOW_POSTGRES_WORKER_CONCURRENCY || '10', 10) ||
|
|
20
19
|
10,
|
|
21
20
|
}) {
|
|
22
|
-
const boss = new PgBoss({
|
|
23
|
-
connectionString: config.connectionString,
|
|
24
|
-
});
|
|
25
21
|
const postgres = createPostgres(config.connectionString);
|
|
26
22
|
const drizzle = createClient(postgres);
|
|
27
|
-
const queue = createQueue(
|
|
23
|
+
const queue = createQueue(config, postgres);
|
|
28
24
|
const storage = createStorage(drizzle);
|
|
29
25
|
const streamer = createStreamer(postgres, drizzle);
|
|
30
26
|
return {
|
|
@@ -34,6 +30,22 @@ export function createWorld(config = {
|
|
|
34
30
|
async start() {
|
|
35
31
|
await queue.start();
|
|
36
32
|
},
|
|
33
|
+
async close() {
|
|
34
|
+
await streamer.close();
|
|
35
|
+
await queue.close();
|
|
36
|
+
await postgres.end();
|
|
37
|
+
// Force-destroy any TCP sockets that survived postgres.end().
|
|
38
|
+
// postgres.js's terminate() calls socket.end() (graceful TCP FIN)
|
|
39
|
+
// rather than socket.destroy(), leaving sockets in FIN_WAIT state
|
|
40
|
+
// that prevent the process from exiting on slower networks (e.g.
|
|
41
|
+
// CI Docker containers).
|
|
42
|
+
// See: https://github.com/porsager/postgres/issues/1022
|
|
43
|
+
for (const h of process._getActiveHandles?.() ?? []) {
|
|
44
|
+
if (h?.constructor?.name === 'Socket' && !h._type && !h.destroyed) {
|
|
45
|
+
h.destroy();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
},
|
|
37
49
|
};
|
|
38
50
|
}
|
|
39
51
|
export * from './drizzle/schema.js';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,cAAc,MAAM,UAAU,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAgB,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,SAAS,aAAa,CAAC,OAAgB;IACrC,OAAO;QACL,IAAI,EAAE,iBAAiB,CAAC,OAAO,CAAC;QAChC,MAAM,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACpC,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC;QAClC,KAAK,EAAE,kBAAkB,CAAC,OAAO,CAAC;KACnC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,SAA8B;IAC5B,gBAAgB,EACd,OAAO,CAAC,GAAG,CAAC,qBAAqB;QACjC,6CAA6C;IAC/C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,4BAA4B;IACnD,gBAAgB,EACd,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,IAAI,EAAE,EAAE,CAAC;QACtE,EAAE;CACL;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEnD,OAAO;QACL,GAAG,OAAO;QACV,GAAG,QAAQ;QACX,GAAG,KAAK;QACR,KAAK,CAAC,KAAK;YACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,MAAM,QAAQ,CAAC,GAAG,EAAE,CAAC;YACrB,8DAA8D;YAC9D,kEAAkE;YAClE,kEAAkE;YAClE,iEAAiE;YACjE,yBAAyB;YACzB,wDAAwD;YACxD,KAAK,MAAM,CAAC,IAAK,OAAe,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC7D,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;oBACjE,CAAY,CAAC,OAAO,EAAE,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAID,cAAc,qBAAqB,CAAC"}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import * as z from 'zod';
|
|
2
2
|
/**
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
* graphile-worker is using JSON under the hood, so we need to base64 encode
|
|
4
|
+
* the body to ensure binary safety
|
|
5
|
+
* maybe later we can have a `blobs` table for larger payloads
|
|
6
|
+
*/
|
|
7
7
|
export declare const MessageData: z.ZodObject<{
|
|
8
8
|
attempt: z.ZodNumber;
|
|
9
|
-
messageId: z.core.$ZodBranded<z.ZodString, "MessageId">;
|
|
9
|
+
messageId: z.core.$ZodBranded<z.ZodString, "MessageId", "out">;
|
|
10
10
|
idempotencyKey: z.ZodOptional<z.ZodString>;
|
|
11
11
|
id: z.ZodString;
|
|
12
12
|
data: z.ZodCodec<z.ZodBase64, z.ZodCustom<Buffer<ArrayBufferLike>, Buffer<ArrayBufferLike>>>;
|
|
13
13
|
}, z.core.$strip>;
|
|
14
14
|
export type MessageData = z.infer<typeof MessageData>;
|
|
15
|
-
//# sourceMappingURL=
|
|
15
|
+
//# sourceMappingURL=message.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"message.d.ts","sourceRoot":"","sources":["../src/message.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAGzB;;;;GAIG;AACH,eAAO,MAAM,WAAW;;;;;;iBAUtB,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC"}
|
|
@@ -2,10 +2,10 @@ import { MessageId } from '@workflow/world';
|
|
|
2
2
|
import * as z from 'zod';
|
|
3
3
|
import { Base64Buffer } from './zod.js';
|
|
4
4
|
/**
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
* graphile-worker is using JSON under the hood, so we need to base64 encode
|
|
6
|
+
* the body to ensure binary safety
|
|
7
|
+
* maybe later we can have a `blobs` table for larger payloads
|
|
8
|
+
*/
|
|
9
9
|
export const MessageData = z.object({
|
|
10
10
|
attempt: z.number().describe('The attempt number of the message'),
|
|
11
11
|
messageId: MessageId.describe('The unique ID of the message'),
|
|
@@ -15,4 +15,4 @@ export const MessageData = z.object({
|
|
|
15
15
|
.describe("The ID of the sub-queue. For workflows, it's the workflow name. For steps, it's the step name."),
|
|
16
16
|
data: Base64Buffer.describe('The message that was sent'),
|
|
17
17
|
});
|
|
18
|
-
//# sourceMappingURL=
|
|
18
|
+
//# sourceMappingURL=message.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"message.js","sourceRoot":"","sources":["../src/message.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC;;;;GAIG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IACjE,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAC7D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,EAAE,EAAE,CAAC;SACF,MAAM,EAAE;SACR,QAAQ,CACP,gGAAgG,CACjG;IACH,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,2BAA2B,CAAC;CACzD,CAAC,CAAC"}
|
package/dist/queue.d.ts
CHANGED
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { type Queue } from '@workflow/world';
|
|
2
|
-
import type
|
|
2
|
+
import type Postgres from 'postgres';
|
|
3
3
|
import type { PostgresWorldConfig } from './config.js';
|
|
4
4
|
/**
|
|
5
|
-
* The Postgres queue works by creating two job types in
|
|
5
|
+
* The Postgres queue works by creating two job types in graphile-worker:
|
|
6
6
|
* - `workflow` for workflow jobs
|
|
7
7
|
* - `step` for step jobs
|
|
8
8
|
*
|
|
9
|
-
* When a message is queued, it is sent to
|
|
10
|
-
* When a job is processed, it is deserialized and then re-queued into the
|
|
11
|
-
* we can reuse the
|
|
9
|
+
* When a message is queued, it is sent to graphile-worker with the appropriate job type.
|
|
10
|
+
* When a job is processed, it is deserialized and then re-queued into the _local world_, showing that
|
|
11
|
+
* we can reuse the local world, mix and match worlds to build
|
|
12
12
|
* hybrid architectures, and even migrate between worlds.
|
|
13
13
|
*/
|
|
14
|
-
export
|
|
14
|
+
export type PostgresQueue = Queue & {
|
|
15
15
|
start(): Promise<void>;
|
|
16
|
+
close(): Promise<void>;
|
|
16
17
|
};
|
|
18
|
+
export declare function createQueue(config: PostgresWorldConfig, postgres: Postgres.Sql): PostgresQueue;
|
|
17
19
|
//# sourceMappingURL=queue.d.ts.map
|
package/dist/queue.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,KAAK,EAIX,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,KAAK,EAIX,MAAM,iBAAiB,CAAC;AASzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAuBvD;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG;IAClC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAgB,WAAW,CACzB,MAAM,EAAE,mBAAmB,EAC3B,QAAQ,EAAE,QAAQ,CAAC,GAAG,GACrB,aAAa,CAyKf"}
|
package/dist/queue.js
CHANGED
|
@@ -1,22 +1,31 @@
|
|
|
1
1
|
import * as Stream from 'node:stream';
|
|
2
2
|
import { JsonTransport } from '@vercel/queue';
|
|
3
3
|
import { MessageId, QueuePayloadSchema, } from '@workflow/world';
|
|
4
|
-
import {
|
|
4
|
+
import { createLocalWorld } from '@workflow/world-local';
|
|
5
|
+
import { Logger, makeWorkerUtils, run, } from 'graphile-worker';
|
|
5
6
|
import { monotonicFactory } from 'ulid';
|
|
6
|
-
import { MessageData } from './
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
7
|
+
import { MessageData } from './message.js';
|
|
8
|
+
function createGraphileLogger() {
|
|
9
|
+
const isJsonMode = () => process.env.WORKFLOW_JSON_MODE === '1';
|
|
10
|
+
const isVerbose = () => Boolean(process.env.DEBUG);
|
|
11
|
+
return new Logger(() => (level, message, meta) => {
|
|
12
|
+
if (isJsonMode())
|
|
13
|
+
return;
|
|
14
|
+
if ((level === 'debug' || level === 'info') && !isVerbose())
|
|
15
|
+
return;
|
|
16
|
+
const pipe = level === 'error' ? process.stderr : process.stdout;
|
|
17
|
+
if (meta) {
|
|
18
|
+
pipe.write(`[Graphile Worker] ${message} ${JSON.stringify(meta, null, 2)}\n`);
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
pipe.write(`[Graphile Worker] ${message}\n`);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
const graphileLogger = createGraphileLogger();
|
|
26
|
+
export function createQueue(config, postgres) {
|
|
18
27
|
const port = process.env.PORT ? Number(process.env.PORT) : undefined;
|
|
19
|
-
const
|
|
28
|
+
const localWorld = createLocalWorld({ dataDir: undefined, port });
|
|
20
29
|
const transport = new JsonTransport();
|
|
21
30
|
const generateMessageId = monotonicFactory();
|
|
22
31
|
const prefix = config.jobPrefix || 'workflow_';
|
|
@@ -24,70 +33,140 @@ export function createQueue(boss, config) {
|
|
|
24
33
|
__wkf_workflow_: `${prefix}flows`,
|
|
25
34
|
__wkf_step_: `${prefix}steps`,
|
|
26
35
|
};
|
|
27
|
-
const createQueueHandler =
|
|
36
|
+
const createQueueHandler = localWorld.createQueueHandler;
|
|
28
37
|
const getDeploymentId = async () => {
|
|
29
38
|
return 'postgres';
|
|
30
39
|
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
40
|
+
let workerUtils = null;
|
|
41
|
+
let runner = null;
|
|
42
|
+
let startPromise = null;
|
|
43
|
+
async function migratePgBossJobs(utils) {
|
|
44
|
+
// Scenario A: Drizzle migration already ran — staging table exists
|
|
45
|
+
const hasStaging = await postgres `
|
|
46
|
+
SELECT EXISTS (
|
|
47
|
+
SELECT 1 FROM information_schema.tables
|
|
48
|
+
WHERE table_schema = 'workflow'
|
|
49
|
+
AND table_name = '_pgboss_pending_jobs'
|
|
50
|
+
) AS exists
|
|
51
|
+
`;
|
|
52
|
+
if (hasStaging[0].exists) {
|
|
53
|
+
const jobs = await postgres `
|
|
54
|
+
SELECT name, data, singleton_key, retry_limit
|
|
55
|
+
FROM "workflow"."_pgboss_pending_jobs"
|
|
56
|
+
`;
|
|
57
|
+
for (const job of jobs) {
|
|
58
|
+
await utils.addJob(job.name, job.data, {
|
|
59
|
+
jobKey: job.singleton_key ?? undefined,
|
|
60
|
+
maxAttempts: job.retry_limit ?? 3,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
await postgres `DROP TABLE "workflow"."_pgboss_pending_jobs"`;
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
// Scenario B: Drizzle migration didn't run — pgboss schema still exists
|
|
67
|
+
const hasPgBoss = await postgres `
|
|
68
|
+
SELECT EXISTS (
|
|
69
|
+
SELECT 1 FROM information_schema.schemata
|
|
70
|
+
WHERE schema_name = 'pgboss'
|
|
71
|
+
) AS exists
|
|
72
|
+
`;
|
|
73
|
+
if (hasPgBoss[0].exists) {
|
|
74
|
+
const jobs = await postgres `
|
|
75
|
+
SELECT name, data, singleton_key, retry_limit
|
|
76
|
+
FROM pgboss.job
|
|
77
|
+
WHERE state IN ('created', 'retry')
|
|
78
|
+
`;
|
|
79
|
+
for (const job of jobs) {
|
|
80
|
+
await utils.addJob(job.name, job.data, {
|
|
81
|
+
jobKey: job.singleton_key ?? undefined,
|
|
82
|
+
maxAttempts: job.retry_limit ?? 3,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
await postgres `DROP SCHEMA pgboss CASCADE`;
|
|
37
86
|
}
|
|
38
|
-
|
|
87
|
+
}
|
|
88
|
+
async function start() {
|
|
89
|
+
if (!startPromise) {
|
|
90
|
+
startPromise = (async () => {
|
|
91
|
+
try {
|
|
92
|
+
workerUtils = await makeWorkerUtils({
|
|
93
|
+
connectionString: config.connectionString,
|
|
94
|
+
logger: graphileLogger,
|
|
95
|
+
});
|
|
96
|
+
await workerUtils.migrate();
|
|
97
|
+
await migratePgBossJobs(workerUtils);
|
|
98
|
+
await setupListeners();
|
|
99
|
+
}
|
|
100
|
+
catch (err) {
|
|
101
|
+
startPromise = null;
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
})();
|
|
105
|
+
}
|
|
106
|
+
await startPromise;
|
|
39
107
|
}
|
|
40
108
|
const queue = async (queue, message, opts) => {
|
|
41
|
-
await
|
|
109
|
+
await start();
|
|
42
110
|
const [prefix, queueId] = parseQueueName(queue);
|
|
43
111
|
const jobName = Queues[prefix];
|
|
44
|
-
await createQueue(jobName);
|
|
45
112
|
const body = transport.serialize(message);
|
|
46
113
|
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
47
|
-
await
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
attempt: 1,
|
|
57
|
-
messageId,
|
|
58
|
-
idempotencyKey: opts?.idempotencyKey,
|
|
59
|
-
}),
|
|
114
|
+
await workerUtils.addJob(jobName, MessageData.encode({
|
|
115
|
+
id: queueId,
|
|
116
|
+
data: body,
|
|
117
|
+
attempt: 1,
|
|
118
|
+
messageId,
|
|
119
|
+
idempotencyKey: opts?.idempotencyKey,
|
|
120
|
+
}), {
|
|
121
|
+
jobKey: opts?.idempotencyKey ?? messageId,
|
|
122
|
+
maxAttempts: 3,
|
|
60
123
|
});
|
|
61
124
|
return { messageId };
|
|
62
125
|
};
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
await boss.work(jobName, work);
|
|
67
|
-
}));
|
|
68
|
-
async function work([job]) {
|
|
69
|
-
const messageData = MessageData.parse(job.data);
|
|
126
|
+
function createTaskHandler(queue) {
|
|
127
|
+
return async (payload) => {
|
|
128
|
+
const messageData = MessageData.parse(payload);
|
|
70
129
|
const bodyStream = Stream.Readable.toWeb(Stream.Readable.from([messageData.data]));
|
|
71
130
|
const body = await transport.deserialize(bodyStream);
|
|
72
131
|
const message = QueuePayloadSchema.parse(body);
|
|
73
132
|
const queueName = `${queue}${messageData.id}`;
|
|
74
|
-
|
|
133
|
+
// TODO: Custom headers from opts.headers are not propagated into MessageData.
|
|
134
|
+
// To support this, MessageData schema would need to include a headers field
|
|
135
|
+
// and the headers would need to be stored/retrieved from graphile-worker job data.
|
|
136
|
+
await localWorld.queue(queueName, message, {
|
|
75
137
|
idempotencyKey: messageData.idempotencyKey,
|
|
76
138
|
});
|
|
77
|
-
}
|
|
139
|
+
};
|
|
78
140
|
}
|
|
79
141
|
async function setupListeners() {
|
|
142
|
+
const taskList = {};
|
|
80
143
|
for (const [prefix, jobName] of Object.entries(Queues)) {
|
|
81
|
-
|
|
144
|
+
taskList[jobName] = createTaskHandler(prefix);
|
|
82
145
|
}
|
|
146
|
+
runner = await run({
|
|
147
|
+
connectionString: config.connectionString,
|
|
148
|
+
concurrency: config.queueConcurrency || 10,
|
|
149
|
+
logger: graphileLogger,
|
|
150
|
+
pollInterval: 500, // 500ms = 0.5s (graphile-worker uses LISTEN/NOTIFY when available)
|
|
151
|
+
taskList,
|
|
152
|
+
});
|
|
83
153
|
}
|
|
84
154
|
return {
|
|
85
155
|
createQueueHandler,
|
|
86
156
|
getDeploymentId,
|
|
87
157
|
queue,
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
158
|
+
start,
|
|
159
|
+
async close() {
|
|
160
|
+
if (runner) {
|
|
161
|
+
await runner.stop();
|
|
162
|
+
runner = null;
|
|
163
|
+
}
|
|
164
|
+
if (workerUtils) {
|
|
165
|
+
await workerUtils.release();
|
|
166
|
+
workerUtils = null;
|
|
167
|
+
}
|
|
168
|
+
startPromise = null;
|
|
169
|
+
await localWorld.close?.();
|
|
91
170
|
},
|
|
92
171
|
};
|
|
93
172
|
}
|
package/dist/queue.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,SAAS,EAET,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EACL,SAAS,EAET,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EACL,MAAM,EACN,eAAe,EAEf,GAAG,GAEJ,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,SAAS,oBAAoB;IAC3B,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC;IAChE,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAEnD,OAAO,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,KAAa,EAAE,OAAe,EAAE,IAAc,EAAE,EAAE;QACzE,IAAI,UAAU,EAAE;YAAE,OAAO;QACzB,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACpE,MAAM,IAAI,GAAG,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACjE,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,CACR,qBAAqB,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAClE,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,qBAAqB,OAAO,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,cAAc,GAAG,oBAAoB,EAAE,CAAC;AAiB9C,MAAM,UAAU,WAAW,CACzB,MAA2B,EAC3B,QAAsB;IAEtB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,UAAU,GAAG,gBAAgB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAElE,MAAM,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;IACtC,MAAM,iBAAiB,GAAG,gBAAgB,EAAE,CAAC;IAE7C,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,IAAI,WAAW,CAAC;IAC/C,MAAM,MAAM,GAAG;QACb,eAAe,EAAE,GAAG,MAAM,OAAO;QACjC,WAAW,EAAE,GAAG,MAAM,OAAO;KACiB,CAAC;IAEjD,MAAM,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;IAEzD,MAAM,eAAe,GAA6B,KAAK,IAAI,EAAE;QAC3D,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;IAEF,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,YAAY,GAAyB,IAAI,CAAC;IAE9C,KAAK,UAAU,iBAAiB,CAAC,KAAkB;QACjD,mEAAmE;QACnE,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAA;;;;;;KAMhC,CAAC;QACF,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAA;;;OAG1B,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAA+B,EAAE;oBAChE,MAAM,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;oBACtC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC;iBAClC,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,CAAA,8CAA8C,CAAC;YAC7D,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAA;;;;;KAK/B,CAAC;QACF,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAA;;;;OAI1B,CAAC;YACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAA+B,EAAE;oBAChE,MAAM,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;oBACtC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,CAAC;iBAClC,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,CAAA,4BAA4B,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,KAAK,UAAU,KAAK;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;gBACzB,IAAI,CAAC;oBACH,WAAW,GAAG,MAAM,eAAe,CAAC;wBAClC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;wBACzC,MAAM,EAAE,cAAc;qBACvB,CAAC,CAAC;oBACH,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;oBAC5B,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;oBACrC,MAAM,cAAc,EAAE,CAAC;gBACzB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QACD,MAAM,YAAY,CAAC;IACrB,CAAC;IAED,MAAM,KAAK,GAAmB,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC3D,MAAM,KAAK,EAAE,CAAC;QACd,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAChE,MAAM,WAAY,CAAC,MAAM,CACvB,OAAO,EACP,WAAW,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,IAAI;YACV,OAAO,EAAE,CAAC;YACV,SAAS;YACT,cAAc,EAAE,IAAI,EAAE,cAAc;SACrC,CAAC,EACF;YACE,MAAM,EAAE,IAAI,EAAE,cAAc,IAAI,SAAS;YACzC,WAAW,EAAE,CAAC;SACf,CACF,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,CAAC;IACvB,CAAC,CAAC;IAEF,SAAS,iBAAiB,CAAC,KAAkB;QAC3C,OAAO,KAAK,EAAE,OAAgB,EAAE,EAAE;YAChC,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CACtC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CACzC,CAAC;YACF,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,WAAW,CACtC,UAAwC,CACzC,CAAC;YACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,SAAS,GAAG,GAAG,KAAK,GAAG,WAAW,CAAC,EAAE,EAAW,CAAC;YACvD,8EAA8E;YAC9E,4EAA4E;YAC5E,mFAAmF;YACnF,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,EAAE;gBACzC,cAAc,EAAE,WAAW,CAAC,cAAc;aAC3C,CAAC,CAAC;QACL,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,UAAU,cAAc;QAC3B,MAAM,QAAQ,GAAwD,EAAE,CAAC;QACzE,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAGlD,EAAE,CAAC;YACJ,QAAQ,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,GAAG,MAAM,GAAG,CAAC;YACjB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,WAAW,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;YAC1C,MAAM,EAAE,cAAc;YACtB,YAAY,EAAE,GAAG,EAAE,mEAAmE;YACtF,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,kBAAkB;QAClB,eAAe;QACf,KAAK;QACL,KAAK;QACL,KAAK,CAAC,KAAK;YACT,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBACpB,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC5B,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;QAC7B,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,cAAc,GAAG,CAAC,IAAoB,EAAyB,EAAE;IACrE,MAAM,QAAQ,GAAkB,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IACnE,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC"}
|
package/dist/storage.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAUV,OAAO,EAKR,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,KAAK,OAAO,EAAU,MAAM,oBAAoB,CAAC;AAiE1D,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CA2DnE;AAmGD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CA42BvE;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAiErE;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CA8DrE"}
|