@pattern-stack/codegen 0.15.3 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/consumer-skills/integration/change-sources-and-sinks.md +1 -1
- package/dist/{chunk-GCYKMF22.js → chunk-24WXSC3C.js} +6 -6
- package/dist/{chunk-32DOFN3T.js → chunk-2WDX6I7T.js} +2 -2
- package/dist/{chunk-WWGYCIJX.js → chunk-43SBT72G.js} +2 -2
- package/dist/{chunk-FBGHYQIZ.js → chunk-5LXOJGO2.js} +6 -6
- package/dist/{chunk-32BMMV4H.js → chunk-5RT7JGKT.js} +5 -5
- package/dist/{chunk-3NMCDN7L.js → chunk-5TK7MEN4.js} +2 -2
- package/dist/chunk-5TK7MEN4.js.map +1 -0
- package/dist/{chunk-4H3PETLM.js → chunk-AYC2HEAL.js} +12 -9
- package/dist/chunk-AYC2HEAL.js.map +1 -0
- package/dist/{chunk-27ETSJ2X.js → chunk-COGHTKXY.js} +2 -2
- package/dist/{chunk-IYNSRIGR.js → chunk-CRBVI4GE.js} +5 -5
- package/dist/{chunk-J7JMVS2B.js → chunk-CZQUOIDY.js} +4 -4
- package/dist/{chunk-O37C3YE6.js → chunk-DGYTSCKN.js} +14 -8
- package/dist/chunk-DGYTSCKN.js.map +1 -0
- package/dist/{chunk-L7BNNRGI.js → chunk-DLG62MQY.js} +26 -6
- package/dist/chunk-DLG62MQY.js.map +1 -0
- package/dist/{chunk-TNXH7BJS.js → chunk-E45CSC33.js} +2 -2
- package/dist/{chunk-4JLJYWJC.js → chunk-H6FO2ZDJ.js} +99 -11
- package/dist/chunk-H6FO2ZDJ.js.map +1 -0
- package/dist/{chunk-5Y7W3XR6.js → chunk-IT6FRTEW.js} +30 -11
- package/dist/chunk-IT6FRTEW.js.map +1 -0
- package/dist/{chunk-RC23QROE.js → chunk-JM3T27ZW.js} +78 -4
- package/dist/chunk-JM3T27ZW.js.map +1 -0
- package/dist/{chunk-Z7PQCAVK.js → chunk-LQ6PYFU6.js} +4 -4
- package/dist/chunk-MYQIQ27N.js +118 -0
- package/dist/chunk-MYQIQ27N.js.map +1 -0
- package/dist/{chunk-YTN6BKWA.js → chunk-NXNVTXKG.js} +5 -5
- package/dist/{chunk-RDVTWIYY.js → chunk-QSJ3J4HE.js} +5 -5
- package/dist/{chunk-4MVGAMUA.js → chunk-RUSUZZAF.js} +4 -4
- package/dist/{chunk-4RFHUZXU.js → chunk-T4YJRD22.js} +4 -4
- package/dist/{chunk-7YGORYZD.js → chunk-T6C4LFLC.js} +4 -4
- package/dist/{chunk-OGIZXGPY.js → chunk-TDEHU73T.js} +4 -4
- package/dist/{chunk-YLPAPPLW.js → chunk-TIZXQU26.js} +36 -9
- package/dist/chunk-TIZXQU26.js.map +1 -0
- package/dist/{chunk-EOLLMEAH.js → chunk-TKVTEUBD.js} +3 -3
- package/dist/chunk-TKVTEUBD.js.map +1 -0
- package/dist/{chunk-YPWODKD5.js → chunk-W2UIDI3R.js} +5 -5
- package/dist/chunk-W4HOHZVF.js +1 -0
- package/dist/{chunk-DCCZB4UC.js → chunk-XWBK3XJK.js} +4 -4
- package/dist/{chunk-SR7F3TJY.js → chunk-YK5JEVLX.js} +4 -4
- package/dist/{chunk-BIO6F7YI.js → chunk-ZPL74UQN.js} +4 -2
- package/dist/{chunk-BIO6F7YI.js.map → chunk-ZPL74UQN.js.map} +1 -1
- package/dist/runtime/base-classes/index.js +22 -22
- package/dist/runtime/subsystems/analytics/analytics.module.js +2 -2
- package/dist/runtime/subsystems/analytics/index.js +4 -4
- package/dist/runtime/subsystems/auth/auth.module.js +1 -1
- package/dist/runtime/subsystems/auth/index.js +7 -7
- package/dist/runtime/subsystems/bridge/bridge-delivery-handler.js +3 -3
- package/dist/runtime/subsystems/bridge/bridge-delivery.drizzle-backend.js +1 -1
- package/dist/runtime/subsystems/bridge/bridge-outbox-drain-hook.d.ts +2 -1
- package/dist/runtime/subsystems/bridge/bridge-outbox-drain-hook.js +6 -5
- package/dist/runtime/subsystems/bridge/bridge.module.js +16 -15
- package/dist/runtime/subsystems/bridge/event-flow.service.js +3 -3
- package/dist/runtime/subsystems/bridge/index.js +16 -15
- package/dist/runtime/subsystems/cache/cache.drizzle-backend.js +2 -2
- package/dist/runtime/subsystems/cache/cache.module.js +3 -3
- package/dist/runtime/subsystems/cache/index.js +5 -5
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.d.ts +20 -0
- package/dist/runtime/subsystems/events/event-bus.drizzle-backend.js +4 -3
- package/dist/runtime/subsystems/events/event-bus.memory-backend.js +2 -2
- package/dist/runtime/subsystems/events/events.module.d.ts +14 -0
- package/dist/runtime/subsystems/events/events.module.js +6 -5
- package/dist/runtime/subsystems/events/index.js +12 -11
- package/dist/runtime/subsystems/index.js +88 -87
- package/dist/runtime/subsystems/integration/build-change-source.js +2 -2
- package/dist/runtime/subsystems/integration/detection-config.schema.d.ts +23 -15
- package/dist/runtime/subsystems/integration/detection-config.schema.js +1 -1
- package/dist/runtime/subsystems/integration/execute-integration.use-case.js +2 -2
- package/dist/runtime/subsystems/integration/index.js +17 -17
- package/dist/runtime/subsystems/integration/integration-cursor-store.drizzle-backend.js +2 -2
- package/dist/runtime/subsystems/integration/integration-run-recorder.drizzle-backend.js +2 -2
- package/dist/runtime/subsystems/integration/integration.module.js +4 -4
- package/dist/runtime/subsystems/integration/webhook-change-source.d.ts +36 -6
- package/dist/runtime/subsystems/integration/webhook-change-source.js +1 -1
- package/dist/runtime/subsystems/jobs/index.d.ts +2 -1
- package/dist/runtime/subsystems/jobs/index.js +42 -30
- package/dist/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.js +6 -5
- package/dist/runtime/subsystems/jobs/job-orchestrator.bullmq-backend.js.map +1 -1
- package/dist/runtime/subsystems/jobs/job-orchestrator.drizzle-backend.d.ts +2 -1
- package/dist/runtime/subsystems/jobs/job-orchestrator.drizzle-backend.js +4 -3
- package/dist/runtime/subsystems/jobs/job-orchestrator.memory-backend.js +3 -3
- package/dist/runtime/subsystems/jobs/job-run-service.drizzle-backend.js +3 -3
- package/dist/runtime/subsystems/jobs/job-run-service.memory-backend.js +2 -2
- package/dist/runtime/subsystems/jobs/job-worker.bullmq-backend.js +3 -3
- package/dist/runtime/subsystems/jobs/job-worker.d.ts +28 -0
- package/dist/runtime/subsystems/jobs/job-worker.js +4 -3
- package/dist/runtime/subsystems/jobs/job-worker.module.js +11 -10
- package/dist/runtime/subsystems/jobs/jobs-domain.module.d.ts +12 -7
- package/dist/runtime/subsystems/jobs/jobs-domain.module.js +9 -8
- package/dist/runtime/subsystems/jobs/jobs-domain.tokens.d.ts +13 -1
- package/dist/runtime/subsystems/jobs/jobs-domain.tokens.js +3 -1
- package/dist/runtime/subsystems/jobs/pg-notify.d.ts +85 -0
- package/dist/runtime/subsystems/jobs/pg-notify.js +14 -0
- package/dist/runtime/subsystems/jobs/pg-notify.js.map +1 -0
- package/dist/runtime/subsystems/observability/index.js +4 -4
- package/dist/runtime/subsystems/observability/observability.module.js +4 -4
- package/dist/runtime/subsystems/observability/observability.service.js +3 -3
- package/dist/runtime/subsystems/storage/index.js +4 -4
- package/dist/runtime/subsystems/storage/storage.module.js +2 -2
- package/dist/src/cli/index.js +53 -15
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/index.d.ts +11 -11
- package/dist/src/index.js +9 -9
- package/package.json +1 -1
- package/runtime/subsystems/bridge/bridge-outbox-drain-hook.ts +27 -0
- package/runtime/subsystems/events/event-bus.drizzle-backend.ts +108 -4
- package/runtime/subsystems/events/events.module.ts +14 -0
- package/runtime/subsystems/integration/detection-config.schema.ts +64 -54
- package/runtime/subsystems/integration/webhook-change-source.ts +187 -133
- package/runtime/subsystems/jobs/index.ts +10 -0
- package/runtime/subsystems/jobs/job-orchestrator.drizzle-backend.ts +29 -2
- package/runtime/subsystems/jobs/job-worker.module.ts +11 -0
- package/runtime/subsystems/jobs/job-worker.ts +98 -0
- package/runtime/subsystems/jobs/jobs-domain.module.ts +22 -7
- package/runtime/subsystems/jobs/jobs-domain.tokens.ts +13 -0
- package/runtime/subsystems/jobs/pg-notify.ts +216 -0
- package/templates/subsystem/events-config/codegen-config-events-block.ejs.t +14 -0
- package/templates/subsystem/jobs-config/codegen-config-jobs-block.ejs.t +13 -4
- package/dist/chunk-3NMCDN7L.js.map +0 -1
- package/dist/chunk-4H3PETLM.js.map +0 -1
- package/dist/chunk-4JLJYWJC.js.map +0 -1
- package/dist/chunk-5Y7W3XR6.js.map +0 -1
- package/dist/chunk-EOLLMEAH.js.map +0 -1
- package/dist/chunk-L7BNNRGI.js.map +0 -1
- package/dist/chunk-O37C3YE6.js.map +0 -1
- package/dist/chunk-RC23QROE.js.map +0 -1
- package/dist/chunk-UTN4GBPQ.js +0 -1
- package/dist/chunk-YLPAPPLW.js.map +0 -1
- /package/dist/{chunk-GCYKMF22.js.map → chunk-24WXSC3C.js.map} +0 -0
- /package/dist/{chunk-32DOFN3T.js.map → chunk-2WDX6I7T.js.map} +0 -0
- /package/dist/{chunk-WWGYCIJX.js.map → chunk-43SBT72G.js.map} +0 -0
- /package/dist/{chunk-FBGHYQIZ.js.map → chunk-5LXOJGO2.js.map} +0 -0
- /package/dist/{chunk-32BMMV4H.js.map → chunk-5RT7JGKT.js.map} +0 -0
- /package/dist/{chunk-27ETSJ2X.js.map → chunk-COGHTKXY.js.map} +0 -0
- /package/dist/{chunk-IYNSRIGR.js.map → chunk-CRBVI4GE.js.map} +0 -0
- /package/dist/{chunk-J7JMVS2B.js.map → chunk-CZQUOIDY.js.map} +0 -0
- /package/dist/{chunk-TNXH7BJS.js.map → chunk-E45CSC33.js.map} +0 -0
- /package/dist/{chunk-Z7PQCAVK.js.map → chunk-LQ6PYFU6.js.map} +0 -0
- /package/dist/{chunk-YTN6BKWA.js.map → chunk-NXNVTXKG.js.map} +0 -0
- /package/dist/{chunk-RDVTWIYY.js.map → chunk-QSJ3J4HE.js.map} +0 -0
- /package/dist/{chunk-4MVGAMUA.js.map → chunk-RUSUZZAF.js.map} +0 -0
- /package/dist/{chunk-4RFHUZXU.js.map → chunk-T4YJRD22.js.map} +0 -0
- /package/dist/{chunk-7YGORYZD.js.map → chunk-T6C4LFLC.js.map} +0 -0
- /package/dist/{chunk-OGIZXGPY.js.map → chunk-TDEHU73T.js.map} +0 -0
- /package/dist/{chunk-YPWODKD5.js.map → chunk-W2UIDI3R.js.map} +0 -0
- /package/dist/{chunk-UTN4GBPQ.js.map → chunk-W4HOHZVF.js.map} +0 -0
- /package/dist/{chunk-DCCZB4UC.js.map → chunk-XWBK3XJK.js.map} +0 -0
- /package/dist/{chunk-SR7F3TJY.js.map → chunk-YK5JEVLX.js.map} +0 -0
|
@@ -7,8 +7,11 @@
|
|
|
7
7
|
* queue. The primitive owns:
|
|
8
8
|
*
|
|
9
9
|
* - canonical `Change<T>.source = 'webhook'` stamping;
|
|
10
|
-
* - `dedupKey` derivation
|
|
11
|
-
* the
|
|
10
|
+
* - `dedupKey` derivation, preferring the `eventId` yielded alongside the
|
|
11
|
+
* record by the queue iterator, and falling back to the configured
|
|
12
|
+
* `webhook.eventIdField` on the emitted record when no `eventId` is yielded
|
|
13
|
+
* (precedence: yielded `eventId` > `eventIdField` record extraction >
|
|
14
|
+
* undefined `dedupKey`);
|
|
12
15
|
* - `externalId` derivation: the mapping entry whose `target === 'external_id'`
|
|
13
16
|
* names — via its `source` — the field on the emitted record that carries
|
|
14
17
|
* the canonical external id (mirrors `PollChangeSource`);
|
|
@@ -37,16 +40,16 @@
|
|
|
37
40
|
* into either this primitive or the poll primitive.
|
|
38
41
|
*/
|
|
39
42
|
|
|
40
|
-
import type { DetectionConfig } from
|
|
43
|
+
import type { DetectionConfig } from "./detection-config.schema";
|
|
41
44
|
import type {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
} from
|
|
45
|
+
Change,
|
|
46
|
+
IChangeSource,
|
|
47
|
+
IntegrationSubscriptionView,
|
|
48
|
+
} from "./integration-change-source.protocol";
|
|
46
49
|
import type {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
} from
|
|
50
|
+
ChangeIterator,
|
|
51
|
+
ChangeMiddleware,
|
|
52
|
+
} from "./integration-middleware.protocol";
|
|
50
53
|
|
|
51
54
|
// ============================================================================
|
|
52
55
|
// Cursor + queue callback shapes
|
|
@@ -66,16 +69,28 @@ export type WebhookCursor = unknown;
|
|
|
66
69
|
* `userId` / `tenantId`.
|
|
67
70
|
*/
|
|
68
71
|
export interface WebhookFetchContext {
|
|
69
|
-
|
|
70
|
-
|
|
72
|
+
readonly subscription: IntegrationSubscriptionView;
|
|
73
|
+
readonly cursor: WebhookCursor | null;
|
|
71
74
|
}
|
|
72
75
|
|
|
73
76
|
/**
|
|
74
77
|
* Consumer-supplied queue iterator. Returns an async iterable of
|
|
75
|
-
* `{ record }`
|
|
76
|
-
* emits already-mapped canonical records `T`. The primitive
|
|
77
|
-
* `source: 'webhook'` and `dedupKey`
|
|
78
|
-
*
|
|
78
|
+
* `{ record, eventId?, cursor? }` tuples — the consumer drains the inbound
|
|
79
|
+
* staging queue and emits already-mapped canonical records `T`. The primitive
|
|
80
|
+
* stamps `source: 'webhook'` and derives `dedupKey` with this precedence:
|
|
81
|
+
*
|
|
82
|
+
* 1. the yielded `eventId` (vendor delivery metadata — the queue is the
|
|
83
|
+
* right channel for it: a vendor's event id should never need a field
|
|
84
|
+
* on the vendor-neutral canonical record);
|
|
85
|
+
* 2. else the record field named by `webhook.eventIdField`, when configured;
|
|
86
|
+
* 3. else `undefined`.
|
|
87
|
+
*
|
|
88
|
+
* Yielding `eventId` is the safe channel when one canonical record identity
|
|
89
|
+
* (the `external_id`) can recur across distinct vendor events in a single
|
|
90
|
+
* drain batch — e.g. a message create and its later edit share an
|
|
91
|
+
* `external_id` but are different events. Reading dedup identity off the
|
|
92
|
+
* record (`eventIdField`) collapses those into one `dedupKey`; the yielded
|
|
93
|
+
* `eventId` keeps them distinct. The consumer is the one who decided when a
|
|
79
94
|
* staging row is "ready" to drain.
|
|
80
95
|
*
|
|
81
96
|
* Webhook mode has no per-record cursor advance — the staging-row drain
|
|
@@ -84,33 +99,33 @@ export interface WebhookFetchContext {
|
|
|
84
99
|
* is whatever the consumer chooses to surface, if anything.
|
|
85
100
|
*/
|
|
86
101
|
export type WebhookFetchCallback<T> = (
|
|
87
|
-
|
|
88
|
-
) => AsyncIterable<{ record: T; cursor?: WebhookCursor }>;
|
|
102
|
+
ctx: WebhookFetchContext,
|
|
103
|
+
) => AsyncIterable<{ record: T; eventId?: string; cursor?: WebhookCursor }>;
|
|
89
104
|
|
|
90
105
|
// ============================================================================
|
|
91
106
|
// Constructor options
|
|
92
107
|
// ============================================================================
|
|
93
108
|
|
|
94
109
|
export interface WebhookChangeSourceOptions<T> {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
110
|
+
/** Consumer-supplied inbound queue iterator. */
|
|
111
|
+
readonly queue: WebhookFetchCallback<T>;
|
|
112
|
+
/**
|
|
113
|
+
* Parsed detection config. MUST be `mode: 'webhook'`; the constructor
|
|
114
|
+
* throws if a poll config is supplied. Codegen-emitted factories call
|
|
115
|
+
* `DetectionConfigSchema.parse(...)` upstream so this is a safety net,
|
|
116
|
+
* not the primary validation point.
|
|
117
|
+
*/
|
|
118
|
+
readonly config: DetectionConfig;
|
|
119
|
+
/**
|
|
120
|
+
* Optional middleware chain. Same shape and composition rules as
|
|
121
|
+
* `PollChangeSource` — first element is the outermost layer.
|
|
122
|
+
*/
|
|
123
|
+
readonly middlewares?: ReadonlyArray<ChangeMiddleware<T>>;
|
|
124
|
+
/**
|
|
125
|
+
* Optional human label for run logs (e.g. `'stripe-webhook-charge'`).
|
|
126
|
+
* Defaults to a derived label based on the mapping at construction.
|
|
127
|
+
*/
|
|
128
|
+
readonly label?: string;
|
|
114
129
|
}
|
|
115
130
|
|
|
116
131
|
// ============================================================================
|
|
@@ -118,100 +133,139 @@ export interface WebhookChangeSourceOptions<T> {
|
|
|
118
133
|
// ============================================================================
|
|
119
134
|
|
|
120
135
|
export class WebhookChangeSource<T> implements IChangeSource<T> {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
136
|
+
public readonly label: string;
|
|
137
|
+
|
|
138
|
+
private readonly queue: WebhookFetchCallback<T>;
|
|
139
|
+
private readonly externalIdSourceField: string;
|
|
140
|
+
/**
|
|
141
|
+
* Record field carrying the event id, when `webhook.eventIdField` is
|
|
142
|
+
* configured. Used only as the fallback when the queue iterator does NOT
|
|
143
|
+
* yield an `eventId` — see {@link WebhookFetchCallback} for the precedence.
|
|
144
|
+
*/
|
|
145
|
+
private readonly eventIdSourceField: string | undefined;
|
|
146
|
+
private readonly composed: ChangeIterator<T>;
|
|
147
|
+
|
|
148
|
+
constructor(opts: WebhookChangeSourceOptions<T>) {
|
|
149
|
+
if (opts.config.mode !== "webhook") {
|
|
150
|
+
throw new Error(
|
|
151
|
+
`WebhookChangeSource requires DetectionConfig.mode === 'webhook'; got '${(opts.config as { mode: string }).mode}'`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
const config = opts.config;
|
|
155
|
+
|
|
156
|
+
// Field mapping: locate the entry whose canonical `target` is `external_id`
|
|
157
|
+
// — mirrors the poll primitive's contract. Adapters emit records
|
|
158
|
+
// already-mapped; the primitive needs to know which key on T carries the
|
|
159
|
+
// external id so it can stamp `Change.externalId`. That key is the
|
|
160
|
+
// mapping's `source` (the field on the emitted record), NOT its `target`
|
|
161
|
+
// (the canonical column) — they differ whenever the canonical record is
|
|
162
|
+
// vendor-neutral camelCase (e.g. `source: 'externalId'` → `target: 'external_id'`).
|
|
163
|
+
const externalIdMapping = config.mapping.find(
|
|
164
|
+
(m) => m.target === "external_id",
|
|
165
|
+
);
|
|
166
|
+
if (!externalIdMapping) {
|
|
167
|
+
throw new Error(
|
|
168
|
+
"WebhookChangeSource: DetectionConfig.mapping must include an entry with target 'external_id' so emitted Change<T>.externalId can be populated",
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
this.externalIdSourceField = externalIdMapping.source;
|
|
172
|
+
this.eventIdSourceField = config.webhook.eventIdField;
|
|
173
|
+
// `eventIdField` is optional (a callback that always yields `eventId` need
|
|
174
|
+
// not declare one); `undefined` here just disables the fallback extraction.
|
|
175
|
+
|
|
176
|
+
this.queue = opts.queue;
|
|
177
|
+
|
|
178
|
+
this.label =
|
|
179
|
+
opts.label ?? `webhook-change-source:${externalIdMapping.source}`;
|
|
180
|
+
|
|
181
|
+
// Compose middleware chain — same shape as PollChangeSource.
|
|
182
|
+
const inner: ChangeIterator<T> = (sub, cur) => this.fetch(sub, cur);
|
|
183
|
+
const middlewares = opts.middlewares ?? [];
|
|
184
|
+
this.composed = middlewares.reduceRight<ChangeIterator<T>>(
|
|
185
|
+
(next, mw) => mw(next),
|
|
186
|
+
inner,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
listChanges(
|
|
191
|
+
subscription: IntegrationSubscriptionView,
|
|
192
|
+
cursor: unknown | null,
|
|
193
|
+
): AsyncIterable<Change<T>> {
|
|
194
|
+
return this.composed(subscription, cursor);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
private async *fetch(
|
|
198
|
+
subscription: IntegrationSubscriptionView,
|
|
199
|
+
cursor: unknown | null,
|
|
200
|
+
): AsyncIterable<Change<T>> {
|
|
201
|
+
const ctx: WebhookFetchContext = {
|
|
202
|
+
subscription,
|
|
203
|
+
cursor: cursor as WebhookCursor | null,
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
for await (const {
|
|
207
|
+
record,
|
|
208
|
+
eventId: yieldedEventId,
|
|
209
|
+
cursor: nextCursor,
|
|
210
|
+
} of this.queue(ctx)) {
|
|
211
|
+
const externalIdRaw = (record as Record<string, unknown>)[
|
|
212
|
+
this.externalIdSourceField
|
|
213
|
+
];
|
|
214
|
+
if (typeof externalIdRaw !== "string" || externalIdRaw.length === 0) {
|
|
215
|
+
throw new Error(
|
|
216
|
+
`WebhookChangeSource: record missing string '${this.externalIdSourceField}' — emitted records MUST carry the canonical external id keyed by the mapping source`,
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// dedupKey precedence: yielded `eventId` > `eventIdField` record
|
|
221
|
+
// extraction > undefined. The yielded id is vendor delivery metadata
|
|
222
|
+
// (the right channel for it), and keeps distinct vendor events for the
|
|
223
|
+
// same `external_id` (e.g. a message and its edit) from collapsing to
|
|
224
|
+
// one dedupKey — which a record-field extraction would do.
|
|
225
|
+
const dedupKey = this.deriveDedupKey(yieldedEventId, record);
|
|
226
|
+
|
|
227
|
+
const change: Change<T> = {
|
|
228
|
+
externalId: externalIdRaw,
|
|
229
|
+
// Webhook mode cannot distinguish create vs. update vs. delete on
|
|
230
|
+
// its own — the orchestrator's diff stage handles classification.
|
|
231
|
+
// Tombstone / soft-delete detection is consumer-driven (same as
|
|
232
|
+
// poll mode — see ADR-033).
|
|
233
|
+
operation: "updated",
|
|
234
|
+
record,
|
|
235
|
+
cursor: nextCursor ?? null,
|
|
236
|
+
source: "webhook",
|
|
237
|
+
dedupKey,
|
|
238
|
+
};
|
|
239
|
+
yield change;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Resolve `Change<T>.dedupKey` with the precedence: yielded `eventId` >
|
|
245
|
+
* `webhook.eventIdField` record extraction > `undefined`. A non-empty
|
|
246
|
+
* yielded `eventId` always wins; otherwise the configured field is read off
|
|
247
|
+
* the record (and must be a non-empty string when the field is configured);
|
|
248
|
+
* with neither, `dedupKey` is `undefined` (the orchestrator then has no
|
|
249
|
+
* delivery-level dedup signal for this change).
|
|
250
|
+
*/
|
|
251
|
+
private deriveDedupKey(
|
|
252
|
+
yieldedEventId: string | undefined,
|
|
253
|
+
record: T,
|
|
254
|
+
): string | undefined {
|
|
255
|
+
if (yieldedEventId !== undefined && yieldedEventId.length > 0) {
|
|
256
|
+
return yieldedEventId;
|
|
257
|
+
}
|
|
258
|
+
if (this.eventIdSourceField === undefined) {
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
const eventIdRaw = (record as Record<string, unknown>)[
|
|
262
|
+
this.eventIdSourceField
|
|
263
|
+
];
|
|
264
|
+
if (typeof eventIdRaw !== "string" || eventIdRaw.length === 0) {
|
|
265
|
+
throw new Error(
|
|
266
|
+
`WebhookChangeSource: record missing string '${this.eventIdSourceField}' — a webhook record MUST carry the event id (DetectionConfig.webhook.eventIdField) so Change<T>.dedupKey can be populated, unless the queue iterator yields an 'eventId' alongside the record`,
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
return eventIdRaw;
|
|
270
|
+
}
|
|
217
271
|
}
|
|
@@ -22,6 +22,7 @@ export {
|
|
|
22
22
|
JOB_RUN_SERVICE,
|
|
23
23
|
JOB_STEP_SERVICE,
|
|
24
24
|
JOBS_MULTI_TENANT,
|
|
25
|
+
JOBS_LISTEN_NOTIFY,
|
|
25
26
|
} from './jobs-domain.tokens';
|
|
26
27
|
|
|
27
28
|
// ─── JOB-2: orchestrator protocol ──────────────────────────────────────────
|
|
@@ -111,6 +112,15 @@ export {
|
|
|
111
112
|
buildStaleSweepQuery,
|
|
112
113
|
} from './job-worker';
|
|
113
114
|
export type { JobWorkerOptions } from './job-worker';
|
|
115
|
+
|
|
116
|
+
// ─── LISTEN-NOTIFY-1: Postgres LISTEN/NOTIFY wakeups ───────────────────────
|
|
117
|
+
export {
|
|
118
|
+
PgNotifyListener,
|
|
119
|
+
pgNotify,
|
|
120
|
+
JOBS_WAKE_CHANNEL,
|
|
121
|
+
EVENTS_WAKE_CHANNEL,
|
|
122
|
+
} from './pg-notify';
|
|
123
|
+
export type { PgNotifyListenerOptions } from './pg-notify';
|
|
114
124
|
export {
|
|
115
125
|
JobCollisionError,
|
|
116
126
|
JobNotReplayableError,
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* No `job_queue` table, no executor port. See `docs/specs/JOB-3.md`.
|
|
8
8
|
*/
|
|
9
9
|
import { randomUUID } from 'node:crypto';
|
|
10
|
-
import { Inject, Injectable, Logger } from '@nestjs/common';
|
|
10
|
+
import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
|
|
11
11
|
import { and, desc, eq, gt, inArray, isNotNull, ne, notInArray, sql } from 'drizzle-orm';
|
|
12
12
|
import type { DrizzleClient } from '../../types/drizzle';
|
|
13
13
|
import type { DrizzleTransaction } from '../events/event-bus.protocol';
|
|
@@ -34,7 +34,8 @@ import {
|
|
|
34
34
|
MissingTenantIdError,
|
|
35
35
|
} from './jobs-errors';
|
|
36
36
|
import { jobSteps } from './job-orchestration.schema';
|
|
37
|
-
import { JOBS_MULTI_TENANT } from './jobs-domain.tokens';
|
|
37
|
+
import { JOBS_MULTI_TENANT, JOBS_LISTEN_NOTIFY } from './jobs-domain.tokens';
|
|
38
|
+
import { JOBS_WAKE_CHANNEL, pgNotify } from './pg-notify';
|
|
38
39
|
|
|
39
40
|
/**
|
|
40
41
|
* Terminal statuses — transitions into these are final. Used by `cancel`
|
|
@@ -83,6 +84,13 @@ export class DrizzleJobOrchestrator implements IJobOrchestrator {
|
|
|
83
84
|
constructor(
|
|
84
85
|
@Inject(DRIZZLE) private readonly db: DrizzleClient,
|
|
85
86
|
@Inject(JOBS_MULTI_TENANT) private readonly multiTenant: boolean,
|
|
87
|
+
// LISTEN-NOTIFY-1 — when true, `start()` emits an in-tx
|
|
88
|
+
// `pg_notify(codegen_jobs_wake, <pool>)` so a `listen_notify` worker wakes
|
|
89
|
+
// on enqueue-commit. `@Optional()` defaulting to false so direct
|
|
90
|
+
// construction (integration tests not going through DI) keeps working.
|
|
91
|
+
@Optional()
|
|
92
|
+
@Inject(JOBS_LISTEN_NOTIFY)
|
|
93
|
+
private readonly listenNotify: boolean = false,
|
|
86
94
|
) {}
|
|
87
95
|
|
|
88
96
|
/**
|
|
@@ -251,6 +259,25 @@ export class DrizzleJobOrchestrator implements IJobOrchestrator {
|
|
|
251
259
|
})
|
|
252
260
|
.returning();
|
|
253
261
|
|
|
262
|
+
// LISTEN-NOTIFY-1 — wake a listening worker the instant this enqueue
|
|
263
|
+
// commits. Emitted through the SAME `client` (the caller's tx when one was
|
|
264
|
+
// passed, else the pool) so delivery is gated on commit — a rolled-back
|
|
265
|
+
// enqueue emits no phantom wake (D2). The pool name is the payload; the
|
|
266
|
+
// worker re-runs its own pool-filtered claim query on wake. Polling is the
|
|
267
|
+
// fallback, so a failed notify is non-fatal: log + continue.
|
|
268
|
+
if (this.listenNotify) {
|
|
269
|
+
const wakePool = (inserted as JobRunRow).pool;
|
|
270
|
+
try {
|
|
271
|
+
await pgNotify(client, JOBS_WAKE_CHANNEL, wakePool);
|
|
272
|
+
} catch (err) {
|
|
273
|
+
this.logger.warn(
|
|
274
|
+
`pg_notify(${JOBS_WAKE_CHANNEL}, ${wakePool}) failed for run ` +
|
|
275
|
+
`${(inserted as JobRunRow).id}: ${(err as Error).message} ` +
|
|
276
|
+
`(non-fatal — interval polling still claims the run).`,
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
254
281
|
return inserted as JobRun;
|
|
255
282
|
}
|
|
256
283
|
|
|
@@ -245,11 +245,22 @@ export class JobWorkerOrchestrator implements OnModuleInit, OnModuleDestroy {
|
|
|
245
245
|
// naming; it MUST NOT be passed as the claim-filter pool, or the
|
|
246
246
|
// worker will never match any row and the pool silently never
|
|
247
247
|
// drains. See v0.4.4 fix notes.
|
|
248
|
+
// LISTEN-NOTIFY-1 — thread the drizzle extension knobs into each spawned
|
|
249
|
+
// worker. `pollIntervalMs` was always honored by JobWorker but never
|
|
250
|
+
// received a config value; `listenNotify` is the new wake opt-in. Only
|
|
251
|
+
// the drizzle backend reads these (bullmq has native wakeups + its own
|
|
252
|
+
// queue topology), so we ignore them under `backend: 'bullmq'`.
|
|
253
|
+
const drizzleExt =
|
|
254
|
+
backend === 'drizzle'
|
|
255
|
+
? this.options.domainModuleExtensions?.drizzle
|
|
256
|
+
: undefined;
|
|
248
257
|
const workerOptions: JobWorkerOptions = {
|
|
249
258
|
pool: poolName,
|
|
250
259
|
concurrency: def.concurrency,
|
|
251
260
|
shutdownTimeoutMs:
|
|
252
261
|
this.options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS,
|
|
262
|
+
pollIntervalMs: drizzleExt?.pollIntervalMs,
|
|
263
|
+
listenNotify: drizzleExt?.listenNotify,
|
|
253
264
|
};
|
|
254
265
|
const worker = this.options.workerFactory
|
|
255
266
|
? this.options.workerFactory(workerOptions)
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
type SpawnChildOptions,
|
|
38
38
|
type StepOptions,
|
|
39
39
|
} from './job-handler.base';
|
|
40
|
+
import { JOBS_WAKE_CHANNEL, PgNotifyListener } from './pg-notify';
|
|
40
41
|
|
|
41
42
|
/**
|
|
42
43
|
* Options accepted by `JobWorker`. JOB-5 threads these through module
|
|
@@ -59,6 +60,14 @@ export interface JobWorkerOptions {
|
|
|
59
60
|
staleThresholdMs?: number;
|
|
60
61
|
/** Max ms to wait for in-flight drain on SIGTERM. Default 30_000. */
|
|
61
62
|
shutdownTimeoutMs?: number;
|
|
63
|
+
/**
|
|
64
|
+
* LISTEN-NOTIFY-1 — when true, hold a dedicated listener connection and
|
|
65
|
+
* LISTEN on `codegen_jobs_wake`. A notification naming this worker's `pool`
|
|
66
|
+
* triggers an immediate (debounced) claim cycle, so an enqueue is claimed in
|
|
67
|
+
* milliseconds instead of waiting for the next `pollIntervalMs` tick. Polling
|
|
68
|
+
* continues unchanged as the fallback heartbeat. Default false.
|
|
69
|
+
*/
|
|
70
|
+
listenNotify?: boolean;
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
// ADR-037: namespaced `Symbol.for(...)` (via `tokenKey()`) — matches by value
|
|
@@ -192,6 +201,15 @@ export class JobWorker implements OnModuleInit, OnModuleDestroy {
|
|
|
192
201
|
private readonly staleThresholdMs: number;
|
|
193
202
|
private readonly shutdownTimeoutMs: number;
|
|
194
203
|
|
|
204
|
+
// LISTEN-NOTIFY-1 — dedicated listener + debounce state. `null` when
|
|
205
|
+
// `listenNotify` is off (the common case); polling is the only driver then.
|
|
206
|
+
private readonly listenNotifyEnabled: boolean;
|
|
207
|
+
private notifyListener: PgNotifyListener | null = null;
|
|
208
|
+
/** True while a wake-driven claim cycle is in flight (debounce gate). */
|
|
209
|
+
private wakeDraining = false;
|
|
210
|
+
/** A notify arrived mid-cycle → re-check once when the cycle ends. */
|
|
211
|
+
private wakeRecheckPending = false;
|
|
212
|
+
|
|
195
213
|
constructor(
|
|
196
214
|
@Inject(DRIZZLE) private readonly db: DrizzleClient,
|
|
197
215
|
@Inject(JOB_ORCHESTRATOR) private readonly orchestrator: IJobOrchestrator,
|
|
@@ -206,6 +224,7 @@ export class JobWorker implements OnModuleInit, OnModuleDestroy {
|
|
|
206
224
|
this.staleThresholdMs = options.staleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
|
207
225
|
this.shutdownTimeoutMs =
|
|
208
226
|
options.shutdownTimeoutMs ?? DEFAULT_SHUTDOWN_TIMEOUT_MS;
|
|
227
|
+
this.listenNotifyEnabled = options.listenNotify ?? false;
|
|
209
228
|
|
|
210
229
|
this.sigtermHandler = () => {
|
|
211
230
|
if (this.sigtermHandled) return;
|
|
@@ -227,6 +246,74 @@ export class JobWorker implements OnModuleInit, OnModuleDestroy {
|
|
|
227
246
|
void this.sweepStaleClaims();
|
|
228
247
|
}, this.staleSweeperIntervalMs);
|
|
229
248
|
process.on('SIGTERM', this.sigtermHandler);
|
|
249
|
+
|
|
250
|
+
// LISTEN-NOTIFY-1 — start the wake listener ALONGSIDE the poll timer (never
|
|
251
|
+
// instead). A notify for this worker's pool drives an immediate claim cycle;
|
|
252
|
+
// the interval timer above stays the durability heartbeat. Listener startup
|
|
253
|
+
// is fire-and-forget: a connect failure self-heals via the listener's own
|
|
254
|
+
// backoff, and until it's up the poll loop is the sole driver.
|
|
255
|
+
if (this.listenNotifyEnabled) {
|
|
256
|
+
// The DRIZZLE provider wraps a `pg.Pool`, exposed by drizzle as `$client`.
|
|
257
|
+
const pool = (this.db as unknown as { $client?: unknown }).$client;
|
|
258
|
+
if (!pool || typeof (pool as { connect?: unknown }).connect !== 'function') {
|
|
259
|
+
this.logger.warn(
|
|
260
|
+
`listen_notify enabled but the Drizzle client exposes no pg Pool ` +
|
|
261
|
+
`($client.connect missing) — falling back to interval polling only.`,
|
|
262
|
+
);
|
|
263
|
+
} else {
|
|
264
|
+
this.notifyListener = new PgNotifyListener({
|
|
265
|
+
channel: JOBS_WAKE_CHANNEL,
|
|
266
|
+
pool: pool as { connect(): Promise<never> },
|
|
267
|
+
label: `jobs:${this.options.pool}`,
|
|
268
|
+
onNotify: (payload) => this.onWake(payload),
|
|
269
|
+
});
|
|
270
|
+
void this.notifyListener.start();
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Wake handler — a `codegen_jobs_wake` notification arrived. Only payloads
|
|
277
|
+
* naming THIS worker's pool are relevant (other pools have their own workers).
|
|
278
|
+
* Debounced: if a claim cycle is already running we just flag a re-check so a
|
|
279
|
+
* burst of N enqueues collapses to at most one extra cycle (D3).
|
|
280
|
+
*/
|
|
281
|
+
private onWake(payload: string): void {
|
|
282
|
+
if (this.shuttingDown) return;
|
|
283
|
+
if (payload !== this.options.pool) return;
|
|
284
|
+
if (this.wakeDraining) {
|
|
285
|
+
this.wakeRecheckPending = true;
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
void this.drainOnWake();
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Claim-until-empty on a wake. Unlike the interval `pollAndProcess` (one
|
|
293
|
+
* claim per tick), a wake drains greedily up to the concurrency ceiling so a
|
|
294
|
+
* burst that arrived together is dispatched without waiting for N ticks. The
|
|
295
|
+
* `wakeRecheckPending` flag coalesces notifies that land mid-drain.
|
|
296
|
+
*/
|
|
297
|
+
private async drainOnWake(): Promise<void> {
|
|
298
|
+
this.wakeDraining = true;
|
|
299
|
+
try {
|
|
300
|
+
do {
|
|
301
|
+
this.wakeRecheckPending = false;
|
|
302
|
+
// Claim while there's capacity; pollAndProcess no-ops at the ceiling.
|
|
303
|
+
let progressed = true;
|
|
304
|
+
while (
|
|
305
|
+
progressed &&
|
|
306
|
+
!this.shuttingDown &&
|
|
307
|
+
this.inFlight.size < this.options.concurrency
|
|
308
|
+
) {
|
|
309
|
+
const before = this.inFlight.size;
|
|
310
|
+
await this.pollAndProcess();
|
|
311
|
+
progressed = this.inFlight.size > before;
|
|
312
|
+
}
|
|
313
|
+
} while (this.wakeRecheckPending && !this.shuttingDown);
|
|
314
|
+
} finally {
|
|
315
|
+
this.wakeDraining = false;
|
|
316
|
+
}
|
|
230
317
|
}
|
|
231
318
|
|
|
232
319
|
async onModuleDestroy(): Promise<void> {
|
|
@@ -246,6 +333,17 @@ export class JobWorker implements OnModuleInit, OnModuleDestroy {
|
|
|
246
333
|
}
|
|
247
334
|
process.removeListener('SIGTERM', this.sigtermHandler);
|
|
248
335
|
|
|
336
|
+
// LISTEN-NOTIFY-1 — release the listener connection so the process can exit
|
|
337
|
+
// cleanly. Best-effort; a failure here doesn't block the drain.
|
|
338
|
+
if (this.notifyListener) {
|
|
339
|
+
try {
|
|
340
|
+
await this.notifyListener.stop();
|
|
341
|
+
} catch (err) {
|
|
342
|
+
this.logger.error(`notify listener stop failed: ${(err as Error).message}`);
|
|
343
|
+
}
|
|
344
|
+
this.notifyListener = null;
|
|
345
|
+
}
|
|
346
|
+
|
|
249
347
|
await this.drainInFlight();
|
|
250
348
|
|
|
251
349
|
// Any rows still `running` past timeout → release back to pending.
|