@secondlayer/subgraphs 3.7.4 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +30 -1
- package/dist/src/index.js +414 -83
- package/dist/src/index.js.map +9 -7
- package/dist/src/runtime/block-processor.js +40 -2
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.js +40 -2
- package/dist/src/runtime/catchup.js.map +3 -3
- package/dist/src/runtime/emitter.d.ts +18 -0
- package/dist/src/runtime/emitter.js +773 -0
- package/dist/src/runtime/emitter.js.map +19 -0
- package/dist/src/runtime/processor.js +101 -52
- package/dist/src/runtime/processor.js.map +4 -4
- package/dist/src/runtime/reindex.js +101 -52
- package/dist/src/runtime/reindex.js.map +4 -4
- package/dist/src/runtime/reorg.js +40 -2
- package/dist/src/runtime/reorg.js.map +3 -3
- package/dist/src/runtime/replay.js +33 -2
- package/dist/src/runtime/replay.js.map +4 -4
- package/dist/src/schema/index.d.ts +2 -1
- package/dist/src/schema/index.js +75 -82
- package/dist/src/schema/index.js.map +5 -5
- package/dist/src/service.js +101 -52
- package/dist/src/service.js.map +4 -4
- package/dist/src/validate.d.ts +2 -1
- package/dist/src/validate.js +2 -1
- package/dist/src/validate.js.map +3 -3
- package/package.json +6 -2
|
@@ -0,0 +1,773 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
3
|
+
|
|
4
|
+
// src/runtime/emitter.ts
|
|
5
|
+
import { randomUUID } from "node:crypto";
|
|
6
|
+
import {
|
|
7
|
+
getTargetDb
|
|
8
|
+
} from "@secondlayer/shared/db";
|
|
9
|
+
import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
|
|
10
|
+
import { logger as logger2 } from "@secondlayer/shared/logger";
|
|
11
|
+
import { listen, targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
12
|
+
import { sql as sql2 } from "kysely";
|
|
13
|
+
|
|
14
|
+
// src/runtime/formats/index.ts
|
|
15
|
+
import { signSecondlayerWebhook } from "@secondlayer/shared/crypto/secondlayer-webhook";
|
|
16
|
+
import { logger } from "@secondlayer/shared/logger";
|
|
17
|
+
|
|
18
|
+
// src/runtime/formats/cloudevents.ts
|
|
19
|
+
function buildCloudEvents(outboxRow, _sub) {
|
|
20
|
+
const event = {
|
|
21
|
+
specversion: "1.0",
|
|
22
|
+
type: outboxRow.event_type,
|
|
23
|
+
source: `secondlayer:${outboxRow.subgraph_name}`,
|
|
24
|
+
id: outboxRow.id,
|
|
25
|
+
time: new Date(outboxRow.created_at).toISOString(),
|
|
26
|
+
datacontenttype: "application/json",
|
|
27
|
+
data: outboxRow.payload
|
|
28
|
+
};
|
|
29
|
+
return {
|
|
30
|
+
body: JSON.stringify(event),
|
|
31
|
+
headers: {
|
|
32
|
+
"content-type": "application/cloudevents+json; charset=utf-8"
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// src/runtime/formats/cloudflare.ts
|
|
38
|
+
import { decryptSecret } from "@secondlayer/shared/crypto/secrets";
|
|
39
|
+
function resolveBearer(sub) {
|
|
40
|
+
const cfg = sub.auth_config;
|
|
41
|
+
if (cfg.tokenEnc) {
|
|
42
|
+
return decryptSecret(Buffer.from(cfg.tokenEnc, "base64"));
|
|
43
|
+
}
|
|
44
|
+
return cfg.token ?? null;
|
|
45
|
+
}
|
|
46
|
+
function buildCloudflare(outboxRow, sub) {
|
|
47
|
+
const body = JSON.stringify({
|
|
48
|
+
params: {
|
|
49
|
+
...outboxRow.payload,
|
|
50
|
+
_type: outboxRow.event_type,
|
|
51
|
+
_outboxId: outboxRow.id
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
const headers = {
|
|
55
|
+
"content-type": "application/json"
|
|
56
|
+
};
|
|
57
|
+
const token = resolveBearer(sub);
|
|
58
|
+
if (token)
|
|
59
|
+
headers.authorization = `Bearer ${token}`;
|
|
60
|
+
return { body, headers };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// src/runtime/formats/inngest.ts
|
|
64
|
+
var INNGEST_VERSION = "2026-04-23.v1";
|
|
65
|
+
function buildInngest(outboxRow) {
|
|
66
|
+
const event = {
|
|
67
|
+
name: outboxRow.event_type,
|
|
68
|
+
data: outboxRow.payload,
|
|
69
|
+
id: outboxRow.id,
|
|
70
|
+
ts: new Date(outboxRow.created_at).getTime(),
|
|
71
|
+
v: INNGEST_VERSION
|
|
72
|
+
};
|
|
73
|
+
return {
|
|
74
|
+
body: JSON.stringify([event]),
|
|
75
|
+
headers: {
|
|
76
|
+
"content-type": "application/json"
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// src/runtime/formats/raw.ts
|
|
82
|
+
function buildRaw(outboxRow, sub) {
|
|
83
|
+
const cfg = sub.auth_config;
|
|
84
|
+
const headers = {
|
|
85
|
+
"content-type": cfg.contentType ?? "application/json",
|
|
86
|
+
...cfg.headers ?? {}
|
|
87
|
+
};
|
|
88
|
+
if (cfg.authType === "bearer" && cfg.token) {
|
|
89
|
+
headers.authorization = `Bearer ${cfg.token}`;
|
|
90
|
+
} else if (cfg.authType === "basic" && cfg.basicAuth) {
|
|
91
|
+
headers.authorization = `Basic ${cfg.basicAuth}`;
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
body: JSON.stringify(outboxRow.payload),
|
|
95
|
+
headers
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// src/runtime/formats/standard-webhooks.ts
|
|
100
|
+
import { sign } from "@secondlayer/shared/crypto/standard-webhooks";
|
|
101
|
+
function buildStandardWebhooks(outboxRow, signingSecret) {
|
|
102
|
+
const nowSeconds = Math.floor(Date.now() / 1000);
|
|
103
|
+
const payload = {
|
|
104
|
+
type: outboxRow.event_type,
|
|
105
|
+
timestamp: new Date(nowSeconds * 1000).toISOString(),
|
|
106
|
+
data: outboxRow.payload
|
|
107
|
+
};
|
|
108
|
+
const body = JSON.stringify(payload);
|
|
109
|
+
const sigHeaders = sign(body, signingSecret, {
|
|
110
|
+
id: outboxRow.id,
|
|
111
|
+
timestampSeconds: nowSeconds
|
|
112
|
+
});
|
|
113
|
+
return {
|
|
114
|
+
body,
|
|
115
|
+
headers: {
|
|
116
|
+
"content-type": "application/json",
|
|
117
|
+
...sigHeaders
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// src/runtime/formats/trigger.ts
|
|
123
|
+
import { decryptSecret as decryptSecret2 } from "@secondlayer/shared/crypto/secrets";
|
|
124
|
+
function resolveBearer2(sub) {
|
|
125
|
+
const cfg = sub.auth_config;
|
|
126
|
+
if (cfg.tokenEnc) {
|
|
127
|
+
return decryptSecret2(Buffer.from(cfg.tokenEnc, "base64"));
|
|
128
|
+
}
|
|
129
|
+
return cfg.token ?? null;
|
|
130
|
+
}
|
|
131
|
+
function buildTrigger(outboxRow, sub) {
|
|
132
|
+
const body = JSON.stringify({
|
|
133
|
+
payload: outboxRow.payload,
|
|
134
|
+
options: {
|
|
135
|
+
idempotencyKey: outboxRow.id
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
const headers = {
|
|
139
|
+
"content-type": "application/json"
|
|
140
|
+
};
|
|
141
|
+
const token = resolveBearer2(sub);
|
|
142
|
+
if (token)
|
|
143
|
+
headers.authorization = `Bearer ${token}`;
|
|
144
|
+
return { body, headers };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// src/runtime/formats/index.ts
|
|
148
|
+
function buildBody(outboxRow, sub, signingSecret) {
|
|
149
|
+
switch (sub.format) {
|
|
150
|
+
case "inngest":
|
|
151
|
+
return buildInngest(outboxRow);
|
|
152
|
+
case "trigger":
|
|
153
|
+
return buildTrigger(outboxRow, sub);
|
|
154
|
+
case "cloudflare":
|
|
155
|
+
return buildCloudflare(outboxRow, sub);
|
|
156
|
+
case "cloudevents":
|
|
157
|
+
return buildCloudEvents(outboxRow, sub);
|
|
158
|
+
case "raw":
|
|
159
|
+
return buildRaw(outboxRow, sub);
|
|
160
|
+
case "standard-webhooks":
|
|
161
|
+
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
162
|
+
default:
|
|
163
|
+
logger.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
164
|
+
format: sub.format,
|
|
165
|
+
subscriptionId: sub.id
|
|
166
|
+
});
|
|
167
|
+
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
function buildForFormat(outboxRow, sub, signingSecret) {
|
|
171
|
+
const result = buildBody(outboxRow, sub, signingSecret);
|
|
172
|
+
const sigHeaders = signSecondlayerWebhook(outboxRow.id, result.body);
|
|
173
|
+
if (sigHeaders) {
|
|
174
|
+
result.headers = { ...result.headers, ...sigHeaders };
|
|
175
|
+
}
|
|
176
|
+
return result;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// src/runtime/subscription-state.ts
|
|
180
|
+
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
181
|
+
import { sql } from "kysely";
|
|
182
|
+
|
|
183
|
+
// src/runtime/emitter-matcher.ts
|
|
184
|
+
function isPrimitive(v) {
|
|
185
|
+
const t = typeof v;
|
|
186
|
+
return t === "string" || t === "number" || t === "boolean";
|
|
187
|
+
}
|
|
188
|
+
function coerceBigInt(v) {
|
|
189
|
+
if (typeof v === "bigint")
|
|
190
|
+
return v;
|
|
191
|
+
if (typeof v === "number") {
|
|
192
|
+
if (!Number.isFinite(v))
|
|
193
|
+
return null;
|
|
194
|
+
if (!Number.isInteger(v))
|
|
195
|
+
return null;
|
|
196
|
+
return BigInt(v);
|
|
197
|
+
}
|
|
198
|
+
if (typeof v === "string" && /^-?\d+$/.test(v)) {
|
|
199
|
+
try {
|
|
200
|
+
return BigInt(v);
|
|
201
|
+
} catch {
|
|
202
|
+
return null;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
function coerceFloat(v) {
|
|
208
|
+
if (typeof v === "number" && Number.isFinite(v))
|
|
209
|
+
return v;
|
|
210
|
+
if (typeof v === "bigint")
|
|
211
|
+
return Number(v);
|
|
212
|
+
if (typeof v === "string" && v !== "" && !Number.isNaN(Number(v))) {
|
|
213
|
+
return Number(v);
|
|
214
|
+
}
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
function compareNumeric(a, b) {
|
|
218
|
+
const ba = coerceBigInt(a);
|
|
219
|
+
const bb = coerceBigInt(b);
|
|
220
|
+
if (ba !== null && bb !== null) {
|
|
221
|
+
if (ba === bb)
|
|
222
|
+
return 0;
|
|
223
|
+
return ba > bb ? 1 : -1;
|
|
224
|
+
}
|
|
225
|
+
const fa = coerceFloat(a);
|
|
226
|
+
const fb = coerceFloat(b);
|
|
227
|
+
if (fa === null || fb === null)
|
|
228
|
+
return null;
|
|
229
|
+
if (fa === fb)
|
|
230
|
+
return 0;
|
|
231
|
+
return fa > fb ? 1 : -1;
|
|
232
|
+
}
|
|
233
|
+
function matchClause(rowValue, clause) {
|
|
234
|
+
const rowIsPrimitive = isPrimitive(rowValue) || typeof rowValue === "bigint";
|
|
235
|
+
if (isPrimitive(clause)) {
|
|
236
|
+
if (!rowIsPrimitive)
|
|
237
|
+
return false;
|
|
238
|
+
if (typeof clause === "number" || typeof rowValue === "number" || typeof rowValue === "bigint") {
|
|
239
|
+
const cmp = compareNumeric(rowValue, clause);
|
|
240
|
+
if (cmp !== null)
|
|
241
|
+
return cmp === 0;
|
|
242
|
+
}
|
|
243
|
+
return rowValue === clause || String(rowValue) === String(clause);
|
|
244
|
+
}
|
|
245
|
+
if (clause === null || typeof clause !== "object" || Array.isArray(clause)) {
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
const keys = Object.keys(clause);
|
|
249
|
+
if (keys.length !== 1)
|
|
250
|
+
return false;
|
|
251
|
+
const op = keys[0];
|
|
252
|
+
const c = clause;
|
|
253
|
+
switch (op) {
|
|
254
|
+
case "eq":
|
|
255
|
+
return matchClause(rowValue, c.eq);
|
|
256
|
+
case "neq":
|
|
257
|
+
return !matchClause(rowValue, c.neq);
|
|
258
|
+
case "gt":
|
|
259
|
+
case "gte":
|
|
260
|
+
case "lt":
|
|
261
|
+
case "lte": {
|
|
262
|
+
if (!rowIsPrimitive)
|
|
263
|
+
return false;
|
|
264
|
+
const cmp = compareNumeric(rowValue, c[op]);
|
|
265
|
+
if (cmp === null)
|
|
266
|
+
return false;
|
|
267
|
+
if (op === "gt")
|
|
268
|
+
return cmp > 0;
|
|
269
|
+
if (op === "gte")
|
|
270
|
+
return cmp >= 0;
|
|
271
|
+
if (op === "lt")
|
|
272
|
+
return cmp < 0;
|
|
273
|
+
return cmp <= 0;
|
|
274
|
+
}
|
|
275
|
+
case "in": {
|
|
276
|
+
const list = c.in;
|
|
277
|
+
if (!Array.isArray(list))
|
|
278
|
+
return false;
|
|
279
|
+
if (!rowIsPrimitive)
|
|
280
|
+
return false;
|
|
281
|
+
return list.some((item) => isPrimitive(item) && matchClause(rowValue, item));
|
|
282
|
+
}
|
|
283
|
+
default:
|
|
284
|
+
return false;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function matchesFilter(filter, row) {
|
|
288
|
+
if (!filter || Object.keys(filter).length === 0)
|
|
289
|
+
return true;
|
|
290
|
+
for (const [col, clause] of Object.entries(filter)) {
|
|
291
|
+
if (!matchClause(row[col], clause))
|
|
292
|
+
return false;
|
|
293
|
+
}
|
|
294
|
+
return true;
|
|
295
|
+
}
|
|
296
|
+
function key(subgraphName, tableName) {
|
|
297
|
+
return `${subgraphName}\x00${tableName}`;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
class SubscriptionMatcher {
|
|
301
|
+
byKey = new Map;
|
|
302
|
+
byId = new Map;
|
|
303
|
+
setAll(subs) {
|
|
304
|
+
this.byKey.clear();
|
|
305
|
+
this.byId.clear();
|
|
306
|
+
for (const sub of subs) {
|
|
307
|
+
if (sub.status !== "active")
|
|
308
|
+
continue;
|
|
309
|
+
if (sub.kind !== "subgraph" || !sub.subgraph_name || !sub.table_name)
|
|
310
|
+
continue;
|
|
311
|
+
this.byId.set(sub.id, sub);
|
|
312
|
+
const k = key(sub.subgraph_name, sub.table_name);
|
|
313
|
+
const arr = this.byKey.get(k);
|
|
314
|
+
if (arr)
|
|
315
|
+
arr.push(sub);
|
|
316
|
+
else
|
|
317
|
+
this.byKey.set(k, [sub]);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
match(subgraphName, tableName, row) {
|
|
321
|
+
const bucket = this.byKey.get(key(subgraphName, tableName));
|
|
322
|
+
if (!bucket)
|
|
323
|
+
return [];
|
|
324
|
+
const hits = [];
|
|
325
|
+
for (const sub of bucket) {
|
|
326
|
+
if (matchesFilter(sub.filter, row))
|
|
327
|
+
hits.push(sub);
|
|
328
|
+
}
|
|
329
|
+
return hits;
|
|
330
|
+
}
|
|
331
|
+
has(subgraphName, tableName) {
|
|
332
|
+
return this.byKey.has(key(subgraphName, tableName));
|
|
333
|
+
}
|
|
334
|
+
size() {
|
|
335
|
+
return this.byId.size;
|
|
336
|
+
}
|
|
337
|
+
get(id) {
|
|
338
|
+
return this.byId.get(id);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// src/runtime/subscription-state.ts
|
|
343
|
+
var matcher = new SubscriptionMatcher;
|
|
344
|
+
async function refreshMatcher(db) {
|
|
345
|
+
const rows = await sql`
|
|
346
|
+
SELECT * FROM subscriptions WHERE status = 'active'
|
|
347
|
+
`.execute(db);
|
|
348
|
+
matcher.setAll(rows.rows);
|
|
349
|
+
return matcher.size();
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// src/runtime/emitter.ts
|
|
353
|
+
var BATCH_SIZE = 50;
|
|
354
|
+
var LIVE_SHARE = 0.9;
|
|
355
|
+
var BACKOFF_SECONDS = [30, 120, 600, 3600, 21600, 86400, 259200];
|
|
356
|
+
var CIRCUIT_THRESHOLD = 20;
|
|
357
|
+
var LOCK_WINDOW_MS = 60000;
|
|
358
|
+
function nextDelaySeconds(attempt) {
|
|
359
|
+
return BACKOFF_SECONDS[Math.min(attempt, BACKOFF_SECONDS.length - 1)];
|
|
360
|
+
}
|
|
361
|
+
var PRIVATE_V4_PATTERNS = [
|
|
362
|
+
/^127\./,
|
|
363
|
+
/^10\./,
|
|
364
|
+
/^172\.(1[6-9]|2\d|3[01])\./,
|
|
365
|
+
/^192\.168\./,
|
|
366
|
+
/^169\.254\./,
|
|
367
|
+
/^0\./,
|
|
368
|
+
/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./
|
|
369
|
+
];
|
|
370
|
+
function isPrivateEgress(url) {
|
|
371
|
+
let parsed;
|
|
372
|
+
try {
|
|
373
|
+
parsed = new URL(url);
|
|
374
|
+
} catch {
|
|
375
|
+
return true;
|
|
376
|
+
}
|
|
377
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
378
|
+
return true;
|
|
379
|
+
}
|
|
380
|
+
const raw = parsed.hostname.toLowerCase();
|
|
381
|
+
const host = raw.startsWith("[") && raw.endsWith("]") ? raw.slice(1, -1) : raw;
|
|
382
|
+
if (host === "localhost" || host === "0.0.0.0")
|
|
383
|
+
return true;
|
|
384
|
+
if (host === "::" || host === "::1")
|
|
385
|
+
return true;
|
|
386
|
+
if (/^f[cd][0-9a-f]{2}:/.test(host))
|
|
387
|
+
return true;
|
|
388
|
+
if (/^fe[89ab][0-9a-f]:/.test(host))
|
|
389
|
+
return true;
|
|
390
|
+
const mapped = host.match(/^::ffff:(.+)$/);
|
|
391
|
+
if (mapped) {
|
|
392
|
+
const inner = mapped[1];
|
|
393
|
+
if (/^\d+\.\d+\.\d+\.\d+$/.test(inner)) {
|
|
394
|
+
for (const p of PRIVATE_V4_PATTERNS)
|
|
395
|
+
if (p.test(inner))
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
const hex = inner.match(/^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
|
399
|
+
if (hex) {
|
|
400
|
+
const a = Number.parseInt(hex[1], 16);
|
|
401
|
+
const b = Number.parseInt(hex[2], 16);
|
|
402
|
+
const dotted = `${a >> 8 & 255}.${a & 255}.${b >> 8 & 255}.${b & 255}`;
|
|
403
|
+
for (const p of PRIVATE_V4_PATTERNS)
|
|
404
|
+
if (p.test(dotted))
|
|
405
|
+
return true;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
for (const p of PRIVATE_V4_PATTERNS) {
|
|
409
|
+
if (p.test(host))
|
|
410
|
+
return true;
|
|
411
|
+
}
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
function allowPrivateEgress() {
|
|
415
|
+
return process.env.SECONDLAYER_ALLOW_PRIVATE_EGRESS === "true";
|
|
416
|
+
}
|
|
417
|
+
async function postToSubscription(url, body, headers, timeoutMs) {
|
|
418
|
+
if (isPrivateEgress(url) && !allowPrivateEgress()) {
|
|
419
|
+
logger2.warn("[emitter] refused private egress", { url });
|
|
420
|
+
return {
|
|
421
|
+
ok: false,
|
|
422
|
+
statusCode: null,
|
|
423
|
+
error: "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)",
|
|
424
|
+
durationMs: 0,
|
|
425
|
+
responseBody: null,
|
|
426
|
+
responseHeaders: null
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
const start = performance.now();
|
|
430
|
+
let statusCode = null;
|
|
431
|
+
let error = null;
|
|
432
|
+
let ok = false;
|
|
433
|
+
let responseBody = "";
|
|
434
|
+
let responseHeaders = {};
|
|
435
|
+
try {
|
|
436
|
+
const res = await fetch(url, {
|
|
437
|
+
method: "POST",
|
|
438
|
+
headers,
|
|
439
|
+
body,
|
|
440
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
441
|
+
});
|
|
442
|
+
statusCode = res.status;
|
|
443
|
+
ok = res.ok;
|
|
444
|
+
const buf = await res.arrayBuffer();
|
|
445
|
+
const truncated = buf.byteLength > 8192 ? buf.slice(0, 8192) : buf;
|
|
446
|
+
responseBody = Buffer.from(truncated).toString("utf8");
|
|
447
|
+
responseHeaders = Object.fromEntries(res.headers.entries());
|
|
448
|
+
} catch (err) {
|
|
449
|
+
error = err instanceof Error ? err.message : String(err);
|
|
450
|
+
}
|
|
451
|
+
return {
|
|
452
|
+
ok,
|
|
453
|
+
statusCode,
|
|
454
|
+
error,
|
|
455
|
+
durationMs: Math.round(performance.now() - start),
|
|
456
|
+
responseBody: responseBody || null,
|
|
457
|
+
responseHeaders
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
async function dispatchOne(db, outboxRow, sub) {
|
|
461
|
+
const { body, headers } = buildForFormat(outboxRow, sub, getSubscriptionSigningSecret(sub));
|
|
462
|
+
const r = await postToSubscription(sub.url, body, headers, sub.timeout_ms);
|
|
463
|
+
const attempt = outboxRow.attempt + 1;
|
|
464
|
+
await db.insertInto("subscription_deliveries").values({
|
|
465
|
+
outbox_id: outboxRow.id,
|
|
466
|
+
subscription_id: outboxRow.subscription_id,
|
|
467
|
+
attempt,
|
|
468
|
+
status_code: r.statusCode,
|
|
469
|
+
response_headers: r.responseHeaders,
|
|
470
|
+
response_body: r.responseBody,
|
|
471
|
+
error_message: r.error,
|
|
472
|
+
duration_ms: r.durationMs
|
|
473
|
+
}).execute();
|
|
474
|
+
return {
|
|
475
|
+
ok: r.ok,
|
|
476
|
+
statusCode: r.statusCode,
|
|
477
|
+
error: r.error,
|
|
478
|
+
durationMs: r.durationMs
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
function buildTestOutboxRow(sub) {
|
|
482
|
+
const now = new Date;
|
|
483
|
+
return {
|
|
484
|
+
id: randomUUID(),
|
|
485
|
+
subscription_id: sub.id,
|
|
486
|
+
kind: sub.kind,
|
|
487
|
+
subgraph_name: sub.subgraph_name ?? null,
|
|
488
|
+
table_name: sub.table_name ?? null,
|
|
489
|
+
block_height: 0,
|
|
490
|
+
tx_id: null,
|
|
491
|
+
row_pk: null,
|
|
492
|
+
event_type: sub.kind === "chain" ? "chain.test.apply" : `${sub.subgraph_name ?? "subgraph"}.${sub.table_name ?? "test"}.created`,
|
|
493
|
+
payload: {
|
|
494
|
+
test: true,
|
|
495
|
+
message: "Secondlayer test delivery",
|
|
496
|
+
subscription_id: sub.id,
|
|
497
|
+
sent_at: now.toISOString()
|
|
498
|
+
},
|
|
499
|
+
dedup_key: `test:${sub.id}:${now.getTime()}`,
|
|
500
|
+
attempt: 0,
|
|
501
|
+
next_attempt_at: now,
|
|
502
|
+
status: "pending",
|
|
503
|
+
is_replay: false,
|
|
504
|
+
delivered_at: null,
|
|
505
|
+
failed_at: null,
|
|
506
|
+
locked_by: null,
|
|
507
|
+
locked_until: null,
|
|
508
|
+
created_at: now
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
async function deliverTestEvent(db, sub) {
|
|
512
|
+
const testRow = buildTestOutboxRow(sub);
|
|
513
|
+
const { body, headers } = buildForFormat(testRow, sub, getSubscriptionSigningSecret(sub));
|
|
514
|
+
const r = await postToSubscription(sub.url, body, headers, sub.timeout_ms);
|
|
515
|
+
const inserted = await db.insertInto("subscription_deliveries").values({
|
|
516
|
+
outbox_id: null,
|
|
517
|
+
subscription_id: sub.id,
|
|
518
|
+
attempt: 1,
|
|
519
|
+
status_code: r.statusCode,
|
|
520
|
+
response_headers: r.responseHeaders,
|
|
521
|
+
response_body: r.responseBody,
|
|
522
|
+
error_message: r.error,
|
|
523
|
+
duration_ms: r.durationMs
|
|
524
|
+
}).returning("id").executeTakeFirstOrThrow();
|
|
525
|
+
return {
|
|
526
|
+
ok: r.ok,
|
|
527
|
+
statusCode: r.statusCode,
|
|
528
|
+
error: r.error,
|
|
529
|
+
durationMs: r.durationMs,
|
|
530
|
+
deliveryId: inserted.id
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
async function settleDelivered(db, outboxRow) {
|
|
534
|
+
await db.transaction().execute(async (tx) => {
|
|
535
|
+
await tx.updateTable("subscription_outbox").set({
|
|
536
|
+
status: "delivered",
|
|
537
|
+
delivered_at: new Date,
|
|
538
|
+
attempt: outboxRow.attempt + 1,
|
|
539
|
+
locked_by: null,
|
|
540
|
+
locked_until: null
|
|
541
|
+
}).where("id", "=", outboxRow.id).execute();
|
|
542
|
+
await tx.updateTable("subscriptions").set({
|
|
543
|
+
last_delivery_at: new Date,
|
|
544
|
+
last_success_at: new Date,
|
|
545
|
+
circuit_failures: 0,
|
|
546
|
+
last_error: null,
|
|
547
|
+
updated_at: new Date
|
|
548
|
+
}).where("id", "=", outboxRow.subscription_id).execute();
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
async function settleFailed(db, outboxRow, sub, errText) {
|
|
552
|
+
const attempt = outboxRow.attempt + 1;
|
|
553
|
+
const isDead = attempt >= sub.max_retries;
|
|
554
|
+
const nextAt = isDead ? null : new Date(Date.now() + nextDelaySeconds(outboxRow.attempt) * 1000);
|
|
555
|
+
await db.transaction().execute(async (tx) => {
|
|
556
|
+
await tx.updateTable("subscription_outbox").set({
|
|
557
|
+
attempt,
|
|
558
|
+
next_attempt_at: nextAt ?? new Date,
|
|
559
|
+
status: isDead ? "dead" : "pending",
|
|
560
|
+
failed_at: isDead ? new Date : null,
|
|
561
|
+
locked_by: null,
|
|
562
|
+
locked_until: null
|
|
563
|
+
}).where("id", "=", outboxRow.id).execute();
|
|
564
|
+
const incResult = await sql2`
|
|
565
|
+
UPDATE subscriptions
|
|
566
|
+
SET circuit_failures = circuit_failures + 1,
|
|
567
|
+
last_delivery_at = NOW(),
|
|
568
|
+
last_error = ${errText.slice(0, 500)},
|
|
569
|
+
updated_at = NOW()
|
|
570
|
+
WHERE id = ${sub.id}
|
|
571
|
+
RETURNING circuit_failures
|
|
572
|
+
`.execute(tx);
|
|
573
|
+
const newFailures = incResult.rows[0]?.circuit_failures ?? sub.circuit_failures + 1;
|
|
574
|
+
const shouldTripCircuit = newFailures >= CIRCUIT_THRESHOLD;
|
|
575
|
+
if (shouldTripCircuit) {
|
|
576
|
+
await tx.updateTable("subscriptions").set({
|
|
577
|
+
status: "paused",
|
|
578
|
+
circuit_opened_at: new Date,
|
|
579
|
+
updated_at: new Date
|
|
580
|
+
}).where("id", "=", sub.id).execute();
|
|
581
|
+
logger2.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
582
|
+
subscription: sub.name,
|
|
583
|
+
failures: newFailures
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
async function claimAndDrain(db, state, emitterId) {
|
|
589
|
+
if (state.claimInFlight)
|
|
590
|
+
return 0;
|
|
591
|
+
state.claimInFlight = true;
|
|
592
|
+
try {
|
|
593
|
+
const liveLimit = Math.max(1, Math.round(BATCH_SIZE * LIVE_SHARE));
|
|
594
|
+
const replayLimit = BATCH_SIZE - liveLimit;
|
|
595
|
+
const claimed = await db.transaction().execute(async (tx) => {
|
|
596
|
+
const live = await sql2`
|
|
597
|
+
SELECT * FROM subscription_outbox
|
|
598
|
+
WHERE status = 'pending'
|
|
599
|
+
AND next_attempt_at <= NOW()
|
|
600
|
+
AND is_replay = FALSE
|
|
601
|
+
ORDER BY next_attempt_at ASC
|
|
602
|
+
FOR UPDATE SKIP LOCKED
|
|
603
|
+
LIMIT ${sql2.lit(liveLimit)}
|
|
604
|
+
`.execute(tx);
|
|
605
|
+
const replay = await sql2`
|
|
606
|
+
SELECT * FROM subscription_outbox
|
|
607
|
+
WHERE status = 'pending'
|
|
608
|
+
AND next_attempt_at <= NOW()
|
|
609
|
+
AND is_replay = TRUE
|
|
610
|
+
ORDER BY next_attempt_at ASC
|
|
611
|
+
FOR UPDATE SKIP LOCKED
|
|
612
|
+
LIMIT ${sql2.lit(replayLimit)}
|
|
613
|
+
`.execute(tx);
|
|
614
|
+
const combined = [...live.rows, ...replay.rows];
|
|
615
|
+
if (combined.length === 0)
|
|
616
|
+
return [];
|
|
617
|
+
const now = new Date;
|
|
618
|
+
const lockUntil = new Date(now.getTime() + LOCK_WINDOW_MS);
|
|
619
|
+
await tx.updateTable("subscription_outbox").set({
|
|
620
|
+
locked_by: emitterId,
|
|
621
|
+
locked_until: lockUntil,
|
|
622
|
+
next_attempt_at: lockUntil
|
|
623
|
+
}).where("id", "in", combined.map((r) => r.id)).execute();
|
|
624
|
+
return combined;
|
|
625
|
+
});
|
|
626
|
+
if (claimed.length === 0)
|
|
627
|
+
return 0;
|
|
628
|
+
const bySubId = new Map;
|
|
629
|
+
for (const row of claimed) {
|
|
630
|
+
const arr = bySubId.get(row.subscription_id);
|
|
631
|
+
if (arr)
|
|
632
|
+
arr.push(row);
|
|
633
|
+
else
|
|
634
|
+
bySubId.set(row.subscription_id, [row]);
|
|
635
|
+
}
|
|
636
|
+
const subIds = Array.from(bySubId.keys());
|
|
637
|
+
const subs = await db.selectFrom("subscriptions").selectAll().where("id", "in", subIds).execute();
|
|
638
|
+
const subById = new Map(subs.map((s) => [s.id, s]));
|
|
639
|
+
await Promise.all(subIds.map((subId) => drainForSub(db, state, subById.get(subId), bySubId.get(subId))));
|
|
640
|
+
return claimed.length;
|
|
641
|
+
} finally {
|
|
642
|
+
state.claimInFlight = false;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
async function drainForSub(db, state, sub, rows) {
|
|
646
|
+
const cap = sub.concurrency || 4;
|
|
647
|
+
const counter = () => state.inFlightBySub.get(sub.id) ?? 0;
|
|
648
|
+
const inc = () => state.inFlightBySub.set(sub.id, counter() + 1);
|
|
649
|
+
const dec = () => state.inFlightBySub.set(sub.id, Math.max(0, counter() - 1));
|
|
650
|
+
const queue = [...rows];
|
|
651
|
+
const workers = [];
|
|
652
|
+
const slots = Math.min(cap, queue.length);
|
|
653
|
+
for (let i = 0;i < slots; i++) {
|
|
654
|
+
workers.push((async () => {
|
|
655
|
+
while (state.running && queue.length > 0) {
|
|
656
|
+
const row = queue.shift();
|
|
657
|
+
if (!row)
|
|
658
|
+
break;
|
|
659
|
+
inc();
|
|
660
|
+
try {
|
|
661
|
+
const result = await dispatchOne(db, row, sub);
|
|
662
|
+
if (result.ok) {
|
|
663
|
+
await settleDelivered(db, row);
|
|
664
|
+
} else {
|
|
665
|
+
const err = result.error ?? `HTTP ${result.statusCode ?? "?"}`;
|
|
666
|
+
await settleFailed(db, row, sub, err);
|
|
667
|
+
}
|
|
668
|
+
} catch (err) {
|
|
669
|
+
logger2.error("Emitter dispatch crashed", {
|
|
670
|
+
outboxId: row.id,
|
|
671
|
+
error: err instanceof Error ? err.message : String(err)
|
|
672
|
+
});
|
|
673
|
+
await settleFailed(db, row, sub, err instanceof Error ? err.message : String(err));
|
|
674
|
+
} finally {
|
|
675
|
+
dec();
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
})());
|
|
679
|
+
}
|
|
680
|
+
await Promise.all(workers);
|
|
681
|
+
}
|
|
682
|
+
async function runRetention(db) {
|
|
683
|
+
await sql2`
|
|
684
|
+
DELETE FROM subscription_outbox
|
|
685
|
+
WHERE status = 'delivered' AND delivered_at < NOW() - interval '7 days'
|
|
686
|
+
`.execute(db);
|
|
687
|
+
await sql2`
|
|
688
|
+
DELETE FROM subscription_deliveries
|
|
689
|
+
WHERE dispatched_at < NOW() - interval '30 days'
|
|
690
|
+
`.execute(db);
|
|
691
|
+
await sql2`
|
|
692
|
+
DELETE FROM subscription_outbox
|
|
693
|
+
WHERE status = 'dead' AND failed_at < NOW() - interval '90 days'
|
|
694
|
+
`.execute(db);
|
|
695
|
+
}
|
|
696
|
+
async function startEmitter(opts) {
|
|
697
|
+
const emitterId = `emitter-${Math.random().toString(36).slice(2, 10)}`;
|
|
698
|
+
const db = getTargetDb();
|
|
699
|
+
const state = {
|
|
700
|
+
running: true,
|
|
701
|
+
inFlightBySub: new Map,
|
|
702
|
+
claimInFlight: false
|
|
703
|
+
};
|
|
704
|
+
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
705
|
+
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
706
|
+
logger2.info("[emitter] started", { id: emitterId });
|
|
707
|
+
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
708
|
+
let lastErr = null;
|
|
709
|
+
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
710
|
+
try {
|
|
711
|
+
await refreshMatcher(db);
|
|
712
|
+
lastErr = null;
|
|
713
|
+
break;
|
|
714
|
+
} catch (err) {
|
|
715
|
+
lastErr = err;
|
|
716
|
+
const delayMs = 500 * 2 ** i;
|
|
717
|
+
logger2.warn("[emitter] matcher refresh failed, retrying", {
|
|
718
|
+
attempt: i + 1,
|
|
719
|
+
delayMs,
|
|
720
|
+
error: err instanceof Error ? err.message : String(err)
|
|
721
|
+
});
|
|
722
|
+
await new Promise((r) => setTimeout(r, delayMs));
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
if (lastErr) {
|
|
726
|
+
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
727
|
+
}
|
|
728
|
+
const listenUrl = targetListenerUrl();
|
|
729
|
+
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
730
|
+
if (!state.running)
|
|
731
|
+
return;
|
|
732
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger2.error("[emitter] claim failed", {
|
|
733
|
+
error: err instanceof Error ? err.message : String(err)
|
|
734
|
+
}));
|
|
735
|
+
}, { connectionString: listenUrl });
|
|
736
|
+
const stopChanged = await listen("subscriptions:changed", () => {
|
|
737
|
+
if (!state.running)
|
|
738
|
+
return;
|
|
739
|
+
refreshMatcher(db).catch((err) => logger2.error("[emitter] matcher refresh failed", {
|
|
740
|
+
error: err instanceof Error ? err.message : String(err)
|
|
741
|
+
}));
|
|
742
|
+
}, { connectionString: listenUrl });
|
|
743
|
+
const poll = setInterval(() => {
|
|
744
|
+
if (!state.running)
|
|
745
|
+
return;
|
|
746
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger2.error("[emitter] poll claim failed", {
|
|
747
|
+
error: err instanceof Error ? err.message : String(err)
|
|
748
|
+
}));
|
|
749
|
+
}, pollIntervalMs);
|
|
750
|
+
claimAndDrain(db, state, emitterId);
|
|
751
|
+
const retention = setInterval(() => {
|
|
752
|
+
if (!state.running)
|
|
753
|
+
return;
|
|
754
|
+
runRetention(db).catch((err) => logger2.error("[emitter] retention failed", {
|
|
755
|
+
error: err instanceof Error ? err.message : String(err)
|
|
756
|
+
}));
|
|
757
|
+
}, retentionIntervalMs);
|
|
758
|
+
return async () => {
|
|
759
|
+
state.running = false;
|
|
760
|
+
clearInterval(poll);
|
|
761
|
+
clearInterval(retention);
|
|
762
|
+
await stopNew();
|
|
763
|
+
await stopChanged();
|
|
764
|
+
logger2.info("[emitter] stopped", { id: emitterId });
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
export {
|
|
768
|
+
startEmitter,
|
|
769
|
+
deliverTestEvent
|
|
770
|
+
};
|
|
771
|
+
|
|
772
|
+
//# debugId=1CD71540D1B0DBC964756E2164756E21
|
|
773
|
+
//# sourceMappingURL=emitter.js.map
|