@secondlayer/subgraphs 4.2.1 → 4.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/src/index.js +11 -11
- package/dist/src/index.js.map +9 -9
- package/dist/src/runtime/block-processor.js +11 -11
- package/dist/src/runtime/block-processor.js.map +9 -9
- package/dist/src/runtime/catchup.js +11 -11
- package/dist/src/runtime/catchup.js.map +9 -9
- package/dist/src/runtime/context.d.ts +1 -1
- package/dist/src/runtime/context.js.map +1 -1
- package/dist/src/runtime/emitter.d.ts +10 -10
- package/dist/src/runtime/emitter.js +47 -47
- package/dist/src/runtime/emitter.js.map +14 -14
- package/dist/src/runtime/processor.js +19 -19
- package/dist/src/runtime/processor.js.map +13 -13
- package/dist/src/runtime/reindex.js +11 -11
- package/dist/src/runtime/reindex.js.map +9 -9
- package/dist/src/runtime/reorg.d.ts +2 -2
- package/dist/src/runtime/reorg.js +17 -17
- package/dist/src/runtime/reorg.js.map +10 -10
- package/dist/src/runtime/replay.d.ts +6 -6
- package/dist/src/runtime/replay.js +26 -26
- package/dist/src/runtime/replay.js.map +4 -4
- package/dist/src/runtime/runner.d.ts +1 -1
- package/dist/src/service.js +19 -19
- package/dist/src/service.js.map +14 -14
- package/dist/src/testing/index.js.map +1 -1
- package/dist/src/triggers/index.d.ts +2 -2
- package/dist/src/triggers/index.js.map +1 -1
- package/package.json +3 -3
|
@@ -7,7 +7,7 @@ import { lookup as dnsLookup } from "node:dns/promises";
|
|
|
7
7
|
import {
|
|
8
8
|
getTargetDb
|
|
9
9
|
} from "@secondlayer/shared/db";
|
|
10
|
-
import {
|
|
10
|
+
import { getWebhookSigningSecret } from "@secondlayer/shared/db/queries/webhooks";
|
|
11
11
|
import { logger as logger2 } from "@secondlayer/shared/logger";
|
|
12
12
|
import { listen, targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
13
13
|
import { sql as sql2 } from "kysely";
|
|
@@ -161,9 +161,9 @@ function buildBody(outboxRow, sub, signingSecret) {
|
|
|
161
161
|
case "standard-webhooks":
|
|
162
162
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
163
163
|
default:
|
|
164
|
-
logger.warn("Unknown
|
|
164
|
+
logger.warn("Unknown webhook format, falling back to standard-webhooks", {
|
|
165
165
|
format: sub.format,
|
|
166
|
-
|
|
166
|
+
webhookId: sub.id
|
|
167
167
|
});
|
|
168
168
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
169
169
|
}
|
|
@@ -189,16 +189,16 @@ async function meterBlocksProcessed(accountId, blocks, subgraphName) {
|
|
|
189
189
|
return true;
|
|
190
190
|
return hooks.onBlocksProcessed(accountId, blocks, subgraphName);
|
|
191
191
|
}
|
|
192
|
-
async function meterDeliveryAttempt(accountId,
|
|
192
|
+
async function meterDeliveryAttempt(accountId, webhookId) {
|
|
193
193
|
if (!accountId)
|
|
194
194
|
return true;
|
|
195
195
|
if (!hooks.onDeliveryAttempt)
|
|
196
196
|
return true;
|
|
197
|
-
return hooks.onDeliveryAttempt(accountId,
|
|
197
|
+
return hooks.onDeliveryAttempt(accountId, webhookId);
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
-
// src/runtime/
|
|
201
|
-
import {
|
|
200
|
+
// src/runtime/webhook-state.ts
|
|
201
|
+
import { listWebhooks } from "@secondlayer/shared/db/queries/webhooks";
|
|
202
202
|
import { sql } from "kysely";
|
|
203
203
|
|
|
204
204
|
// src/runtime/emitter-matcher.ts
|
|
@@ -318,7 +318,7 @@ function key(subgraphName, tableName) {
|
|
|
318
318
|
return `${subgraphName}\x00${tableName}`;
|
|
319
319
|
}
|
|
320
320
|
|
|
321
|
-
class
|
|
321
|
+
class WebhookMatcher {
|
|
322
322
|
byKey = new Map;
|
|
323
323
|
byId = new Map;
|
|
324
324
|
setAll(subs) {
|
|
@@ -360,11 +360,11 @@ class SubscriptionMatcher {
|
|
|
360
360
|
}
|
|
361
361
|
}
|
|
362
362
|
|
|
363
|
-
// src/runtime/
|
|
364
|
-
var matcher = new
|
|
363
|
+
// src/runtime/webhook-state.ts
|
|
364
|
+
var matcher = new WebhookMatcher;
|
|
365
365
|
async function refreshMatcher(db) {
|
|
366
366
|
const rows = await sql`
|
|
367
|
-
SELECT * FROM
|
|
367
|
+
SELECT * FROM webhooks WHERE status = 'active'
|
|
368
368
|
`.execute(db);
|
|
369
369
|
matcher.setAll(rows.rows);
|
|
370
370
|
return matcher.size();
|
|
@@ -375,8 +375,8 @@ var BATCH_SIZE = 50;
|
|
|
375
375
|
var LIVE_SHARE = 0.9;
|
|
376
376
|
var BACKOFF_SECONDS = [30, 120, 600, 3600, 21600, 86400, 259200];
|
|
377
377
|
var CIRCUIT_THRESHOLD = 20;
|
|
378
|
-
var
|
|
379
|
-
var LOCK_WINDOW_MS =
|
|
378
|
+
var MAX_WEBHOOK_TIMEOUT_MS = 300000;
|
|
379
|
+
var LOCK_WINDOW_MS = MAX_WEBHOOK_TIMEOUT_MS + 60000;
|
|
380
380
|
function nextDelaySeconds(attempt) {
|
|
381
381
|
return BACKOFF_SECONDS[Math.min(attempt, BACKOFF_SECONDS.length - 1)];
|
|
382
382
|
}
|
|
@@ -484,7 +484,7 @@ async function checkEgressAllowed(url) {
|
|
|
484
484
|
return null;
|
|
485
485
|
}
|
|
486
486
|
var MAX_REDIRECT_HOPS = 3;
|
|
487
|
-
async function
|
|
487
|
+
async function postToWebhook(url, body, headers, timeoutMs) {
|
|
488
488
|
const start = performance.now();
|
|
489
489
|
let target = url;
|
|
490
490
|
let statusCode = null;
|
|
@@ -553,14 +553,14 @@ async function postToSubscription(url, body, headers, timeoutMs) {
|
|
|
553
553
|
responseHeaders
|
|
554
554
|
};
|
|
555
555
|
}
|
|
556
|
-
var
|
|
556
|
+
var __postToWebhookForTest = postToWebhook;
|
|
557
557
|
async function dispatchOne(db, outboxRow, sub) {
|
|
558
|
-
const { body, headers } = buildForFormat(outboxRow, sub,
|
|
559
|
-
const r = await
|
|
558
|
+
const { body, headers } = buildForFormat(outboxRow, sub, getWebhookSigningSecret(sub));
|
|
559
|
+
const r = await postToWebhook(sub.url, body, headers, sub.timeout_ms);
|
|
560
560
|
const attempt = outboxRow.attempt + 1;
|
|
561
|
-
await db.insertInto("
|
|
561
|
+
await db.insertInto("webhook_deliveries").values({
|
|
562
562
|
outbox_id: outboxRow.id,
|
|
563
|
-
|
|
563
|
+
webhook_id: outboxRow.webhook_id,
|
|
564
564
|
attempt,
|
|
565
565
|
status_code: r.statusCode,
|
|
566
566
|
response_headers: r.responseHeaders,
|
|
@@ -580,7 +580,7 @@ function buildTestOutboxRow(sub) {
|
|
|
580
580
|
const now = new Date;
|
|
581
581
|
return {
|
|
582
582
|
id: randomUUID(),
|
|
583
|
-
|
|
583
|
+
webhook_id: sub.id,
|
|
584
584
|
kind: sub.kind,
|
|
585
585
|
subgraph_name: sub.subgraph_name ?? null,
|
|
586
586
|
table_name: sub.table_name ?? null,
|
|
@@ -591,7 +591,7 @@ function buildTestOutboxRow(sub) {
|
|
|
591
591
|
payload: {
|
|
592
592
|
test: true,
|
|
593
593
|
message: "Secondlayer test delivery",
|
|
594
|
-
|
|
594
|
+
webhook_id: sub.id,
|
|
595
595
|
sent_at: now.toISOString()
|
|
596
596
|
},
|
|
597
597
|
dedup_key: `test:${sub.id}:${now.getTime()}`,
|
|
@@ -608,11 +608,11 @@ function buildTestOutboxRow(sub) {
|
|
|
608
608
|
}
|
|
609
609
|
async function deliverTestEvent(db, sub) {
|
|
610
610
|
const testRow = buildTestOutboxRow(sub);
|
|
611
|
-
const { body, headers } = buildForFormat(testRow, sub,
|
|
612
|
-
const r = await
|
|
613
|
-
const inserted = await db.insertInto("
|
|
611
|
+
const { body, headers } = buildForFormat(testRow, sub, getWebhookSigningSecret(sub));
|
|
612
|
+
const r = await postToWebhook(sub.url, body, headers, sub.timeout_ms);
|
|
613
|
+
const inserted = await db.insertInto("webhook_deliveries").values({
|
|
614
614
|
outbox_id: null,
|
|
615
|
-
|
|
615
|
+
webhook_id: sub.id,
|
|
616
616
|
attempt: 1,
|
|
617
617
|
status_code: r.statusCode,
|
|
618
618
|
response_headers: r.responseHeaders,
|
|
@@ -630,20 +630,20 @@ async function deliverTestEvent(db, sub) {
|
|
|
630
630
|
}
|
|
631
631
|
async function settleDelivered(db, outboxRow) {
|
|
632
632
|
await db.transaction().execute(async (tx) => {
|
|
633
|
-
await tx.updateTable("
|
|
633
|
+
await tx.updateTable("webhook_outbox").set({
|
|
634
634
|
status: "delivered",
|
|
635
635
|
delivered_at: new Date,
|
|
636
636
|
attempt: outboxRow.attempt + 1,
|
|
637
637
|
locked_by: null,
|
|
638
638
|
locked_until: null
|
|
639
639
|
}).where("id", "=", outboxRow.id).execute();
|
|
640
|
-
await tx.updateTable("
|
|
640
|
+
await tx.updateTable("webhooks").set({
|
|
641
641
|
last_delivery_at: new Date,
|
|
642
642
|
last_success_at: new Date,
|
|
643
643
|
circuit_failures: 0,
|
|
644
644
|
last_error: null,
|
|
645
645
|
updated_at: new Date
|
|
646
|
-
}).where("id", "=", outboxRow.
|
|
646
|
+
}).where("id", "=", outboxRow.webhook_id).execute();
|
|
647
647
|
});
|
|
648
648
|
}
|
|
649
649
|
async function settleFailed(db, outboxRow, sub, errText) {
|
|
@@ -651,7 +651,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
651
651
|
const isDead = attempt >= sub.max_retries;
|
|
652
652
|
const nextAt = isDead ? null : new Date(Date.now() + nextDelaySeconds(outboxRow.attempt) * 1000);
|
|
653
653
|
await db.transaction().execute(async (tx) => {
|
|
654
|
-
await tx.updateTable("
|
|
654
|
+
await tx.updateTable("webhook_outbox").set({
|
|
655
655
|
attempt,
|
|
656
656
|
next_attempt_at: nextAt ?? new Date,
|
|
657
657
|
status: isDead ? "dead" : "pending",
|
|
@@ -660,7 +660,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
660
660
|
locked_until: null
|
|
661
661
|
}).where("id", "=", outboxRow.id).execute();
|
|
662
662
|
const incResult = await sql2`
|
|
663
|
-
UPDATE
|
|
663
|
+
UPDATE webhooks
|
|
664
664
|
SET circuit_failures = circuit_failures + 1,
|
|
665
665
|
last_delivery_at = NOW(),
|
|
666
666
|
last_error = ${errText.slice(0, 500)},
|
|
@@ -671,13 +671,13 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
671
671
|
const newFailures = incResult.rows[0]?.circuit_failures ?? sub.circuit_failures + 1;
|
|
672
672
|
const shouldTripCircuit = newFailures >= CIRCUIT_THRESHOLD;
|
|
673
673
|
if (shouldTripCircuit) {
|
|
674
|
-
await tx.updateTable("
|
|
674
|
+
await tx.updateTable("webhooks").set({
|
|
675
675
|
status: "paused",
|
|
676
676
|
circuit_opened_at: new Date,
|
|
677
677
|
updated_at: new Date
|
|
678
678
|
}).where("id", "=", sub.id).execute();
|
|
679
|
-
logger2.warn("
|
|
680
|
-
|
|
679
|
+
logger2.warn("Webhook circuit tripped — paused after consecutive failures", {
|
|
680
|
+
webhook: sub.name,
|
|
681
681
|
failures: newFailures
|
|
682
682
|
});
|
|
683
683
|
}
|
|
@@ -692,7 +692,7 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
692
692
|
const replayLimit = BATCH_SIZE - liveLimit;
|
|
693
693
|
const claimed = await db.transaction().execute(async (tx) => {
|
|
694
694
|
const live = await sql2`
|
|
695
|
-
SELECT * FROM
|
|
695
|
+
SELECT * FROM webhook_outbox
|
|
696
696
|
WHERE status = 'pending'
|
|
697
697
|
AND next_attempt_at <= NOW()
|
|
698
698
|
AND is_replay = FALSE
|
|
@@ -701,7 +701,7 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
701
701
|
LIMIT ${sql2.lit(liveLimit)}
|
|
702
702
|
`.execute(tx);
|
|
703
703
|
const replay = await sql2`
|
|
704
|
-
SELECT * FROM
|
|
704
|
+
SELECT * FROM webhook_outbox
|
|
705
705
|
WHERE status = 'pending'
|
|
706
706
|
AND next_attempt_at <= NOW()
|
|
707
707
|
AND is_replay = TRUE
|
|
@@ -714,7 +714,7 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
714
714
|
return [];
|
|
715
715
|
const now = new Date;
|
|
716
716
|
const lockUntil = new Date(now.getTime() + LOCK_WINDOW_MS);
|
|
717
|
-
await tx.updateTable("
|
|
717
|
+
await tx.updateTable("webhook_outbox").set({
|
|
718
718
|
locked_by: emitterId,
|
|
719
719
|
locked_until: lockUntil,
|
|
720
720
|
next_attempt_at: lockUntil
|
|
@@ -725,14 +725,14 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
725
725
|
return 0;
|
|
726
726
|
const bySubId = new Map;
|
|
727
727
|
for (const row of claimed) {
|
|
728
|
-
const arr = bySubId.get(row.
|
|
728
|
+
const arr = bySubId.get(row.webhook_id);
|
|
729
729
|
if (arr)
|
|
730
730
|
arr.push(row);
|
|
731
731
|
else
|
|
732
|
-
bySubId.set(row.
|
|
732
|
+
bySubId.set(row.webhook_id, [row]);
|
|
733
733
|
}
|
|
734
734
|
const subIds = Array.from(bySubId.keys());
|
|
735
|
-
const subs = await db.selectFrom("
|
|
735
|
+
const subs = await db.selectFrom("webhooks").selectAll().where("id", "in", subIds).execute();
|
|
736
736
|
const subById = new Map(subs.map((s) => [s.id, s]));
|
|
737
737
|
await Promise.all(subIds.map((subId) => drainForSub(db, state, subById.get(subId), bySubId.get(subId))));
|
|
738
738
|
return claimed.length;
|
|
@@ -781,15 +781,15 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
781
781
|
}
|
|
782
782
|
async function runRetention(db) {
|
|
783
783
|
await sql2`
|
|
784
|
-
DELETE FROM
|
|
784
|
+
DELETE FROM webhook_outbox
|
|
785
785
|
WHERE status = 'delivered' AND delivered_at < NOW() - interval '7 days'
|
|
786
786
|
`.execute(db);
|
|
787
787
|
await sql2`
|
|
788
|
-
DELETE FROM
|
|
788
|
+
DELETE FROM webhook_deliveries
|
|
789
789
|
WHERE dispatched_at < NOW() - interval '30 days'
|
|
790
790
|
`.execute(db);
|
|
791
791
|
await sql2`
|
|
792
|
-
DELETE FROM
|
|
792
|
+
DELETE FROM webhook_outbox
|
|
793
793
|
WHERE status = 'dead' AND failed_at < NOW() - interval '90 days'
|
|
794
794
|
`.execute(db);
|
|
795
795
|
}
|
|
@@ -826,14 +826,14 @@ async function startEmitter(opts) {
|
|
|
826
826
|
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
827
827
|
}
|
|
828
828
|
const listenUrl = targetListenerUrl();
|
|
829
|
-
const stopNew = await listen("
|
|
829
|
+
const stopNew = await listen("webhooks:new_outbox", () => {
|
|
830
830
|
if (!state.running)
|
|
831
831
|
return;
|
|
832
832
|
claimAndDrain(db, state, emitterId).catch((err) => logger2.error("[emitter] claim failed", {
|
|
833
833
|
error: err instanceof Error ? err.message : String(err)
|
|
834
834
|
}));
|
|
835
835
|
}, { connectionString: listenUrl });
|
|
836
|
-
const stopChanged = await listen("
|
|
836
|
+
const stopChanged = await listen("webhooks:changed", () => {
|
|
837
837
|
if (!state.running)
|
|
838
838
|
return;
|
|
839
839
|
refreshMatcher(db).catch((err) => logger2.error("[emitter] matcher refresh failed", {
|
|
@@ -869,11 +869,11 @@ export {
|
|
|
869
869
|
deliverTestEvent,
|
|
870
870
|
checkEgressAllowed,
|
|
871
871
|
__setDnsLookupForTest,
|
|
872
|
-
|
|
873
|
-
|
|
872
|
+
__postToWebhookForTest,
|
|
873
|
+
MAX_WEBHOOK_TIMEOUT_MS,
|
|
874
874
|
MAX_REDIRECT_HOPS,
|
|
875
875
|
LOCK_WINDOW_MS
|
|
876
876
|
};
|
|
877
877
|
|
|
878
|
-
//# debugId=
|
|
878
|
+
//# debugId=15D49DA18C7A55F864756E2164756E21
|
|
879
879
|
//# sourceMappingURL=emitter.js.map
|