@secondlayer/subgraphs 4.2.1 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/src/index.js +11 -11
- package/dist/src/index.js.map +9 -9
- package/dist/src/runtime/block-processor.js +11 -11
- package/dist/src/runtime/block-processor.js.map +9 -9
- package/dist/src/runtime/catchup.js +11 -11
- package/dist/src/runtime/catchup.js.map +9 -9
- package/dist/src/runtime/context.d.ts +1 -1
- package/dist/src/runtime/context.js.map +1 -1
- package/dist/src/runtime/emitter.d.ts +10 -10
- package/dist/src/runtime/emitter.js +47 -46
- package/dist/src/runtime/emitter.js.map +14 -14
- package/dist/src/runtime/processor.js +19 -19
- package/dist/src/runtime/processor.js.map +13 -13
- package/dist/src/runtime/reindex.js +11 -11
- package/dist/src/runtime/reindex.js.map +9 -9
- package/dist/src/runtime/reorg.d.ts +2 -2
- package/dist/src/runtime/reorg.js +17 -17
- package/dist/src/runtime/reorg.js.map +10 -10
- package/dist/src/runtime/replay.d.ts +6 -6
- package/dist/src/runtime/replay.js +26 -26
- package/dist/src/runtime/replay.js.map +4 -4
- package/dist/src/runtime/runner.d.ts +1 -1
- package/dist/src/service.js +19 -19
- package/dist/src/service.js.map +14 -14
- package/dist/src/testing/index.js.map +1 -1
- package/dist/src/triggers/index.d.ts +2 -2
- package/dist/src/triggers/index.js.map +1 -1
- package/package.json +3 -3
|
@@ -7,7 +7,7 @@ import { lookup as dnsLookup } from "node:dns/promises";
|
|
|
7
7
|
import {
|
|
8
8
|
getTargetDb
|
|
9
9
|
} from "@secondlayer/shared/db";
|
|
10
|
-
import {
|
|
10
|
+
import { getWebhookSigningSecret } from "@secondlayer/shared/db/queries/webhooks";
|
|
11
11
|
import { logger as logger2 } from "@secondlayer/shared/logger";
|
|
12
12
|
import { listen, targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
13
13
|
import { sql as sql2 } from "kysely";
|
|
@@ -161,9 +161,9 @@ function buildBody(outboxRow, sub, signingSecret) {
|
|
|
161
161
|
case "standard-webhooks":
|
|
162
162
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
163
163
|
default:
|
|
164
|
-
logger.warn("Unknown
|
|
164
|
+
logger.warn("Unknown webhook format, falling back to standard-webhooks", {
|
|
165
165
|
format: sub.format,
|
|
166
|
-
|
|
166
|
+
webhookId: sub.id
|
|
167
167
|
});
|
|
168
168
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
169
169
|
}
|
|
@@ -189,16 +189,16 @@ async function meterBlocksProcessed(accountId, blocks, subgraphName) {
|
|
|
189
189
|
return true;
|
|
190
190
|
return hooks.onBlocksProcessed(accountId, blocks, subgraphName);
|
|
191
191
|
}
|
|
192
|
-
async function meterDeliveryAttempt(accountId,
|
|
192
|
+
async function meterDeliveryAttempt(accountId, webhookId) {
|
|
193
193
|
if (!accountId)
|
|
194
194
|
return true;
|
|
195
195
|
if (!hooks.onDeliveryAttempt)
|
|
196
196
|
return true;
|
|
197
|
-
return hooks.onDeliveryAttempt(accountId,
|
|
197
|
+
return hooks.onDeliveryAttempt(accountId, webhookId);
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
-
// src/runtime/
|
|
201
|
-
import {
|
|
200
|
+
// src/runtime/webhook-state.ts
|
|
201
|
+
import { listWebhooks } from "@secondlayer/shared/db/queries/webhooks";
|
|
202
202
|
import { sql } from "kysely";
|
|
203
203
|
|
|
204
204
|
// src/runtime/emitter-matcher.ts
|
|
@@ -318,7 +318,7 @@ function key(subgraphName, tableName) {
|
|
|
318
318
|
return `${subgraphName}\x00${tableName}`;
|
|
319
319
|
}
|
|
320
320
|
|
|
321
|
-
class
|
|
321
|
+
class WebhookMatcher {
|
|
322
322
|
byKey = new Map;
|
|
323
323
|
byId = new Map;
|
|
324
324
|
setAll(subs) {
|
|
@@ -360,11 +360,11 @@ class SubscriptionMatcher {
|
|
|
360
360
|
}
|
|
361
361
|
}
|
|
362
362
|
|
|
363
|
-
// src/runtime/
|
|
364
|
-
var matcher = new
|
|
363
|
+
// src/runtime/webhook-state.ts
|
|
364
|
+
var matcher = new WebhookMatcher;
|
|
365
365
|
async function refreshMatcher(db) {
|
|
366
366
|
const rows = await sql`
|
|
367
|
-
SELECT * FROM
|
|
367
|
+
SELECT * FROM webhooks WHERE status = 'active'
|
|
368
368
|
`.execute(db);
|
|
369
369
|
matcher.setAll(rows.rows);
|
|
370
370
|
return matcher.size();
|
|
@@ -375,8 +375,8 @@ var BATCH_SIZE = 50;
|
|
|
375
375
|
var LIVE_SHARE = 0.9;
|
|
376
376
|
var BACKOFF_SECONDS = [30, 120, 600, 3600, 21600, 86400, 259200];
|
|
377
377
|
var CIRCUIT_THRESHOLD = 20;
|
|
378
|
-
var
|
|
379
|
-
var LOCK_WINDOW_MS =
|
|
378
|
+
var MAX_WEBHOOK_TIMEOUT_MS = 300000;
|
|
379
|
+
var LOCK_WINDOW_MS = MAX_WEBHOOK_TIMEOUT_MS + 60000;
|
|
380
380
|
function nextDelaySeconds(attempt) {
|
|
381
381
|
return BACKOFF_SECONDS[Math.min(attempt, BACKOFF_SECONDS.length - 1)];
|
|
382
382
|
}
|
|
@@ -484,7 +484,7 @@ async function checkEgressAllowed(url) {
|
|
|
484
484
|
return null;
|
|
485
485
|
}
|
|
486
486
|
var MAX_REDIRECT_HOPS = 3;
|
|
487
|
-
async function
|
|
487
|
+
async function postToWebhook(url, body, headers, timeoutMs) {
|
|
488
488
|
const start = performance.now();
|
|
489
489
|
let target = url;
|
|
490
490
|
let statusCode = null;
|
|
@@ -553,14 +553,14 @@ async function postToSubscription(url, body, headers, timeoutMs) {
|
|
|
553
553
|
responseHeaders
|
|
554
554
|
};
|
|
555
555
|
}
|
|
556
|
-
var
|
|
556
|
+
var __postToWebhookForTest = postToWebhook;
|
|
557
557
|
async function dispatchOne(db, outboxRow, sub) {
|
|
558
|
-
const { body, headers } = buildForFormat(outboxRow, sub,
|
|
559
|
-
const r = await
|
|
558
|
+
const { body, headers } = buildForFormat(outboxRow, sub, getWebhookSigningSecret(sub));
|
|
559
|
+
const r = await postToWebhook(sub.url, body, headers, sub.timeout_ms);
|
|
560
560
|
const attempt = outboxRow.attempt + 1;
|
|
561
|
-
await db.insertInto("
|
|
561
|
+
await db.insertInto("webhook_deliveries").values({
|
|
562
562
|
outbox_id: outboxRow.id,
|
|
563
|
-
|
|
563
|
+
webhook_id: outboxRow.webhook_id,
|
|
564
564
|
attempt,
|
|
565
565
|
status_code: r.statusCode,
|
|
566
566
|
response_headers: r.responseHeaders,
|
|
@@ -580,7 +580,7 @@ function buildTestOutboxRow(sub) {
|
|
|
580
580
|
const now = new Date;
|
|
581
581
|
return {
|
|
582
582
|
id: randomUUID(),
|
|
583
|
-
|
|
583
|
+
webhook_id: sub.id,
|
|
584
584
|
kind: sub.kind,
|
|
585
585
|
subgraph_name: sub.subgraph_name ?? null,
|
|
586
586
|
table_name: sub.table_name ?? null,
|
|
@@ -591,6 +591,7 @@ function buildTestOutboxRow(sub) {
|
|
|
591
591
|
payload: {
|
|
592
592
|
test: true,
|
|
593
593
|
message: "Secondlayer test delivery",
|
|
594
|
+
webhook_id: sub.id,
|
|
594
595
|
subscription_id: sub.id,
|
|
595
596
|
sent_at: now.toISOString()
|
|
596
597
|
},
|
|
@@ -608,11 +609,11 @@ function buildTestOutboxRow(sub) {
|
|
|
608
609
|
}
|
|
609
610
|
async function deliverTestEvent(db, sub) {
|
|
610
611
|
const testRow = buildTestOutboxRow(sub);
|
|
611
|
-
const { body, headers } = buildForFormat(testRow, sub,
|
|
612
|
-
const r = await
|
|
613
|
-
const inserted = await db.insertInto("
|
|
612
|
+
const { body, headers } = buildForFormat(testRow, sub, getWebhookSigningSecret(sub));
|
|
613
|
+
const r = await postToWebhook(sub.url, body, headers, sub.timeout_ms);
|
|
614
|
+
const inserted = await db.insertInto("webhook_deliveries").values({
|
|
614
615
|
outbox_id: null,
|
|
615
|
-
|
|
616
|
+
webhook_id: sub.id,
|
|
616
617
|
attempt: 1,
|
|
617
618
|
status_code: r.statusCode,
|
|
618
619
|
response_headers: r.responseHeaders,
|
|
@@ -630,20 +631,20 @@ async function deliverTestEvent(db, sub) {
|
|
|
630
631
|
}
|
|
631
632
|
async function settleDelivered(db, outboxRow) {
|
|
632
633
|
await db.transaction().execute(async (tx) => {
|
|
633
|
-
await tx.updateTable("
|
|
634
|
+
await tx.updateTable("webhook_outbox").set({
|
|
634
635
|
status: "delivered",
|
|
635
636
|
delivered_at: new Date,
|
|
636
637
|
attempt: outboxRow.attempt + 1,
|
|
637
638
|
locked_by: null,
|
|
638
639
|
locked_until: null
|
|
639
640
|
}).where("id", "=", outboxRow.id).execute();
|
|
640
|
-
await tx.updateTable("
|
|
641
|
+
await tx.updateTable("webhooks").set({
|
|
641
642
|
last_delivery_at: new Date,
|
|
642
643
|
last_success_at: new Date,
|
|
643
644
|
circuit_failures: 0,
|
|
644
645
|
last_error: null,
|
|
645
646
|
updated_at: new Date
|
|
646
|
-
}).where("id", "=", outboxRow.
|
|
647
|
+
}).where("id", "=", outboxRow.webhook_id).execute();
|
|
647
648
|
});
|
|
648
649
|
}
|
|
649
650
|
async function settleFailed(db, outboxRow, sub, errText) {
|
|
@@ -651,7 +652,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
651
652
|
const isDead = attempt >= sub.max_retries;
|
|
652
653
|
const nextAt = isDead ? null : new Date(Date.now() + nextDelaySeconds(outboxRow.attempt) * 1000);
|
|
653
654
|
await db.transaction().execute(async (tx) => {
|
|
654
|
-
await tx.updateTable("
|
|
655
|
+
await tx.updateTable("webhook_outbox").set({
|
|
655
656
|
attempt,
|
|
656
657
|
next_attempt_at: nextAt ?? new Date,
|
|
657
658
|
status: isDead ? "dead" : "pending",
|
|
@@ -660,7 +661,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
660
661
|
locked_until: null
|
|
661
662
|
}).where("id", "=", outboxRow.id).execute();
|
|
662
663
|
const incResult = await sql2`
|
|
663
|
-
UPDATE
|
|
664
|
+
UPDATE webhooks
|
|
664
665
|
SET circuit_failures = circuit_failures + 1,
|
|
665
666
|
last_delivery_at = NOW(),
|
|
666
667
|
last_error = ${errText.slice(0, 500)},
|
|
@@ -671,13 +672,13 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
671
672
|
const newFailures = incResult.rows[0]?.circuit_failures ?? sub.circuit_failures + 1;
|
|
672
673
|
const shouldTripCircuit = newFailures >= CIRCUIT_THRESHOLD;
|
|
673
674
|
if (shouldTripCircuit) {
|
|
674
|
-
await tx.updateTable("
|
|
675
|
+
await tx.updateTable("webhooks").set({
|
|
675
676
|
status: "paused",
|
|
676
677
|
circuit_opened_at: new Date,
|
|
677
678
|
updated_at: new Date
|
|
678
679
|
}).where("id", "=", sub.id).execute();
|
|
679
|
-
logger2.warn("
|
|
680
|
-
|
|
680
|
+
logger2.warn("Webhook circuit tripped — paused after consecutive failures", {
|
|
681
|
+
webhook: sub.name,
|
|
681
682
|
failures: newFailures
|
|
682
683
|
});
|
|
683
684
|
}
|
|
@@ -692,7 +693,7 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
692
693
|
const replayLimit = BATCH_SIZE - liveLimit;
|
|
693
694
|
const claimed = await db.transaction().execute(async (tx) => {
|
|
694
695
|
const live = await sql2`
|
|
695
|
-
SELECT * FROM
|
|
696
|
+
SELECT * FROM webhook_outbox
|
|
696
697
|
WHERE status = 'pending'
|
|
697
698
|
AND next_attempt_at <= NOW()
|
|
698
699
|
AND is_replay = FALSE
|
|
@@ -701,7 +702,7 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
701
702
|
LIMIT ${sql2.lit(liveLimit)}
|
|
702
703
|
`.execute(tx);
|
|
703
704
|
const replay = await sql2`
|
|
704
|
-
SELECT * FROM
|
|
705
|
+
SELECT * FROM webhook_outbox
|
|
705
706
|
WHERE status = 'pending'
|
|
706
707
|
AND next_attempt_at <= NOW()
|
|
707
708
|
AND is_replay = TRUE
|
|
@@ -714,7 +715,7 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
714
715
|
return [];
|
|
715
716
|
const now = new Date;
|
|
716
717
|
const lockUntil = new Date(now.getTime() + LOCK_WINDOW_MS);
|
|
717
|
-
await tx.updateTable("
|
|
718
|
+
await tx.updateTable("webhook_outbox").set({
|
|
718
719
|
locked_by: emitterId,
|
|
719
720
|
locked_until: lockUntil,
|
|
720
721
|
next_attempt_at: lockUntil
|
|
@@ -725,14 +726,14 @@ async function claimAndDrain(db, state, emitterId) {
|
|
|
725
726
|
return 0;
|
|
726
727
|
const bySubId = new Map;
|
|
727
728
|
for (const row of claimed) {
|
|
728
|
-
const arr = bySubId.get(row.
|
|
729
|
+
const arr = bySubId.get(row.webhook_id);
|
|
729
730
|
if (arr)
|
|
730
731
|
arr.push(row);
|
|
731
732
|
else
|
|
732
|
-
bySubId.set(row.
|
|
733
|
+
bySubId.set(row.webhook_id, [row]);
|
|
733
734
|
}
|
|
734
735
|
const subIds = Array.from(bySubId.keys());
|
|
735
|
-
const subs = await db.selectFrom("
|
|
736
|
+
const subs = await db.selectFrom("webhooks").selectAll().where("id", "in", subIds).execute();
|
|
736
737
|
const subById = new Map(subs.map((s) => [s.id, s]));
|
|
737
738
|
await Promise.all(subIds.map((subId) => drainForSub(db, state, subById.get(subId), bySubId.get(subId))));
|
|
738
739
|
return claimed.length;
|
|
@@ -781,15 +782,15 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
781
782
|
}
|
|
782
783
|
async function runRetention(db) {
|
|
783
784
|
await sql2`
|
|
784
|
-
DELETE FROM
|
|
785
|
+
DELETE FROM webhook_outbox
|
|
785
786
|
WHERE status = 'delivered' AND delivered_at < NOW() - interval '7 days'
|
|
786
787
|
`.execute(db);
|
|
787
788
|
await sql2`
|
|
788
|
-
DELETE FROM
|
|
789
|
+
DELETE FROM webhook_deliveries
|
|
789
790
|
WHERE dispatched_at < NOW() - interval '30 days'
|
|
790
791
|
`.execute(db);
|
|
791
792
|
await sql2`
|
|
792
|
-
DELETE FROM
|
|
793
|
+
DELETE FROM webhook_outbox
|
|
793
794
|
WHERE status = 'dead' AND failed_at < NOW() - interval '90 days'
|
|
794
795
|
`.execute(db);
|
|
795
796
|
}
|
|
@@ -826,14 +827,14 @@ async function startEmitter(opts) {
|
|
|
826
827
|
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
827
828
|
}
|
|
828
829
|
const listenUrl = targetListenerUrl();
|
|
829
|
-
const stopNew = await listen("
|
|
830
|
+
const stopNew = await listen("webhooks:new_outbox", () => {
|
|
830
831
|
if (!state.running)
|
|
831
832
|
return;
|
|
832
833
|
claimAndDrain(db, state, emitterId).catch((err) => logger2.error("[emitter] claim failed", {
|
|
833
834
|
error: err instanceof Error ? err.message : String(err)
|
|
834
835
|
}));
|
|
835
836
|
}, { connectionString: listenUrl });
|
|
836
|
-
const stopChanged = await listen("
|
|
837
|
+
const stopChanged = await listen("webhooks:changed", () => {
|
|
837
838
|
if (!state.running)
|
|
838
839
|
return;
|
|
839
840
|
refreshMatcher(db).catch((err) => logger2.error("[emitter] matcher refresh failed", {
|
|
@@ -869,11 +870,11 @@ export {
|
|
|
869
870
|
deliverTestEvent,
|
|
870
871
|
checkEgressAllowed,
|
|
871
872
|
__setDnsLookupForTest,
|
|
872
|
-
|
|
873
|
-
|
|
873
|
+
__postToWebhookForTest,
|
|
874
|
+
MAX_WEBHOOK_TIMEOUT_MS,
|
|
874
875
|
MAX_REDIRECT_HOPS,
|
|
875
876
|
LOCK_WINDOW_MS
|
|
876
877
|
};
|
|
877
878
|
|
|
878
|
-
//# debugId=
|
|
879
|
+
//# debugId=51FFE04872B2ADAD64756E2164756E21
|
|
879
880
|
//# sourceMappingURL=emitter.js.map
|