@secondlayer/subgraphs 3.7.2 → 3.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +7 -11
- package/dist/src/index.js.map +3 -3
- package/dist/src/runtime/block-processor.js +7 -11
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.js +7 -11
- package/dist/src/runtime/catchup.js.map +3 -3
- package/dist/src/runtime/processor.js +78 -855
- package/dist/src/runtime/processor.js.map +7 -17
- package/dist/src/runtime/reindex.js +7 -11
- package/dist/src/runtime/reindex.js.map +3 -3
- package/dist/src/runtime/reorg.js +7 -11
- package/dist/src/runtime/reorg.js.map +3 -3
- package/dist/src/runtime/replay.js +686 -4
- package/dist/src/runtime/replay.js.map +10 -5
- package/dist/src/service.js +83 -860
- package/dist/src/service.js.map +7 -17
- package/package.json +2 -2
package/dist/src/service.js
CHANGED
|
@@ -1242,6 +1242,11 @@ function resolveBlockSource(subgraph) {
|
|
|
1242
1242
|
import { createHash } from "node:crypto";
|
|
1243
1243
|
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
1244
1244
|
var loggedKillSwitch = false;
|
|
1245
|
+
var OP_VERB = {
|
|
1246
|
+
insert: "created",
|
|
1247
|
+
update: "updated",
|
|
1248
|
+
delete: "deleted"
|
|
1249
|
+
};
|
|
1245
1250
|
function isEmitOutboxEnabled() {
|
|
1246
1251
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
1247
1252
|
}
|
|
@@ -1269,12 +1274,10 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
1269
1274
|
return 0;
|
|
1270
1275
|
const rows = [];
|
|
1271
1276
|
for (const write of manifest.writes) {
|
|
1272
|
-
if (write.op !== "insert")
|
|
1273
|
-
continue;
|
|
1274
1277
|
const subs = matcher.match(subgraphName, write.table, write.row);
|
|
1275
1278
|
if (subs.length === 0)
|
|
1276
1279
|
continue;
|
|
1277
|
-
const eventType = `${subgraphName}.${write.table}.
|
|
1280
|
+
const eventType = `${subgraphName}.${write.table}.${OP_VERB[write.op]}`;
|
|
1278
1281
|
for (const s of subs) {
|
|
1279
1282
|
rows.push({
|
|
1280
1283
|
subscription_id: s.id,
|
|
@@ -1460,13 +1463,6 @@ class SubscriptionMatcher {
|
|
|
1460
1463
|
|
|
1461
1464
|
// src/runtime/subscription-state.ts
|
|
1462
1465
|
var matcher = new SubscriptionMatcher;
|
|
1463
|
-
async function refreshMatcher(db) {
|
|
1464
|
-
const rows = await sql2`
|
|
1465
|
-
SELECT * FROM subscriptions WHERE status = 'active'
|
|
1466
|
-
`.execute(db);
|
|
1467
|
-
matcher.setAll(rows.rows);
|
|
1468
|
-
return matcher.size();
|
|
1469
|
-
}
|
|
1470
1466
|
|
|
1471
1467
|
// src/runtime/block-processor.ts
|
|
1472
1468
|
var routeCache = new Map;
|
|
@@ -2496,8 +2492,8 @@ import { randomUUID } from "node:crypto";
|
|
|
2496
2492
|
import { hostname } from "node:os";
|
|
2497
2493
|
import { resolve } from "node:path";
|
|
2498
2494
|
import { pathToFileURL } from "node:url";
|
|
2499
|
-
import { getErrorMessage as
|
|
2500
|
-
import { getTargetDb as
|
|
2495
|
+
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
2496
|
+
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
2501
2497
|
import {
|
|
2502
2498
|
cancelSubgraphOperation,
|
|
2503
2499
|
claimSubgraphOperation,
|
|
@@ -2515,607 +2511,56 @@ import {
|
|
|
2515
2511
|
pgSchemaName as pgSchemaName2,
|
|
2516
2512
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
2517
2513
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2518
|
-
import { logger as
|
|
2514
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2519
2515
|
import {
|
|
2520
|
-
listen
|
|
2516
|
+
listen,
|
|
2521
2517
|
sourceListenerUrl,
|
|
2522
2518
|
targetListenerUrl as targetListenerUrl2
|
|
2523
2519
|
} from "@secondlayer/shared/queue/listener";
|
|
2524
2520
|
|
|
2525
|
-
// src/runtime/
|
|
2526
|
-
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
2527
|
-
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2528
|
-
var MAX_ORPHANED_PER_SUB = 500;
|
|
2529
|
-
async function handleChainReorg(forkHeight, db = getTargetDb5()) {
|
|
2530
|
-
await db.deleteFrom("subscription_outbox").where("kind", "=", "chain").where("block_height", ">=", forkHeight).where("status", "=", "pending").where("event_type", "like", "chain.%.apply").execute();
|
|
2531
|
-
const delivered = await db.selectFrom("subscription_outbox").select(["subscription_id", "tx_id", "payload"]).where("kind", "=", "chain").where("block_height", ">=", forkHeight).where("status", "=", "delivered").where("event_type", "like", "chain.%.apply").orderBy("block_height").orderBy("id").execute();
|
|
2532
|
-
const bySub = new Map;
|
|
2533
|
-
for (const row of delivered) {
|
|
2534
|
-
const list = bySub.get(row.subscription_id) ?? [];
|
|
2535
|
-
const payload = row.payload;
|
|
2536
|
-
list.push({ tx_id: row.tx_id, event: payload?.event ?? null });
|
|
2537
|
-
bySub.set(row.subscription_id, list);
|
|
2538
|
-
}
|
|
2539
|
-
if (bySub.size > 0) {
|
|
2540
|
-
const rows = [];
|
|
2541
|
-
for (const [subscriptionId, entries] of bySub) {
|
|
2542
|
-
const truncated = entries.length > MAX_ORPHANED_PER_SUB;
|
|
2543
|
-
const payload = {
|
|
2544
|
-
action: "rollback",
|
|
2545
|
-
fork_point_height: forkHeight,
|
|
2546
|
-
orphaned: truncated ? entries.slice(0, MAX_ORPHANED_PER_SUB) : entries,
|
|
2547
|
-
truncated
|
|
2548
|
-
};
|
|
2549
|
-
rows.push({
|
|
2550
|
-
subscription_id: subscriptionId,
|
|
2551
|
-
kind: "chain",
|
|
2552
|
-
subgraph_name: null,
|
|
2553
|
-
table_name: null,
|
|
2554
|
-
block_height: forkHeight,
|
|
2555
|
-
tx_id: null,
|
|
2556
|
-
row_pk: { fork_point_height: forkHeight },
|
|
2557
|
-
event_type: "chain.reorg.rollback",
|
|
2558
|
-
payload,
|
|
2559
|
-
dedup_key: `chainreorg:${subscriptionId}:${forkHeight}`
|
|
2560
|
-
});
|
|
2561
|
-
}
|
|
2562
|
-
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
|
|
2563
|
-
logger9.info("Chain reorg — emitted rollbacks", {
|
|
2564
|
-
forkPointHeight: forkHeight,
|
|
2565
|
-
subscriptions: bySub.size
|
|
2566
|
-
});
|
|
2567
|
-
}
|
|
2568
|
-
await db.transaction().execute(async (trx) => {
|
|
2569
|
-
const cur = await trx.selectFrom("trigger_evaluator_state").select("last_processed_block").where("id", "=", true).forUpdate().executeTakeFirst();
|
|
2570
|
-
if (cur && Number(cur.last_processed_block) >= forkHeight) {
|
|
2571
|
-
await trx.updateTable("trigger_evaluator_state").set({ last_processed_block: forkHeight - 1, updated_at: new Date }).where("id", "=", true).execute();
|
|
2572
|
-
}
|
|
2573
|
-
});
|
|
2574
|
-
}
|
|
2575
|
-
|
|
2576
|
-
// src/runtime/emitter.ts
|
|
2521
|
+
// src/runtime/catchup-leader.ts
|
|
2577
2522
|
import {
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
import {
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
};
|
|
2599
|
-
return {
|
|
2600
|
-
body: JSON.stringify(event),
|
|
2601
|
-
headers: {
|
|
2602
|
-
"content-type": "application/cloudevents+json; charset=utf-8"
|
|
2603
|
-
}
|
|
2604
|
-
};
|
|
2605
|
-
}
|
|
2606
|
-
|
|
2607
|
-
// src/runtime/formats/cloudflare.ts
|
|
2608
|
-
import { decryptSecret } from "@secondlayer/shared/crypto/secrets";
|
|
2609
|
-
function resolveBearer(sub) {
|
|
2610
|
-
const cfg = sub.auth_config;
|
|
2611
|
-
if (cfg.tokenEnc) {
|
|
2612
|
-
return decryptSecret(Buffer.from(cfg.tokenEnc, "base64"));
|
|
2613
|
-
}
|
|
2614
|
-
return cfg.token ?? null;
|
|
2615
|
-
}
|
|
2616
|
-
function buildCloudflare(outboxRow, sub) {
|
|
2617
|
-
const body = JSON.stringify({
|
|
2618
|
-
params: {
|
|
2619
|
-
...outboxRow.payload,
|
|
2620
|
-
_type: outboxRow.event_type,
|
|
2621
|
-
_outboxId: outboxRow.id
|
|
2622
|
-
}
|
|
2623
|
-
});
|
|
2624
|
-
const headers = {
|
|
2625
|
-
"content-type": "application/json"
|
|
2626
|
-
};
|
|
2627
|
-
const token = resolveBearer(sub);
|
|
2628
|
-
if (token)
|
|
2629
|
-
headers.authorization = `Bearer ${token}`;
|
|
2630
|
-
return { body, headers };
|
|
2631
|
-
}
|
|
2632
|
-
|
|
2633
|
-
// src/runtime/formats/inngest.ts
|
|
2634
|
-
var INNGEST_VERSION = "2026-04-23.v1";
|
|
2635
|
-
function buildInngest(outboxRow) {
|
|
2636
|
-
const event = {
|
|
2637
|
-
name: outboxRow.event_type,
|
|
2638
|
-
data: outboxRow.payload,
|
|
2639
|
-
id: outboxRow.id,
|
|
2640
|
-
ts: new Date(outboxRow.created_at).getTime(),
|
|
2641
|
-
v: INNGEST_VERSION
|
|
2642
|
-
};
|
|
2643
|
-
return {
|
|
2644
|
-
body: JSON.stringify([event]),
|
|
2645
|
-
headers: {
|
|
2646
|
-
"content-type": "application/json"
|
|
2647
|
-
}
|
|
2648
|
-
};
|
|
2649
|
-
}
|
|
2650
|
-
|
|
2651
|
-
// src/runtime/formats/raw.ts
|
|
2652
|
-
function buildRaw(outboxRow, sub) {
|
|
2653
|
-
const cfg = sub.auth_config;
|
|
2654
|
-
const headers = {
|
|
2655
|
-
"content-type": cfg.contentType ?? "application/json",
|
|
2656
|
-
...cfg.headers ?? {}
|
|
2657
|
-
};
|
|
2658
|
-
if (cfg.authType === "bearer" && cfg.token) {
|
|
2659
|
-
headers.authorization = `Bearer ${cfg.token}`;
|
|
2660
|
-
} else if (cfg.authType === "basic" && cfg.basicAuth) {
|
|
2661
|
-
headers.authorization = `Basic ${cfg.basicAuth}`;
|
|
2662
|
-
}
|
|
2663
|
-
return {
|
|
2664
|
-
body: JSON.stringify(outboxRow.payload),
|
|
2665
|
-
headers
|
|
2666
|
-
};
|
|
2667
|
-
}
|
|
2668
|
-
|
|
2669
|
-
// src/runtime/formats/standard-webhooks.ts
|
|
2670
|
-
import { sign } from "@secondlayer/shared/crypto/standard-webhooks";
|
|
2671
|
-
function buildStandardWebhooks(outboxRow, signingSecret) {
|
|
2672
|
-
const nowSeconds = Math.floor(Date.now() / 1000);
|
|
2673
|
-
const payload = {
|
|
2674
|
-
type: outboxRow.event_type,
|
|
2675
|
-
timestamp: new Date(nowSeconds * 1000).toISOString(),
|
|
2676
|
-
data: outboxRow.payload
|
|
2677
|
-
};
|
|
2678
|
-
const body = JSON.stringify(payload);
|
|
2679
|
-
const sigHeaders = sign(body, signingSecret, {
|
|
2680
|
-
id: outboxRow.id,
|
|
2681
|
-
timestampSeconds: nowSeconds
|
|
2682
|
-
});
|
|
2683
|
-
return {
|
|
2684
|
-
body,
|
|
2685
|
-
headers: {
|
|
2686
|
-
"content-type": "application/json",
|
|
2687
|
-
...sigHeaders
|
|
2688
|
-
}
|
|
2689
|
-
};
|
|
2690
|
-
}
|
|
2691
|
-
|
|
2692
|
-
// src/runtime/formats/trigger.ts
|
|
2693
|
-
import { decryptSecret as decryptSecret2 } from "@secondlayer/shared/crypto/secrets";
|
|
2694
|
-
function resolveBearer2(sub) {
|
|
2695
|
-
const cfg = sub.auth_config;
|
|
2696
|
-
if (cfg.tokenEnc) {
|
|
2697
|
-
return decryptSecret2(Buffer.from(cfg.tokenEnc, "base64"));
|
|
2698
|
-
}
|
|
2699
|
-
return cfg.token ?? null;
|
|
2700
|
-
}
|
|
2701
|
-
function buildTrigger(outboxRow, sub) {
|
|
2702
|
-
const body = JSON.stringify({
|
|
2703
|
-
payload: outboxRow.payload,
|
|
2704
|
-
options: {
|
|
2705
|
-
idempotencyKey: outboxRow.id
|
|
2706
|
-
}
|
|
2707
|
-
});
|
|
2708
|
-
const headers = {
|
|
2709
|
-
"content-type": "application/json"
|
|
2710
|
-
};
|
|
2711
|
-
const token = resolveBearer2(sub);
|
|
2712
|
-
if (token)
|
|
2713
|
-
headers.authorization = `Bearer ${token}`;
|
|
2714
|
-
return { body, headers };
|
|
2715
|
-
}
|
|
2716
|
-
|
|
2717
|
-
// src/runtime/formats/index.ts
|
|
2718
|
-
function buildForFormat(outboxRow, sub, signingSecret) {
|
|
2719
|
-
switch (sub.format) {
|
|
2720
|
-
case "inngest":
|
|
2721
|
-
return buildInngest(outboxRow);
|
|
2722
|
-
case "trigger":
|
|
2723
|
-
return buildTrigger(outboxRow, sub);
|
|
2724
|
-
case "cloudflare":
|
|
2725
|
-
return buildCloudflare(outboxRow, sub);
|
|
2726
|
-
case "cloudevents":
|
|
2727
|
-
return buildCloudEvents(outboxRow, sub);
|
|
2728
|
-
case "raw":
|
|
2729
|
-
return buildRaw(outboxRow, sub);
|
|
2730
|
-
case "standard-webhooks":
|
|
2731
|
-
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2732
|
-
default:
|
|
2733
|
-
logger10.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
2734
|
-
format: sub.format,
|
|
2735
|
-
subscriptionId: sub.id
|
|
2736
|
-
});
|
|
2737
|
-
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2738
|
-
}
|
|
2739
|
-
}
|
|
2740
|
-
|
|
2741
|
-
// src/runtime/emitter.ts
|
|
2742
|
-
var BATCH_SIZE = 50;
|
|
2743
|
-
var LIVE_SHARE = 0.9;
|
|
2744
|
-
var BACKOFF_SECONDS = [30, 120, 600, 3600, 21600, 86400, 259200];
|
|
2745
|
-
var CIRCUIT_THRESHOLD = 20;
|
|
2746
|
-
var LOCK_WINDOW_MS = 60000;
|
|
2747
|
-
function nextDelaySeconds(attempt) {
|
|
2748
|
-
return BACKOFF_SECONDS[Math.min(attempt, BACKOFF_SECONDS.length - 1)];
|
|
2749
|
-
}
|
|
2750
|
-
var PRIVATE_V4_PATTERNS = [
|
|
2751
|
-
/^127\./,
|
|
2752
|
-
/^10\./,
|
|
2753
|
-
/^172\.(1[6-9]|2\d|3[01])\./,
|
|
2754
|
-
/^192\.168\./,
|
|
2755
|
-
/^169\.254\./,
|
|
2756
|
-
/^0\./,
|
|
2757
|
-
/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./
|
|
2758
|
-
];
|
|
2759
|
-
function isPrivateEgress(url) {
|
|
2760
|
-
let parsed;
|
|
2761
|
-
try {
|
|
2762
|
-
parsed = new URL(url);
|
|
2763
|
-
} catch {
|
|
2764
|
-
return true;
|
|
2765
|
-
}
|
|
2766
|
-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
2767
|
-
return true;
|
|
2768
|
-
}
|
|
2769
|
-
const raw = parsed.hostname.toLowerCase();
|
|
2770
|
-
const host = raw.startsWith("[") && raw.endsWith("]") ? raw.slice(1, -1) : raw;
|
|
2771
|
-
if (host === "localhost" || host === "0.0.0.0")
|
|
2772
|
-
return true;
|
|
2773
|
-
if (host === "::" || host === "::1")
|
|
2774
|
-
return true;
|
|
2775
|
-
if (/^f[cd][0-9a-f]{2}:/.test(host))
|
|
2776
|
-
return true;
|
|
2777
|
-
if (/^fe[89ab][0-9a-f]:/.test(host))
|
|
2778
|
-
return true;
|
|
2779
|
-
const mapped = host.match(/^::ffff:(.+)$/);
|
|
2780
|
-
if (mapped) {
|
|
2781
|
-
const inner = mapped[1];
|
|
2782
|
-
if (/^\d+\.\d+\.\d+\.\d+$/.test(inner)) {
|
|
2783
|
-
for (const p of PRIVATE_V4_PATTERNS)
|
|
2784
|
-
if (p.test(inner))
|
|
2785
|
-
return true;
|
|
2786
|
-
}
|
|
2787
|
-
const hex = inner.match(/^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
|
2788
|
-
if (hex) {
|
|
2789
|
-
const a = Number.parseInt(hex[1], 16);
|
|
2790
|
-
const b = Number.parseInt(hex[2], 16);
|
|
2791
|
-
const dotted = `${a >> 8 & 255}.${a & 255}.${b >> 8 & 255}.${b & 255}`;
|
|
2792
|
-
for (const p of PRIVATE_V4_PATTERNS)
|
|
2793
|
-
if (p.test(dotted))
|
|
2794
|
-
return true;
|
|
2795
|
-
}
|
|
2796
|
-
}
|
|
2797
|
-
for (const p of PRIVATE_V4_PATTERNS) {
|
|
2798
|
-
if (p.test(host))
|
|
2799
|
-
return true;
|
|
2800
|
-
}
|
|
2801
|
-
return false;
|
|
2802
|
-
}
|
|
2803
|
-
function allowPrivateEgress() {
|
|
2804
|
-
return process.env.SECONDLAYER_ALLOW_PRIVATE_EGRESS === "true";
|
|
2805
|
-
}
|
|
2806
|
-
async function dispatchOne(db, outboxRow, sub) {
|
|
2807
|
-
const { body, headers } = buildForFormat(outboxRow, sub, getSubscriptionSigningSecret(sub));
|
|
2808
|
-
const start = performance.now();
|
|
2809
|
-
let statusCode = null;
|
|
2810
|
-
let error = null;
|
|
2811
|
-
let ok = false;
|
|
2812
|
-
let responseBody = "";
|
|
2813
|
-
let responseHeaders = {};
|
|
2814
|
-
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
2815
|
-
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
2816
|
-
logger11.warn("[emitter] refused private egress", {
|
|
2817
|
-
subscription: sub.name,
|
|
2818
|
-
url: sub.url
|
|
2819
|
-
});
|
|
2820
|
-
const durationMs2 = 0;
|
|
2821
|
-
const attempt2 = outboxRow.attempt + 1;
|
|
2822
|
-
await db.insertInto("subscription_deliveries").values({
|
|
2823
|
-
outbox_id: outboxRow.id,
|
|
2824
|
-
subscription_id: outboxRow.subscription_id,
|
|
2825
|
-
attempt: attempt2,
|
|
2826
|
-
status_code: null,
|
|
2827
|
-
response_headers: null,
|
|
2828
|
-
response_body: null,
|
|
2829
|
-
error_message: error,
|
|
2830
|
-
duration_ms: durationMs2
|
|
2831
|
-
}).execute();
|
|
2832
|
-
return { ok: false, statusCode: null, error, durationMs: durationMs2 };
|
|
2833
|
-
}
|
|
2834
|
-
try {
|
|
2835
|
-
const res = await fetch(sub.url, {
|
|
2836
|
-
method: "POST",
|
|
2837
|
-
headers,
|
|
2838
|
-
body,
|
|
2839
|
-
signal: AbortSignal.timeout(sub.timeout_ms)
|
|
2840
|
-
});
|
|
2841
|
-
statusCode = res.status;
|
|
2842
|
-
ok = res.ok;
|
|
2843
|
-
const buf = await res.arrayBuffer();
|
|
2844
|
-
const truncated = buf.byteLength > 8192 ? buf.slice(0, 8192) : buf;
|
|
2845
|
-
responseBody = Buffer.from(truncated).toString("utf8");
|
|
2846
|
-
responseHeaders = Object.fromEntries(res.headers.entries());
|
|
2847
|
-
} catch (err) {
|
|
2848
|
-
error = err instanceof Error ? err.message : String(err);
|
|
2849
|
-
}
|
|
2850
|
-
const durationMs = Math.round(performance.now() - start);
|
|
2851
|
-
const attempt = outboxRow.attempt + 1;
|
|
2852
|
-
await db.insertInto("subscription_deliveries").values({
|
|
2853
|
-
outbox_id: outboxRow.id,
|
|
2854
|
-
subscription_id: outboxRow.subscription_id,
|
|
2855
|
-
attempt,
|
|
2856
|
-
status_code: statusCode,
|
|
2857
|
-
response_headers: responseHeaders,
|
|
2858
|
-
response_body: responseBody || null,
|
|
2859
|
-
error_message: error,
|
|
2860
|
-
duration_ms: durationMs
|
|
2861
|
-
}).execute();
|
|
2862
|
-
return { ok, statusCode, error, durationMs };
|
|
2863
|
-
}
|
|
2864
|
-
async function settleDelivered(db, outboxRow) {
|
|
2865
|
-
await db.transaction().execute(async (tx) => {
|
|
2866
|
-
await tx.updateTable("subscription_outbox").set({
|
|
2867
|
-
status: "delivered",
|
|
2868
|
-
delivered_at: new Date,
|
|
2869
|
-
attempt: outboxRow.attempt + 1,
|
|
2870
|
-
locked_by: null,
|
|
2871
|
-
locked_until: null
|
|
2872
|
-
}).where("id", "=", outboxRow.id).execute();
|
|
2873
|
-
await tx.updateTable("subscriptions").set({
|
|
2874
|
-
last_delivery_at: new Date,
|
|
2875
|
-
last_success_at: new Date,
|
|
2876
|
-
circuit_failures: 0,
|
|
2877
|
-
last_error: null,
|
|
2878
|
-
updated_at: new Date
|
|
2879
|
-
}).where("id", "=", outboxRow.subscription_id).execute();
|
|
2880
|
-
});
|
|
2881
|
-
}
|
|
2882
|
-
async function settleFailed(db, outboxRow, sub, errText) {
|
|
2883
|
-
const attempt = outboxRow.attempt + 1;
|
|
2884
|
-
const isDead = attempt >= sub.max_retries;
|
|
2885
|
-
const nextAt = isDead ? null : new Date(Date.now() + nextDelaySeconds(outboxRow.attempt) * 1000);
|
|
2886
|
-
await db.transaction().execute(async (tx) => {
|
|
2887
|
-
await tx.updateTable("subscription_outbox").set({
|
|
2888
|
-
attempt,
|
|
2889
|
-
next_attempt_at: nextAt ?? new Date,
|
|
2890
|
-
status: isDead ? "dead" : "pending",
|
|
2891
|
-
failed_at: isDead ? new Date : null,
|
|
2892
|
-
locked_by: null,
|
|
2893
|
-
locked_until: null
|
|
2894
|
-
}).where("id", "=", outboxRow.id).execute();
|
|
2895
|
-
const incResult = await sql4`
|
|
2896
|
-
UPDATE subscriptions
|
|
2897
|
-
SET circuit_failures = circuit_failures + 1,
|
|
2898
|
-
last_delivery_at = NOW(),
|
|
2899
|
-
last_error = ${errText.slice(0, 500)},
|
|
2900
|
-
updated_at = NOW()
|
|
2901
|
-
WHERE id = ${sub.id}
|
|
2902
|
-
RETURNING circuit_failures
|
|
2903
|
-
`.execute(tx);
|
|
2904
|
-
const newFailures = incResult.rows[0]?.circuit_failures ?? sub.circuit_failures + 1;
|
|
2905
|
-
const shouldTripCircuit = newFailures >= CIRCUIT_THRESHOLD;
|
|
2906
|
-
if (shouldTripCircuit) {
|
|
2907
|
-
await tx.updateTable("subscriptions").set({
|
|
2908
|
-
status: "paused",
|
|
2909
|
-
circuit_opened_at: new Date,
|
|
2910
|
-
updated_at: new Date
|
|
2911
|
-
}).where("id", "=", sub.id).execute();
|
|
2912
|
-
logger11.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
2913
|
-
subscription: sub.name,
|
|
2914
|
-
failures: newFailures
|
|
2915
|
-
});
|
|
2916
|
-
}
|
|
2523
|
+
SUBGRAPH_CATCHUP_LOCK_KEY,
|
|
2524
|
+
createPostgresLeaderBackend,
|
|
2525
|
+
withLeaderLock
|
|
2526
|
+
} from "@secondlayer/shared/leader";
|
|
2527
|
+
import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
2528
|
+
var catchUpLeader = false;
|
|
2529
|
+
function isCatchUpLeader() {
|
|
2530
|
+
return catchUpLeader;
|
|
2531
|
+
}
|
|
2532
|
+
function startCatchUpLeader(opts = {}) {
|
|
2533
|
+
return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
|
|
2534
|
+
catchUpLeader = true;
|
|
2535
|
+
await opts.onAcquire?.();
|
|
2536
|
+
return () => {
|
|
2537
|
+
catchUpLeader = false;
|
|
2538
|
+
};
|
|
2539
|
+
}, {
|
|
2540
|
+
pollMs: opts.pollMs,
|
|
2541
|
+
heartbeatMs: opts.heartbeatMs,
|
|
2542
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
|
|
2917
2543
|
});
|
|
2918
2544
|
}
|
|
2919
|
-
async function claimAndDrain(db, state, emitterId) {
|
|
2920
|
-
if (state.claimInFlight)
|
|
2921
|
-
return 0;
|
|
2922
|
-
state.claimInFlight = true;
|
|
2923
|
-
try {
|
|
2924
|
-
const liveLimit = Math.max(1, Math.round(BATCH_SIZE * LIVE_SHARE));
|
|
2925
|
-
const replayLimit = BATCH_SIZE - liveLimit;
|
|
2926
|
-
const claimed = await db.transaction().execute(async (tx) => {
|
|
2927
|
-
const live = await sql4`
|
|
2928
|
-
SELECT * FROM subscription_outbox
|
|
2929
|
-
WHERE status = 'pending'
|
|
2930
|
-
AND next_attempt_at <= NOW()
|
|
2931
|
-
AND is_replay = FALSE
|
|
2932
|
-
ORDER BY next_attempt_at ASC
|
|
2933
|
-
FOR UPDATE SKIP LOCKED
|
|
2934
|
-
LIMIT ${sql4.lit(liveLimit)}
|
|
2935
|
-
`.execute(tx);
|
|
2936
|
-
const replay = await sql4`
|
|
2937
|
-
SELECT * FROM subscription_outbox
|
|
2938
|
-
WHERE status = 'pending'
|
|
2939
|
-
AND next_attempt_at <= NOW()
|
|
2940
|
-
AND is_replay = TRUE
|
|
2941
|
-
ORDER BY next_attempt_at ASC
|
|
2942
|
-
FOR UPDATE SKIP LOCKED
|
|
2943
|
-
LIMIT ${sql4.lit(replayLimit)}
|
|
2944
|
-
`.execute(tx);
|
|
2945
|
-
const combined = [...live.rows, ...replay.rows];
|
|
2946
|
-
if (combined.length === 0)
|
|
2947
|
-
return [];
|
|
2948
|
-
const now = new Date;
|
|
2949
|
-
const lockUntil = new Date(now.getTime() + LOCK_WINDOW_MS);
|
|
2950
|
-
await tx.updateTable("subscription_outbox").set({
|
|
2951
|
-
locked_by: emitterId,
|
|
2952
|
-
locked_until: lockUntil,
|
|
2953
|
-
next_attempt_at: lockUntil
|
|
2954
|
-
}).where("id", "in", combined.map((r) => r.id)).execute();
|
|
2955
|
-
return combined;
|
|
2956
|
-
});
|
|
2957
|
-
if (claimed.length === 0)
|
|
2958
|
-
return 0;
|
|
2959
|
-
const bySubId = new Map;
|
|
2960
|
-
for (const row of claimed) {
|
|
2961
|
-
const arr = bySubId.get(row.subscription_id);
|
|
2962
|
-
if (arr)
|
|
2963
|
-
arr.push(row);
|
|
2964
|
-
else
|
|
2965
|
-
bySubId.set(row.subscription_id, [row]);
|
|
2966
|
-
}
|
|
2967
|
-
const subIds = Array.from(bySubId.keys());
|
|
2968
|
-
const subs = await db.selectFrom("subscriptions").selectAll().where("id", "in", subIds).execute();
|
|
2969
|
-
const subById = new Map(subs.map((s) => [s.id, s]));
|
|
2970
|
-
await Promise.all(subIds.map((subId) => drainForSub(db, state, subById.get(subId), bySubId.get(subId))));
|
|
2971
|
-
return claimed.length;
|
|
2972
|
-
} finally {
|
|
2973
|
-
state.claimInFlight = false;
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
async function drainForSub(db, state, sub, rows) {
|
|
2977
|
-
const cap = sub.concurrency || 4;
|
|
2978
|
-
const counter = () => state.inFlightBySub.get(sub.id) ?? 0;
|
|
2979
|
-
const inc = () => state.inFlightBySub.set(sub.id, counter() + 1);
|
|
2980
|
-
const dec = () => state.inFlightBySub.set(sub.id, Math.max(0, counter() - 1));
|
|
2981
|
-
const queue = [...rows];
|
|
2982
|
-
const workers = [];
|
|
2983
|
-
const slots = Math.min(cap, queue.length);
|
|
2984
|
-
for (let i = 0;i < slots; i++) {
|
|
2985
|
-
workers.push((async () => {
|
|
2986
|
-
while (state.running && queue.length > 0) {
|
|
2987
|
-
const row = queue.shift();
|
|
2988
|
-
if (!row)
|
|
2989
|
-
break;
|
|
2990
|
-
inc();
|
|
2991
|
-
try {
|
|
2992
|
-
const result = await dispatchOne(db, row, sub);
|
|
2993
|
-
if (result.ok) {
|
|
2994
|
-
await settleDelivered(db, row);
|
|
2995
|
-
} else {
|
|
2996
|
-
const err = result.error ?? `HTTP ${result.statusCode ?? "?"}`;
|
|
2997
|
-
await settleFailed(db, row, sub, err);
|
|
2998
|
-
}
|
|
2999
|
-
} catch (err) {
|
|
3000
|
-
logger11.error("Emitter dispatch crashed", {
|
|
3001
|
-
outboxId: row.id,
|
|
3002
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3003
|
-
});
|
|
3004
|
-
await settleFailed(db, row, sub, err instanceof Error ? err.message : String(err));
|
|
3005
|
-
} finally {
|
|
3006
|
-
dec();
|
|
3007
|
-
}
|
|
3008
|
-
}
|
|
3009
|
-
})());
|
|
3010
|
-
}
|
|
3011
|
-
await Promise.all(workers);
|
|
3012
|
-
}
|
|
3013
|
-
async function runRetention(db) {
|
|
3014
|
-
await sql4`
|
|
3015
|
-
DELETE FROM subscription_outbox
|
|
3016
|
-
WHERE status = 'delivered' AND delivered_at < NOW() - interval '7 days'
|
|
3017
|
-
`.execute(db);
|
|
3018
|
-
await sql4`
|
|
3019
|
-
DELETE FROM subscription_deliveries
|
|
3020
|
-
WHERE dispatched_at < NOW() - interval '30 days'
|
|
3021
|
-
`.execute(db);
|
|
3022
|
-
await sql4`
|
|
3023
|
-
DELETE FROM subscription_outbox
|
|
3024
|
-
WHERE status = 'dead' AND failed_at < NOW() - interval '90 days'
|
|
3025
|
-
`.execute(db);
|
|
3026
|
-
}
|
|
3027
|
-
async function startEmitter(opts) {
|
|
3028
|
-
const emitterId = `emitter-${Math.random().toString(36).slice(2, 10)}`;
|
|
3029
|
-
const db = getTargetDb6();
|
|
3030
|
-
const state = {
|
|
3031
|
-
running: true,
|
|
3032
|
-
inFlightBySub: new Map,
|
|
3033
|
-
claimInFlight: false
|
|
3034
|
-
};
|
|
3035
|
-
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
3036
|
-
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
3037
|
-
logger11.info("[emitter] started", { id: emitterId });
|
|
3038
|
-
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
3039
|
-
let lastErr = null;
|
|
3040
|
-
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
3041
|
-
try {
|
|
3042
|
-
await refreshMatcher(db);
|
|
3043
|
-
lastErr = null;
|
|
3044
|
-
break;
|
|
3045
|
-
} catch (err) {
|
|
3046
|
-
lastErr = err;
|
|
3047
|
-
const delayMs = 500 * 2 ** i;
|
|
3048
|
-
logger11.warn("[emitter] matcher refresh failed, retrying", {
|
|
3049
|
-
attempt: i + 1,
|
|
3050
|
-
delayMs,
|
|
3051
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3052
|
-
});
|
|
3053
|
-
await new Promise((r) => setTimeout(r, delayMs));
|
|
3054
|
-
}
|
|
3055
|
-
}
|
|
3056
|
-
if (lastErr) {
|
|
3057
|
-
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
3058
|
-
}
|
|
3059
|
-
const listenUrl = targetListenerUrl();
|
|
3060
|
-
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
3061
|
-
if (!state.running)
|
|
3062
|
-
return;
|
|
3063
|
-
claimAndDrain(db, state, emitterId).catch((err) => logger11.error("[emitter] claim failed", {
|
|
3064
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3065
|
-
}));
|
|
3066
|
-
}, { connectionString: listenUrl });
|
|
3067
|
-
const stopChanged = await listen("subscriptions:changed", () => {
|
|
3068
|
-
if (!state.running)
|
|
3069
|
-
return;
|
|
3070
|
-
refreshMatcher(db).catch((err) => logger11.error("[emitter] matcher refresh failed", {
|
|
3071
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3072
|
-
}));
|
|
3073
|
-
}, { connectionString: listenUrl });
|
|
3074
|
-
const poll = setInterval(() => {
|
|
3075
|
-
if (!state.running)
|
|
3076
|
-
return;
|
|
3077
|
-
claimAndDrain(db, state, emitterId).catch((err) => logger11.error("[emitter] poll claim failed", {
|
|
3078
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3079
|
-
}));
|
|
3080
|
-
}, pollIntervalMs);
|
|
3081
|
-
claimAndDrain(db, state, emitterId);
|
|
3082
|
-
const retention = setInterval(() => {
|
|
3083
|
-
if (!state.running)
|
|
3084
|
-
return;
|
|
3085
|
-
runRetention(db).catch((err) => logger11.error("[emitter] retention failed", {
|
|
3086
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3087
|
-
}));
|
|
3088
|
-
}, retentionIntervalMs);
|
|
3089
|
-
return async () => {
|
|
3090
|
-
state.running = false;
|
|
3091
|
-
clearInterval(poll);
|
|
3092
|
-
clearInterval(retention);
|
|
3093
|
-
await stopNew();
|
|
3094
|
-
await stopChanged();
|
|
3095
|
-
logger11.info("[emitter] stopped", { id: emitterId });
|
|
3096
|
-
};
|
|
3097
|
-
}
|
|
3098
2545
|
|
|
3099
2546
|
// src/runtime/streams-reorg-poll.ts
|
|
3100
2547
|
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
3101
2548
|
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3102
|
-
import { logger as
|
|
2549
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
3103
2550
|
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3104
2551
|
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3105
|
-
async function pollReorgsOnce(http, cursor,
|
|
2552
|
+
async function pollReorgsOnce(http, cursor, onReorg) {
|
|
3106
2553
|
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
3107
2554
|
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3108
2555
|
for (const r of sorted) {
|
|
3109
|
-
|
|
2556
|
+
logger9.info("Streams reorg — rewinding", {
|
|
3110
2557
|
forkPointHeight: r.fork_point_height
|
|
3111
2558
|
});
|
|
3112
|
-
await
|
|
3113
|
-
if (handleChainReorg2)
|
|
3114
|
-
await handleChainReorg2(r.fork_point_height);
|
|
2559
|
+
await onReorg(r.fork_point_height);
|
|
3115
2560
|
}
|
|
3116
2561
|
return next_since ?? cursor;
|
|
3117
2562
|
}
|
|
3118
|
-
function startStreamsReorgPoll(
|
|
2563
|
+
function startStreamsReorgPoll(onReorg) {
|
|
3119
2564
|
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
3120
2565
|
const http = new IndexHttpClient2({
|
|
3121
2566
|
indexBaseUrl: baseUrl,
|
|
@@ -3129,9 +2574,9 @@ function startStreamsReorgPoll(handleReorg, loadDef, handleChainReorg2) {
|
|
|
3129
2574
|
if (!running)
|
|
3130
2575
|
return;
|
|
3131
2576
|
try {
|
|
3132
|
-
since = await pollReorgsOnce(http, since,
|
|
2577
|
+
since = await pollReorgsOnce(http, since, onReorg);
|
|
3133
2578
|
} catch (err) {
|
|
3134
|
-
|
|
2579
|
+
logger9.error("Streams reorg poll failed", {
|
|
3135
2580
|
error: getErrorMessage4(err)
|
|
3136
2581
|
});
|
|
3137
2582
|
}
|
|
@@ -3139,222 +2584,7 @@ function startStreamsReorgPoll(handleReorg, loadDef, handleChainReorg2) {
|
|
|
3139
2584
|
timer = setTimeout(tick, POLL_MS);
|
|
3140
2585
|
};
|
|
3141
2586
|
timer = setTimeout(tick, POLL_MS);
|
|
3142
|
-
|
|
3143
|
-
return () => {
|
|
3144
|
-
running = false;
|
|
3145
|
-
if (timer)
|
|
3146
|
-
clearTimeout(timer);
|
|
3147
|
-
};
|
|
3148
|
-
}
|
|
3149
|
-
|
|
3150
|
-
// src/runtime/trigger-evaluator-loop.ts
|
|
3151
|
-
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
3152
|
-
import { getTargetDb as getTargetDb7 } from "@secondlayer/shared/db";
|
|
3153
|
-
import { listActiveChainSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
3154
|
-
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3155
|
-
|
|
3156
|
-
// src/runtime/trigger-evaluator.ts
|
|
3157
|
-
import { resolveTraitContractIds as resolveTraitContractIds2 } from "@secondlayer/shared/db/queries/contracts";
|
|
3158
|
-
var TX_LEVEL_TRIGGER_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
3159
|
-
function sourceKey(subscriptionId, triggerIndex) {
|
|
3160
|
-
return `${subscriptionId}#${triggerIndex}`;
|
|
3161
|
-
}
|
|
3162
|
-
function toAmount(v) {
|
|
3163
|
-
return v === undefined ? undefined : BigInt(v);
|
|
3164
|
-
}
|
|
3165
|
-
function chainTriggerToFilter(trigger) {
|
|
3166
|
-
const t = trigger;
|
|
3167
|
-
const filter = { ...trigger };
|
|
3168
|
-
const minAmount = toAmount(t.minAmount);
|
|
3169
|
-
const maxAmount = toAmount(t.maxAmount);
|
|
3170
|
-
if (minAmount !== undefined)
|
|
3171
|
-
filter.minAmount = minAmount;
|
|
3172
|
-
if (maxAmount !== undefined)
|
|
3173
|
-
filter.maxAmount = maxAmount;
|
|
3174
|
-
return filter;
|
|
3175
|
-
}
|
|
3176
|
-
function triggersOf(sub) {
|
|
3177
|
-
return sub.triggers ?? [];
|
|
3178
|
-
}
|
|
3179
|
-
function buildSourcesMap(chainSubs) {
|
|
3180
|
-
const sources = {};
|
|
3181
|
-
const keyMeta = new Map;
|
|
3182
|
-
for (const sub of chainSubs) {
|
|
3183
|
-
triggersOf(sub).forEach((trigger, triggerIndex) => {
|
|
3184
|
-
const key2 = sourceKey(sub.id, triggerIndex);
|
|
3185
|
-
sources[key2] = chainTriggerToFilter(trigger);
|
|
3186
|
-
keyMeta.set(key2, {
|
|
3187
|
-
subscriptionId: sub.id,
|
|
3188
|
-
triggerIndex,
|
|
3189
|
-
triggerType: trigger.type
|
|
3190
|
-
});
|
|
3191
|
-
});
|
|
3192
|
-
}
|
|
3193
|
-
return { sources, keyMeta };
|
|
3194
|
-
}
|
|
3195
|
-
function referencedEventTypes(chainSubs) {
|
|
3196
|
-
const filterTypes = new Set;
|
|
3197
|
-
for (const sub of chainSubs) {
|
|
3198
|
-
for (const trigger of triggersOf(sub))
|
|
3199
|
-
filterTypes.add(trigger.type);
|
|
3200
|
-
}
|
|
3201
|
-
return indexEventTypesForFilterTypes([...filterTypes]);
|
|
3202
|
-
}
|
|
3203
|
-
function referencedTraits(chainSubs) {
|
|
3204
|
-
const traits = new Set;
|
|
3205
|
-
for (const sub of chainSubs) {
|
|
3206
|
-
for (const trigger of triggersOf(sub)) {
|
|
3207
|
-
const trait = trigger.trait;
|
|
3208
|
-
if (trait)
|
|
3209
|
-
traits.add(trait);
|
|
3210
|
-
}
|
|
3211
|
-
}
|
|
3212
|
-
return [...traits];
|
|
3213
|
-
}
|
|
3214
|
-
async function buildTraitContracts(db, chainSubs, asOfBlock) {
|
|
3215
|
-
const resolved = new Map;
|
|
3216
|
-
for (const trait of referencedTraits(chainSubs)) {
|
|
3217
|
-
const ids = await resolveTraitContractIds2(db, trait, asOfBlock);
|
|
3218
|
-
resolved.set(trait, new Set(ids));
|
|
3219
|
-
}
|
|
3220
|
-
return resolved;
|
|
3221
|
-
}
|
|
3222
|
-
function evaluateBlock(block, sources, traitContracts) {
|
|
3223
|
-
return matchSources(sources, block.txs, block.events, traitContracts);
|
|
3224
|
-
}
|
|
3225
|
-
function chainDedupKey(subscriptionId, txId, eventIndex, blockHash) {
|
|
3226
|
-
return `chain:${subscriptionId}:${txId}:${eventIndex}:${blockHash}`;
|
|
3227
|
-
}
|
|
3228
|
-
function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event) {
|
|
3229
|
-
const payload = {
|
|
3230
|
-
action: "apply",
|
|
3231
|
-
block_hash: blockHash,
|
|
3232
|
-
block_height: blockHeight,
|
|
3233
|
-
tx_id: txId,
|
|
3234
|
-
canonical: true,
|
|
3235
|
-
trigger: meta.triggerType,
|
|
3236
|
-
event
|
|
3237
|
-
};
|
|
3238
|
-
return {
|
|
3239
|
-
subscription_id: meta.subscriptionId,
|
|
3240
|
-
kind: "chain",
|
|
3241
|
-
subgraph_name: null,
|
|
3242
|
-
table_name: null,
|
|
3243
|
-
block_height: blockHeight,
|
|
3244
|
-
tx_id: txId,
|
|
3245
|
-
row_pk: { tx_id: txId, event_index: eventIndex },
|
|
3246
|
-
event_type: `chain.${meta.triggerType}.apply`,
|
|
3247
|
-
payload,
|
|
3248
|
-
dedup_key: chainDedupKey(meta.subscriptionId, txId, eventIndex, blockHash)
|
|
3249
|
-
};
|
|
3250
|
-
}
|
|
3251
|
-
async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash) {
|
|
3252
|
-
const rows = [];
|
|
3253
|
-
for (const match of matches) {
|
|
3254
|
-
const meta = keyMeta.get(match.sourceName);
|
|
3255
|
-
if (!meta)
|
|
3256
|
-
continue;
|
|
3257
|
-
const txId = match.tx.tx_id;
|
|
3258
|
-
if (TX_LEVEL_TRIGGER_TYPES.has(meta.triggerType)) {
|
|
3259
|
-
rows.push(applyRow(meta, blockHeight, blockHash, txId, -1, {
|
|
3260
|
-
tx_id: txId,
|
|
3261
|
-
type: match.tx.type,
|
|
3262
|
-
sender: match.tx.sender,
|
|
3263
|
-
status: match.tx.status,
|
|
3264
|
-
contract_id: match.tx.contract_id ?? null,
|
|
3265
|
-
function_name: match.tx.function_name ?? null,
|
|
3266
|
-
function_args: match.tx.function_args ?? null,
|
|
3267
|
-
result_hex: match.tx.raw_result ?? null
|
|
3268
|
-
}));
|
|
3269
|
-
} else {
|
|
3270
|
-
for (const event of match.events) {
|
|
3271
|
-
rows.push(applyRow(meta, blockHeight, blockHash, txId, event.event_index, {
|
|
3272
|
-
tx_id: txId,
|
|
3273
|
-
type: event.type,
|
|
3274
|
-
event_index: event.event_index,
|
|
3275
|
-
data: event.data
|
|
3276
|
-
}));
|
|
3277
|
-
}
|
|
3278
|
-
}
|
|
3279
|
-
}
|
|
3280
|
-
if (rows.length === 0)
|
|
3281
|
-
return 0;
|
|
3282
|
-
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
|
|
3283
|
-
return rows.length;
|
|
3284
|
-
}
|
|
3285
|
-
|
|
3286
|
-
// src/runtime/trigger-evaluator-loop.ts
|
|
3287
|
-
var POLL_MS2 = Number(process.env.TRIGGER_EVALUATOR_POLL_MS) || 5000;
|
|
3288
|
-
var BATCH = Number(process.env.TRIGGER_EVALUATOR_BATCH) || 200;
|
|
3289
|
-
var MAX_BLOCKS_PER_TICK = Number(process.env.TRIGGER_EVALUATOR_MAX_BLOCKS) || 2000;
|
|
3290
|
-
async function readCursor(db) {
|
|
3291
|
-
const row = await db.selectFrom("trigger_evaluator_state").select("last_processed_block").where("id", "=", true).executeTakeFirst();
|
|
3292
|
-
return row ? Number(row.last_processed_block) : 0;
|
|
3293
|
-
}
|
|
3294
|
-
async function advanceCursor(db, to) {
|
|
3295
|
-
await db.transaction().execute(async (trx) => {
|
|
3296
|
-
const cur = await trx.selectFrom("trigger_evaluator_state").select("last_processed_block").where("id", "=", true).forUpdate().executeTakeFirst();
|
|
3297
|
-
if (cur && Number(cur.last_processed_block) < to) {
|
|
3298
|
-
await trx.updateTable("trigger_evaluator_state").set({ last_processed_block: to, updated_at: new Date }).where("id", "=", true).execute();
|
|
3299
|
-
}
|
|
3300
|
-
});
|
|
3301
|
-
}
|
|
3302
|
-
async function runEvaluatorOnce(db = getTargetDb7()) {
|
|
3303
|
-
const chainSubs = await listActiveChainSubscriptions(db);
|
|
3304
|
-
const source = new PublicApiBlockSource(buildHttpClient(), referencedEventTypes(chainSubs));
|
|
3305
|
-
const tip = await source.getTip();
|
|
3306
|
-
if (tip <= 0)
|
|
3307
|
-
return 0;
|
|
3308
|
-
const cursor = await readCursor(db);
|
|
3309
|
-
if (cursor === 0 || chainSubs.length === 0) {
|
|
3310
|
-
await advanceCursor(db, tip);
|
|
3311
|
-
return 0;
|
|
3312
|
-
}
|
|
3313
|
-
if (cursor >= tip)
|
|
3314
|
-
return 0;
|
|
3315
|
-
const { sources, keyMeta } = buildSourcesMap(chainSubs);
|
|
3316
|
-
const target = Math.min(tip, cursor + MAX_BLOCKS_PER_TICK);
|
|
3317
|
-
let emitted = 0;
|
|
3318
|
-
for (let from = cursor + 1;from <= target; from = from + BATCH) {
|
|
3319
|
-
const to = Math.min(from + BATCH - 1, target);
|
|
3320
|
-
const blocks = await source.loadBlockRange(from, to);
|
|
3321
|
-
const traitContracts = await buildTraitContracts(db, chainSubs, to);
|
|
3322
|
-
for (let h = from;h <= to; h++) {
|
|
3323
|
-
const bd = blocks.get(h);
|
|
3324
|
-
if (!bd)
|
|
3325
|
-
continue;
|
|
3326
|
-
const matches = evaluateBlock(bd, sources, traitContracts);
|
|
3327
|
-
if (matches.length === 0)
|
|
3328
|
-
continue;
|
|
3329
|
-
emitted += await emitChainOutbox(db, matches, keyMeta, h, bd.block.hash);
|
|
3330
|
-
}
|
|
3331
|
-
await advanceCursor(db, to);
|
|
3332
|
-
}
|
|
3333
|
-
return emitted;
|
|
3334
|
-
}
|
|
3335
|
-
function startTriggerEvaluator() {
|
|
3336
|
-
let running = true;
|
|
3337
|
-
let timer;
|
|
3338
|
-
const tick = async () => {
|
|
3339
|
-
if (!running)
|
|
3340
|
-
return;
|
|
3341
|
-
try {
|
|
3342
|
-
const emitted = await runEvaluatorOnce();
|
|
3343
|
-
if (emitted > 0) {
|
|
3344
|
-
logger13.info("Trigger evaluator emitted chain deliveries", {
|
|
3345
|
-
count: emitted
|
|
3346
|
-
});
|
|
3347
|
-
}
|
|
3348
|
-
} catch (err) {
|
|
3349
|
-
logger13.error("Trigger evaluator tick failed", {
|
|
3350
|
-
error: getErrorMessage5(err)
|
|
3351
|
-
});
|
|
3352
|
-
}
|
|
3353
|
-
if (running)
|
|
3354
|
-
timer = setTimeout(tick, POLL_MS2);
|
|
3355
|
-
};
|
|
3356
|
-
timer = setTimeout(tick, POLL_MS2);
|
|
3357
|
-
logger13.info("Trigger evaluator started", { pollMs: POLL_MS2 });
|
|
2587
|
+
logger9.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3358
2588
|
return () => {
|
|
3359
2589
|
running = false;
|
|
3360
2590
|
if (timer)
|
|
@@ -3381,11 +2611,11 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3381
2611
|
const def = await loadSubgraphDefinition(sg);
|
|
3382
2612
|
await catchUpSubgraph(def, sg.name);
|
|
3383
2613
|
} catch (err) {
|
|
3384
|
-
const msg =
|
|
2614
|
+
const msg = getErrorMessage5(err);
|
|
3385
2615
|
if (isHandlerNotFoundError(err)) {
|
|
3386
2616
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3387
2617
|
}
|
|
3388
|
-
|
|
2618
|
+
logger10.error("Subgraph catch-up failed", {
|
|
3389
2619
|
subgraph: sg.name,
|
|
3390
2620
|
error: msg
|
|
3391
2621
|
});
|
|
@@ -3425,7 +2655,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3425
2655
|
definitionCache.set(sg.name, def);
|
|
3426
2656
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3427
2657
|
invalidateSubgraphRoute(sg.name);
|
|
3428
|
-
|
|
2658
|
+
logger10.info("Subgraph handler reloaded", {
|
|
3429
2659
|
subgraph: sg.name,
|
|
3430
2660
|
from: prevVersion,
|
|
3431
2661
|
to: sg.version
|
|
@@ -3444,7 +2674,7 @@ function cleanupCaches(active) {
|
|
|
3444
2674
|
}
|
|
3445
2675
|
}
|
|
3446
2676
|
async function synthesizeLegacyReindexOperations() {
|
|
3447
|
-
const db =
|
|
2677
|
+
const db = getTargetDb5();
|
|
3448
2678
|
const stale = (await listSubgraphs2(db)).filter((sg) => sg.status === "reindexing");
|
|
3449
2679
|
for (const sg of stale) {
|
|
3450
2680
|
const active = await findActiveSubgraphOperation(db, sg.id);
|
|
@@ -3459,7 +2689,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3459
2689
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3460
2690
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3461
2691
|
});
|
|
3462
|
-
|
|
2692
|
+
logger10.info("Queued legacy reindex resume operation", {
|
|
3463
2693
|
subgraph: sg.name
|
|
3464
2694
|
});
|
|
3465
2695
|
} catch (err) {
|
|
@@ -3473,7 +2703,7 @@ async function runSubgraphOperation(operation, signal) {
|
|
|
3473
2703
|
if (operation.cancel_requested) {
|
|
3474
2704
|
return 0;
|
|
3475
2705
|
}
|
|
3476
|
-
const db =
|
|
2706
|
+
const db = getTargetDb5();
|
|
3477
2707
|
const subgraph = await db.selectFrom("subgraphs").selectAll().where("id", "=", operation.subgraph_id).executeTakeFirst();
|
|
3478
2708
|
if (!subgraph)
|
|
3479
2709
|
throw new Error(`Subgraph not found: ${operation.subgraph_id}`);
|
|
@@ -3509,13 +2739,13 @@ async function runSubgraphOperation(operation, signal) {
|
|
|
3509
2739
|
}
|
|
3510
2740
|
async function startSubgraphOperationRunner(opts) {
|
|
3511
2741
|
const concurrency = opts?.concurrency ?? DEFAULT_OPERATION_CONCURRENCY;
|
|
3512
|
-
const db =
|
|
2742
|
+
const db = getTargetDb5();
|
|
3513
2743
|
const lockedBy = `${hostname()}:${process.pid}:${randomUUID()}`;
|
|
3514
2744
|
const active = new Map;
|
|
3515
2745
|
const activeRuns = new Map;
|
|
3516
2746
|
let running = true;
|
|
3517
2747
|
let draining = false;
|
|
3518
|
-
|
|
2748
|
+
logger10.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3519
2749
|
const startOperation = (operation) => {
|
|
3520
2750
|
const controller = new AbortController;
|
|
3521
2751
|
active.set(operation.id, controller);
|
|
@@ -3523,9 +2753,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3523
2753
|
if (!running)
|
|
3524
2754
|
return;
|
|
3525
2755
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3526
|
-
|
|
2756
|
+
logger10.warn("Subgraph operation heartbeat failed", {
|
|
3527
2757
|
operationId: operation.id,
|
|
3528
|
-
error:
|
|
2758
|
+
error: getErrorMessage5(err)
|
|
3529
2759
|
});
|
|
3530
2760
|
});
|
|
3531
2761
|
}, HEARTBEAT_INTERVAL_MS);
|
|
@@ -3535,9 +2765,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3535
2765
|
controller.abort("user-cancelled");
|
|
3536
2766
|
}
|
|
3537
2767
|
}).catch((err) => {
|
|
3538
|
-
|
|
2768
|
+
logger10.warn("Subgraph operation cancel poll failed", {
|
|
3539
2769
|
operationId: operation.id,
|
|
3540
|
-
error:
|
|
2770
|
+
error: getErrorMessage5(err)
|
|
3541
2771
|
});
|
|
3542
2772
|
});
|
|
3543
2773
|
}, CANCEL_POLL_INTERVAL_MS);
|
|
@@ -3552,14 +2782,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3552
2782
|
const reason = String(controller.signal.reason ?? "");
|
|
3553
2783
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3554
2784
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3555
|
-
|
|
2785
|
+
logger10.info("Subgraph operation cancelled", {
|
|
3556
2786
|
operationId: operation.id,
|
|
3557
2787
|
subgraph: operation.subgraph_name
|
|
3558
2788
|
});
|
|
3559
2789
|
return;
|
|
3560
2790
|
}
|
|
3561
2791
|
if (controller.signal.aborted) {
|
|
3562
|
-
|
|
2792
|
+
logger10.info("Subgraph operation interrupted", {
|
|
3563
2793
|
operationId: operation.id,
|
|
3564
2794
|
subgraph: operation.subgraph_name,
|
|
3565
2795
|
reason
|
|
@@ -3567,7 +2797,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3567
2797
|
return;
|
|
3568
2798
|
}
|
|
3569
2799
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3570
|
-
|
|
2800
|
+
logger10.info("Subgraph operation completed", {
|
|
3571
2801
|
operationId: operation.id,
|
|
3572
2802
|
subgraph: operation.subgraph_name,
|
|
3573
2803
|
processed
|
|
@@ -3575,7 +2805,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3575
2805
|
} catch (err) {
|
|
3576
2806
|
const reason = String(controller.signal.reason ?? "");
|
|
3577
2807
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3578
|
-
|
|
2808
|
+
logger10.info("Subgraph operation interrupted by shutdown", {
|
|
3579
2809
|
operationId: operation.id,
|
|
3580
2810
|
subgraph: operation.subgraph_name
|
|
3581
2811
|
});
|
|
@@ -3585,11 +2815,11 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3585
2815
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3586
2816
|
return;
|
|
3587
2817
|
}
|
|
3588
|
-
await failSubgraphOperation(db, operation.id, lockedBy,
|
|
3589
|
-
|
|
2818
|
+
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage5(err), processed);
|
|
2819
|
+
logger10.error("Subgraph operation failed", {
|
|
3590
2820
|
operationId: operation.id,
|
|
3591
2821
|
subgraph: operation.subgraph_name,
|
|
3592
|
-
error:
|
|
2822
|
+
error: getErrorMessage5(err)
|
|
3593
2823
|
});
|
|
3594
2824
|
} finally {
|
|
3595
2825
|
clearInterval(heartbeat);
|
|
@@ -3619,7 +2849,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3619
2849
|
};
|
|
3620
2850
|
await synthesizeLegacyReindexOperations();
|
|
3621
2851
|
await drain();
|
|
3622
|
-
const stopListening = await
|
|
2852
|
+
const stopListening = await listen(CHANNEL_SUBGRAPH_OPERATIONS, () => {
|
|
3623
2853
|
drain();
|
|
3624
2854
|
}, { connectionString: targetListenerUrl2() });
|
|
3625
2855
|
const pollInterval = setInterval(() => {
|
|
@@ -3633,28 +2863,29 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3633
2863
|
controller.abort("shutdown");
|
|
3634
2864
|
}
|
|
3635
2865
|
await Promise.allSettled(activeRuns.values());
|
|
3636
|
-
|
|
2866
|
+
logger10.info("Subgraph operation runner stopped");
|
|
3637
2867
|
};
|
|
3638
2868
|
}
|
|
3639
2869
|
async function startSubgraphProcessor(opts) {
|
|
3640
2870
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3641
2871
|
let running = true;
|
|
3642
|
-
|
|
2872
|
+
logger10.info("Starting subgraph processor", { concurrency });
|
|
3643
2873
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3644
2874
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3645
2875
|
});
|
|
3646
|
-
const
|
|
3647
|
-
|
|
3648
|
-
await catchUpAll(activeSubgraphs, targetDb, concurrency);
|
|
3649
|
-
const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
|
|
3650
|
-
if (!running)
|
|
2876
|
+
const runCatchUp = async () => {
|
|
2877
|
+
if (!running || !isCatchUpLeader())
|
|
3651
2878
|
return;
|
|
3652
|
-
const db =
|
|
2879
|
+
const db = getTargetDb5();
|
|
3653
2880
|
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3654
2881
|
cleanupCaches(subgraphs);
|
|
3655
2882
|
await catchUpAll(subgraphs, db, concurrency);
|
|
2883
|
+
};
|
|
2884
|
+
const stopCatchUpLeader = startCatchUpLeader({ onAcquire: runCatchUp });
|
|
2885
|
+
const stopListening = await listen(CHANNEL_NEW_BLOCK, async () => {
|
|
2886
|
+
await runCatchUp();
|
|
3656
2887
|
}, { connectionString: sourceListenerUrl() });
|
|
3657
|
-
const stopReorgListening = await
|
|
2888
|
+
const stopReorgListening = await listen("subgraph_reorg", async (payload) => {
|
|
3658
2889
|
if (!running)
|
|
3659
2890
|
return;
|
|
3660
2891
|
try {
|
|
@@ -3664,47 +2895,39 @@ async function startSubgraphProcessor(opts) {
|
|
|
3664
2895
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3665
2896
|
}
|
|
3666
2897
|
} catch (err) {
|
|
3667
|
-
|
|
3668
|
-
error:
|
|
2898
|
+
logger10.error("Subgraph reorg handling failed", {
|
|
2899
|
+
error: getErrorMessage5(err)
|
|
3669
2900
|
});
|
|
3670
2901
|
}
|
|
3671
2902
|
}, { connectionString: sourceListenerUrl() });
|
|
3672
|
-
const pollInterval = setInterval(
|
|
3673
|
-
|
|
3674
|
-
return;
|
|
3675
|
-
const db = getTargetDb8();
|
|
3676
|
-
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3677
|
-
cleanupCaches(subgraphs);
|
|
3678
|
-
await catchUpAll(subgraphs, db, concurrency);
|
|
2903
|
+
const pollInterval = setInterval(() => {
|
|
2904
|
+
runCatchUp();
|
|
3679
2905
|
}, POLL_INTERVAL_MS);
|
|
3680
|
-
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll(
|
|
3681
|
-
|
|
3682
|
-
const stopEmitter = await startEmitter();
|
|
3683
|
-
logger14.info("Subgraph processor ready");
|
|
2906
|
+
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
2907
|
+
logger10.info("Subgraph processor ready");
|
|
3684
2908
|
return async () => {
|
|
3685
2909
|
running = false;
|
|
3686
2910
|
clearInterval(pollInterval);
|
|
2911
|
+
await stopCatchUpLeader();
|
|
3687
2912
|
await stopListening();
|
|
3688
2913
|
await stopReorgListening();
|
|
3689
2914
|
stopStreamsReorgPoll?.();
|
|
3690
|
-
stopTriggerEvaluator?.();
|
|
3691
2915
|
await stopOperations();
|
|
3692
|
-
|
|
3693
|
-
logger14.info("Subgraph processor stopped");
|
|
2916
|
+
logger10.info("Subgraph processor stopped");
|
|
3694
2917
|
};
|
|
3695
2918
|
}
|
|
3696
2919
|
|
|
3697
2920
|
// src/service.ts
|
|
3698
2921
|
import { assertDbSplit, getDb } from "@secondlayer/shared/db";
|
|
3699
|
-
import { logger as
|
|
3700
|
-
import { sql as
|
|
2922
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
2923
|
+
import { sql as sql4 } from "kysely";
|
|
3701
2924
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
3702
2925
|
var SERVICE_NAME = "subgraph-processor";
|
|
3703
2926
|
async function writeHeartbeat() {
|
|
3704
2927
|
try {
|
|
3705
|
-
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at:
|
|
2928
|
+
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql4`now()` })).execute();
|
|
3706
2929
|
} catch (err) {
|
|
3707
|
-
|
|
2930
|
+
logger11.warn("subgraph-processor heartbeat write failed", {
|
|
3708
2931
|
error: err instanceof Error ? err.message : String(err)
|
|
3709
2932
|
});
|
|
3710
2933
|
}
|
|
@@ -3716,7 +2939,7 @@ var processor = await startSubgraphProcessor({
|
|
|
3716
2939
|
await writeHeartbeat();
|
|
3717
2940
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
3718
2941
|
var shutdown = async () => {
|
|
3719
|
-
|
|
2942
|
+
logger11.info("Shutting down subgraph processor...");
|
|
3720
2943
|
clearInterval(heartbeatInterval);
|
|
3721
2944
|
await processor();
|
|
3722
2945
|
process.exit(0);
|
|
@@ -3724,5 +2947,5 @@ var shutdown = async () => {
|
|
|
3724
2947
|
process.on("SIGINT", shutdown);
|
|
3725
2948
|
process.on("SIGTERM", shutdown);
|
|
3726
2949
|
|
|
3727
|
-
//# debugId=
|
|
2950
|
+
//# debugId=FAA88387F9E565F664756E2164756E21
|
|
3728
2951
|
//# sourceMappingURL=service.js.map
|