@secondlayer/subgraphs 3.7.3 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +30 -1
- package/dist/src/index.js +407 -83
- package/dist/src/index.js.map +9 -7
- package/dist/src/runtime/block-processor.js +33 -2
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.js +33 -2
- package/dist/src/runtime/catchup.js.map +3 -3
- package/dist/src/runtime/emitter.d.ts +18 -0
- package/dist/src/runtime/emitter.js +773 -0
- package/dist/src/runtime/emitter.js.map +19 -0
- package/dist/src/runtime/processor.js +129 -943
- package/dist/src/runtime/processor.js.map +7 -20
- package/dist/src/runtime/reindex.js +94 -52
- package/dist/src/runtime/reindex.js.map +4 -4
- package/dist/src/runtime/reorg.js +33 -2
- package/dist/src/runtime/reorg.js.map +3 -3
- package/dist/src/runtime/replay.js +33 -2
- package/dist/src/runtime/replay.js.map +4 -4
- package/dist/src/schema/index.d.ts +2 -1
- package/dist/src/schema/index.js +75 -82
- package/dist/src/schema/index.js.map +5 -5
- package/dist/src/service.js +134 -948
- package/dist/src/service.js.map +6 -19
- package/dist/src/validate.d.ts +2 -1
- package/dist/src/validate.js +2 -1
- package/dist/src/validate.js.map +3 -3
- package/package.json +6 -2
package/dist/src/service.js
CHANGED
|
@@ -1217,6 +1217,37 @@ class PublicApiBlockSource {
|
|
|
1217
1217
|
return map;
|
|
1218
1218
|
}
|
|
1219
1219
|
}
|
|
1220
|
+
|
|
1221
|
+
class FallbackBlockSource {
|
|
1222
|
+
primary;
|
|
1223
|
+
fallback;
|
|
1224
|
+
constructor(primary, fallback) {
|
|
1225
|
+
this.primary = primary;
|
|
1226
|
+
this.fallback = fallback;
|
|
1227
|
+
}
|
|
1228
|
+
async getTip() {
|
|
1229
|
+
try {
|
|
1230
|
+
return await this.primary.getTip();
|
|
1231
|
+
} catch (err) {
|
|
1232
|
+
logger3.warn("block source primary getTip failed — using DB tap", {
|
|
1233
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1234
|
+
});
|
|
1235
|
+
return this.fallback.getTip();
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
async loadBlockRange(fromHeight, toHeight) {
|
|
1239
|
+
try {
|
|
1240
|
+
return await this.primary.loadBlockRange(fromHeight, toHeight);
|
|
1241
|
+
} catch (err) {
|
|
1242
|
+
logger3.warn("block source primary loadBlockRange failed — using DB tap", {
|
|
1243
|
+
from: fromHeight,
|
|
1244
|
+
to: toHeight,
|
|
1245
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1246
|
+
});
|
|
1247
|
+
return this.fallback.loadBlockRange(fromHeight, toHeight);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1220
1251
|
var postgresBlockSource = new PostgresBlockSource;
|
|
1221
1252
|
function buildHttpClient() {
|
|
1222
1253
|
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
@@ -1228,7 +1259,7 @@ function buildHttpClient() {
|
|
|
1228
1259
|
}
|
|
1229
1260
|
function resolveBlockSource(subgraph) {
|
|
1230
1261
|
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
|
|
1231
|
-
return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
|
|
1262
|
+
return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph)), postgresBlockSource);
|
|
1232
1263
|
}
|
|
1233
1264
|
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
|
|
1234
1265
|
logger3.debug("Subgraph not streams-index eligible, using DB tap", {
|
|
@@ -1737,6 +1768,65 @@ function escapeLiteralDefault(value) {
|
|
|
1737
1768
|
return value ? "TRUE" : "FALSE";
|
|
1738
1769
|
return `'${String(value).replace(/'/g, "''")}'`;
|
|
1739
1770
|
}
|
|
1771
|
+
function tableNeedsTrgm(tableDef) {
|
|
1772
|
+
return Object.values(tableDef.columns).some((col) => col.search);
|
|
1773
|
+
}
|
|
1774
|
+
function emitTableDDL(schemaName, tableName, tableDef) {
|
|
1775
|
+
const qualifiedName = `${schemaName}.${tableName}`;
|
|
1776
|
+
const statements = [];
|
|
1777
|
+
const columnDefs = [
|
|
1778
|
+
"_id BIGSERIAL PRIMARY KEY",
|
|
1779
|
+
"_block_height BIGINT NOT NULL",
|
|
1780
|
+
"_tx_id TEXT NOT NULL",
|
|
1781
|
+
"_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
|
|
1782
|
+
];
|
|
1783
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1784
|
+
const sqlType = TYPE_MAP[col.type];
|
|
1785
|
+
const nullable = col.nullable ? "" : " NOT NULL";
|
|
1786
|
+
let colDef = `${colName} ${sqlType}${nullable}`;
|
|
1787
|
+
if (col.default !== undefined) {
|
|
1788
|
+
colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
|
|
1789
|
+
}
|
|
1790
|
+
columnDefs.push(colDef);
|
|
1791
|
+
}
|
|
1792
|
+
statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
|
|
1793
|
+
${columnDefs.join(`,
|
|
1794
|
+
`)}
|
|
1795
|
+
)`);
|
|
1796
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
|
|
1797
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
|
|
1798
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1799
|
+
if (col.indexed) {
|
|
1800
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1804
|
+
if (col.search) {
|
|
1805
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
if (tableDef.indexes) {
|
|
1809
|
+
for (let i = 0;i < tableDef.indexes.length; i++) {
|
|
1810
|
+
const cols = tableDef.indexes[i];
|
|
1811
|
+
const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
|
|
1812
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
if (tableDef.uniqueKeys) {
|
|
1816
|
+
for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
|
|
1817
|
+
const cols = tableDef.uniqueKeys[i];
|
|
1818
|
+
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
|
|
1819
|
+
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
return statements;
|
|
1823
|
+
}
|
|
1824
|
+
function emitForeignKeyDDL(schemaName, tableName, tableDef) {
|
|
1825
|
+
return (tableDef.relations ?? []).map((rel) => {
|
|
1826
|
+
const constraintName = `fk_${schemaName}_${tableName}_${rel.name}`;
|
|
1827
|
+
return `ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`;
|
|
1828
|
+
});
|
|
1829
|
+
}
|
|
1740
1830
|
function generateSubgraphSQL(def, schemaNameOverride) {
|
|
1741
1831
|
const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
|
|
1742
1832
|
const statements = [];
|
|
@@ -1746,58 +1836,10 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
1746
1836
|
}
|
|
1747
1837
|
statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
|
|
1748
1838
|
for (const [tableName, tableDef] of Object.entries(def.schema)) {
|
|
1749
|
-
|
|
1750
|
-
const columnDefs = [
|
|
1751
|
-
"_id BIGSERIAL PRIMARY KEY",
|
|
1752
|
-
"_block_height BIGINT NOT NULL",
|
|
1753
|
-
"_tx_id TEXT NOT NULL",
|
|
1754
|
-
"_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
|
|
1755
|
-
];
|
|
1756
|
-
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1757
|
-
const sqlType = TYPE_MAP[col.type];
|
|
1758
|
-
const nullable = col.nullable ? "" : " NOT NULL";
|
|
1759
|
-
let colDef = `${colName} ${sqlType}${nullable}`;
|
|
1760
|
-
if (col.default !== undefined) {
|
|
1761
|
-
colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
|
|
1762
|
-
}
|
|
1763
|
-
columnDefs.push(colDef);
|
|
1764
|
-
}
|
|
1765
|
-
statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
|
|
1766
|
-
${columnDefs.join(`,
|
|
1767
|
-
`)}
|
|
1768
|
-
)`);
|
|
1769
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
|
|
1770
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
|
|
1771
|
-
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1772
|
-
if (col.indexed) {
|
|
1773
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1777
|
-
if (col.search) {
|
|
1778
|
-
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
if (tableDef.indexes) {
|
|
1782
|
-
for (let i = 0;i < tableDef.indexes.length; i++) {
|
|
1783
|
-
const cols = tableDef.indexes[i];
|
|
1784
|
-
const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
|
|
1785
|
-
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
|
|
1786
|
-
}
|
|
1787
|
-
}
|
|
1788
|
-
if (tableDef.uniqueKeys) {
|
|
1789
|
-
for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
|
|
1790
|
-
const cols = tableDef.uniqueKeys[i];
|
|
1791
|
-
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
|
|
1792
|
-
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
|
|
1793
|
-
}
|
|
1794
|
-
}
|
|
1839
|
+
statements.push(...emitTableDDL(schemaName, tableName, tableDef));
|
|
1795
1840
|
}
|
|
1796
1841
|
for (const [tableName, tableDef] of Object.entries(def.schema)) {
|
|
1797
|
-
|
|
1798
|
-
const constraintName = `fk_${schemaName}_${tableName}_${rel.name}`;
|
|
1799
|
-
statements.push(`ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`);
|
|
1800
|
-
}
|
|
1842
|
+
statements.push(...emitForeignKeyDDL(schemaName, tableName, tableDef));
|
|
1801
1843
|
}
|
|
1802
1844
|
const hashInput = JSON.stringify({
|
|
1803
1845
|
name: def.name,
|
|
@@ -2499,8 +2541,8 @@ import { randomUUID } from "node:crypto";
|
|
|
2499
2541
|
import { hostname } from "node:os";
|
|
2500
2542
|
import { resolve } from "node:path";
|
|
2501
2543
|
import { pathToFileURL } from "node:url";
|
|
2502
|
-
import { getErrorMessage as
|
|
2503
|
-
import { getTargetDb as
|
|
2544
|
+
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
2545
|
+
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
2504
2546
|
import {
|
|
2505
2547
|
cancelSubgraphOperation,
|
|
2506
2548
|
claimSubgraphOperation,
|
|
@@ -2518,11 +2560,11 @@ import {
|
|
|
2518
2560
|
pgSchemaName as pgSchemaName2,
|
|
2519
2561
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
2520
2562
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2521
|
-
import { logger as
|
|
2563
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2522
2564
|
import {
|
|
2523
|
-
listen
|
|
2565
|
+
listen,
|
|
2524
2566
|
sourceListenerUrl,
|
|
2525
|
-
targetListenerUrl as
|
|
2567
|
+
targetListenerUrl as targetListenerUrl2
|
|
2526
2568
|
} from "@secondlayer/shared/queue/listener";
|
|
2527
2569
|
|
|
2528
2570
|
// src/runtime/catchup-leader.ts
|
|
@@ -2599,860 +2641,6 @@ function startStreamsReorgPoll(onReorg) {
|
|
|
2599
2641
|
};
|
|
2600
2642
|
}
|
|
2601
2643
|
|
|
2602
|
-
// src/runtime/subscription-plane.ts
|
|
2603
|
-
import { logger as logger14 } from "@secondlayer/shared/logger";
|
|
2604
|
-
|
|
2605
|
-
// src/runtime/chain-reorg.ts
|
|
2606
|
-
import { getTargetDb as getTargetDb5 } from "@secondlayer/shared/db";
|
|
2607
|
-
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2608
|
-
var MAX_ORPHANED_PER_SUB = 500;
|
|
2609
|
-
async function handleChainReorg(forkHeight, db = getTargetDb5()) {
|
|
2610
|
-
await db.deleteFrom("subscription_outbox").where("kind", "=", "chain").where("block_height", ">=", forkHeight).where("status", "=", "pending").where("event_type", "like", "chain.%.apply").execute();
|
|
2611
|
-
const delivered = await db.selectFrom("subscription_outbox").select(["subscription_id", "tx_id", "payload"]).where("kind", "=", "chain").where("block_height", ">=", forkHeight).where("status", "=", "delivered").where("event_type", "like", "chain.%.apply").orderBy("block_height").orderBy("id").execute();
|
|
2612
|
-
const bySub = new Map;
|
|
2613
|
-
for (const row of delivered) {
|
|
2614
|
-
const list = bySub.get(row.subscription_id) ?? [];
|
|
2615
|
-
const payload = row.payload;
|
|
2616
|
-
list.push({ tx_id: row.tx_id, event: payload?.event ?? null });
|
|
2617
|
-
bySub.set(row.subscription_id, list);
|
|
2618
|
-
}
|
|
2619
|
-
if (bySub.size > 0) {
|
|
2620
|
-
const rows = [];
|
|
2621
|
-
for (const [subscriptionId, entries] of bySub) {
|
|
2622
|
-
const truncated = entries.length > MAX_ORPHANED_PER_SUB;
|
|
2623
|
-
const payload = {
|
|
2624
|
-
action: "rollback",
|
|
2625
|
-
fork_point_height: forkHeight,
|
|
2626
|
-
orphaned: truncated ? entries.slice(0, MAX_ORPHANED_PER_SUB) : entries,
|
|
2627
|
-
truncated
|
|
2628
|
-
};
|
|
2629
|
-
rows.push({
|
|
2630
|
-
subscription_id: subscriptionId,
|
|
2631
|
-
kind: "chain",
|
|
2632
|
-
subgraph_name: null,
|
|
2633
|
-
table_name: null,
|
|
2634
|
-
block_height: forkHeight,
|
|
2635
|
-
tx_id: null,
|
|
2636
|
-
row_pk: { fork_point_height: forkHeight },
|
|
2637
|
-
event_type: "chain.reorg.rollback",
|
|
2638
|
-
payload,
|
|
2639
|
-
dedup_key: `chainreorg:${subscriptionId}:${forkHeight}`
|
|
2640
|
-
});
|
|
2641
|
-
}
|
|
2642
|
-
await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
|
|
2643
|
-
logger10.info("Chain reorg — emitted rollbacks", {
|
|
2644
|
-
forkPointHeight: forkHeight,
|
|
2645
|
-
subscriptions: bySub.size
|
|
2646
|
-
});
|
|
2647
|
-
}
|
|
2648
|
-
await db.transaction().execute(async (trx) => {
|
|
2649
|
-
const cur = await trx.selectFrom("trigger_evaluator_state").select("last_processed_block").where("id", "=", true).forUpdate().executeTakeFirst();
|
|
2650
|
-
if (cur && Number(cur.last_processed_block) >= forkHeight) {
|
|
2651
|
-
await trx.updateTable("trigger_evaluator_state").set({ last_processed_block: forkHeight - 1, updated_at: new Date }).where("id", "=", true).execute();
|
|
2652
|
-
}
|
|
2653
|
-
});
|
|
2654
|
-
}
|
|
2655
|
-
|
|
2656
|
-
// src/runtime/emitter.ts
|
|
2657
|
-
import {
|
|
2658
|
-
getTargetDb as getTargetDb6
|
|
2659
|
-
} from "@secondlayer/shared/db";
|
|
2660
|
-
import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2661
|
-
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
2662
|
-
import { listen, targetListenerUrl as targetListenerUrl2 } from "@secondlayer/shared/queue/listener";
|
|
2663
|
-
import { sql as sql4 } from "kysely";
|
|
2664
|
-
|
|
2665
|
-
// src/runtime/formats/index.ts
|
|
2666
|
-
import { signSecondlayerWebhook } from "@secondlayer/shared/crypto/secondlayer-webhook";
|
|
2667
|
-
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
2668
|
-
|
|
2669
|
-
// src/runtime/formats/cloudevents.ts
|
|
2670
|
-
function buildCloudEvents(outboxRow, _sub) {
|
|
2671
|
-
const event = {
|
|
2672
|
-
specversion: "1.0",
|
|
2673
|
-
type: outboxRow.event_type,
|
|
2674
|
-
source: `secondlayer:${outboxRow.subgraph_name}`,
|
|
2675
|
-
id: outboxRow.id,
|
|
2676
|
-
time: new Date(outboxRow.created_at).toISOString(),
|
|
2677
|
-
datacontenttype: "application/json",
|
|
2678
|
-
data: outboxRow.payload
|
|
2679
|
-
};
|
|
2680
|
-
return {
|
|
2681
|
-
body: JSON.stringify(event),
|
|
2682
|
-
headers: {
|
|
2683
|
-
"content-type": "application/cloudevents+json; charset=utf-8"
|
|
2684
|
-
}
|
|
2685
|
-
};
|
|
2686
|
-
}
|
|
2687
|
-
|
|
2688
|
-
// src/runtime/formats/cloudflare.ts
|
|
2689
|
-
import { decryptSecret } from "@secondlayer/shared/crypto/secrets";
|
|
2690
|
-
function resolveBearer(sub) {
|
|
2691
|
-
const cfg = sub.auth_config;
|
|
2692
|
-
if (cfg.tokenEnc) {
|
|
2693
|
-
return decryptSecret(Buffer.from(cfg.tokenEnc, "base64"));
|
|
2694
|
-
}
|
|
2695
|
-
return cfg.token ?? null;
|
|
2696
|
-
}
|
|
2697
|
-
function buildCloudflare(outboxRow, sub) {
|
|
2698
|
-
const body = JSON.stringify({
|
|
2699
|
-
params: {
|
|
2700
|
-
...outboxRow.payload,
|
|
2701
|
-
_type: outboxRow.event_type,
|
|
2702
|
-
_outboxId: outboxRow.id
|
|
2703
|
-
}
|
|
2704
|
-
});
|
|
2705
|
-
const headers = {
|
|
2706
|
-
"content-type": "application/json"
|
|
2707
|
-
};
|
|
2708
|
-
const token = resolveBearer(sub);
|
|
2709
|
-
if (token)
|
|
2710
|
-
headers.authorization = `Bearer ${token}`;
|
|
2711
|
-
return { body, headers };
|
|
2712
|
-
}
|
|
2713
|
-
|
|
2714
|
-
// src/runtime/formats/inngest.ts
|
|
2715
|
-
var INNGEST_VERSION = "2026-04-23.v1";
|
|
2716
|
-
function buildInngest(outboxRow) {
|
|
2717
|
-
const event = {
|
|
2718
|
-
name: outboxRow.event_type,
|
|
2719
|
-
data: outboxRow.payload,
|
|
2720
|
-
id: outboxRow.id,
|
|
2721
|
-
ts: new Date(outboxRow.created_at).getTime(),
|
|
2722
|
-
v: INNGEST_VERSION
|
|
2723
|
-
};
|
|
2724
|
-
return {
|
|
2725
|
-
body: JSON.stringify([event]),
|
|
2726
|
-
headers: {
|
|
2727
|
-
"content-type": "application/json"
|
|
2728
|
-
}
|
|
2729
|
-
};
|
|
2730
|
-
}
|
|
2731
|
-
|
|
2732
|
-
// src/runtime/formats/raw.ts
|
|
2733
|
-
function buildRaw(outboxRow, sub) {
|
|
2734
|
-
const cfg = sub.auth_config;
|
|
2735
|
-
const headers = {
|
|
2736
|
-
"content-type": cfg.contentType ?? "application/json",
|
|
2737
|
-
...cfg.headers ?? {}
|
|
2738
|
-
};
|
|
2739
|
-
if (cfg.authType === "bearer" && cfg.token) {
|
|
2740
|
-
headers.authorization = `Bearer ${cfg.token}`;
|
|
2741
|
-
} else if (cfg.authType === "basic" && cfg.basicAuth) {
|
|
2742
|
-
headers.authorization = `Basic ${cfg.basicAuth}`;
|
|
2743
|
-
}
|
|
2744
|
-
return {
|
|
2745
|
-
body: JSON.stringify(outboxRow.payload),
|
|
2746
|
-
headers
|
|
2747
|
-
};
|
|
2748
|
-
}
|
|
2749
|
-
|
|
2750
|
-
// src/runtime/formats/standard-webhooks.ts
|
|
2751
|
-
import { sign } from "@secondlayer/shared/crypto/standard-webhooks";
|
|
2752
|
-
function buildStandardWebhooks(outboxRow, signingSecret) {
|
|
2753
|
-
const nowSeconds = Math.floor(Date.now() / 1000);
|
|
2754
|
-
const payload = {
|
|
2755
|
-
type: outboxRow.event_type,
|
|
2756
|
-
timestamp: new Date(nowSeconds * 1000).toISOString(),
|
|
2757
|
-
data: outboxRow.payload
|
|
2758
|
-
};
|
|
2759
|
-
const body = JSON.stringify(payload);
|
|
2760
|
-
const sigHeaders = sign(body, signingSecret, {
|
|
2761
|
-
id: outboxRow.id,
|
|
2762
|
-
timestampSeconds: nowSeconds
|
|
2763
|
-
});
|
|
2764
|
-
return {
|
|
2765
|
-
body,
|
|
2766
|
-
headers: {
|
|
2767
|
-
"content-type": "application/json",
|
|
2768
|
-
...sigHeaders
|
|
2769
|
-
}
|
|
2770
|
-
};
|
|
2771
|
-
}
|
|
2772
|
-
|
|
2773
|
-
// src/runtime/formats/trigger.ts
|
|
2774
|
-
import { decryptSecret as decryptSecret2 } from "@secondlayer/shared/crypto/secrets";
|
|
2775
|
-
function resolveBearer2(sub) {
|
|
2776
|
-
const cfg = sub.auth_config;
|
|
2777
|
-
if (cfg.tokenEnc) {
|
|
2778
|
-
return decryptSecret2(Buffer.from(cfg.tokenEnc, "base64"));
|
|
2779
|
-
}
|
|
2780
|
-
return cfg.token ?? null;
|
|
2781
|
-
}
|
|
2782
|
-
function buildTrigger(outboxRow, sub) {
|
|
2783
|
-
const body = JSON.stringify({
|
|
2784
|
-
payload: outboxRow.payload,
|
|
2785
|
-
options: {
|
|
2786
|
-
idempotencyKey: outboxRow.id
|
|
2787
|
-
}
|
|
2788
|
-
});
|
|
2789
|
-
const headers = {
|
|
2790
|
-
"content-type": "application/json"
|
|
2791
|
-
};
|
|
2792
|
-
const token = resolveBearer2(sub);
|
|
2793
|
-
if (token)
|
|
2794
|
-
headers.authorization = `Bearer ${token}`;
|
|
2795
|
-
return { body, headers };
|
|
2796
|
-
}
|
|
2797
|
-
|
|
2798
|
-
// src/runtime/formats/index.ts
|
|
2799
|
-
function buildBody(outboxRow, sub, signingSecret) {
|
|
2800
|
-
switch (sub.format) {
|
|
2801
|
-
case "inngest":
|
|
2802
|
-
return buildInngest(outboxRow);
|
|
2803
|
-
case "trigger":
|
|
2804
|
-
return buildTrigger(outboxRow, sub);
|
|
2805
|
-
case "cloudflare":
|
|
2806
|
-
return buildCloudflare(outboxRow, sub);
|
|
2807
|
-
case "cloudevents":
|
|
2808
|
-
return buildCloudEvents(outboxRow, sub);
|
|
2809
|
-
case "raw":
|
|
2810
|
-
return buildRaw(outboxRow, sub);
|
|
2811
|
-
case "standard-webhooks":
|
|
2812
|
-
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2813
|
-
default:
|
|
2814
|
-
logger11.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
2815
|
-
format: sub.format,
|
|
2816
|
-
subscriptionId: sub.id
|
|
2817
|
-
});
|
|
2818
|
-
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2819
|
-
}
|
|
2820
|
-
}
|
|
2821
|
-
function buildForFormat(outboxRow, sub, signingSecret) {
|
|
2822
|
-
const result = buildBody(outboxRow, sub, signingSecret);
|
|
2823
|
-
const sigHeaders = signSecondlayerWebhook(outboxRow.id, result.body);
|
|
2824
|
-
if (sigHeaders) {
|
|
2825
|
-
result.headers = { ...result.headers, ...sigHeaders };
|
|
2826
|
-
}
|
|
2827
|
-
return result;
|
|
2828
|
-
}
|
|
2829
|
-
|
|
2830
|
-
// src/runtime/emitter.ts
|
|
2831
|
-
var BATCH_SIZE = 50;
|
|
2832
|
-
var LIVE_SHARE = 0.9;
|
|
2833
|
-
var BACKOFF_SECONDS = [30, 120, 600, 3600, 21600, 86400, 259200];
|
|
2834
|
-
var CIRCUIT_THRESHOLD = 20;
|
|
2835
|
-
var LOCK_WINDOW_MS = 60000;
|
|
2836
|
-
function nextDelaySeconds(attempt) {
|
|
2837
|
-
return BACKOFF_SECONDS[Math.min(attempt, BACKOFF_SECONDS.length - 1)];
|
|
2838
|
-
}
|
|
2839
|
-
var PRIVATE_V4_PATTERNS = [
|
|
2840
|
-
/^127\./,
|
|
2841
|
-
/^10\./,
|
|
2842
|
-
/^172\.(1[6-9]|2\d|3[01])\./,
|
|
2843
|
-
/^192\.168\./,
|
|
2844
|
-
/^169\.254\./,
|
|
2845
|
-
/^0\./,
|
|
2846
|
-
/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./
|
|
2847
|
-
];
|
|
2848
|
-
function isPrivateEgress(url) {
|
|
2849
|
-
let parsed;
|
|
2850
|
-
try {
|
|
2851
|
-
parsed = new URL(url);
|
|
2852
|
-
} catch {
|
|
2853
|
-
return true;
|
|
2854
|
-
}
|
|
2855
|
-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
2856
|
-
return true;
|
|
2857
|
-
}
|
|
2858
|
-
const raw = parsed.hostname.toLowerCase();
|
|
2859
|
-
const host = raw.startsWith("[") && raw.endsWith("]") ? raw.slice(1, -1) : raw;
|
|
2860
|
-
if (host === "localhost" || host === "0.0.0.0")
|
|
2861
|
-
return true;
|
|
2862
|
-
if (host === "::" || host === "::1")
|
|
2863
|
-
return true;
|
|
2864
|
-
if (/^f[cd][0-9a-f]{2}:/.test(host))
|
|
2865
|
-
return true;
|
|
2866
|
-
if (/^fe[89ab][0-9a-f]:/.test(host))
|
|
2867
|
-
return true;
|
|
2868
|
-
const mapped = host.match(/^::ffff:(.+)$/);
|
|
2869
|
-
if (mapped) {
|
|
2870
|
-
const inner = mapped[1];
|
|
2871
|
-
if (/^\d+\.\d+\.\d+\.\d+$/.test(inner)) {
|
|
2872
|
-
for (const p of PRIVATE_V4_PATTERNS)
|
|
2873
|
-
if (p.test(inner))
|
|
2874
|
-
return true;
|
|
2875
|
-
}
|
|
2876
|
-
const hex = inner.match(/^([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
|
2877
|
-
if (hex) {
|
|
2878
|
-
const a = Number.parseInt(hex[1], 16);
|
|
2879
|
-
const b = Number.parseInt(hex[2], 16);
|
|
2880
|
-
const dotted = `${a >> 8 & 255}.${a & 255}.${b >> 8 & 255}.${b & 255}`;
|
|
2881
|
-
for (const p of PRIVATE_V4_PATTERNS)
|
|
2882
|
-
if (p.test(dotted))
|
|
2883
|
-
return true;
|
|
2884
|
-
}
|
|
2885
|
-
}
|
|
2886
|
-
for (const p of PRIVATE_V4_PATTERNS) {
|
|
2887
|
-
if (p.test(host))
|
|
2888
|
-
return true;
|
|
2889
|
-
}
|
|
2890
|
-
return false;
|
|
2891
|
-
}
|
|
2892
|
-
function allowPrivateEgress() {
|
|
2893
|
-
return process.env.SECONDLAYER_ALLOW_PRIVATE_EGRESS === "true";
|
|
2894
|
-
}
|
|
2895
|
-
async function dispatchOne(db, outboxRow, sub) {
|
|
2896
|
-
const { body, headers } = buildForFormat(outboxRow, sub, getSubscriptionSigningSecret(sub));
|
|
2897
|
-
const start = performance.now();
|
|
2898
|
-
let statusCode = null;
|
|
2899
|
-
let error = null;
|
|
2900
|
-
let ok = false;
|
|
2901
|
-
let responseBody = "";
|
|
2902
|
-
let responseHeaders = {};
|
|
2903
|
-
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
2904
|
-
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
2905
|
-
logger12.warn("[emitter] refused private egress", {
|
|
2906
|
-
subscription: sub.name,
|
|
2907
|
-
url: sub.url
|
|
2908
|
-
});
|
|
2909
|
-
const durationMs2 = 0;
|
|
2910
|
-
const attempt2 = outboxRow.attempt + 1;
|
|
2911
|
-
await db.insertInto("subscription_deliveries").values({
|
|
2912
|
-
outbox_id: outboxRow.id,
|
|
2913
|
-
subscription_id: outboxRow.subscription_id,
|
|
2914
|
-
attempt: attempt2,
|
|
2915
|
-
status_code: null,
|
|
2916
|
-
response_headers: null,
|
|
2917
|
-
response_body: null,
|
|
2918
|
-
error_message: error,
|
|
2919
|
-
duration_ms: durationMs2
|
|
2920
|
-
}).execute();
|
|
2921
|
-
return { ok: false, statusCode: null, error, durationMs: durationMs2 };
|
|
2922
|
-
}
|
|
2923
|
-
try {
|
|
2924
|
-
const res = await fetch(sub.url, {
|
|
2925
|
-
method: "POST",
|
|
2926
|
-
headers,
|
|
2927
|
-
body,
|
|
2928
|
-
signal: AbortSignal.timeout(sub.timeout_ms)
|
|
2929
|
-
});
|
|
2930
|
-
statusCode = res.status;
|
|
2931
|
-
ok = res.ok;
|
|
2932
|
-
const buf = await res.arrayBuffer();
|
|
2933
|
-
const truncated = buf.byteLength > 8192 ? buf.slice(0, 8192) : buf;
|
|
2934
|
-
responseBody = Buffer.from(truncated).toString("utf8");
|
|
2935
|
-
responseHeaders = Object.fromEntries(res.headers.entries());
|
|
2936
|
-
} catch (err) {
|
|
2937
|
-
error = err instanceof Error ? err.message : String(err);
|
|
2938
|
-
}
|
|
2939
|
-
const durationMs = Math.round(performance.now() - start);
|
|
2940
|
-
const attempt = outboxRow.attempt + 1;
|
|
2941
|
-
await db.insertInto("subscription_deliveries").values({
|
|
2942
|
-
outbox_id: outboxRow.id,
|
|
2943
|
-
subscription_id: outboxRow.subscription_id,
|
|
2944
|
-
attempt,
|
|
2945
|
-
status_code: statusCode,
|
|
2946
|
-
response_headers: responseHeaders,
|
|
2947
|
-
response_body: responseBody || null,
|
|
2948
|
-
error_message: error,
|
|
2949
|
-
duration_ms: durationMs
|
|
2950
|
-
}).execute();
|
|
2951
|
-
return { ok, statusCode, error, durationMs };
|
|
2952
|
-
}
|
|
2953
|
-
async function settleDelivered(db, outboxRow) {
|
|
2954
|
-
await db.transaction().execute(async (tx) => {
|
|
2955
|
-
await tx.updateTable("subscription_outbox").set({
|
|
2956
|
-
status: "delivered",
|
|
2957
|
-
delivered_at: new Date,
|
|
2958
|
-
attempt: outboxRow.attempt + 1,
|
|
2959
|
-
locked_by: null,
|
|
2960
|
-
locked_until: null
|
|
2961
|
-
}).where("id", "=", outboxRow.id).execute();
|
|
2962
|
-
await tx.updateTable("subscriptions").set({
|
|
2963
|
-
last_delivery_at: new Date,
|
|
2964
|
-
last_success_at: new Date,
|
|
2965
|
-
circuit_failures: 0,
|
|
2966
|
-
last_error: null,
|
|
2967
|
-
updated_at: new Date
|
|
2968
|
-
}).where("id", "=", outboxRow.subscription_id).execute();
|
|
2969
|
-
});
|
|
2970
|
-
}
|
|
2971
|
-
async function settleFailed(db, outboxRow, sub, errText) {
|
|
2972
|
-
const attempt = outboxRow.attempt + 1;
|
|
2973
|
-
const isDead = attempt >= sub.max_retries;
|
|
2974
|
-
const nextAt = isDead ? null : new Date(Date.now() + nextDelaySeconds(outboxRow.attempt) * 1000);
|
|
2975
|
-
await db.transaction().execute(async (tx) => {
|
|
2976
|
-
await tx.updateTable("subscription_outbox").set({
|
|
2977
|
-
attempt,
|
|
2978
|
-
next_attempt_at: nextAt ?? new Date,
|
|
2979
|
-
status: isDead ? "dead" : "pending",
|
|
2980
|
-
failed_at: isDead ? new Date : null,
|
|
2981
|
-
locked_by: null,
|
|
2982
|
-
locked_until: null
|
|
2983
|
-
}).where("id", "=", outboxRow.id).execute();
|
|
2984
|
-
const incResult = await sql4`
|
|
2985
|
-
UPDATE subscriptions
|
|
2986
|
-
SET circuit_failures = circuit_failures + 1,
|
|
2987
|
-
last_delivery_at = NOW(),
|
|
2988
|
-
last_error = ${errText.slice(0, 500)},
|
|
2989
|
-
updated_at = NOW()
|
|
2990
|
-
WHERE id = ${sub.id}
|
|
2991
|
-
RETURNING circuit_failures
|
|
2992
|
-
`.execute(tx);
|
|
2993
|
-
const newFailures = incResult.rows[0]?.circuit_failures ?? sub.circuit_failures + 1;
|
|
2994
|
-
const shouldTripCircuit = newFailures >= CIRCUIT_THRESHOLD;
|
|
2995
|
-
if (shouldTripCircuit) {
|
|
2996
|
-
await tx.updateTable("subscriptions").set({
|
|
2997
|
-
status: "paused",
|
|
2998
|
-
circuit_opened_at: new Date,
|
|
2999
|
-
updated_at: new Date
|
|
3000
|
-
}).where("id", "=", sub.id).execute();
|
|
3001
|
-
logger12.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
3002
|
-
subscription: sub.name,
|
|
3003
|
-
failures: newFailures
|
|
3004
|
-
});
|
|
3005
|
-
}
|
|
3006
|
-
});
|
|
3007
|
-
}
|
|
3008
|
-
async function claimAndDrain(db, state, emitterId) {
|
|
3009
|
-
if (state.claimInFlight)
|
|
3010
|
-
return 0;
|
|
3011
|
-
state.claimInFlight = true;
|
|
3012
|
-
try {
|
|
3013
|
-
const liveLimit = Math.max(1, Math.round(BATCH_SIZE * LIVE_SHARE));
|
|
3014
|
-
const replayLimit = BATCH_SIZE - liveLimit;
|
|
3015
|
-
const claimed = await db.transaction().execute(async (tx) => {
|
|
3016
|
-
const live = await sql4`
|
|
3017
|
-
SELECT * FROM subscription_outbox
|
|
3018
|
-
WHERE status = 'pending'
|
|
3019
|
-
AND next_attempt_at <= NOW()
|
|
3020
|
-
AND is_replay = FALSE
|
|
3021
|
-
ORDER BY next_attempt_at ASC
|
|
3022
|
-
FOR UPDATE SKIP LOCKED
|
|
3023
|
-
LIMIT ${sql4.lit(liveLimit)}
|
|
3024
|
-
`.execute(tx);
|
|
3025
|
-
const replay = await sql4`
|
|
3026
|
-
SELECT * FROM subscription_outbox
|
|
3027
|
-
WHERE status = 'pending'
|
|
3028
|
-
AND next_attempt_at <= NOW()
|
|
3029
|
-
AND is_replay = TRUE
|
|
3030
|
-
ORDER BY next_attempt_at ASC
|
|
3031
|
-
FOR UPDATE SKIP LOCKED
|
|
3032
|
-
LIMIT ${sql4.lit(replayLimit)}
|
|
3033
|
-
`.execute(tx);
|
|
3034
|
-
const combined = [...live.rows, ...replay.rows];
|
|
3035
|
-
if (combined.length === 0)
|
|
3036
|
-
return [];
|
|
3037
|
-
const now = new Date;
|
|
3038
|
-
const lockUntil = new Date(now.getTime() + LOCK_WINDOW_MS);
|
|
3039
|
-
await tx.updateTable("subscription_outbox").set({
|
|
3040
|
-
locked_by: emitterId,
|
|
3041
|
-
locked_until: lockUntil,
|
|
3042
|
-
next_attempt_at: lockUntil
|
|
3043
|
-
}).where("id", "in", combined.map((r) => r.id)).execute();
|
|
3044
|
-
return combined;
|
|
3045
|
-
});
|
|
3046
|
-
if (claimed.length === 0)
|
|
3047
|
-
return 0;
|
|
3048
|
-
const bySubId = new Map;
|
|
3049
|
-
for (const row of claimed) {
|
|
3050
|
-
const arr = bySubId.get(row.subscription_id);
|
|
3051
|
-
if (arr)
|
|
3052
|
-
arr.push(row);
|
|
3053
|
-
else
|
|
3054
|
-
bySubId.set(row.subscription_id, [row]);
|
|
3055
|
-
}
|
|
3056
|
-
const subIds = Array.from(bySubId.keys());
|
|
3057
|
-
const subs = await db.selectFrom("subscriptions").selectAll().where("id", "in", subIds).execute();
|
|
3058
|
-
const subById = new Map(subs.map((s) => [s.id, s]));
|
|
3059
|
-
await Promise.all(subIds.map((subId) => drainForSub(db, state, subById.get(subId), bySubId.get(subId))));
|
|
3060
|
-
return claimed.length;
|
|
3061
|
-
} finally {
|
|
3062
|
-
state.claimInFlight = false;
|
|
3063
|
-
}
|
|
3064
|
-
}
|
|
3065
|
-
async function drainForSub(db, state, sub, rows) {
|
|
3066
|
-
const cap = sub.concurrency || 4;
|
|
3067
|
-
const counter = () => state.inFlightBySub.get(sub.id) ?? 0;
|
|
3068
|
-
const inc = () => state.inFlightBySub.set(sub.id, counter() + 1);
|
|
3069
|
-
const dec = () => state.inFlightBySub.set(sub.id, Math.max(0, counter() - 1));
|
|
3070
|
-
const queue = [...rows];
|
|
3071
|
-
const workers = [];
|
|
3072
|
-
const slots = Math.min(cap, queue.length);
|
|
3073
|
-
for (let i = 0;i < slots; i++) {
|
|
3074
|
-
workers.push((async () => {
|
|
3075
|
-
while (state.running && queue.length > 0) {
|
|
3076
|
-
const row = queue.shift();
|
|
3077
|
-
if (!row)
|
|
3078
|
-
break;
|
|
3079
|
-
inc();
|
|
3080
|
-
try {
|
|
3081
|
-
const result = await dispatchOne(db, row, sub);
|
|
3082
|
-
if (result.ok) {
|
|
3083
|
-
await settleDelivered(db, row);
|
|
3084
|
-
} else {
|
|
3085
|
-
const err = result.error ?? `HTTP ${result.statusCode ?? "?"}`;
|
|
3086
|
-
await settleFailed(db, row, sub, err);
|
|
3087
|
-
}
|
|
3088
|
-
} catch (err) {
|
|
3089
|
-
logger12.error("Emitter dispatch crashed", {
|
|
3090
|
-
outboxId: row.id,
|
|
3091
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3092
|
-
});
|
|
3093
|
-
await settleFailed(db, row, sub, err instanceof Error ? err.message : String(err));
|
|
3094
|
-
} finally {
|
|
3095
|
-
dec();
|
|
3096
|
-
}
|
|
3097
|
-
}
|
|
3098
|
-
})());
|
|
3099
|
-
}
|
|
3100
|
-
await Promise.all(workers);
|
|
3101
|
-
}
|
|
3102
|
-
async function runRetention(db) {
|
|
3103
|
-
await sql4`
|
|
3104
|
-
DELETE FROM subscription_outbox
|
|
3105
|
-
WHERE status = 'delivered' AND delivered_at < NOW() - interval '7 days'
|
|
3106
|
-
`.execute(db);
|
|
3107
|
-
await sql4`
|
|
3108
|
-
DELETE FROM subscription_deliveries
|
|
3109
|
-
WHERE dispatched_at < NOW() - interval '30 days'
|
|
3110
|
-
`.execute(db);
|
|
3111
|
-
await sql4`
|
|
3112
|
-
DELETE FROM subscription_outbox
|
|
3113
|
-
WHERE status = 'dead' AND failed_at < NOW() - interval '90 days'
|
|
3114
|
-
`.execute(db);
|
|
3115
|
-
}
|
|
3116
|
-
async function startEmitter(opts) {
|
|
3117
|
-
const emitterId = `emitter-${Math.random().toString(36).slice(2, 10)}`;
|
|
3118
|
-
const db = getTargetDb6();
|
|
3119
|
-
const state = {
|
|
3120
|
-
running: true,
|
|
3121
|
-
inFlightBySub: new Map,
|
|
3122
|
-
claimInFlight: false
|
|
3123
|
-
};
|
|
3124
|
-
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
3125
|
-
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
3126
|
-
logger12.info("[emitter] started", { id: emitterId });
|
|
3127
|
-
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
3128
|
-
let lastErr = null;
|
|
3129
|
-
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
3130
|
-
try {
|
|
3131
|
-
await refreshMatcher(db);
|
|
3132
|
-
lastErr = null;
|
|
3133
|
-
break;
|
|
3134
|
-
} catch (err) {
|
|
3135
|
-
lastErr = err;
|
|
3136
|
-
const delayMs = 500 * 2 ** i;
|
|
3137
|
-
logger12.warn("[emitter] matcher refresh failed, retrying", {
|
|
3138
|
-
attempt: i + 1,
|
|
3139
|
-
delayMs,
|
|
3140
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3141
|
-
});
|
|
3142
|
-
await new Promise((r) => setTimeout(r, delayMs));
|
|
3143
|
-
}
|
|
3144
|
-
}
|
|
3145
|
-
if (lastErr) {
|
|
3146
|
-
throw new Error(`[emitter] matcher refresh failed ${MATCHER_BOOT_ATTEMPTS}×; aborting boot: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`);
|
|
3147
|
-
}
|
|
3148
|
-
const listenUrl = targetListenerUrl2();
|
|
3149
|
-
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
3150
|
-
if (!state.running)
|
|
3151
|
-
return;
|
|
3152
|
-
claimAndDrain(db, state, emitterId).catch((err) => logger12.error("[emitter] claim failed", {
|
|
3153
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3154
|
-
}));
|
|
3155
|
-
}, { connectionString: listenUrl });
|
|
3156
|
-
const stopChanged = await listen("subscriptions:changed", () => {
|
|
3157
|
-
if (!state.running)
|
|
3158
|
-
return;
|
|
3159
|
-
refreshMatcher(db).catch((err) => logger12.error("[emitter] matcher refresh failed", {
|
|
3160
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3161
|
-
}));
|
|
3162
|
-
}, { connectionString: listenUrl });
|
|
3163
|
-
const poll = setInterval(() => {
|
|
3164
|
-
if (!state.running)
|
|
3165
|
-
return;
|
|
3166
|
-
claimAndDrain(db, state, emitterId).catch((err) => logger12.error("[emitter] poll claim failed", {
|
|
3167
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3168
|
-
}));
|
|
3169
|
-
}, pollIntervalMs);
|
|
3170
|
-
claimAndDrain(db, state, emitterId);
|
|
3171
|
-
const retention = setInterval(() => {
|
|
3172
|
-
if (!state.running)
|
|
3173
|
-
return;
|
|
3174
|
-
runRetention(db).catch((err) => logger12.error("[emitter] retention failed", {
|
|
3175
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3176
|
-
}));
|
|
3177
|
-
}, retentionIntervalMs);
|
|
3178
|
-
return async () => {
|
|
3179
|
-
state.running = false;
|
|
3180
|
-
clearInterval(poll);
|
|
3181
|
-
clearInterval(retention);
|
|
3182
|
-
await stopNew();
|
|
3183
|
-
await stopChanged();
|
|
3184
|
-
logger12.info("[emitter] stopped", { id: emitterId });
|
|
3185
|
-
};
|
|
3186
|
-
}
|
|
3187
|
-
|
|
3188
|
-
// src/runtime/subscription-leader.ts
|
|
3189
|
-
import {
|
|
3190
|
-
SUBSCRIPTION_EVALUATOR_LOCK_KEY,
|
|
3191
|
-
createPostgresLeaderBackend as createPostgresLeaderBackend2,
|
|
3192
|
-
withLeaderLock as withLeaderLock2
|
|
3193
|
-
} from "@secondlayer/shared/leader";
|
|
3194
|
-
import { targetListenerUrl as targetListenerUrl3 } from "@secondlayer/shared/queue/listener";
|
|
3195
|
-
|
|
3196
|
-
// src/runtime/trigger-evaluator-loop.ts
|
|
3197
|
-
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
3198
|
-
import { getTargetDb as getTargetDb7 } from "@secondlayer/shared/db";
|
|
3199
|
-
import { listActiveChainSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
3200
|
-
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3201
|
-
|
|
3202
|
-
// src/runtime/trigger-evaluator.ts
|
|
3203
|
-
import { resolveTraitContractIds as resolveTraitContractIds2 } from "@secondlayer/shared/db/queries/contracts";
|
|
3204
|
-
var TX_LEVEL_TRIGGER_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
3205
|
-
function sourceKey(subscriptionId, triggerIndex) {
|
|
3206
|
-
return `${subscriptionId}#${triggerIndex}`;
|
|
3207
|
-
}
|
|
3208
|
-
function toAmount(v) {
|
|
3209
|
-
return v === undefined ? undefined : BigInt(v);
|
|
3210
|
-
}
|
|
3211
|
-
function chainTriggerToFilter(trigger) {
|
|
3212
|
-
const t = trigger;
|
|
3213
|
-
const filter = { ...trigger };
|
|
3214
|
-
const minAmount = toAmount(t.minAmount);
|
|
3215
|
-
const maxAmount = toAmount(t.maxAmount);
|
|
3216
|
-
if (minAmount !== undefined)
|
|
3217
|
-
filter.minAmount = minAmount;
|
|
3218
|
-
if (maxAmount !== undefined)
|
|
3219
|
-
filter.maxAmount = maxAmount;
|
|
3220
|
-
return filter;
|
|
3221
|
-
}
|
|
3222
|
-
function triggersOf(sub) {
|
|
3223
|
-
return sub.triggers ?? [];
|
|
3224
|
-
}
|
|
3225
|
-
function buildSourcesMap(chainSubs) {
|
|
3226
|
-
const sources = {};
|
|
3227
|
-
const keyMeta = new Map;
|
|
3228
|
-
for (const sub of chainSubs) {
|
|
3229
|
-
triggersOf(sub).forEach((trigger, triggerIndex) => {
|
|
3230
|
-
const key2 = sourceKey(sub.id, triggerIndex);
|
|
3231
|
-
sources[key2] = chainTriggerToFilter(trigger);
|
|
3232
|
-
keyMeta.set(key2, {
|
|
3233
|
-
subscriptionId: sub.id,
|
|
3234
|
-
triggerIndex,
|
|
3235
|
-
triggerType: trigger.type
|
|
3236
|
-
});
|
|
3237
|
-
});
|
|
3238
|
-
}
|
|
3239
|
-
return { sources, keyMeta };
|
|
3240
|
-
}
|
|
3241
|
-
function referencedEventTypes(chainSubs) {
|
|
3242
|
-
const filterTypes = new Set;
|
|
3243
|
-
for (const sub of chainSubs) {
|
|
3244
|
-
for (const trigger of triggersOf(sub))
|
|
3245
|
-
filterTypes.add(trigger.type);
|
|
3246
|
-
}
|
|
3247
|
-
return indexEventTypesForFilterTypes([...filterTypes]);
|
|
3248
|
-
}
|
|
3249
|
-
function referencedTraits(chainSubs) {
|
|
3250
|
-
const traits = new Set;
|
|
3251
|
-
for (const sub of chainSubs) {
|
|
3252
|
-
for (const trigger of triggersOf(sub)) {
|
|
3253
|
-
const trait = trigger.trait;
|
|
3254
|
-
if (trait)
|
|
3255
|
-
traits.add(trait);
|
|
3256
|
-
}
|
|
3257
|
-
}
|
|
3258
|
-
return [...traits];
|
|
3259
|
-
}
|
|
3260
|
-
async function buildTraitContracts(db, chainSubs, asOfBlock) {
|
|
3261
|
-
const resolved = new Map;
|
|
3262
|
-
for (const trait of referencedTraits(chainSubs)) {
|
|
3263
|
-
const ids = await resolveTraitContractIds2(db, trait, asOfBlock);
|
|
3264
|
-
resolved.set(trait, new Set(ids));
|
|
3265
|
-
}
|
|
3266
|
-
return resolved;
|
|
3267
|
-
}
|
|
3268
|
-
function evaluateBlock(block, sources, traitContracts) {
|
|
3269
|
-
return matchSources(sources, block.txs, block.events, traitContracts);
|
|
3270
|
-
}
|
|
3271
|
-
function chainDedupKey(subscriptionId, txId, eventIndex, blockHash, replayId) {
|
|
3272
|
-
const base = `chain:${subscriptionId}:${txId}:${eventIndex}:${blockHash}`;
|
|
3273
|
-
return replayId ? `replay:${replayId}:${base}` : base;
|
|
3274
|
-
}
|
|
3275
|
-
function applyRow(meta, blockHeight, blockHash, txId, eventIndex, event, replayId) {
|
|
3276
|
-
const payload = {
|
|
3277
|
-
action: "apply",
|
|
3278
|
-
block_hash: blockHash,
|
|
3279
|
-
block_height: blockHeight,
|
|
3280
|
-
tx_id: txId,
|
|
3281
|
-
canonical: true,
|
|
3282
|
-
trigger: meta.triggerType,
|
|
3283
|
-
event
|
|
3284
|
-
};
|
|
3285
|
-
return {
|
|
3286
|
-
subscription_id: meta.subscriptionId,
|
|
3287
|
-
kind: "chain",
|
|
3288
|
-
subgraph_name: null,
|
|
3289
|
-
table_name: null,
|
|
3290
|
-
block_height: blockHeight,
|
|
3291
|
-
tx_id: txId,
|
|
3292
|
-
row_pk: { tx_id: txId, event_index: eventIndex },
|
|
3293
|
-
event_type: `chain.${meta.triggerType}.apply`,
|
|
3294
|
-
payload,
|
|
3295
|
-
dedup_key: chainDedupKey(meta.subscriptionId, txId, eventIndex, blockHash, replayId),
|
|
3296
|
-
...replayId ? { is_replay: true } : {}
|
|
3297
|
-
};
|
|
3298
|
-
}
|
|
3299
|
-
async function emitChainOutbox(db, matches, keyMeta, blockHeight, blockHash, opts) {
|
|
3300
|
-
const replayId = opts?.replayId;
|
|
3301
|
-
const rows = [];
|
|
3302
|
-
for (const match of matches) {
|
|
3303
|
-
const meta = keyMeta.get(match.sourceName);
|
|
3304
|
-
if (!meta)
|
|
3305
|
-
continue;
|
|
3306
|
-
const txId = match.tx.tx_id;
|
|
3307
|
-
if (TX_LEVEL_TRIGGER_TYPES.has(meta.triggerType)) {
|
|
3308
|
-
rows.push(applyRow(meta, blockHeight, blockHash, txId, -1, {
|
|
3309
|
-
tx_id: txId,
|
|
3310
|
-
type: match.tx.type,
|
|
3311
|
-
sender: match.tx.sender,
|
|
3312
|
-
status: match.tx.status,
|
|
3313
|
-
contract_id: match.tx.contract_id ?? null,
|
|
3314
|
-
function_name: match.tx.function_name ?? null,
|
|
3315
|
-
function_args: match.tx.function_args ?? null,
|
|
3316
|
-
result_hex: match.tx.raw_result ?? null
|
|
3317
|
-
}, replayId));
|
|
3318
|
-
} else {
|
|
3319
|
-
for (const event of match.events) {
|
|
3320
|
-
rows.push(applyRow(meta, blockHeight, blockHash, txId, event.event_index, {
|
|
3321
|
-
tx_id: txId,
|
|
3322
|
-
type: event.type,
|
|
3323
|
-
event_index: event.event_index,
|
|
3324
|
-
data: event.data
|
|
3325
|
-
}, replayId));
|
|
3326
|
-
}
|
|
3327
|
-
}
|
|
3328
|
-
}
|
|
3329
|
-
if (rows.length === 0)
|
|
3330
|
-
return 0;
|
|
3331
|
-
const result = await db.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).executeTakeFirst();
|
|
3332
|
-
return Number(result.numInsertedOrUpdatedRows ?? 0);
|
|
3333
|
-
}
|
|
3334
|
-
|
|
3335
|
-
// src/runtime/trigger-evaluator-loop.ts
|
|
3336
|
-
var POLL_MS2 = Number(process.env.TRIGGER_EVALUATOR_POLL_MS) || 5000;
|
|
3337
|
-
var BATCH = Number(process.env.TRIGGER_EVALUATOR_BATCH) || 200;
|
|
3338
|
-
var MAX_BLOCKS_PER_TICK = Number(process.env.TRIGGER_EVALUATOR_MAX_BLOCKS) || 2000;
|
|
3339
|
-
async function readCursor(db) {
|
|
3340
|
-
const row = await db.selectFrom("trigger_evaluator_state").select("last_processed_block").where("id", "=", true).executeTakeFirst();
|
|
3341
|
-
return row ? Number(row.last_processed_block) : 0;
|
|
3342
|
-
}
|
|
3343
|
-
async function advanceCursor(db, to) {
|
|
3344
|
-
await db.transaction().execute(async (trx) => {
|
|
3345
|
-
const cur = await trx.selectFrom("trigger_evaluator_state").select("last_processed_block").where("id", "=", true).forUpdate().executeTakeFirst();
|
|
3346
|
-
if (cur && Number(cur.last_processed_block) < to) {
|
|
3347
|
-
await trx.updateTable("trigger_evaluator_state").set({ last_processed_block: to, updated_at: new Date }).where("id", "=", true).execute();
|
|
3348
|
-
}
|
|
3349
|
-
});
|
|
3350
|
-
}
|
|
3351
|
-
async function runEvaluatorOnce(db = getTargetDb7()) {
|
|
3352
|
-
const chainSubs = await listActiveChainSubscriptions(db);
|
|
3353
|
-
const source = new PublicApiBlockSource(buildHttpClient(), referencedEventTypes(chainSubs));
|
|
3354
|
-
const tip = await source.getTip();
|
|
3355
|
-
if (tip <= 0)
|
|
3356
|
-
return 0;
|
|
3357
|
-
const cursor = await readCursor(db);
|
|
3358
|
-
if (cursor === 0 || chainSubs.length === 0) {
|
|
3359
|
-
await advanceCursor(db, tip);
|
|
3360
|
-
return 0;
|
|
3361
|
-
}
|
|
3362
|
-
if (cursor >= tip)
|
|
3363
|
-
return 0;
|
|
3364
|
-
const { sources, keyMeta } = buildSourcesMap(chainSubs);
|
|
3365
|
-
const target = Math.min(tip, cursor + MAX_BLOCKS_PER_TICK);
|
|
3366
|
-
let emitted = 0;
|
|
3367
|
-
for (let from = cursor + 1;from <= target; from = from + BATCH) {
|
|
3368
|
-
const to = Math.min(from + BATCH - 1, target);
|
|
3369
|
-
const blocks = await source.loadBlockRange(from, to);
|
|
3370
|
-
const traitContracts = await buildTraitContracts(db, chainSubs, to);
|
|
3371
|
-
for (let h = from;h <= to; h++) {
|
|
3372
|
-
const bd = blocks.get(h);
|
|
3373
|
-
if (!bd)
|
|
3374
|
-
continue;
|
|
3375
|
-
const matches = evaluateBlock(bd, sources, traitContracts);
|
|
3376
|
-
if (matches.length === 0)
|
|
3377
|
-
continue;
|
|
3378
|
-
emitted += await emitChainOutbox(db, matches, keyMeta, h, bd.block.hash);
|
|
3379
|
-
}
|
|
3380
|
-
await advanceCursor(db, to);
|
|
3381
|
-
}
|
|
3382
|
-
return emitted;
|
|
3383
|
-
}
|
|
3384
|
-
function startTriggerEvaluator() {
|
|
3385
|
-
let running = true;
|
|
3386
|
-
let timer;
|
|
3387
|
-
const tick = async () => {
|
|
3388
|
-
if (!running)
|
|
3389
|
-
return;
|
|
3390
|
-
try {
|
|
3391
|
-
const emitted = await runEvaluatorOnce();
|
|
3392
|
-
if (emitted > 0) {
|
|
3393
|
-
logger13.info("Trigger evaluator emitted chain deliveries", {
|
|
3394
|
-
count: emitted
|
|
3395
|
-
});
|
|
3396
|
-
}
|
|
3397
|
-
} catch (err) {
|
|
3398
|
-
logger13.error("Trigger evaluator tick failed", {
|
|
3399
|
-
error: getErrorMessage5(err)
|
|
3400
|
-
});
|
|
3401
|
-
}
|
|
3402
|
-
if (running)
|
|
3403
|
-
timer = setTimeout(tick, POLL_MS2);
|
|
3404
|
-
};
|
|
3405
|
-
timer = setTimeout(tick, POLL_MS2);
|
|
3406
|
-
logger13.info("Trigger evaluator started", { pollMs: POLL_MS2 });
|
|
3407
|
-
return () => {
|
|
3408
|
-
running = false;
|
|
3409
|
-
if (timer)
|
|
3410
|
-
clearTimeout(timer);
|
|
3411
|
-
};
|
|
3412
|
-
}
|
|
3413
|
-
|
|
3414
|
-
// src/runtime/subscription-leader.ts
|
|
3415
|
-
var evaluatorLeader = false;
|
|
3416
|
-
function isEvaluatorLeader() {
|
|
3417
|
-
return evaluatorLeader;
|
|
3418
|
-
}
|
|
3419
|
-
function gateChainReorgOnLeader(handler, isLeader = isEvaluatorLeader) {
|
|
3420
|
-
return async (forkHeight) => {
|
|
3421
|
-
if (!isLeader())
|
|
3422
|
-
return;
|
|
3423
|
-
await handler(forkHeight);
|
|
3424
|
-
};
|
|
3425
|
-
}
|
|
3426
|
-
function startTriggerEvaluatorLeader(opts = {}) {
|
|
3427
|
-
const startWork = opts.startWork ?? startTriggerEvaluator;
|
|
3428
|
-
return withLeaderLock2(SUBSCRIPTION_EVALUATOR_LOCK_KEY, async () => {
|
|
3429
|
-
evaluatorLeader = true;
|
|
3430
|
-
const stop = await startWork();
|
|
3431
|
-
return () => {
|
|
3432
|
-
evaluatorLeader = false;
|
|
3433
|
-
stop();
|
|
3434
|
-
};
|
|
3435
|
-
}, {
|
|
3436
|
-
pollMs: opts.pollMs,
|
|
3437
|
-
heartbeatMs: opts.heartbeatMs,
|
|
3438
|
-
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend2(targetListenerUrl3()))
|
|
3439
|
-
});
|
|
3440
|
-
}
|
|
3441
|
-
|
|
3442
|
-
// src/runtime/subscription-plane.ts
|
|
3443
|
-
async function startSubscriptionPlane() {
|
|
3444
|
-
const streamsIndex = process.env.SUBGRAPH_SOURCE === "streams-index";
|
|
3445
|
-
const stopChainReorgPoll = streamsIndex ? startStreamsReorgPoll(gateChainReorgOnLeader((forkHeight) => handleChainReorg(forkHeight))) : undefined;
|
|
3446
|
-
const stopTriggerEvaluator = streamsIndex ? startTriggerEvaluatorLeader() : undefined;
|
|
3447
|
-
const stopEmitter = await startEmitter();
|
|
3448
|
-
logger14.info("Subscription plane ready", { streamsIndex });
|
|
3449
|
-
return async () => {
|
|
3450
|
-
stopChainReorgPoll?.();
|
|
3451
|
-
await stopTriggerEvaluator?.();
|
|
3452
|
-
await stopEmitter();
|
|
3453
|
-
};
|
|
3454
|
-
}
|
|
3455
|
-
|
|
3456
2644
|
// src/runtime/processor.ts
|
|
3457
2645
|
var CHANNEL_NEW_BLOCK = "indexer:new_block";
|
|
3458
2646
|
var CHANNEL_SUBGRAPH_OPERATIONS = "subgraph_operations:new";
|
|
@@ -3472,11 +2660,11 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
3472
2660
|
const def = await loadSubgraphDefinition(sg);
|
|
3473
2661
|
await catchUpSubgraph(def, sg.name);
|
|
3474
2662
|
} catch (err) {
|
|
3475
|
-
const msg =
|
|
2663
|
+
const msg = getErrorMessage5(err);
|
|
3476
2664
|
if (isHandlerNotFoundError(err)) {
|
|
3477
2665
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
3478
2666
|
}
|
|
3479
|
-
|
|
2667
|
+
logger10.error("Subgraph catch-up failed", {
|
|
3480
2668
|
subgraph: sg.name,
|
|
3481
2669
|
error: msg
|
|
3482
2670
|
});
|
|
@@ -3516,7 +2704,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
3516
2704
|
definitionCache.set(sg.name, def);
|
|
3517
2705
|
if (prevVersion && prevVersion !== sg.version) {
|
|
3518
2706
|
invalidateSubgraphRoute(sg.name);
|
|
3519
|
-
|
|
2707
|
+
logger10.info("Subgraph handler reloaded", {
|
|
3520
2708
|
subgraph: sg.name,
|
|
3521
2709
|
from: prevVersion,
|
|
3522
2710
|
to: sg.version
|
|
@@ -3535,7 +2723,7 @@ function cleanupCaches(active) {
|
|
|
3535
2723
|
}
|
|
3536
2724
|
}
|
|
3537
2725
|
async function synthesizeLegacyReindexOperations() {
|
|
3538
|
-
const db =
|
|
2726
|
+
const db = getTargetDb5();
|
|
3539
2727
|
const stale = (await listSubgraphs2(db)).filter((sg) => sg.status === "reindexing");
|
|
3540
2728
|
for (const sg of stale) {
|
|
3541
2729
|
const active = await findActiveSubgraphOperation(db, sg.id);
|
|
@@ -3550,7 +2738,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
3550
2738
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
3551
2739
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
3552
2740
|
});
|
|
3553
|
-
|
|
2741
|
+
logger10.info("Queued legacy reindex resume operation", {
|
|
3554
2742
|
subgraph: sg.name
|
|
3555
2743
|
});
|
|
3556
2744
|
} catch (err) {
|
|
@@ -3564,7 +2752,7 @@ async function runSubgraphOperation(operation, signal) {
|
|
|
3564
2752
|
if (operation.cancel_requested) {
|
|
3565
2753
|
return 0;
|
|
3566
2754
|
}
|
|
3567
|
-
const db =
|
|
2755
|
+
const db = getTargetDb5();
|
|
3568
2756
|
const subgraph = await db.selectFrom("subgraphs").selectAll().where("id", "=", operation.subgraph_id).executeTakeFirst();
|
|
3569
2757
|
if (!subgraph)
|
|
3570
2758
|
throw new Error(`Subgraph not found: ${operation.subgraph_id}`);
|
|
@@ -3600,13 +2788,13 @@ async function runSubgraphOperation(operation, signal) {
|
|
|
3600
2788
|
}
|
|
3601
2789
|
async function startSubgraphOperationRunner(opts) {
|
|
3602
2790
|
const concurrency = opts?.concurrency ?? DEFAULT_OPERATION_CONCURRENCY;
|
|
3603
|
-
const db =
|
|
2791
|
+
const db = getTargetDb5();
|
|
3604
2792
|
const lockedBy = `${hostname()}:${process.pid}:${randomUUID()}`;
|
|
3605
2793
|
const active = new Map;
|
|
3606
2794
|
const activeRuns = new Map;
|
|
3607
2795
|
let running = true;
|
|
3608
2796
|
let draining = false;
|
|
3609
|
-
|
|
2797
|
+
logger10.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
3610
2798
|
const startOperation = (operation) => {
|
|
3611
2799
|
const controller = new AbortController;
|
|
3612
2800
|
active.set(operation.id, controller);
|
|
@@ -3614,9 +2802,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3614
2802
|
if (!running)
|
|
3615
2803
|
return;
|
|
3616
2804
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
3617
|
-
|
|
2805
|
+
logger10.warn("Subgraph operation heartbeat failed", {
|
|
3618
2806
|
operationId: operation.id,
|
|
3619
|
-
error:
|
|
2807
|
+
error: getErrorMessage5(err)
|
|
3620
2808
|
});
|
|
3621
2809
|
});
|
|
3622
2810
|
}, HEARTBEAT_INTERVAL_MS);
|
|
@@ -3626,9 +2814,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3626
2814
|
controller.abort("user-cancelled");
|
|
3627
2815
|
}
|
|
3628
2816
|
}).catch((err) => {
|
|
3629
|
-
|
|
2817
|
+
logger10.warn("Subgraph operation cancel poll failed", {
|
|
3630
2818
|
operationId: operation.id,
|
|
3631
|
-
error:
|
|
2819
|
+
error: getErrorMessage5(err)
|
|
3632
2820
|
});
|
|
3633
2821
|
});
|
|
3634
2822
|
}, CANCEL_POLL_INTERVAL_MS);
|
|
@@ -3643,14 +2831,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3643
2831
|
const reason = String(controller.signal.reason ?? "");
|
|
3644
2832
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3645
2833
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3646
|
-
|
|
2834
|
+
logger10.info("Subgraph operation cancelled", {
|
|
3647
2835
|
operationId: operation.id,
|
|
3648
2836
|
subgraph: operation.subgraph_name
|
|
3649
2837
|
});
|
|
3650
2838
|
return;
|
|
3651
2839
|
}
|
|
3652
2840
|
if (controller.signal.aborted) {
|
|
3653
|
-
|
|
2841
|
+
logger10.info("Subgraph operation interrupted", {
|
|
3654
2842
|
operationId: operation.id,
|
|
3655
2843
|
subgraph: operation.subgraph_name,
|
|
3656
2844
|
reason
|
|
@@ -3658,7 +2846,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3658
2846
|
return;
|
|
3659
2847
|
}
|
|
3660
2848
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3661
|
-
|
|
2849
|
+
logger10.info("Subgraph operation completed", {
|
|
3662
2850
|
operationId: operation.id,
|
|
3663
2851
|
subgraph: operation.subgraph_name,
|
|
3664
2852
|
processed
|
|
@@ -3666,7 +2854,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3666
2854
|
} catch (err) {
|
|
3667
2855
|
const reason = String(controller.signal.reason ?? "");
|
|
3668
2856
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3669
|
-
|
|
2857
|
+
logger10.info("Subgraph operation interrupted by shutdown", {
|
|
3670
2858
|
operationId: operation.id,
|
|
3671
2859
|
subgraph: operation.subgraph_name
|
|
3672
2860
|
});
|
|
@@ -3676,11 +2864,11 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3676
2864
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3677
2865
|
return;
|
|
3678
2866
|
}
|
|
3679
|
-
await failSubgraphOperation(db, operation.id, lockedBy,
|
|
3680
|
-
|
|
2867
|
+
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage5(err), processed);
|
|
2868
|
+
logger10.error("Subgraph operation failed", {
|
|
3681
2869
|
operationId: operation.id,
|
|
3682
2870
|
subgraph: operation.subgraph_name,
|
|
3683
|
-
error:
|
|
2871
|
+
error: getErrorMessage5(err)
|
|
3684
2872
|
});
|
|
3685
2873
|
} finally {
|
|
3686
2874
|
clearInterval(heartbeat);
|
|
@@ -3710,9 +2898,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3710
2898
|
};
|
|
3711
2899
|
await synthesizeLegacyReindexOperations();
|
|
3712
2900
|
await drain();
|
|
3713
|
-
const stopListening = await
|
|
2901
|
+
const stopListening = await listen(CHANNEL_SUBGRAPH_OPERATIONS, () => {
|
|
3714
2902
|
drain();
|
|
3715
|
-
}, { connectionString:
|
|
2903
|
+
}, { connectionString: targetListenerUrl2() });
|
|
3716
2904
|
const pollInterval = setInterval(() => {
|
|
3717
2905
|
drain();
|
|
3718
2906
|
}, POLL_INTERVAL_MS);
|
|
@@ -3724,29 +2912,29 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3724
2912
|
controller.abort("shutdown");
|
|
3725
2913
|
}
|
|
3726
2914
|
await Promise.allSettled(activeRuns.values());
|
|
3727
|
-
|
|
2915
|
+
logger10.info("Subgraph operation runner stopped");
|
|
3728
2916
|
};
|
|
3729
2917
|
}
|
|
3730
2918
|
async function startSubgraphProcessor(opts) {
|
|
3731
2919
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3732
2920
|
let running = true;
|
|
3733
|
-
|
|
2921
|
+
logger10.info("Starting subgraph processor", { concurrency });
|
|
3734
2922
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3735
2923
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3736
2924
|
});
|
|
3737
2925
|
const runCatchUp = async () => {
|
|
3738
2926
|
if (!running || !isCatchUpLeader())
|
|
3739
2927
|
return;
|
|
3740
|
-
const db =
|
|
2928
|
+
const db = getTargetDb5();
|
|
3741
2929
|
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
3742
2930
|
cleanupCaches(subgraphs);
|
|
3743
2931
|
await catchUpAll(subgraphs, db, concurrency);
|
|
3744
2932
|
};
|
|
3745
2933
|
const stopCatchUpLeader = startCatchUpLeader({ onAcquire: runCatchUp });
|
|
3746
|
-
const stopListening = await
|
|
2934
|
+
const stopListening = await listen(CHANNEL_NEW_BLOCK, async () => {
|
|
3747
2935
|
await runCatchUp();
|
|
3748
2936
|
}, { connectionString: sourceListenerUrl() });
|
|
3749
|
-
const stopReorgListening = await
|
|
2937
|
+
const stopReorgListening = await listen("subgraph_reorg", async (payload) => {
|
|
3750
2938
|
if (!running)
|
|
3751
2939
|
return;
|
|
3752
2940
|
try {
|
|
@@ -3756,8 +2944,8 @@ async function startSubgraphProcessor(opts) {
|
|
|
3756
2944
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3757
2945
|
}
|
|
3758
2946
|
} catch (err) {
|
|
3759
|
-
|
|
3760
|
-
error:
|
|
2947
|
+
logger10.error("Subgraph reorg handling failed", {
|
|
2948
|
+
error: getErrorMessage5(err)
|
|
3761
2949
|
});
|
|
3762
2950
|
}
|
|
3763
2951
|
}, { connectionString: sourceListenerUrl() });
|
|
@@ -3765,8 +2953,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
3765
2953
|
runCatchUp();
|
|
3766
2954
|
}, POLL_INTERVAL_MS);
|
|
3767
2955
|
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll((forkHeight) => handleSubgraphReorg(forkHeight, loadSubgraphDefinition)) : undefined;
|
|
3768
|
-
|
|
3769
|
-
logger15.info("Subgraph processor ready");
|
|
2956
|
+
logger10.info("Subgraph processor ready");
|
|
3770
2957
|
return async () => {
|
|
3771
2958
|
running = false;
|
|
3772
2959
|
clearInterval(pollInterval);
|
|
@@ -3774,23 +2961,22 @@ async function startSubgraphProcessor(opts) {
|
|
|
3774
2961
|
await stopListening();
|
|
3775
2962
|
await stopReorgListening();
|
|
3776
2963
|
stopStreamsReorgPoll?.();
|
|
3777
|
-
await stopSubscriptionPlane();
|
|
3778
2964
|
await stopOperations();
|
|
3779
|
-
|
|
2965
|
+
logger10.info("Subgraph processor stopped");
|
|
3780
2966
|
};
|
|
3781
2967
|
}
|
|
3782
2968
|
|
|
3783
2969
|
// src/service.ts
|
|
3784
2970
|
import { assertDbSplit, getDb } from "@secondlayer/shared/db";
|
|
3785
|
-
import { logger as
|
|
3786
|
-
import { sql as
|
|
2971
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
2972
|
+
import { sql as sql4 } from "kysely";
|
|
3787
2973
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
3788
2974
|
var SERVICE_NAME = "subgraph-processor";
|
|
3789
2975
|
async function writeHeartbeat() {
|
|
3790
2976
|
try {
|
|
3791
|
-
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at:
|
|
2977
|
+
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql4`now()` })).execute();
|
|
3792
2978
|
} catch (err) {
|
|
3793
|
-
|
|
2979
|
+
logger11.warn("subgraph-processor heartbeat write failed", {
|
|
3794
2980
|
error: err instanceof Error ? err.message : String(err)
|
|
3795
2981
|
});
|
|
3796
2982
|
}
|
|
@@ -3802,7 +2988,7 @@ var processor = await startSubgraphProcessor({
|
|
|
3802
2988
|
await writeHeartbeat();
|
|
3803
2989
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
3804
2990
|
var shutdown = async () => {
|
|
3805
|
-
|
|
2991
|
+
logger11.info("Shutting down subgraph processor...");
|
|
3806
2992
|
clearInterval(heartbeatInterval);
|
|
3807
2993
|
await processor();
|
|
3808
2994
|
process.exit(0);
|
|
@@ -3810,5 +2996,5 @@ var shutdown = async () => {
|
|
|
3810
2996
|
process.on("SIGINT", shutdown);
|
|
3811
2997
|
process.on("SIGTERM", shutdown);
|
|
3812
2998
|
|
|
3813
|
-
//# debugId=
|
|
2999
|
+
//# debugId=9428CDEA9E276AF264756E2164756E21
|
|
3814
3000
|
//# sourceMappingURL=service.js.map
|