@secondlayer/subgraphs 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +16 -1
- package/dist/src/index.js +325 -112
- package/dist/src/index.js.map +12 -10
- package/dist/src/runtime/block-processor.js +290 -29
- package/dist/src/runtime/block-processor.js.map +8 -5
- package/dist/src/runtime/catchup.js +304 -90
- package/dist/src/runtime/catchup.js.map +9 -7
- package/dist/src/runtime/processor.js +431 -170
- package/dist/src/runtime/processor.js.map +14 -11
- package/dist/src/runtime/reindex.js +323 -110
- package/dist/src/runtime/reindex.js.map +10 -8
- package/dist/src/runtime/reorg.js +296 -35
- package/dist/src/runtime/reorg.js.map +8 -5
- package/dist/src/runtime/runner.d.ts +6 -1
- package/dist/src/runtime/runner.js +7 -6
- package/dist/src/runtime/runner.js.map +3 -3
- package/dist/src/schema/index.js.map +1 -1
- package/dist/src/service.js +434 -173
- package/dist/src/service.js.map +13 -10
- package/package.json +2 -2
package/dist/src/service.js
CHANGED
|
@@ -518,21 +518,21 @@ function buildEventPayload(filter, tx, event) {
|
|
|
518
518
|
return {
|
|
519
519
|
sender: decoded.sender,
|
|
520
520
|
recipient: decoded.recipient,
|
|
521
|
-
tokenId: decoded.value,
|
|
521
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
522
522
|
assetIdentifier: decoded.asset_identifier,
|
|
523
523
|
tx: txMeta
|
|
524
524
|
};
|
|
525
525
|
case "nft_mint":
|
|
526
526
|
return {
|
|
527
527
|
recipient: decoded.recipient,
|
|
528
|
-
tokenId: decoded.value,
|
|
528
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
529
529
|
assetIdentifier: decoded.asset_identifier,
|
|
530
530
|
tx: txMeta
|
|
531
531
|
};
|
|
532
532
|
case "nft_burn":
|
|
533
533
|
return {
|
|
534
534
|
sender: decoded.sender,
|
|
535
|
-
tokenId: decoded.value,
|
|
535
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
536
536
|
assetIdentifier: decoded.asset_identifier,
|
|
537
537
|
tx: txMeta
|
|
538
538
|
};
|
|
@@ -564,8 +564,8 @@ function buildEventPayload(filter, tx, event) {
|
|
|
564
564
|
tx: txMeta
|
|
565
565
|
};
|
|
566
566
|
case "print_event": {
|
|
567
|
-
const
|
|
568
|
-
const rawValue =
|
|
567
|
+
const rawHex = event.data?.raw_value;
|
|
568
|
+
const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
|
|
569
569
|
const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
|
|
570
570
|
const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
|
|
571
571
|
const { topic: _, ...rest } = clarityObj ?? {};
|
|
@@ -949,10 +949,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
|
|
|
949
949
|
}
|
|
950
950
|
|
|
951
951
|
// src/runtime/block-processor.ts
|
|
952
|
-
import {
|
|
953
|
-
getSourceDb,
|
|
954
|
-
getTargetDb
|
|
955
|
-
} from "@secondlayer/shared/db";
|
|
952
|
+
import { getTargetDb } from "@secondlayer/shared/db";
|
|
956
953
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
957
954
|
import {
|
|
958
955
|
isByoSubgraph,
|
|
@@ -960,15 +957,287 @@ import {
|
|
|
960
957
|
resolveSubgraphDb,
|
|
961
958
|
updateSubgraphStatus
|
|
962
959
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
963
|
-
import { logger as
|
|
960
|
+
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
964
961
|
import { sql as sql3 } from "kysely";
|
|
965
962
|
|
|
966
963
|
// src/schema/utils.ts
|
|
967
964
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
968
965
|
|
|
966
|
+
// src/runtime/block-source.ts
|
|
967
|
+
import { getSourceDb } from "@secondlayer/shared/db";
|
|
968
|
+
import { IndexHttpClient } from "@secondlayer/shared/index-http";
|
|
969
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
970
|
+
|
|
971
|
+
// src/runtime/batch-loader.ts
|
|
972
|
+
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
973
|
+
const [blocks, txs, events] = await Promise.all([
|
|
974
|
+
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
975
|
+
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
976
|
+
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
977
|
+
]);
|
|
978
|
+
const txsByHeight = new Map;
|
|
979
|
+
for (const tx of txs) {
|
|
980
|
+
const h = Number(tx.block_height);
|
|
981
|
+
const list = txsByHeight.get(h) ?? [];
|
|
982
|
+
list.push(tx);
|
|
983
|
+
txsByHeight.set(h, list);
|
|
984
|
+
}
|
|
985
|
+
const eventsByHeight = new Map;
|
|
986
|
+
for (const evt of events) {
|
|
987
|
+
const h = Number(evt.block_height);
|
|
988
|
+
const list = eventsByHeight.get(h) ?? [];
|
|
989
|
+
list.push(evt);
|
|
990
|
+
eventsByHeight.set(h, list);
|
|
991
|
+
}
|
|
992
|
+
const result = new Map;
|
|
993
|
+
for (const block of blocks) {
|
|
994
|
+
const h = Number(block.height);
|
|
995
|
+
result.set(h, {
|
|
996
|
+
block,
|
|
997
|
+
txs: txsByHeight.get(h) ?? [],
|
|
998
|
+
events: eventsByHeight.get(h) ?? []
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
return result;
|
|
1002
|
+
}
|
|
1003
|
+
function avgEventsPerBlock(batch) {
|
|
1004
|
+
if (batch.size === 0)
|
|
1005
|
+
return 0;
|
|
1006
|
+
let totalEvents = 0;
|
|
1007
|
+
for (const data of batch.values()) {
|
|
1008
|
+
totalEvents += data.events.length;
|
|
1009
|
+
}
|
|
1010
|
+
return totalEvents / batch.size;
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
// src/runtime/reconstruct.ts
|
|
1014
|
+
function isoToUnixSeconds(iso) {
|
|
1015
|
+
if (!iso)
|
|
1016
|
+
return 0;
|
|
1017
|
+
return Math.floor(new Date(iso).getTime() / 1000);
|
|
1018
|
+
}
|
|
1019
|
+
function reconstructBlock(b) {
|
|
1020
|
+
return {
|
|
1021
|
+
height: b.block_height,
|
|
1022
|
+
hash: b.block_hash,
|
|
1023
|
+
parent_hash: b.parent_hash,
|
|
1024
|
+
burn_block_height: b.burn_block_height,
|
|
1025
|
+
burn_block_hash: b.burn_block_hash,
|
|
1026
|
+
timestamp: isoToUnixSeconds(b.block_time),
|
|
1027
|
+
canonical: true,
|
|
1028
|
+
created_at: new Date(0)
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
function reconstructTransaction(t) {
|
|
1032
|
+
return {
|
|
1033
|
+
tx_id: t.tx_id,
|
|
1034
|
+
block_height: t.block_height,
|
|
1035
|
+
tx_index: t.tx_index,
|
|
1036
|
+
type: t.tx_type,
|
|
1037
|
+
sender: t.sender,
|
|
1038
|
+
status: t.status,
|
|
1039
|
+
contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
|
|
1040
|
+
function_name: t.contract_call?.function_name ?? null,
|
|
1041
|
+
function_args: t.contract_call?.function_args_hex ?? [],
|
|
1042
|
+
raw_result: t.contract_call?.result_hex ?? null,
|
|
1043
|
+
raw_tx: "",
|
|
1044
|
+
created_at: new Date(0)
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
function reconstructEvent(e) {
|
|
1048
|
+
const base = {
|
|
1049
|
+
id: `${e.tx_id}#${e.event_index}`,
|
|
1050
|
+
tx_id: e.tx_id,
|
|
1051
|
+
block_height: e.block_height,
|
|
1052
|
+
event_index: e.event_index,
|
|
1053
|
+
created_at: new Date(0)
|
|
1054
|
+
};
|
|
1055
|
+
switch (e.event_type) {
|
|
1056
|
+
case "ft_transfer":
|
|
1057
|
+
case "ft_mint":
|
|
1058
|
+
case "ft_burn":
|
|
1059
|
+
return {
|
|
1060
|
+
...base,
|
|
1061
|
+
type: `${e.event_type}_event`,
|
|
1062
|
+
data: {
|
|
1063
|
+
asset_identifier: e.asset_identifier,
|
|
1064
|
+
sender: e.sender,
|
|
1065
|
+
recipient: e.recipient,
|
|
1066
|
+
amount: e.amount
|
|
1067
|
+
}
|
|
1068
|
+
};
|
|
1069
|
+
case "nft_transfer":
|
|
1070
|
+
case "nft_mint":
|
|
1071
|
+
case "nft_burn":
|
|
1072
|
+
return {
|
|
1073
|
+
...base,
|
|
1074
|
+
type: `${e.event_type}_event`,
|
|
1075
|
+
data: {
|
|
1076
|
+
asset_identifier: e.asset_identifier,
|
|
1077
|
+
sender: e.sender,
|
|
1078
|
+
recipient: e.recipient,
|
|
1079
|
+
raw_value: e.value
|
|
1080
|
+
}
|
|
1081
|
+
};
|
|
1082
|
+
case "stx_transfer":
|
|
1083
|
+
case "stx_mint":
|
|
1084
|
+
case "stx_burn":
|
|
1085
|
+
return {
|
|
1086
|
+
...base,
|
|
1087
|
+
type: `${e.event_type}_event`,
|
|
1088
|
+
data: {
|
|
1089
|
+
sender: e.sender,
|
|
1090
|
+
recipient: e.recipient,
|
|
1091
|
+
amount: e.amount,
|
|
1092
|
+
..."memo" in e ? { memo: e.memo ?? undefined } : {}
|
|
1093
|
+
}
|
|
1094
|
+
};
|
|
1095
|
+
case "stx_lock":
|
|
1096
|
+
return {
|
|
1097
|
+
...base,
|
|
1098
|
+
type: "stx_lock_event",
|
|
1099
|
+
data: {
|
|
1100
|
+
locked_address: e.sender,
|
|
1101
|
+
locked_amount: e.amount,
|
|
1102
|
+
unlock_height: e.payload.unlock_height
|
|
1103
|
+
}
|
|
1104
|
+
};
|
|
1105
|
+
case "print":
|
|
1106
|
+
return {
|
|
1107
|
+
...base,
|
|
1108
|
+
type: "contract_event",
|
|
1109
|
+
data: {
|
|
1110
|
+
topic: e.payload.topic,
|
|
1111
|
+
contract_id: e.contract_id,
|
|
1112
|
+
contract_identifier: e.contract_id,
|
|
1113
|
+
value: e.payload.value,
|
|
1114
|
+
raw_value: e.payload.raw_value
|
|
1115
|
+
}
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// src/runtime/block-source.ts
|
|
1121
|
+
class PostgresBlockSource {
|
|
1122
|
+
async getTip() {
|
|
1123
|
+
const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1124
|
+
return progress ? Number(progress.highest_seen_block) : 0;
|
|
1125
|
+
}
|
|
1126
|
+
loadBlockRange(fromHeight, toHeight) {
|
|
1127
|
+
return loadBlockRange(getSourceDb(), fromHeight, toHeight);
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
var EVENT_FILTER_TO_INDEX_TYPE = {
|
|
1131
|
+
stx_transfer: "stx_transfer",
|
|
1132
|
+
stx_mint: "stx_mint",
|
|
1133
|
+
stx_burn: "stx_burn",
|
|
1134
|
+
stx_lock: "stx_lock",
|
|
1135
|
+
ft_transfer: "ft_transfer",
|
|
1136
|
+
ft_mint: "ft_mint",
|
|
1137
|
+
ft_burn: "ft_burn",
|
|
1138
|
+
nft_transfer: "nft_transfer",
|
|
1139
|
+
nft_mint: "nft_mint",
|
|
1140
|
+
nft_burn: "nft_burn",
|
|
1141
|
+
print_event: "print"
|
|
1142
|
+
};
|
|
1143
|
+
var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
1144
|
+
var ALL_INDEX_EVENT_TYPES = [
|
|
1145
|
+
...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
|
|
1146
|
+
];
|
|
1147
|
+
function sourceFilters(subgraph) {
|
|
1148
|
+
const sources = subgraph.sources;
|
|
1149
|
+
return Array.isArray(sources) ? sources : Object.values(sources);
|
|
1150
|
+
}
|
|
1151
|
+
function referencedIndexEventTypes(subgraph) {
|
|
1152
|
+
const filters = sourceFilters(subgraph);
|
|
1153
|
+
if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
|
|
1154
|
+
return ALL_INDEX_EVENT_TYPES;
|
|
1155
|
+
}
|
|
1156
|
+
const types = new Set;
|
|
1157
|
+
for (const f of filters) {
|
|
1158
|
+
const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
|
|
1159
|
+
if (t)
|
|
1160
|
+
types.add(t);
|
|
1161
|
+
}
|
|
1162
|
+
return [...types];
|
|
1163
|
+
}
|
|
1164
|
+
function isStreamsIndexEligible(subgraph) {
|
|
1165
|
+
if (Array.isArray(subgraph.sources))
|
|
1166
|
+
return false;
|
|
1167
|
+
const filters = sourceFilters(subgraph);
|
|
1168
|
+
if (filters.length === 0)
|
|
1169
|
+
return false;
|
|
1170
|
+
for (const f of filters) {
|
|
1171
|
+
const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
|
|
1172
|
+
if (!known)
|
|
1173
|
+
return false;
|
|
1174
|
+
}
|
|
1175
|
+
return true;
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
class PublicApiBlockSource {
|
|
1179
|
+
http;
|
|
1180
|
+
eventTypes;
|
|
1181
|
+
constructor(http, eventTypes) {
|
|
1182
|
+
this.http = http;
|
|
1183
|
+
this.eventTypes = eventTypes;
|
|
1184
|
+
}
|
|
1185
|
+
getTip() {
|
|
1186
|
+
return this.http.getIndexTip();
|
|
1187
|
+
}
|
|
1188
|
+
async loadBlockRange(fromHeight, toHeight) {
|
|
1189
|
+
const [blocks, txs, eventLists] = await Promise.all([
|
|
1190
|
+
this.http.walkBlocks(fromHeight, toHeight),
|
|
1191
|
+
this.http.walkTransactions(fromHeight, toHeight),
|
|
1192
|
+
Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
|
|
1193
|
+
]);
|
|
1194
|
+
const map = new Map;
|
|
1195
|
+
for (const b of blocks) {
|
|
1196
|
+
map.set(b.block_height, {
|
|
1197
|
+
block: reconstructBlock(b),
|
|
1198
|
+
txs: [],
|
|
1199
|
+
events: []
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
for (const t of txs) {
|
|
1203
|
+
map.get(t.block_height)?.txs.push(reconstructTransaction(t));
|
|
1204
|
+
}
|
|
1205
|
+
for (const list of eventLists) {
|
|
1206
|
+
for (const e of list) {
|
|
1207
|
+
map.get(e.block_height)?.events.push(reconstructEvent(e));
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
for (const bd of map.values()) {
|
|
1211
|
+
bd.txs.sort((a, b) => a.tx_index - b.tx_index);
|
|
1212
|
+
bd.events.sort((a, b) => a.event_index - b.event_index);
|
|
1213
|
+
}
|
|
1214
|
+
return map;
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
var postgresBlockSource = new PostgresBlockSource;
|
|
1218
|
+
function buildHttpClient() {
|
|
1219
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
1220
|
+
return new IndexHttpClient({
|
|
1221
|
+
indexBaseUrl: baseUrl,
|
|
1222
|
+
streamsBaseUrl: baseUrl,
|
|
1223
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
1224
|
+
});
|
|
1225
|
+
}
|
|
1226
|
+
function resolveBlockSource(subgraph) {
|
|
1227
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
|
|
1228
|
+
return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
|
|
1229
|
+
}
|
|
1230
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
|
|
1231
|
+
logger3.debug("Subgraph not streams-index eligible, using DB tap", {
|
|
1232
|
+
subgraph: subgraph.name
|
|
1233
|
+
});
|
|
1234
|
+
}
|
|
1235
|
+
return postgresBlockSource;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
969
1238
|
// src/runtime/outbox-emit.ts
|
|
970
1239
|
import { createHash } from "node:crypto";
|
|
971
|
-
import { logger as
|
|
1240
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
972
1241
|
var loggedKillSwitch = false;
|
|
973
1242
|
function isEmitOutboxEnabled() {
|
|
974
1243
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
@@ -987,7 +1256,7 @@ function stableStringify(obj) {
|
|
|
987
1256
|
async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
988
1257
|
if (!isEmitOutboxEnabled()) {
|
|
989
1258
|
if (!loggedKillSwitch) {
|
|
990
|
-
|
|
1259
|
+
logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
991
1260
|
loggedKillSwitch = true;
|
|
992
1261
|
}
|
|
993
1262
|
return 0;
|
|
@@ -1228,7 +1497,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
|
|
|
1228
1497
|
return resolved;
|
|
1229
1498
|
}
|
|
1230
1499
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
1231
|
-
const sourceDb = getSourceDb();
|
|
1232
1500
|
const targetDb = getTargetDb();
|
|
1233
1501
|
const blockStart = performance.now();
|
|
1234
1502
|
const result = {
|
|
@@ -1246,25 +1514,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1246
1514
|
txs = opts.preloaded.txs;
|
|
1247
1515
|
evts = opts.preloaded.events;
|
|
1248
1516
|
} else {
|
|
1249
|
-
|
|
1250
|
-
if (!
|
|
1251
|
-
|
|
1517
|
+
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
1518
|
+
if (!data) {
|
|
1519
|
+
logger5.debug("Block not found or non-canonical for subgraph processing", {
|
|
1252
1520
|
subgraph: subgraphName,
|
|
1253
1521
|
blockHeight
|
|
1254
1522
|
});
|
|
1255
1523
|
result.skipped = true;
|
|
1256
1524
|
return result;
|
|
1257
1525
|
}
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
blockHeight
|
|
1262
|
-
});
|
|
1263
|
-
result.skipped = true;
|
|
1264
|
-
return result;
|
|
1265
|
-
}
|
|
1266
|
-
txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
|
|
1267
|
-
evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
|
|
1526
|
+
block = data.block;
|
|
1527
|
+
txs = data.txs;
|
|
1528
|
+
evts = data.events;
|
|
1268
1529
|
}
|
|
1269
1530
|
const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
|
|
1270
1531
|
const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
|
|
@@ -1355,7 +1616,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1355
1616
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
1356
1617
|
const count = Number(rows[0]?.count ?? 0);
|
|
1357
1618
|
if (count >= 1e7) {
|
|
1358
|
-
|
|
1619
|
+
logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
1359
1620
|
subgraph: subgraphName,
|
|
1360
1621
|
table,
|
|
1361
1622
|
count
|
|
@@ -1363,7 +1624,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1363
1624
|
}
|
|
1364
1625
|
}
|
|
1365
1626
|
} catch (err) {
|
|
1366
|
-
|
|
1627
|
+
logger5.debug("Row count sample failed", {
|
|
1367
1628
|
subgraph: subgraphName,
|
|
1368
1629
|
error: err instanceof Error ? err.message : String(err)
|
|
1369
1630
|
});
|
|
@@ -1437,7 +1698,7 @@ class StatsAccumulator {
|
|
|
1437
1698
|
|
|
1438
1699
|
// src/runtime/reindex.ts
|
|
1439
1700
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
1440
|
-
import { getRawClient,
|
|
1701
|
+
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
1441
1702
|
import {
|
|
1442
1703
|
recordGapBatch,
|
|
1443
1704
|
resolveGaps
|
|
@@ -1446,7 +1707,7 @@ import {
|
|
|
1446
1707
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1447
1708
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
1448
1709
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1449
|
-
import { logger as
|
|
1710
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
1450
1711
|
|
|
1451
1712
|
// src/schema/generator.ts
|
|
1452
1713
|
import { createHash as createHash2 } from "node:crypto";
|
|
@@ -1539,48 +1800,6 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
1539
1800
|
return { statements, hash };
|
|
1540
1801
|
}
|
|
1541
1802
|
|
|
1542
|
-
// src/runtime/batch-loader.ts
|
|
1543
|
-
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1544
|
-
const [blocks, txs, events] = await Promise.all([
|
|
1545
|
-
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1546
|
-
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1547
|
-
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1548
|
-
]);
|
|
1549
|
-
const txsByHeight = new Map;
|
|
1550
|
-
for (const tx of txs) {
|
|
1551
|
-
const h = Number(tx.block_height);
|
|
1552
|
-
const list = txsByHeight.get(h) ?? [];
|
|
1553
|
-
list.push(tx);
|
|
1554
|
-
txsByHeight.set(h, list);
|
|
1555
|
-
}
|
|
1556
|
-
const eventsByHeight = new Map;
|
|
1557
|
-
for (const evt of events) {
|
|
1558
|
-
const h = Number(evt.block_height);
|
|
1559
|
-
const list = eventsByHeight.get(h) ?? [];
|
|
1560
|
-
list.push(evt);
|
|
1561
|
-
eventsByHeight.set(h, list);
|
|
1562
|
-
}
|
|
1563
|
-
const result = new Map;
|
|
1564
|
-
for (const block of blocks) {
|
|
1565
|
-
const h = Number(block.height);
|
|
1566
|
-
result.set(h, {
|
|
1567
|
-
block,
|
|
1568
|
-
txs: txsByHeight.get(h) ?? [],
|
|
1569
|
-
events: eventsByHeight.get(h) ?? []
|
|
1570
|
-
});
|
|
1571
|
-
}
|
|
1572
|
-
return result;
|
|
1573
|
-
}
|
|
1574
|
-
function avgEventsPerBlock(batch) {
|
|
1575
|
-
if (batch.size === 0)
|
|
1576
|
-
return 0;
|
|
1577
|
-
let totalEvents = 0;
|
|
1578
|
-
for (const data of batch.values()) {
|
|
1579
|
-
totalEvents += data.events.length;
|
|
1580
|
-
}
|
|
1581
|
-
return totalEvents / batch.size;
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
1803
|
// src/runtime/reindex.ts
|
|
1585
1804
|
var LOG_INTERVAL = 1000;
|
|
1586
1805
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
@@ -1641,7 +1860,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
1641
1860
|
return ranges;
|
|
1642
1861
|
}
|
|
1643
1862
|
async function processBlockRange(def, opts) {
|
|
1644
|
-
const
|
|
1863
|
+
const source = resolveBlockSource(def);
|
|
1645
1864
|
const targetDb = getTargetDb2();
|
|
1646
1865
|
const subgraphName = def.name;
|
|
1647
1866
|
const { fromBlock, toBlock, status } = opts;
|
|
@@ -1671,11 +1890,11 @@ async function processBlockRange(def, opts) {
|
|
|
1671
1890
|
lastHealthFlushAt = Date.now();
|
|
1672
1891
|
};
|
|
1673
1892
|
let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
1674
|
-
let nextBatchPromise = loadBlockRange(
|
|
1893
|
+
let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
|
|
1675
1894
|
while (currentHeight <= toBlock) {
|
|
1676
1895
|
if (opts.signal?.aborted) {
|
|
1677
1896
|
aborted = true;
|
|
1678
|
-
|
|
1897
|
+
logger6.info("Block processing aborted", {
|
|
1679
1898
|
subgraph: subgraphName,
|
|
1680
1899
|
currentBlock: currentHeight,
|
|
1681
1900
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -1687,7 +1906,7 @@ async function processBlockRange(def, opts) {
|
|
|
1687
1906
|
const nextStart = batchEnd + 1;
|
|
1688
1907
|
if (nextStart <= toBlock) {
|
|
1689
1908
|
nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
1690
|
-
nextBatchPromise = loadBlockRange(
|
|
1909
|
+
nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
|
|
1691
1910
|
}
|
|
1692
1911
|
const batchFailedBlocks = [];
|
|
1693
1912
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -1705,7 +1924,7 @@ async function processBlockRange(def, opts) {
|
|
|
1705
1924
|
});
|
|
1706
1925
|
} catch (err) {
|
|
1707
1926
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1708
|
-
|
|
1927
|
+
logger6.error("Block processing error", {
|
|
1709
1928
|
subgraph: subgraphName,
|
|
1710
1929
|
blockHeight: height,
|
|
1711
1930
|
error: errorMsg
|
|
@@ -1739,7 +1958,7 @@ async function processBlockRange(def, opts) {
|
|
|
1739
1958
|
lastProgressFlushAt = now;
|
|
1740
1959
|
}
|
|
1741
1960
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1742
|
-
|
|
1961
|
+
logger6.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1743
1962
|
subgraph: subgraphName,
|
|
1744
1963
|
processed: blocksProcessed,
|
|
1745
1964
|
total: totalBlocks,
|
|
@@ -1758,7 +1977,7 @@ async function processBlockRange(def, opts) {
|
|
|
1758
1977
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1759
1978
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1760
1979
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1761
|
-
|
|
1980
|
+
logger6.warn("Failed to record subgraph gaps", {
|
|
1762
1981
|
subgraph: subgraphName,
|
|
1763
1982
|
error: err instanceof Error ? err.message : String(err)
|
|
1764
1983
|
});
|
|
@@ -1775,32 +1994,26 @@ async function processBlockRange(def, opts) {
|
|
|
1775
1994
|
await flushHealth();
|
|
1776
1995
|
return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
|
|
1777
1996
|
}
|
|
1778
|
-
async function resolveBlockRange(
|
|
1997
|
+
async function resolveBlockRange(source, def, opts) {
|
|
1779
1998
|
const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
|
|
1780
|
-
|
|
1781
|
-
if (opts?.toBlock != null) {
|
|
1782
|
-
toBlock = opts.toBlock;
|
|
1783
|
-
} else {
|
|
1784
|
-
const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1785
|
-
toBlock = progress?.highest_seen_block ?? 0;
|
|
1786
|
-
}
|
|
1999
|
+
const toBlock = opts?.toBlock != null ? opts.toBlock : await source.getTip();
|
|
1787
2000
|
return { fromBlock, toBlock };
|
|
1788
2001
|
}
|
|
1789
2002
|
async function clearReindexMetadata(db, subgraphName) {
|
|
1790
2003
|
await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
|
|
1791
2004
|
}
|
|
1792
2005
|
async function reindexSubgraph(def, opts) {
|
|
1793
|
-
const
|
|
2006
|
+
const source = resolveBlockSource(def);
|
|
1794
2007
|
const targetDb = getTargetDb2();
|
|
1795
2008
|
const client = getRawClient("target");
|
|
1796
2009
|
const subgraphName = def.name;
|
|
1797
2010
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1798
2011
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1799
|
-
|
|
2012
|
+
logger6.info("Reindex starting", { subgraph: subgraphName });
|
|
1800
2013
|
try {
|
|
1801
|
-
const { fromBlock, toBlock } = await resolveBlockRange(
|
|
2014
|
+
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
1802
2015
|
if (fromBlock > toBlock) {
|
|
1803
|
-
|
|
2016
|
+
logger6.info("No blocks to reindex", {
|
|
1804
2017
|
subgraph: subgraphName,
|
|
1805
2018
|
fromBlock,
|
|
1806
2019
|
toBlock
|
|
@@ -1813,7 +2026,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1813
2026
|
for (const stmt of statements) {
|
|
1814
2027
|
await client.unsafe(stmt);
|
|
1815
2028
|
}
|
|
1816
|
-
|
|
2029
|
+
logger6.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1817
2030
|
await targetDb.updateTable("subgraphs").set({
|
|
1818
2031
|
last_processed_block: initialReindexProgressBlock(fromBlock),
|
|
1819
2032
|
reindex_from_block: fromBlock,
|
|
@@ -1824,7 +2037,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1824
2037
|
last_error_at: null,
|
|
1825
2038
|
updated_at: new Date
|
|
1826
2039
|
}).where("name", "=", subgraphName).execute();
|
|
1827
|
-
|
|
2040
|
+
logger6.info("Reindexing blocks", {
|
|
1828
2041
|
subgraph: subgraphName,
|
|
1829
2042
|
fromBlock,
|
|
1830
2043
|
toBlock,
|
|
@@ -1845,9 +2058,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
1845
2058
|
if (reason === "user-cancelled") {
|
|
1846
2059
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1847
2060
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1848
|
-
|
|
2061
|
+
logger6.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1849
2062
|
} else {
|
|
1850
|
-
|
|
2063
|
+
logger6.info("Reindex interrupted by shutdown, will resume", {
|
|
1851
2064
|
subgraph: subgraphName
|
|
1852
2065
|
});
|
|
1853
2066
|
}
|
|
@@ -1855,7 +2068,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1855
2068
|
}
|
|
1856
2069
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1857
2070
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1858
|
-
|
|
2071
|
+
logger6.info("Reindex complete", {
|
|
1859
2072
|
subgraph: subgraphName,
|
|
1860
2073
|
blocks: result.blocksProcessed,
|
|
1861
2074
|
events: result.totalEventsProcessed,
|
|
@@ -1863,7 +2076,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1863
2076
|
});
|
|
1864
2077
|
return { processed: result.blocksProcessed };
|
|
1865
2078
|
} catch (err) {
|
|
1866
|
-
|
|
2079
|
+
logger6.error("Reindex failed", {
|
|
1867
2080
|
subgraph: subgraphName,
|
|
1868
2081
|
error: getErrorMessage2(err)
|
|
1869
2082
|
});
|
|
@@ -1885,7 +2098,7 @@ async function resumeReindex(def, opts) {
|
|
|
1885
2098
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
1886
2099
|
const toBlock = Number(row.reindex_to_block);
|
|
1887
2100
|
if (fromBlock == null) {
|
|
1888
|
-
|
|
2101
|
+
logger6.info("No reindex metadata, starting fresh reindex", {
|
|
1889
2102
|
subgraph: subgraphName
|
|
1890
2103
|
});
|
|
1891
2104
|
return reindexSubgraph(def, {
|
|
@@ -1894,12 +2107,12 @@ async function resumeReindex(def, opts) {
|
|
|
1894
2107
|
});
|
|
1895
2108
|
}
|
|
1896
2109
|
if (fromBlock > toBlock) {
|
|
1897
|
-
|
|
2110
|
+
logger6.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1898
2111
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1899
2112
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1900
2113
|
return { processed: 0 };
|
|
1901
2114
|
}
|
|
1902
|
-
|
|
2115
|
+
logger6.info("Resuming reindex", {
|
|
1903
2116
|
subgraph: subgraphName,
|
|
1904
2117
|
fromBlock,
|
|
1905
2118
|
toBlock,
|
|
@@ -1920,9 +2133,9 @@ async function resumeReindex(def, opts) {
|
|
|
1920
2133
|
if (reason === "user-cancelled") {
|
|
1921
2134
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1922
2135
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1923
|
-
|
|
2136
|
+
logger6.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
1924
2137
|
} else {
|
|
1925
|
-
|
|
2138
|
+
logger6.info("Resume interrupted by shutdown, will resume again", {
|
|
1926
2139
|
subgraph: subgraphName
|
|
1927
2140
|
});
|
|
1928
2141
|
}
|
|
@@ -1930,13 +2143,13 @@ async function resumeReindex(def, opts) {
|
|
|
1930
2143
|
}
|
|
1931
2144
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1932
2145
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1933
|
-
|
|
2146
|
+
logger6.info("Resumed reindex complete", {
|
|
1934
2147
|
subgraph: subgraphName,
|
|
1935
2148
|
blocks: result.blocksProcessed
|
|
1936
2149
|
});
|
|
1937
2150
|
return { processed: result.blocksProcessed };
|
|
1938
2151
|
} catch (err) {
|
|
1939
|
-
|
|
2152
|
+
logger6.error("Resumed reindex failed", {
|
|
1940
2153
|
subgraph: subgraphName,
|
|
1941
2154
|
error: getErrorMessage2(err)
|
|
1942
2155
|
});
|
|
@@ -1947,7 +2160,7 @@ async function resumeReindex(def, opts) {
|
|
|
1947
2160
|
async function backfillSubgraph(def, opts) {
|
|
1948
2161
|
const targetDb = getTargetDb2();
|
|
1949
2162
|
const subgraphName = def.name;
|
|
1950
|
-
|
|
2163
|
+
logger6.info("Backfill starting", {
|
|
1951
2164
|
subgraph: subgraphName,
|
|
1952
2165
|
from: opts.fromBlock,
|
|
1953
2166
|
to: opts.toBlock
|
|
@@ -1964,17 +2177,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
1964
2177
|
signal: opts.signal
|
|
1965
2178
|
});
|
|
1966
2179
|
if (result.aborted) {
|
|
1967
|
-
|
|
2180
|
+
logger6.info("Backfill aborted", { subgraph: subgraphName });
|
|
1968
2181
|
return { processed: result.blocksProcessed };
|
|
1969
2182
|
}
|
|
1970
2183
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
1971
2184
|
if (resolved > 0) {
|
|
1972
|
-
|
|
2185
|
+
logger6.info("Resolved subgraph gaps via backfill", {
|
|
1973
2186
|
subgraph: subgraphName,
|
|
1974
2187
|
resolved
|
|
1975
2188
|
});
|
|
1976
2189
|
}
|
|
1977
|
-
|
|
2190
|
+
logger6.info("Backfill complete", {
|
|
1978
2191
|
subgraph: subgraphName,
|
|
1979
2192
|
blocks: result.blocksProcessed,
|
|
1980
2193
|
events: result.totalEventsProcessed,
|
|
@@ -1982,7 +2195,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
1982
2195
|
});
|
|
1983
2196
|
return { processed: result.blocksProcessed };
|
|
1984
2197
|
} catch (err) {
|
|
1985
|
-
|
|
2198
|
+
logger6.error("Backfill failed", {
|
|
1986
2199
|
subgraph: subgraphName,
|
|
1987
2200
|
error: getErrorMessage2(err)
|
|
1988
2201
|
});
|
|
@@ -1991,12 +2204,12 @@ async function backfillSubgraph(def, opts) {
|
|
|
1991
2204
|
}
|
|
1992
2205
|
|
|
1993
2206
|
// src/runtime/catchup.ts
|
|
1994
|
-
import {
|
|
2207
|
+
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
1995
2208
|
import {
|
|
1996
2209
|
recordGapBatch as recordGapBatch2
|
|
1997
2210
|
} from "@secondlayer/shared/db/queries/subgraph-gaps";
|
|
1998
2211
|
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
1999
|
-
import { logger as
|
|
2212
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
2000
2213
|
var LOG_INTERVAL2 = 1000;
|
|
2001
2214
|
var STANDARD_CATCHUP_BATCH_CONFIG = {
|
|
2002
2215
|
defaultBatchSize: 500,
|
|
@@ -2067,22 +2280,19 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2067
2280
|
return 0;
|
|
2068
2281
|
catchingUp.add(subgraphName);
|
|
2069
2282
|
try {
|
|
2070
|
-
const
|
|
2283
|
+
const source = resolveBlockSource(subgraph);
|
|
2071
2284
|
const targetDb = getTargetDb3();
|
|
2072
2285
|
const subgraphRow = await getSubgraph(targetDb, subgraphName);
|
|
2073
2286
|
if (!subgraphRow)
|
|
2074
2287
|
return 0;
|
|
2075
2288
|
const lastProcessedBlock = Number(subgraphRow.last_processed_block);
|
|
2076
|
-
const
|
|
2077
|
-
if (
|
|
2078
|
-
return 0;
|
|
2079
|
-
const chainTip = Number(progress.highest_seen_block);
|
|
2080
|
-
if (lastProcessedBlock >= chainTip)
|
|
2289
|
+
const chainTip = await source.getTip();
|
|
2290
|
+
if (chainTip <= 0 || lastProcessedBlock >= chainTip)
|
|
2081
2291
|
return 0;
|
|
2082
2292
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
2083
2293
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
2084
2294
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
2085
|
-
|
|
2295
|
+
logger7.info("Subgraph catch-up starting", {
|
|
2086
2296
|
subgraph: subgraphName,
|
|
2087
2297
|
from: startBlock,
|
|
2088
2298
|
to: chainTip,
|
|
@@ -2094,11 +2304,11 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2094
2304
|
let batchSize = batchConfig.defaultBatchSize;
|
|
2095
2305
|
let currentHeight = startBlock;
|
|
2096
2306
|
let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
2097
|
-
let nextBatchPromise = batchConfig.prefetch ? loadBlockRange(
|
|
2307
|
+
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
2098
2308
|
while (currentHeight <= chainTip) {
|
|
2099
2309
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
2100
2310
|
if (!currentRow || currentRow.status !== "active") {
|
|
2101
|
-
|
|
2311
|
+
logger7.info("Subgraph status changed, stopping catch-up", {
|
|
2102
2312
|
subgraph: subgraphName,
|
|
2103
2313
|
status: currentRow?.status ?? "deleted"
|
|
2104
2314
|
});
|
|
@@ -2112,13 +2322,13 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2112
2322
|
const nextStart = batchEnd + 1;
|
|
2113
2323
|
if (nextStart <= chainTip) {
|
|
2114
2324
|
prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
|
|
2115
|
-
nextBatchPromise = loadBlockRange(
|
|
2325
|
+
nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
|
|
2116
2326
|
} else {
|
|
2117
2327
|
nextBatchPromise = undefined;
|
|
2118
2328
|
}
|
|
2119
2329
|
} else {
|
|
2120
2330
|
batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
2121
|
-
batch = await loadBlockRange(
|
|
2331
|
+
batch = await source.loadBlockRange(currentHeight, batchEnd);
|
|
2122
2332
|
}
|
|
2123
2333
|
const batchFailedBlocks = [];
|
|
2124
2334
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -2134,7 +2344,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2134
2344
|
preloaded: blockData
|
|
2135
2345
|
});
|
|
2136
2346
|
} catch (err) {
|
|
2137
|
-
|
|
2347
|
+
logger7.error("Block processing error during catch-up", {
|
|
2138
2348
|
subgraph: subgraphName,
|
|
2139
2349
|
blockHeight: height,
|
|
2140
2350
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -2153,7 +2363,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2153
2363
|
}
|
|
2154
2364
|
}
|
|
2155
2365
|
if (processed % LOG_INTERVAL2 === 0) {
|
|
2156
|
-
|
|
2366
|
+
logger7.info("Subgraph catch-up progress", {
|
|
2157
2367
|
subgraph: subgraphName,
|
|
2158
2368
|
processed,
|
|
2159
2369
|
total: totalBlocks,
|
|
@@ -2165,7 +2375,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2165
2375
|
if (batchFailedBlocks.length > 0) {
|
|
2166
2376
|
const gaps = coalesceGaps(batchFailedBlocks);
|
|
2167
2377
|
await recordGapBatch2(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
|
|
2168
|
-
|
|
2378
|
+
logger7.warn("Failed to record subgraph gaps", {
|
|
2169
2379
|
subgraph: subgraphName,
|
|
2170
2380
|
error: err instanceof Error ? err.message : String(err)
|
|
2171
2381
|
});
|
|
@@ -2176,7 +2386,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2176
2386
|
currentHeight = batchEnd + 1;
|
|
2177
2387
|
}
|
|
2178
2388
|
await stats.flush(targetDb);
|
|
2179
|
-
|
|
2389
|
+
logger7.info("Subgraph catch-up complete", {
|
|
2180
2390
|
subgraph: subgraphName,
|
|
2181
2391
|
processed
|
|
2182
2392
|
});
|
|
@@ -2193,13 +2403,13 @@ import {
|
|
|
2193
2403
|
listSubgraphs,
|
|
2194
2404
|
resolveSubgraphRawClient
|
|
2195
2405
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2196
|
-
import { logger as
|
|
2406
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2197
2407
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
2198
2408
|
const targetDb = getTargetDb4();
|
|
2199
2409
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
2200
2410
|
if (activeSubgraphs.length === 0)
|
|
2201
2411
|
return;
|
|
2202
|
-
|
|
2412
|
+
logger8.info("Propagating reorg to subgraphs", {
|
|
2203
2413
|
blockHeight,
|
|
2204
2414
|
subgraphCount: activeSubgraphs.length
|
|
2205
2415
|
});
|
|
@@ -2247,7 +2457,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
2247
2457
|
}).as("row_pk"),
|
|
2248
2458
|
eb2.val("pending").as("status")
|
|
2249
2459
|
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.column("dedup_key").doNothing()).execute().catch((err) => {
|
|
2250
|
-
|
|
2460
|
+
logger8.warn("Failed to emit revert event for subscriptions", {
|
|
2251
2461
|
subgraph: sg.name,
|
|
2252
2462
|
table: tableName,
|
|
2253
2463
|
blockHeight,
|
|
@@ -2255,19 +2465,19 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
2255
2465
|
});
|
|
2256
2466
|
});
|
|
2257
2467
|
}
|
|
2258
|
-
|
|
2468
|
+
logger8.info("Subgraph reorg cleanup done", {
|
|
2259
2469
|
subgraph: sg.name,
|
|
2260
2470
|
blockHeight,
|
|
2261
2471
|
tablesAffected: Object.keys(revertedByTable).length
|
|
2262
2472
|
});
|
|
2263
2473
|
const def = await loadSubgraphDef(sg);
|
|
2264
2474
|
await processBlock(def, sg.name, blockHeight);
|
|
2265
|
-
|
|
2475
|
+
logger8.info("Subgraph reorg reprocessed", {
|
|
2266
2476
|
subgraph: sg.name,
|
|
2267
2477
|
blockHeight
|
|
2268
2478
|
});
|
|
2269
2479
|
} catch (err) {
|
|
2270
|
-
|
|
2480
|
+
logger8.error("Subgraph reorg handling failed", {
|
|
2271
2481
|
subgraph: sg.name,
|
|
2272
2482
|
blockHeight,
|
|
2273
2483
|
error: getErrorMessage3(err)
|
|
@@ -2281,7 +2491,7 @@ import { randomUUID } from "node:crypto";
|
|
|
2281
2491
|
import { hostname } from "node:os";
|
|
2282
2492
|
import { resolve } from "node:path";
|
|
2283
2493
|
import { pathToFileURL } from "node:url";
|
|
2284
|
-
import { getErrorMessage as
|
|
2494
|
+
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
2285
2495
|
import { getTargetDb as getTargetDb6 } from "@secondlayer/shared/db";
|
|
2286
2496
|
import {
|
|
2287
2497
|
cancelSubgraphOperation,
|
|
@@ -2300,7 +2510,7 @@ import {
|
|
|
2300
2510
|
pgSchemaName as pgSchemaName2,
|
|
2301
2511
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
2302
2512
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2303
|
-
import { logger as
|
|
2513
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
2304
2514
|
import { listen as listen2 } from "@secondlayer/shared/queue/listener";
|
|
2305
2515
|
|
|
2306
2516
|
// src/runtime/emitter.ts
|
|
@@ -2308,12 +2518,12 @@ import {
|
|
|
2308
2518
|
getTargetDb as getTargetDb5
|
|
2309
2519
|
} from "@secondlayer/shared/db";
|
|
2310
2520
|
import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2311
|
-
import { logger as
|
|
2521
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2312
2522
|
import { listen } from "@secondlayer/shared/queue/listener";
|
|
2313
2523
|
import { sql as sql4 } from "kysely";
|
|
2314
2524
|
|
|
2315
2525
|
// src/runtime/formats/index.ts
|
|
2316
|
-
import { logger as
|
|
2526
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2317
2527
|
|
|
2318
2528
|
// src/runtime/formats/cloudevents.ts
|
|
2319
2529
|
function buildCloudEvents(outboxRow, _sub) {
|
|
@@ -2460,7 +2670,7 @@ function buildForFormat(outboxRow, sub, signingSecret) {
|
|
|
2460
2670
|
case "standard-webhooks":
|
|
2461
2671
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2462
2672
|
default:
|
|
2463
|
-
|
|
2673
|
+
logger9.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
2464
2674
|
format: sub.format,
|
|
2465
2675
|
subscriptionId: sub.id
|
|
2466
2676
|
});
|
|
@@ -2543,7 +2753,7 @@ async function dispatchOne(db, outboxRow, sub) {
|
|
|
2543
2753
|
let responseHeaders = {};
|
|
2544
2754
|
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
2545
2755
|
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
2546
|
-
|
|
2756
|
+
logger10.warn("[emitter] refused private egress", {
|
|
2547
2757
|
subscription: sub.name,
|
|
2548
2758
|
url: sub.url
|
|
2549
2759
|
});
|
|
@@ -2639,7 +2849,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
2639
2849
|
circuit_opened_at: new Date,
|
|
2640
2850
|
updated_at: new Date
|
|
2641
2851
|
}).where("id", "=", sub.id).execute();
|
|
2642
|
-
|
|
2852
|
+
logger10.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
2643
2853
|
subscription: sub.name,
|
|
2644
2854
|
failures: newFailures
|
|
2645
2855
|
});
|
|
@@ -2727,7 +2937,7 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
2727
2937
|
await settleFailed(db, row, sub, err);
|
|
2728
2938
|
}
|
|
2729
2939
|
} catch (err) {
|
|
2730
|
-
|
|
2940
|
+
logger10.error("Emitter dispatch crashed", {
|
|
2731
2941
|
outboxId: row.id,
|
|
2732
2942
|
error: err instanceof Error ? err.message : String(err)
|
|
2733
2943
|
});
|
|
@@ -2764,7 +2974,7 @@ async function startEmitter(opts) {
|
|
|
2764
2974
|
};
|
|
2765
2975
|
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
2766
2976
|
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
2767
|
-
|
|
2977
|
+
logger10.info("[emitter] started", { id: emitterId });
|
|
2768
2978
|
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
2769
2979
|
let lastErr = null;
|
|
2770
2980
|
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
@@ -2775,7 +2985,7 @@ async function startEmitter(opts) {
|
|
|
2775
2985
|
} catch (err) {
|
|
2776
2986
|
lastErr = err;
|
|
2777
2987
|
const delayMs = 500 * 2 ** i;
|
|
2778
|
-
|
|
2988
|
+
logger10.warn("[emitter] matcher refresh failed, retrying", {
|
|
2779
2989
|
attempt: i + 1,
|
|
2780
2990
|
delayMs,
|
|
2781
2991
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -2789,21 +2999,21 @@ async function startEmitter(opts) {
|
|
|
2789
2999
|
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
2790
3000
|
if (!state.running)
|
|
2791
3001
|
return;
|
|
2792
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3002
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger10.error("[emitter] claim failed", {
|
|
2793
3003
|
error: err instanceof Error ? err.message : String(err)
|
|
2794
3004
|
}));
|
|
2795
3005
|
});
|
|
2796
3006
|
const stopChanged = await listen("subscriptions:changed", () => {
|
|
2797
3007
|
if (!state.running)
|
|
2798
3008
|
return;
|
|
2799
|
-
refreshMatcher(db).catch((err) =>
|
|
3009
|
+
refreshMatcher(db).catch((err) => logger10.error("[emitter] matcher refresh failed", {
|
|
2800
3010
|
error: err instanceof Error ? err.message : String(err)
|
|
2801
3011
|
}));
|
|
2802
3012
|
});
|
|
2803
3013
|
const poll = setInterval(() => {
|
|
2804
3014
|
if (!state.running)
|
|
2805
3015
|
return;
|
|
2806
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3016
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger10.error("[emitter] poll claim failed", {
|
|
2807
3017
|
error: err instanceof Error ? err.message : String(err)
|
|
2808
3018
|
}));
|
|
2809
3019
|
}, pollIntervalMs);
|
|
@@ -2811,7 +3021,7 @@ async function startEmitter(opts) {
|
|
|
2811
3021
|
const retention = setInterval(() => {
|
|
2812
3022
|
if (!state.running)
|
|
2813
3023
|
return;
|
|
2814
|
-
runRetention(db).catch((err) =>
|
|
3024
|
+
runRetention(db).catch((err) => logger10.error("[emitter] retention failed", {
|
|
2815
3025
|
error: err instanceof Error ? err.message : String(err)
|
|
2816
3026
|
}));
|
|
2817
3027
|
}, retentionIntervalMs);
|
|
@@ -2821,7 +3031,56 @@ async function startEmitter(opts) {
|
|
|
2821
3031
|
clearInterval(retention);
|
|
2822
3032
|
await stopNew();
|
|
2823
3033
|
await stopChanged();
|
|
2824
|
-
|
|
3034
|
+
logger10.info("[emitter] stopped", { id: emitterId });
|
|
3035
|
+
};
|
|
3036
|
+
}
|
|
3037
|
+
|
|
3038
|
+
// src/runtime/streams-reorg-poll.ts
|
|
3039
|
+
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
3040
|
+
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3041
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
3042
|
+
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3043
|
+
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3044
|
+
async function pollReorgsOnce(http, cursor, handleReorg, loadDef) {
|
|
3045
|
+
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
3046
|
+
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3047
|
+
for (const r of sorted) {
|
|
3048
|
+
logger11.info("Streams reorg — rewinding subgraphs", {
|
|
3049
|
+
forkPointHeight: r.fork_point_height
|
|
3050
|
+
});
|
|
3051
|
+
await handleReorg(r.fork_point_height, loadDef);
|
|
3052
|
+
}
|
|
3053
|
+
return next_since ?? cursor;
|
|
3054
|
+
}
|
|
3055
|
+
function startStreamsReorgPoll(handleReorg, loadDef) {
|
|
3056
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
3057
|
+
const http = new IndexHttpClient2({
|
|
3058
|
+
indexBaseUrl: baseUrl,
|
|
3059
|
+
streamsBaseUrl: baseUrl,
|
|
3060
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
3061
|
+
});
|
|
3062
|
+
let since = new Date(Date.now() - STARTUP_MARGIN_MS).toISOString();
|
|
3063
|
+
let running = true;
|
|
3064
|
+
let timer;
|
|
3065
|
+
const tick = async () => {
|
|
3066
|
+
if (!running)
|
|
3067
|
+
return;
|
|
3068
|
+
try {
|
|
3069
|
+
since = await pollReorgsOnce(http, since, handleReorg, loadDef);
|
|
3070
|
+
} catch (err) {
|
|
3071
|
+
logger11.error("Streams reorg poll failed", {
|
|
3072
|
+
error: getErrorMessage4(err)
|
|
3073
|
+
});
|
|
3074
|
+
}
|
|
3075
|
+
if (running)
|
|
3076
|
+
timer = setTimeout(tick, POLL_MS);
|
|
3077
|
+
};
|
|
3078
|
+
timer = setTimeout(tick, POLL_MS);
|
|
3079
|
+
logger11.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3080
|
+
return () => {
|
|
3081
|
+
running = false;
|
|
3082
|
+
if (timer)
|
|
3083
|
+
clearTimeout(timer);
|
|
2825
3084
|
};
|
|
2826
3085
|
}
|
|
2827
3086
|
|
|
@@ -2844,11 +3103,11 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
2844
3103
|
const def = await loadSubgraphDefinition(sg);
|
|
2845
3104
|
await catchUpSubgraph(def, sg.name);
|
|
2846
3105
|
} catch (err) {
|
|
2847
|
-
const msg =
|
|
3106
|
+
const msg = getErrorMessage5(err);
|
|
2848
3107
|
if (isHandlerNotFoundError(err)) {
|
|
2849
3108
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
2850
3109
|
}
|
|
2851
|
-
|
|
3110
|
+
logger12.error("Subgraph catch-up failed", {
|
|
2852
3111
|
subgraph: sg.name,
|
|
2853
3112
|
error: msg
|
|
2854
3113
|
});
|
|
@@ -2894,7 +3153,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
2894
3153
|
definitionCache.set(sg.name, def);
|
|
2895
3154
|
if (prevVersion && prevVersion !== sg.version) {
|
|
2896
3155
|
invalidateSubgraphRoute(sg.name);
|
|
2897
|
-
|
|
3156
|
+
logger12.info("Subgraph handler reloaded", {
|
|
2898
3157
|
subgraph: sg.name,
|
|
2899
3158
|
from: prevVersion,
|
|
2900
3159
|
to: sg.version
|
|
@@ -2928,7 +3187,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
2928
3187
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
2929
3188
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
2930
3189
|
});
|
|
2931
|
-
|
|
3190
|
+
logger12.info("Queued legacy reindex resume operation", {
|
|
2932
3191
|
subgraph: sg.name
|
|
2933
3192
|
});
|
|
2934
3193
|
} catch (err) {
|
|
@@ -2984,7 +3243,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
2984
3243
|
const activeRuns = new Map;
|
|
2985
3244
|
let running = true;
|
|
2986
3245
|
let draining = false;
|
|
2987
|
-
|
|
3246
|
+
logger12.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
2988
3247
|
const startOperation = (operation) => {
|
|
2989
3248
|
const controller = new AbortController;
|
|
2990
3249
|
active.set(operation.id, controller);
|
|
@@ -2992,9 +3251,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
2992
3251
|
if (!running)
|
|
2993
3252
|
return;
|
|
2994
3253
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
2995
|
-
|
|
3254
|
+
logger12.warn("Subgraph operation heartbeat failed", {
|
|
2996
3255
|
operationId: operation.id,
|
|
2997
|
-
error:
|
|
3256
|
+
error: getErrorMessage5(err)
|
|
2998
3257
|
});
|
|
2999
3258
|
});
|
|
3000
3259
|
}, HEARTBEAT_INTERVAL_MS);
|
|
@@ -3004,9 +3263,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3004
3263
|
controller.abort("user-cancelled");
|
|
3005
3264
|
}
|
|
3006
3265
|
}).catch((err) => {
|
|
3007
|
-
|
|
3266
|
+
logger12.warn("Subgraph operation cancel poll failed", {
|
|
3008
3267
|
operationId: operation.id,
|
|
3009
|
-
error:
|
|
3268
|
+
error: getErrorMessage5(err)
|
|
3010
3269
|
});
|
|
3011
3270
|
});
|
|
3012
3271
|
}, CANCEL_POLL_INTERVAL_MS);
|
|
@@ -3021,14 +3280,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3021
3280
|
const reason = String(controller.signal.reason ?? "");
|
|
3022
3281
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3023
3282
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3024
|
-
|
|
3283
|
+
logger12.info("Subgraph operation cancelled", {
|
|
3025
3284
|
operationId: operation.id,
|
|
3026
3285
|
subgraph: operation.subgraph_name
|
|
3027
3286
|
});
|
|
3028
3287
|
return;
|
|
3029
3288
|
}
|
|
3030
3289
|
if (controller.signal.aborted) {
|
|
3031
|
-
|
|
3290
|
+
logger12.info("Subgraph operation interrupted", {
|
|
3032
3291
|
operationId: operation.id,
|
|
3033
3292
|
subgraph: operation.subgraph_name,
|
|
3034
3293
|
reason
|
|
@@ -3036,7 +3295,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3036
3295
|
return;
|
|
3037
3296
|
}
|
|
3038
3297
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3039
|
-
|
|
3298
|
+
logger12.info("Subgraph operation completed", {
|
|
3040
3299
|
operationId: operation.id,
|
|
3041
3300
|
subgraph: operation.subgraph_name,
|
|
3042
3301
|
processed
|
|
@@ -3044,7 +3303,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3044
3303
|
} catch (err) {
|
|
3045
3304
|
const reason = String(controller.signal.reason ?? "");
|
|
3046
3305
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3047
|
-
|
|
3306
|
+
logger12.info("Subgraph operation interrupted by shutdown", {
|
|
3048
3307
|
operationId: operation.id,
|
|
3049
3308
|
subgraph: operation.subgraph_name
|
|
3050
3309
|
});
|
|
@@ -3054,11 +3313,11 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3054
3313
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3055
3314
|
return;
|
|
3056
3315
|
}
|
|
3057
|
-
await failSubgraphOperation(db, operation.id, lockedBy,
|
|
3058
|
-
|
|
3316
|
+
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage5(err), processed);
|
|
3317
|
+
logger12.error("Subgraph operation failed", {
|
|
3059
3318
|
operationId: operation.id,
|
|
3060
3319
|
subgraph: operation.subgraph_name,
|
|
3061
|
-
error:
|
|
3320
|
+
error: getErrorMessage5(err)
|
|
3062
3321
|
});
|
|
3063
3322
|
} finally {
|
|
3064
3323
|
clearInterval(heartbeat);
|
|
@@ -3102,13 +3361,13 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3102
3361
|
controller.abort("shutdown");
|
|
3103
3362
|
}
|
|
3104
3363
|
await Promise.allSettled(activeRuns.values());
|
|
3105
|
-
|
|
3364
|
+
logger12.info("Subgraph operation runner stopped");
|
|
3106
3365
|
};
|
|
3107
3366
|
}
|
|
3108
3367
|
async function startSubgraphProcessor(opts) {
|
|
3109
3368
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3110
3369
|
let running = true;
|
|
3111
|
-
|
|
3370
|
+
logger12.info("Starting subgraph processor", { concurrency });
|
|
3112
3371
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3113
3372
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3114
3373
|
});
|
|
@@ -3133,8 +3392,8 @@ async function startSubgraphProcessor(opts) {
|
|
|
3133
3392
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3134
3393
|
}
|
|
3135
3394
|
} catch (err) {
|
|
3136
|
-
|
|
3137
|
-
error:
|
|
3395
|
+
logger12.error("Subgraph reorg handling failed", {
|
|
3396
|
+
error: getErrorMessage5(err)
|
|
3138
3397
|
});
|
|
3139
3398
|
}
|
|
3140
3399
|
}, { connectionString: sourceListenerUrl() });
|
|
@@ -3146,22 +3405,24 @@ async function startSubgraphProcessor(opts) {
|
|
|
3146
3405
|
cleanupCaches(subgraphs);
|
|
3147
3406
|
await catchUpAll(subgraphs, db, concurrency);
|
|
3148
3407
|
}, POLL_INTERVAL_MS);
|
|
3408
|
+
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll(handleSubgraphReorg, loadSubgraphDefinition) : undefined;
|
|
3149
3409
|
const stopEmitter = await startEmitter();
|
|
3150
|
-
|
|
3410
|
+
logger12.info("Subgraph processor ready");
|
|
3151
3411
|
return async () => {
|
|
3152
3412
|
running = false;
|
|
3153
3413
|
clearInterval(pollInterval);
|
|
3154
3414
|
await stopListening();
|
|
3155
3415
|
await stopReorgListening();
|
|
3416
|
+
stopStreamsReorgPoll?.();
|
|
3156
3417
|
await stopOperations();
|
|
3157
3418
|
await stopEmitter();
|
|
3158
|
-
|
|
3419
|
+
logger12.info("Subgraph processor stopped");
|
|
3159
3420
|
};
|
|
3160
3421
|
}
|
|
3161
3422
|
|
|
3162
3423
|
// src/service.ts
|
|
3163
3424
|
import { getDb } from "@secondlayer/shared/db";
|
|
3164
|
-
import { logger as
|
|
3425
|
+
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3165
3426
|
import { sql as sql5 } from "kysely";
|
|
3166
3427
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
3167
3428
|
var SERVICE_NAME = "subgraph-processor";
|
|
@@ -3169,7 +3430,7 @@ async function writeHeartbeat() {
|
|
|
3169
3430
|
try {
|
|
3170
3431
|
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql5`now()` })).execute();
|
|
3171
3432
|
} catch (err) {
|
|
3172
|
-
|
|
3433
|
+
logger13.warn("subgraph-processor heartbeat write failed", {
|
|
3173
3434
|
error: err instanceof Error ? err.message : String(err)
|
|
3174
3435
|
});
|
|
3175
3436
|
}
|
|
@@ -3180,7 +3441,7 @@ var processor = await startSubgraphProcessor({
|
|
|
3180
3441
|
await writeHeartbeat();
|
|
3181
3442
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
3182
3443
|
var shutdown = async () => {
|
|
3183
|
-
|
|
3444
|
+
logger13.info("Shutting down subgraph processor...");
|
|
3184
3445
|
clearInterval(heartbeatInterval);
|
|
3185
3446
|
await processor();
|
|
3186
3447
|
process.exit(0);
|
|
@@ -3188,5 +3449,5 @@ var shutdown = async () => {
|
|
|
3188
3449
|
process.on("SIGINT", shutdown);
|
|
3189
3450
|
process.on("SIGTERM", shutdown);
|
|
3190
3451
|
|
|
3191
|
-
//# debugId=
|
|
3452
|
+
//# debugId=7F079106165EB8F064756E2164756E21
|
|
3192
3453
|
//# sourceMappingURL=service.js.map
|