@secondlayer/subgraphs 3.5.0 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +327 -113
- package/dist/src/index.js.map +12 -10
- package/dist/src/runtime/block-processor.js +292 -30
- package/dist/src/runtime/block-processor.js.map +8 -5
- package/dist/src/runtime/catchup.js +306 -91
- package/dist/src/runtime/catchup.js.map +9 -7
- package/dist/src/runtime/processor.js +433 -171
- package/dist/src/runtime/processor.js.map +14 -11
- package/dist/src/runtime/reindex.js +325 -111
- package/dist/src/runtime/reindex.js.map +10 -8
- package/dist/src/runtime/reorg.js +298 -36
- package/dist/src/runtime/reorg.js.map +8 -5
- package/dist/src/runtime/runner.d.ts +6 -1
- package/dist/src/runtime/runner.js +10 -7
- package/dist/src/runtime/runner.js.map +3 -3
- package/dist/src/schema/index.js.map +1 -1
- package/dist/src/service.js +436 -174
- package/dist/src/service.js.map +13 -10
- package/package.json +2 -2
package/dist/src/service.js
CHANGED
|
@@ -518,21 +518,21 @@ function buildEventPayload(filter, tx, event) {
|
|
|
518
518
|
return {
|
|
519
519
|
sender: decoded.sender,
|
|
520
520
|
recipient: decoded.recipient,
|
|
521
|
-
tokenId: decoded.value,
|
|
521
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
522
522
|
assetIdentifier: decoded.asset_identifier,
|
|
523
523
|
tx: txMeta
|
|
524
524
|
};
|
|
525
525
|
case "nft_mint":
|
|
526
526
|
return {
|
|
527
527
|
recipient: decoded.recipient,
|
|
528
|
-
tokenId: decoded.value,
|
|
528
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
529
529
|
assetIdentifier: decoded.asset_identifier,
|
|
530
530
|
tx: txMeta
|
|
531
531
|
};
|
|
532
532
|
case "nft_burn":
|
|
533
533
|
return {
|
|
534
534
|
sender: decoded.sender,
|
|
535
|
-
tokenId: decoded.value,
|
|
535
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
536
536
|
assetIdentifier: decoded.asset_identifier,
|
|
537
537
|
tx: txMeta
|
|
538
538
|
};
|
|
@@ -564,8 +564,8 @@ function buildEventPayload(filter, tx, event) {
|
|
|
564
564
|
tx: txMeta
|
|
565
565
|
};
|
|
566
566
|
case "print_event": {
|
|
567
|
-
const
|
|
568
|
-
const rawValue =
|
|
567
|
+
const rawHex = event.data?.raw_value;
|
|
568
|
+
const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
|
|
569
569
|
const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
|
|
570
570
|
const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
|
|
571
571
|
const { topic: _, ...rest } = clarityObj ?? {};
|
|
@@ -578,9 +578,11 @@ function buildEventPayload(filter, tx, event) {
|
|
|
578
578
|
};
|
|
579
579
|
}
|
|
580
580
|
case "contract_call": {
|
|
581
|
+
const ccRawHex = event.data?.raw_value;
|
|
582
|
+
const normalized = typeof ccRawHex === "string" && ccRawHex.startsWith("0x") ? { ...decoded, value: decodeClarityValue(ccRawHex) } : decoded;
|
|
581
583
|
const input = buildContractCallInput(filter, tx);
|
|
582
584
|
return {
|
|
583
|
-
...
|
|
585
|
+
...normalized,
|
|
584
586
|
type: "contract_call",
|
|
585
587
|
_eventType: event.type,
|
|
586
588
|
contractId: tx.contract_id ?? "",
|
|
@@ -949,10 +951,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
|
|
|
949
951
|
}
|
|
950
952
|
|
|
951
953
|
// src/runtime/block-processor.ts
|
|
952
|
-
import {
|
|
953
|
-
getSourceDb,
|
|
954
|
-
getTargetDb
|
|
955
|
-
} from "@secondlayer/shared/db";
|
|
954
|
+
import { getTargetDb } from "@secondlayer/shared/db";
|
|
956
955
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
957
956
|
import {
|
|
958
957
|
isByoSubgraph,
|
|
@@ -960,15 +959,286 @@ import {
|
|
|
960
959
|
resolveSubgraphDb,
|
|
961
960
|
updateSubgraphStatus
|
|
962
961
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
963
|
-
import { logger as
|
|
962
|
+
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
964
963
|
import { sql as sql3 } from "kysely";
|
|
965
964
|
|
|
966
965
|
// src/schema/utils.ts
|
|
967
966
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
968
967
|
|
|
968
|
+
// src/runtime/block-source.ts
|
|
969
|
+
import { getSourceDb } from "@secondlayer/shared/db";
|
|
970
|
+
import { IndexHttpClient } from "@secondlayer/shared/index-http";
|
|
971
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
972
|
+
|
|
973
|
+
// src/runtime/batch-loader.ts
|
|
974
|
+
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
975
|
+
const [blocks, txs, events] = await Promise.all([
|
|
976
|
+
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
977
|
+
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
978
|
+
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
979
|
+
]);
|
|
980
|
+
const txsByHeight = new Map;
|
|
981
|
+
for (const tx of txs) {
|
|
982
|
+
const h = Number(tx.block_height);
|
|
983
|
+
const list = txsByHeight.get(h) ?? [];
|
|
984
|
+
list.push(tx);
|
|
985
|
+
txsByHeight.set(h, list);
|
|
986
|
+
}
|
|
987
|
+
const eventsByHeight = new Map;
|
|
988
|
+
for (const evt of events) {
|
|
989
|
+
const h = Number(evt.block_height);
|
|
990
|
+
const list = eventsByHeight.get(h) ?? [];
|
|
991
|
+
list.push(evt);
|
|
992
|
+
eventsByHeight.set(h, list);
|
|
993
|
+
}
|
|
994
|
+
const result = new Map;
|
|
995
|
+
for (const block of blocks) {
|
|
996
|
+
const h = Number(block.height);
|
|
997
|
+
result.set(h, {
|
|
998
|
+
block,
|
|
999
|
+
txs: txsByHeight.get(h) ?? [],
|
|
1000
|
+
events: eventsByHeight.get(h) ?? []
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
return result;
|
|
1004
|
+
}
|
|
1005
|
+
function avgEventsPerBlock(batch) {
|
|
1006
|
+
if (batch.size === 0)
|
|
1007
|
+
return 0;
|
|
1008
|
+
let totalEvents = 0;
|
|
1009
|
+
for (const data of batch.values()) {
|
|
1010
|
+
totalEvents += data.events.length;
|
|
1011
|
+
}
|
|
1012
|
+
return totalEvents / batch.size;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
// src/runtime/reconstruct.ts
|
|
1016
|
+
function isoToUnixSeconds(iso) {
|
|
1017
|
+
if (!iso)
|
|
1018
|
+
return 0;
|
|
1019
|
+
return Math.floor(new Date(iso).getTime() / 1000);
|
|
1020
|
+
}
|
|
1021
|
+
function reconstructBlock(b) {
|
|
1022
|
+
return {
|
|
1023
|
+
height: b.block_height,
|
|
1024
|
+
hash: b.block_hash,
|
|
1025
|
+
parent_hash: b.parent_hash,
|
|
1026
|
+
burn_block_height: b.burn_block_height,
|
|
1027
|
+
burn_block_hash: b.burn_block_hash,
|
|
1028
|
+
timestamp: isoToUnixSeconds(b.block_time),
|
|
1029
|
+
canonical: true,
|
|
1030
|
+
created_at: new Date(0)
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
function reconstructTransaction(t) {
|
|
1034
|
+
return {
|
|
1035
|
+
tx_id: t.tx_id,
|
|
1036
|
+
block_height: t.block_height,
|
|
1037
|
+
tx_index: t.tx_index,
|
|
1038
|
+
type: t.tx_type,
|
|
1039
|
+
sender: t.sender,
|
|
1040
|
+
status: t.status,
|
|
1041
|
+
contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
|
|
1042
|
+
function_name: t.contract_call?.function_name ?? null,
|
|
1043
|
+
function_args: t.contract_call?.function_args_hex ?? [],
|
|
1044
|
+
raw_result: t.contract_call?.result_hex ?? null,
|
|
1045
|
+
raw_tx: "",
|
|
1046
|
+
created_at: new Date(0)
|
|
1047
|
+
};
|
|
1048
|
+
}
|
|
1049
|
+
function reconstructEvent(e) {
|
|
1050
|
+
const base = {
|
|
1051
|
+
id: `${e.tx_id}#${e.event_index}`,
|
|
1052
|
+
tx_id: e.tx_id,
|
|
1053
|
+
block_height: e.block_height,
|
|
1054
|
+
event_index: e.event_index,
|
|
1055
|
+
created_at: new Date(0)
|
|
1056
|
+
};
|
|
1057
|
+
switch (e.event_type) {
|
|
1058
|
+
case "ft_transfer":
|
|
1059
|
+
case "ft_mint":
|
|
1060
|
+
case "ft_burn":
|
|
1061
|
+
return {
|
|
1062
|
+
...base,
|
|
1063
|
+
type: `${e.event_type}_event`,
|
|
1064
|
+
data: {
|
|
1065
|
+
asset_identifier: e.asset_identifier,
|
|
1066
|
+
sender: e.sender,
|
|
1067
|
+
recipient: e.recipient,
|
|
1068
|
+
amount: e.amount
|
|
1069
|
+
}
|
|
1070
|
+
};
|
|
1071
|
+
case "nft_transfer":
|
|
1072
|
+
case "nft_mint":
|
|
1073
|
+
case "nft_burn":
|
|
1074
|
+
return {
|
|
1075
|
+
...base,
|
|
1076
|
+
type: `${e.event_type}_event`,
|
|
1077
|
+
data: {
|
|
1078
|
+
asset_identifier: e.asset_identifier,
|
|
1079
|
+
sender: e.sender,
|
|
1080
|
+
recipient: e.recipient,
|
|
1081
|
+
raw_value: e.value
|
|
1082
|
+
}
|
|
1083
|
+
};
|
|
1084
|
+
case "stx_transfer":
|
|
1085
|
+
case "stx_mint":
|
|
1086
|
+
case "stx_burn":
|
|
1087
|
+
return {
|
|
1088
|
+
...base,
|
|
1089
|
+
type: `${e.event_type}_event`,
|
|
1090
|
+
data: {
|
|
1091
|
+
sender: e.sender,
|
|
1092
|
+
recipient: e.recipient,
|
|
1093
|
+
amount: e.amount,
|
|
1094
|
+
...e.event_type === "stx_transfer" ? { memo: e.memo ?? "" } : {}
|
|
1095
|
+
}
|
|
1096
|
+
};
|
|
1097
|
+
case "stx_lock":
|
|
1098
|
+
return {
|
|
1099
|
+
...base,
|
|
1100
|
+
type: "stx_lock_event",
|
|
1101
|
+
data: {
|
|
1102
|
+
locked_address: e.sender,
|
|
1103
|
+
locked_amount: e.amount,
|
|
1104
|
+
unlock_height: e.payload.unlock_height
|
|
1105
|
+
}
|
|
1106
|
+
};
|
|
1107
|
+
case "print":
|
|
1108
|
+
return {
|
|
1109
|
+
...base,
|
|
1110
|
+
type: "contract_event",
|
|
1111
|
+
data: {
|
|
1112
|
+
topic: e.payload.topic,
|
|
1113
|
+
contract_identifier: e.contract_id,
|
|
1114
|
+
value: e.payload.value,
|
|
1115
|
+
raw_value: e.payload.raw_value
|
|
1116
|
+
}
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// src/runtime/block-source.ts
|
|
1122
|
+
class PostgresBlockSource {
|
|
1123
|
+
async getTip() {
|
|
1124
|
+
const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1125
|
+
return progress ? Number(progress.highest_seen_block) : 0;
|
|
1126
|
+
}
|
|
1127
|
+
loadBlockRange(fromHeight, toHeight) {
|
|
1128
|
+
return loadBlockRange(getSourceDb(), fromHeight, toHeight);
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
var EVENT_FILTER_TO_INDEX_TYPE = {
|
|
1132
|
+
stx_transfer: "stx_transfer",
|
|
1133
|
+
stx_mint: "stx_mint",
|
|
1134
|
+
stx_burn: "stx_burn",
|
|
1135
|
+
stx_lock: "stx_lock",
|
|
1136
|
+
ft_transfer: "ft_transfer",
|
|
1137
|
+
ft_mint: "ft_mint",
|
|
1138
|
+
ft_burn: "ft_burn",
|
|
1139
|
+
nft_transfer: "nft_transfer",
|
|
1140
|
+
nft_mint: "nft_mint",
|
|
1141
|
+
nft_burn: "nft_burn",
|
|
1142
|
+
print_event: "print"
|
|
1143
|
+
};
|
|
1144
|
+
var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
1145
|
+
var ALL_INDEX_EVENT_TYPES = [
|
|
1146
|
+
...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
|
|
1147
|
+
];
|
|
1148
|
+
function sourceFilters(subgraph) {
|
|
1149
|
+
const sources = subgraph.sources;
|
|
1150
|
+
return Array.isArray(sources) ? sources : Object.values(sources);
|
|
1151
|
+
}
|
|
1152
|
+
function referencedIndexEventTypes(subgraph) {
|
|
1153
|
+
const filters = sourceFilters(subgraph);
|
|
1154
|
+
if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
|
|
1155
|
+
return ALL_INDEX_EVENT_TYPES;
|
|
1156
|
+
}
|
|
1157
|
+
const types = new Set;
|
|
1158
|
+
for (const f of filters) {
|
|
1159
|
+
const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
|
|
1160
|
+
if (t)
|
|
1161
|
+
types.add(t);
|
|
1162
|
+
}
|
|
1163
|
+
return [...types];
|
|
1164
|
+
}
|
|
1165
|
+
function isStreamsIndexEligible(subgraph) {
|
|
1166
|
+
if (Array.isArray(subgraph.sources))
|
|
1167
|
+
return false;
|
|
1168
|
+
const filters = sourceFilters(subgraph);
|
|
1169
|
+
if (filters.length === 0)
|
|
1170
|
+
return false;
|
|
1171
|
+
for (const f of filters) {
|
|
1172
|
+
const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
|
|
1173
|
+
if (!known)
|
|
1174
|
+
return false;
|
|
1175
|
+
}
|
|
1176
|
+
return true;
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
class PublicApiBlockSource {
|
|
1180
|
+
http;
|
|
1181
|
+
eventTypes;
|
|
1182
|
+
constructor(http, eventTypes) {
|
|
1183
|
+
this.http = http;
|
|
1184
|
+
this.eventTypes = eventTypes;
|
|
1185
|
+
}
|
|
1186
|
+
getTip() {
|
|
1187
|
+
return this.http.getIndexTip();
|
|
1188
|
+
}
|
|
1189
|
+
async loadBlockRange(fromHeight, toHeight) {
|
|
1190
|
+
const [blocks, txs, eventLists] = await Promise.all([
|
|
1191
|
+
this.http.walkBlocks(fromHeight, toHeight),
|
|
1192
|
+
this.http.walkTransactions(fromHeight, toHeight),
|
|
1193
|
+
Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
|
|
1194
|
+
]);
|
|
1195
|
+
const map = new Map;
|
|
1196
|
+
for (const b of blocks) {
|
|
1197
|
+
map.set(b.block_height, {
|
|
1198
|
+
block: reconstructBlock(b),
|
|
1199
|
+
txs: [],
|
|
1200
|
+
events: []
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
for (const t of txs) {
|
|
1204
|
+
map.get(t.block_height)?.txs.push(reconstructTransaction(t));
|
|
1205
|
+
}
|
|
1206
|
+
for (const list of eventLists) {
|
|
1207
|
+
for (const e of list) {
|
|
1208
|
+
map.get(e.block_height)?.events.push(reconstructEvent(e));
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
for (const bd of map.values()) {
|
|
1212
|
+
bd.txs.sort((a, b) => a.tx_index - b.tx_index);
|
|
1213
|
+
bd.events.sort((a, b) => a.event_index - b.event_index);
|
|
1214
|
+
}
|
|
1215
|
+
return map;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
var postgresBlockSource = new PostgresBlockSource;
|
|
1219
|
+
function buildHttpClient() {
|
|
1220
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
1221
|
+
return new IndexHttpClient({
|
|
1222
|
+
indexBaseUrl: baseUrl,
|
|
1223
|
+
streamsBaseUrl: baseUrl,
|
|
1224
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
function resolveBlockSource(subgraph) {
|
|
1228
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
|
|
1229
|
+
return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
|
|
1230
|
+
}
|
|
1231
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
|
|
1232
|
+
logger3.debug("Subgraph not streams-index eligible, using DB tap", {
|
|
1233
|
+
subgraph: subgraph.name
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
return postgresBlockSource;
|
|
1237
|
+
}
|
|
1238
|
+
|
|
969
1239
|
// src/runtime/outbox-emit.ts
|
|
970
1240
|
import { createHash } from "node:crypto";
|
|
971
|
-
import { logger as
|
|
1241
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
972
1242
|
var loggedKillSwitch = false;
|
|
973
1243
|
function isEmitOutboxEnabled() {
|
|
974
1244
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
@@ -987,7 +1257,7 @@ function stableStringify(obj) {
|
|
|
987
1257
|
async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
988
1258
|
if (!isEmitOutboxEnabled()) {
|
|
989
1259
|
if (!loggedKillSwitch) {
|
|
990
|
-
|
|
1260
|
+
logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
991
1261
|
loggedKillSwitch = true;
|
|
992
1262
|
}
|
|
993
1263
|
return 0;
|
|
@@ -1228,7 +1498,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
|
|
|
1228
1498
|
return resolved;
|
|
1229
1499
|
}
|
|
1230
1500
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
1231
|
-
const sourceDb = getSourceDb();
|
|
1232
1501
|
const targetDb = getTargetDb();
|
|
1233
1502
|
const blockStart = performance.now();
|
|
1234
1503
|
const result = {
|
|
@@ -1246,25 +1515,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1246
1515
|
txs = opts.preloaded.txs;
|
|
1247
1516
|
evts = opts.preloaded.events;
|
|
1248
1517
|
} else {
|
|
1249
|
-
|
|
1250
|
-
if (!
|
|
1251
|
-
|
|
1518
|
+
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
1519
|
+
if (!data) {
|
|
1520
|
+
logger5.debug("Block not found or non-canonical for subgraph processing", {
|
|
1252
1521
|
subgraph: subgraphName,
|
|
1253
1522
|
blockHeight
|
|
1254
1523
|
});
|
|
1255
1524
|
result.skipped = true;
|
|
1256
1525
|
return result;
|
|
1257
1526
|
}
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
blockHeight
|
|
1262
|
-
});
|
|
1263
|
-
result.skipped = true;
|
|
1264
|
-
return result;
|
|
1265
|
-
}
|
|
1266
|
-
txs = await sourceDb.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
|
|
1267
|
-
evts = await sourceDb.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
|
|
1527
|
+
block = data.block;
|
|
1528
|
+
txs = data.txs;
|
|
1529
|
+
evts = data.events;
|
|
1268
1530
|
}
|
|
1269
1531
|
const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
|
|
1270
1532
|
const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
|
|
@@ -1355,7 +1617,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1355
1617
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
1356
1618
|
const count = Number(rows[0]?.count ?? 0);
|
|
1357
1619
|
if (count >= 1e7) {
|
|
1358
|
-
|
|
1620
|
+
logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
1359
1621
|
subgraph: subgraphName,
|
|
1360
1622
|
table,
|
|
1361
1623
|
count
|
|
@@ -1363,7 +1625,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1363
1625
|
}
|
|
1364
1626
|
}
|
|
1365
1627
|
} catch (err) {
|
|
1366
|
-
|
|
1628
|
+
logger5.debug("Row count sample failed", {
|
|
1367
1629
|
subgraph: subgraphName,
|
|
1368
1630
|
error: err instanceof Error ? err.message : String(err)
|
|
1369
1631
|
});
|
|
@@ -1437,7 +1699,7 @@ class StatsAccumulator {
|
|
|
1437
1699
|
|
|
1438
1700
|
// src/runtime/reindex.ts
|
|
1439
1701
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
1440
|
-
import { getRawClient,
|
|
1702
|
+
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
1441
1703
|
import {
|
|
1442
1704
|
recordGapBatch,
|
|
1443
1705
|
resolveGaps
|
|
@@ -1446,7 +1708,7 @@ import {
|
|
|
1446
1708
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1447
1709
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
1448
1710
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1449
|
-
import { logger as
|
|
1711
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
1450
1712
|
|
|
1451
1713
|
// src/schema/generator.ts
|
|
1452
1714
|
import { createHash as createHash2 } from "node:crypto";
|
|
@@ -1539,48 +1801,6 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
1539
1801
|
return { statements, hash };
|
|
1540
1802
|
}
|
|
1541
1803
|
|
|
1542
|
-
// src/runtime/batch-loader.ts
|
|
1543
|
-
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1544
|
-
const [blocks, txs, events] = await Promise.all([
|
|
1545
|
-
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1546
|
-
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1547
|
-
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1548
|
-
]);
|
|
1549
|
-
const txsByHeight = new Map;
|
|
1550
|
-
for (const tx of txs) {
|
|
1551
|
-
const h = Number(tx.block_height);
|
|
1552
|
-
const list = txsByHeight.get(h) ?? [];
|
|
1553
|
-
list.push(tx);
|
|
1554
|
-
txsByHeight.set(h, list);
|
|
1555
|
-
}
|
|
1556
|
-
const eventsByHeight = new Map;
|
|
1557
|
-
for (const evt of events) {
|
|
1558
|
-
const h = Number(evt.block_height);
|
|
1559
|
-
const list = eventsByHeight.get(h) ?? [];
|
|
1560
|
-
list.push(evt);
|
|
1561
|
-
eventsByHeight.set(h, list);
|
|
1562
|
-
}
|
|
1563
|
-
const result = new Map;
|
|
1564
|
-
for (const block of blocks) {
|
|
1565
|
-
const h = Number(block.height);
|
|
1566
|
-
result.set(h, {
|
|
1567
|
-
block,
|
|
1568
|
-
txs: txsByHeight.get(h) ?? [],
|
|
1569
|
-
events: eventsByHeight.get(h) ?? []
|
|
1570
|
-
});
|
|
1571
|
-
}
|
|
1572
|
-
return result;
|
|
1573
|
-
}
|
|
1574
|
-
function avgEventsPerBlock(batch) {
|
|
1575
|
-
if (batch.size === 0)
|
|
1576
|
-
return 0;
|
|
1577
|
-
let totalEvents = 0;
|
|
1578
|
-
for (const data of batch.values()) {
|
|
1579
|
-
totalEvents += data.events.length;
|
|
1580
|
-
}
|
|
1581
|
-
return totalEvents / batch.size;
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
1804
|
// src/runtime/reindex.ts
|
|
1585
1805
|
var LOG_INTERVAL = 1000;
|
|
1586
1806
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
@@ -1641,7 +1861,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
1641
1861
|
return ranges;
|
|
1642
1862
|
}
|
|
1643
1863
|
async function processBlockRange(def, opts) {
|
|
1644
|
-
const
|
|
1864
|
+
const source = resolveBlockSource(def);
|
|
1645
1865
|
const targetDb = getTargetDb2();
|
|
1646
1866
|
const subgraphName = def.name;
|
|
1647
1867
|
const { fromBlock, toBlock, status } = opts;
|
|
@@ -1671,11 +1891,11 @@ async function processBlockRange(def, opts) {
|
|
|
1671
1891
|
lastHealthFlushAt = Date.now();
|
|
1672
1892
|
};
|
|
1673
1893
|
let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
1674
|
-
let nextBatchPromise = loadBlockRange(
|
|
1894
|
+
let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
|
|
1675
1895
|
while (currentHeight <= toBlock) {
|
|
1676
1896
|
if (opts.signal?.aborted) {
|
|
1677
1897
|
aborted = true;
|
|
1678
|
-
|
|
1898
|
+
logger6.info("Block processing aborted", {
|
|
1679
1899
|
subgraph: subgraphName,
|
|
1680
1900
|
currentBlock: currentHeight,
|
|
1681
1901
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -1687,7 +1907,7 @@ async function processBlockRange(def, opts) {
|
|
|
1687
1907
|
const nextStart = batchEnd + 1;
|
|
1688
1908
|
if (nextStart <= toBlock) {
|
|
1689
1909
|
nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
1690
|
-
nextBatchPromise = loadBlockRange(
|
|
1910
|
+
nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
|
|
1691
1911
|
}
|
|
1692
1912
|
const batchFailedBlocks = [];
|
|
1693
1913
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -1705,7 +1925,7 @@ async function processBlockRange(def, opts) {
|
|
|
1705
1925
|
});
|
|
1706
1926
|
} catch (err) {
|
|
1707
1927
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1708
|
-
|
|
1928
|
+
logger6.error("Block processing error", {
|
|
1709
1929
|
subgraph: subgraphName,
|
|
1710
1930
|
blockHeight: height,
|
|
1711
1931
|
error: errorMsg
|
|
@@ -1739,7 +1959,7 @@ async function processBlockRange(def, opts) {
|
|
|
1739
1959
|
lastProgressFlushAt = now;
|
|
1740
1960
|
}
|
|
1741
1961
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1742
|
-
|
|
1962
|
+
logger6.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1743
1963
|
subgraph: subgraphName,
|
|
1744
1964
|
processed: blocksProcessed,
|
|
1745
1965
|
total: totalBlocks,
|
|
@@ -1758,7 +1978,7 @@ async function processBlockRange(def, opts) {
|
|
|
1758
1978
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1759
1979
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1760
1980
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1761
|
-
|
|
1981
|
+
logger6.warn("Failed to record subgraph gaps", {
|
|
1762
1982
|
subgraph: subgraphName,
|
|
1763
1983
|
error: err instanceof Error ? err.message : String(err)
|
|
1764
1984
|
});
|
|
@@ -1775,32 +1995,26 @@ async function processBlockRange(def, opts) {
|
|
|
1775
1995
|
await flushHealth();
|
|
1776
1996
|
return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
|
|
1777
1997
|
}
|
|
1778
|
-
async function resolveBlockRange(
|
|
1998
|
+
async function resolveBlockRange(source, def, opts) {
|
|
1779
1999
|
const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
|
|
1780
|
-
|
|
1781
|
-
if (opts?.toBlock != null) {
|
|
1782
|
-
toBlock = opts.toBlock;
|
|
1783
|
-
} else {
|
|
1784
|
-
const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1785
|
-
toBlock = progress?.highest_seen_block ?? 0;
|
|
1786
|
-
}
|
|
2000
|
+
const toBlock = opts?.toBlock != null ? opts.toBlock : await source.getTip();
|
|
1787
2001
|
return { fromBlock, toBlock };
|
|
1788
2002
|
}
|
|
1789
2003
|
async function clearReindexMetadata(db, subgraphName) {
|
|
1790
2004
|
await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
|
|
1791
2005
|
}
|
|
1792
2006
|
async function reindexSubgraph(def, opts) {
|
|
1793
|
-
const
|
|
2007
|
+
const source = resolveBlockSource(def);
|
|
1794
2008
|
const targetDb = getTargetDb2();
|
|
1795
2009
|
const client = getRawClient("target");
|
|
1796
2010
|
const subgraphName = def.name;
|
|
1797
2011
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1798
2012
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1799
|
-
|
|
2013
|
+
logger6.info("Reindex starting", { subgraph: subgraphName });
|
|
1800
2014
|
try {
|
|
1801
|
-
const { fromBlock, toBlock } = await resolveBlockRange(
|
|
2015
|
+
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
1802
2016
|
if (fromBlock > toBlock) {
|
|
1803
|
-
|
|
2017
|
+
logger6.info("No blocks to reindex", {
|
|
1804
2018
|
subgraph: subgraphName,
|
|
1805
2019
|
fromBlock,
|
|
1806
2020
|
toBlock
|
|
@@ -1813,7 +2027,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1813
2027
|
for (const stmt of statements) {
|
|
1814
2028
|
await client.unsafe(stmt);
|
|
1815
2029
|
}
|
|
1816
|
-
|
|
2030
|
+
logger6.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1817
2031
|
await targetDb.updateTable("subgraphs").set({
|
|
1818
2032
|
last_processed_block: initialReindexProgressBlock(fromBlock),
|
|
1819
2033
|
reindex_from_block: fromBlock,
|
|
@@ -1824,7 +2038,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1824
2038
|
last_error_at: null,
|
|
1825
2039
|
updated_at: new Date
|
|
1826
2040
|
}).where("name", "=", subgraphName).execute();
|
|
1827
|
-
|
|
2041
|
+
logger6.info("Reindexing blocks", {
|
|
1828
2042
|
subgraph: subgraphName,
|
|
1829
2043
|
fromBlock,
|
|
1830
2044
|
toBlock,
|
|
@@ -1845,9 +2059,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
1845
2059
|
if (reason === "user-cancelled") {
|
|
1846
2060
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1847
2061
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1848
|
-
|
|
2062
|
+
logger6.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1849
2063
|
} else {
|
|
1850
|
-
|
|
2064
|
+
logger6.info("Reindex interrupted by shutdown, will resume", {
|
|
1851
2065
|
subgraph: subgraphName
|
|
1852
2066
|
});
|
|
1853
2067
|
}
|
|
@@ -1855,7 +2069,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1855
2069
|
}
|
|
1856
2070
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1857
2071
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1858
|
-
|
|
2072
|
+
logger6.info("Reindex complete", {
|
|
1859
2073
|
subgraph: subgraphName,
|
|
1860
2074
|
blocks: result.blocksProcessed,
|
|
1861
2075
|
events: result.totalEventsProcessed,
|
|
@@ -1863,7 +2077,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1863
2077
|
});
|
|
1864
2078
|
return { processed: result.blocksProcessed };
|
|
1865
2079
|
} catch (err) {
|
|
1866
|
-
|
|
2080
|
+
logger6.error("Reindex failed", {
|
|
1867
2081
|
subgraph: subgraphName,
|
|
1868
2082
|
error: getErrorMessage2(err)
|
|
1869
2083
|
});
|
|
@@ -1885,7 +2099,7 @@ async function resumeReindex(def, opts) {
|
|
|
1885
2099
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
1886
2100
|
const toBlock = Number(row.reindex_to_block);
|
|
1887
2101
|
if (fromBlock == null) {
|
|
1888
|
-
|
|
2102
|
+
logger6.info("No reindex metadata, starting fresh reindex", {
|
|
1889
2103
|
subgraph: subgraphName
|
|
1890
2104
|
});
|
|
1891
2105
|
return reindexSubgraph(def, {
|
|
@@ -1894,12 +2108,12 @@ async function resumeReindex(def, opts) {
|
|
|
1894
2108
|
});
|
|
1895
2109
|
}
|
|
1896
2110
|
if (fromBlock > toBlock) {
|
|
1897
|
-
|
|
2111
|
+
logger6.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1898
2112
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1899
2113
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1900
2114
|
return { processed: 0 };
|
|
1901
2115
|
}
|
|
1902
|
-
|
|
2116
|
+
logger6.info("Resuming reindex", {
|
|
1903
2117
|
subgraph: subgraphName,
|
|
1904
2118
|
fromBlock,
|
|
1905
2119
|
toBlock,
|
|
@@ -1920,9 +2134,9 @@ async function resumeReindex(def, opts) {
|
|
|
1920
2134
|
if (reason === "user-cancelled") {
|
|
1921
2135
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1922
2136
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1923
|
-
|
|
2137
|
+
logger6.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
1924
2138
|
} else {
|
|
1925
|
-
|
|
2139
|
+
logger6.info("Resume interrupted by shutdown, will resume again", {
|
|
1926
2140
|
subgraph: subgraphName
|
|
1927
2141
|
});
|
|
1928
2142
|
}
|
|
@@ -1930,13 +2144,13 @@ async function resumeReindex(def, opts) {
|
|
|
1930
2144
|
}
|
|
1931
2145
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1932
2146
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1933
|
-
|
|
2147
|
+
logger6.info("Resumed reindex complete", {
|
|
1934
2148
|
subgraph: subgraphName,
|
|
1935
2149
|
blocks: result.blocksProcessed
|
|
1936
2150
|
});
|
|
1937
2151
|
return { processed: result.blocksProcessed };
|
|
1938
2152
|
} catch (err) {
|
|
1939
|
-
|
|
2153
|
+
logger6.error("Resumed reindex failed", {
|
|
1940
2154
|
subgraph: subgraphName,
|
|
1941
2155
|
error: getErrorMessage2(err)
|
|
1942
2156
|
});
|
|
@@ -1947,7 +2161,7 @@ async function resumeReindex(def, opts) {
|
|
|
1947
2161
|
async function backfillSubgraph(def, opts) {
|
|
1948
2162
|
const targetDb = getTargetDb2();
|
|
1949
2163
|
const subgraphName = def.name;
|
|
1950
|
-
|
|
2164
|
+
logger6.info("Backfill starting", {
|
|
1951
2165
|
subgraph: subgraphName,
|
|
1952
2166
|
from: opts.fromBlock,
|
|
1953
2167
|
to: opts.toBlock
|
|
@@ -1964,17 +2178,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
1964
2178
|
signal: opts.signal
|
|
1965
2179
|
});
|
|
1966
2180
|
if (result.aborted) {
|
|
1967
|
-
|
|
2181
|
+
logger6.info("Backfill aborted", { subgraph: subgraphName });
|
|
1968
2182
|
return { processed: result.blocksProcessed };
|
|
1969
2183
|
}
|
|
1970
2184
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
1971
2185
|
if (resolved > 0) {
|
|
1972
|
-
|
|
2186
|
+
logger6.info("Resolved subgraph gaps via backfill", {
|
|
1973
2187
|
subgraph: subgraphName,
|
|
1974
2188
|
resolved
|
|
1975
2189
|
});
|
|
1976
2190
|
}
|
|
1977
|
-
|
|
2191
|
+
logger6.info("Backfill complete", {
|
|
1978
2192
|
subgraph: subgraphName,
|
|
1979
2193
|
blocks: result.blocksProcessed,
|
|
1980
2194
|
events: result.totalEventsProcessed,
|
|
@@ -1982,7 +2196,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
1982
2196
|
});
|
|
1983
2197
|
return { processed: result.blocksProcessed };
|
|
1984
2198
|
} catch (err) {
|
|
1985
|
-
|
|
2199
|
+
logger6.error("Backfill failed", {
|
|
1986
2200
|
subgraph: subgraphName,
|
|
1987
2201
|
error: getErrorMessage2(err)
|
|
1988
2202
|
});
|
|
@@ -1991,12 +2205,12 @@ async function backfillSubgraph(def, opts) {
|
|
|
1991
2205
|
}
|
|
1992
2206
|
|
|
1993
2207
|
// src/runtime/catchup.ts
|
|
1994
|
-
import {
|
|
2208
|
+
import { getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
1995
2209
|
import {
|
|
1996
2210
|
recordGapBatch as recordGapBatch2
|
|
1997
2211
|
} from "@secondlayer/shared/db/queries/subgraph-gaps";
|
|
1998
2212
|
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
1999
|
-
import { logger as
|
|
2213
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
2000
2214
|
var LOG_INTERVAL2 = 1000;
|
|
2001
2215
|
var STANDARD_CATCHUP_BATCH_CONFIG = {
|
|
2002
2216
|
defaultBatchSize: 500,
|
|
@@ -2067,22 +2281,19 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2067
2281
|
return 0;
|
|
2068
2282
|
catchingUp.add(subgraphName);
|
|
2069
2283
|
try {
|
|
2070
|
-
const
|
|
2284
|
+
const source = resolveBlockSource(subgraph);
|
|
2071
2285
|
const targetDb = getTargetDb3();
|
|
2072
2286
|
const subgraphRow = await getSubgraph(targetDb, subgraphName);
|
|
2073
2287
|
if (!subgraphRow)
|
|
2074
2288
|
return 0;
|
|
2075
2289
|
const lastProcessedBlock = Number(subgraphRow.last_processed_block);
|
|
2076
|
-
const
|
|
2077
|
-
if (
|
|
2078
|
-
return 0;
|
|
2079
|
-
const chainTip = Number(progress.highest_seen_block);
|
|
2080
|
-
if (lastProcessedBlock >= chainTip)
|
|
2290
|
+
const chainTip = await source.getTip();
|
|
2291
|
+
if (chainTip <= 0 || lastProcessedBlock >= chainTip)
|
|
2081
2292
|
return 0;
|
|
2082
2293
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
2083
2294
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
2084
2295
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
2085
|
-
|
|
2296
|
+
logger7.info("Subgraph catch-up starting", {
|
|
2086
2297
|
subgraph: subgraphName,
|
|
2087
2298
|
from: startBlock,
|
|
2088
2299
|
to: chainTip,
|
|
@@ -2094,11 +2305,11 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2094
2305
|
let batchSize = batchConfig.defaultBatchSize;
|
|
2095
2306
|
let currentHeight = startBlock;
|
|
2096
2307
|
let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
2097
|
-
let nextBatchPromise = batchConfig.prefetch ? loadBlockRange(
|
|
2308
|
+
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
2098
2309
|
while (currentHeight <= chainTip) {
|
|
2099
2310
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
2100
2311
|
if (!currentRow || currentRow.status !== "active") {
|
|
2101
|
-
|
|
2312
|
+
logger7.info("Subgraph status changed, stopping catch-up", {
|
|
2102
2313
|
subgraph: subgraphName,
|
|
2103
2314
|
status: currentRow?.status ?? "deleted"
|
|
2104
2315
|
});
|
|
@@ -2112,13 +2323,13 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2112
2323
|
const nextStart = batchEnd + 1;
|
|
2113
2324
|
if (nextStart <= chainTip) {
|
|
2114
2325
|
prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
|
|
2115
|
-
nextBatchPromise = loadBlockRange(
|
|
2326
|
+
nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
|
|
2116
2327
|
} else {
|
|
2117
2328
|
nextBatchPromise = undefined;
|
|
2118
2329
|
}
|
|
2119
2330
|
} else {
|
|
2120
2331
|
batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
2121
|
-
batch = await loadBlockRange(
|
|
2332
|
+
batch = await source.loadBlockRange(currentHeight, batchEnd);
|
|
2122
2333
|
}
|
|
2123
2334
|
const batchFailedBlocks = [];
|
|
2124
2335
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -2134,7 +2345,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2134
2345
|
preloaded: blockData
|
|
2135
2346
|
});
|
|
2136
2347
|
} catch (err) {
|
|
2137
|
-
|
|
2348
|
+
logger7.error("Block processing error during catch-up", {
|
|
2138
2349
|
subgraph: subgraphName,
|
|
2139
2350
|
blockHeight: height,
|
|
2140
2351
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -2153,7 +2364,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2153
2364
|
}
|
|
2154
2365
|
}
|
|
2155
2366
|
if (processed % LOG_INTERVAL2 === 0) {
|
|
2156
|
-
|
|
2367
|
+
logger7.info("Subgraph catch-up progress", {
|
|
2157
2368
|
subgraph: subgraphName,
|
|
2158
2369
|
processed,
|
|
2159
2370
|
total: totalBlocks,
|
|
@@ -2165,7 +2376,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2165
2376
|
if (batchFailedBlocks.length > 0) {
|
|
2166
2377
|
const gaps = coalesceGaps(batchFailedBlocks);
|
|
2167
2378
|
await recordGapBatch2(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
|
|
2168
|
-
|
|
2379
|
+
logger7.warn("Failed to record subgraph gaps", {
|
|
2169
2380
|
subgraph: subgraphName,
|
|
2170
2381
|
error: err instanceof Error ? err.message : String(err)
|
|
2171
2382
|
});
|
|
@@ -2176,7 +2387,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
2176
2387
|
currentHeight = batchEnd + 1;
|
|
2177
2388
|
}
|
|
2178
2389
|
await stats.flush(targetDb);
|
|
2179
|
-
|
|
2390
|
+
logger7.info("Subgraph catch-up complete", {
|
|
2180
2391
|
subgraph: subgraphName,
|
|
2181
2392
|
processed
|
|
2182
2393
|
});
|
|
@@ -2193,13 +2404,13 @@ import {
|
|
|
2193
2404
|
listSubgraphs,
|
|
2194
2405
|
resolveSubgraphRawClient
|
|
2195
2406
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2196
|
-
import { logger as
|
|
2407
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
2197
2408
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
2198
2409
|
const targetDb = getTargetDb4();
|
|
2199
2410
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
2200
2411
|
if (activeSubgraphs.length === 0)
|
|
2201
2412
|
return;
|
|
2202
|
-
|
|
2413
|
+
logger8.info("Propagating reorg to subgraphs", {
|
|
2203
2414
|
blockHeight,
|
|
2204
2415
|
subgraphCount: activeSubgraphs.length
|
|
2205
2416
|
});
|
|
@@ -2247,7 +2458,7 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
2247
2458
|
}).as("row_pk"),
|
|
2248
2459
|
eb2.val("pending").as("status")
|
|
2249
2460
|
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.column("dedup_key").doNothing()).execute().catch((err) => {
|
|
2250
|
-
|
|
2461
|
+
logger8.warn("Failed to emit revert event for subscriptions", {
|
|
2251
2462
|
subgraph: sg.name,
|
|
2252
2463
|
table: tableName,
|
|
2253
2464
|
blockHeight,
|
|
@@ -2255,19 +2466,19 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
2255
2466
|
});
|
|
2256
2467
|
});
|
|
2257
2468
|
}
|
|
2258
|
-
|
|
2469
|
+
logger8.info("Subgraph reorg cleanup done", {
|
|
2259
2470
|
subgraph: sg.name,
|
|
2260
2471
|
blockHeight,
|
|
2261
2472
|
tablesAffected: Object.keys(revertedByTable).length
|
|
2262
2473
|
});
|
|
2263
2474
|
const def = await loadSubgraphDef(sg);
|
|
2264
2475
|
await processBlock(def, sg.name, blockHeight);
|
|
2265
|
-
|
|
2476
|
+
logger8.info("Subgraph reorg reprocessed", {
|
|
2266
2477
|
subgraph: sg.name,
|
|
2267
2478
|
blockHeight
|
|
2268
2479
|
});
|
|
2269
2480
|
} catch (err) {
|
|
2270
|
-
|
|
2481
|
+
logger8.error("Subgraph reorg handling failed", {
|
|
2271
2482
|
subgraph: sg.name,
|
|
2272
2483
|
blockHeight,
|
|
2273
2484
|
error: getErrorMessage3(err)
|
|
@@ -2281,7 +2492,7 @@ import { randomUUID } from "node:crypto";
|
|
|
2281
2492
|
import { hostname } from "node:os";
|
|
2282
2493
|
import { resolve } from "node:path";
|
|
2283
2494
|
import { pathToFileURL } from "node:url";
|
|
2284
|
-
import { getErrorMessage as
|
|
2495
|
+
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
2285
2496
|
import { getTargetDb as getTargetDb6 } from "@secondlayer/shared/db";
|
|
2286
2497
|
import {
|
|
2287
2498
|
cancelSubgraphOperation,
|
|
@@ -2300,7 +2511,7 @@ import {
|
|
|
2300
2511
|
pgSchemaName as pgSchemaName2,
|
|
2301
2512
|
updateSubgraphStatus as updateSubgraphStatus3
|
|
2302
2513
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
2303
|
-
import { logger as
|
|
2514
|
+
import { logger as logger12 } from "@secondlayer/shared/logger";
|
|
2304
2515
|
import { listen as listen2 } from "@secondlayer/shared/queue/listener";
|
|
2305
2516
|
|
|
2306
2517
|
// src/runtime/emitter.ts
|
|
@@ -2308,12 +2519,12 @@ import {
|
|
|
2308
2519
|
getTargetDb as getTargetDb5
|
|
2309
2520
|
} from "@secondlayer/shared/db";
|
|
2310
2521
|
import { getSubscriptionSigningSecret } from "@secondlayer/shared/db/queries/subscriptions";
|
|
2311
|
-
import { logger as
|
|
2522
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
2312
2523
|
import { listen } from "@secondlayer/shared/queue/listener";
|
|
2313
2524
|
import { sql as sql4 } from "kysely";
|
|
2314
2525
|
|
|
2315
2526
|
// src/runtime/formats/index.ts
|
|
2316
|
-
import { logger as
|
|
2527
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
2317
2528
|
|
|
2318
2529
|
// src/runtime/formats/cloudevents.ts
|
|
2319
2530
|
function buildCloudEvents(outboxRow, _sub) {
|
|
@@ -2460,7 +2671,7 @@ function buildForFormat(outboxRow, sub, signingSecret) {
|
|
|
2460
2671
|
case "standard-webhooks":
|
|
2461
2672
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
2462
2673
|
default:
|
|
2463
|
-
|
|
2674
|
+
logger9.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
2464
2675
|
format: sub.format,
|
|
2465
2676
|
subscriptionId: sub.id
|
|
2466
2677
|
});
|
|
@@ -2543,7 +2754,7 @@ async function dispatchOne(db, outboxRow, sub) {
|
|
|
2543
2754
|
let responseHeaders = {};
|
|
2544
2755
|
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
2545
2756
|
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
2546
|
-
|
|
2757
|
+
logger10.warn("[emitter] refused private egress", {
|
|
2547
2758
|
subscription: sub.name,
|
|
2548
2759
|
url: sub.url
|
|
2549
2760
|
});
|
|
@@ -2639,7 +2850,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
2639
2850
|
circuit_opened_at: new Date,
|
|
2640
2851
|
updated_at: new Date
|
|
2641
2852
|
}).where("id", "=", sub.id).execute();
|
|
2642
|
-
|
|
2853
|
+
logger10.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
2643
2854
|
subscription: sub.name,
|
|
2644
2855
|
failures: newFailures
|
|
2645
2856
|
});
|
|
@@ -2727,7 +2938,7 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
2727
2938
|
await settleFailed(db, row, sub, err);
|
|
2728
2939
|
}
|
|
2729
2940
|
} catch (err) {
|
|
2730
|
-
|
|
2941
|
+
logger10.error("Emitter dispatch crashed", {
|
|
2731
2942
|
outboxId: row.id,
|
|
2732
2943
|
error: err instanceof Error ? err.message : String(err)
|
|
2733
2944
|
});
|
|
@@ -2764,7 +2975,7 @@ async function startEmitter(opts) {
|
|
|
2764
2975
|
};
|
|
2765
2976
|
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
2766
2977
|
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
2767
|
-
|
|
2978
|
+
logger10.info("[emitter] started", { id: emitterId });
|
|
2768
2979
|
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
2769
2980
|
let lastErr = null;
|
|
2770
2981
|
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
@@ -2775,7 +2986,7 @@ async function startEmitter(opts) {
|
|
|
2775
2986
|
} catch (err) {
|
|
2776
2987
|
lastErr = err;
|
|
2777
2988
|
const delayMs = 500 * 2 ** i;
|
|
2778
|
-
|
|
2989
|
+
logger10.warn("[emitter] matcher refresh failed, retrying", {
|
|
2779
2990
|
attempt: i + 1,
|
|
2780
2991
|
delayMs,
|
|
2781
2992
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -2789,21 +3000,21 @@ async function startEmitter(opts) {
|
|
|
2789
3000
|
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
2790
3001
|
if (!state.running)
|
|
2791
3002
|
return;
|
|
2792
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3003
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger10.error("[emitter] claim failed", {
|
|
2793
3004
|
error: err instanceof Error ? err.message : String(err)
|
|
2794
3005
|
}));
|
|
2795
3006
|
});
|
|
2796
3007
|
const stopChanged = await listen("subscriptions:changed", () => {
|
|
2797
3008
|
if (!state.running)
|
|
2798
3009
|
return;
|
|
2799
|
-
refreshMatcher(db).catch((err) =>
|
|
3010
|
+
refreshMatcher(db).catch((err) => logger10.error("[emitter] matcher refresh failed", {
|
|
2800
3011
|
error: err instanceof Error ? err.message : String(err)
|
|
2801
3012
|
}));
|
|
2802
3013
|
});
|
|
2803
3014
|
const poll = setInterval(() => {
|
|
2804
3015
|
if (!state.running)
|
|
2805
3016
|
return;
|
|
2806
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
3017
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger10.error("[emitter] poll claim failed", {
|
|
2807
3018
|
error: err instanceof Error ? err.message : String(err)
|
|
2808
3019
|
}));
|
|
2809
3020
|
}, pollIntervalMs);
|
|
@@ -2811,7 +3022,7 @@ async function startEmitter(opts) {
|
|
|
2811
3022
|
const retention = setInterval(() => {
|
|
2812
3023
|
if (!state.running)
|
|
2813
3024
|
return;
|
|
2814
|
-
runRetention(db).catch((err) =>
|
|
3025
|
+
runRetention(db).catch((err) => logger10.error("[emitter] retention failed", {
|
|
2815
3026
|
error: err instanceof Error ? err.message : String(err)
|
|
2816
3027
|
}));
|
|
2817
3028
|
}, retentionIntervalMs);
|
|
@@ -2821,7 +3032,56 @@ async function startEmitter(opts) {
|
|
|
2821
3032
|
clearInterval(retention);
|
|
2822
3033
|
await stopNew();
|
|
2823
3034
|
await stopChanged();
|
|
2824
|
-
|
|
3035
|
+
logger10.info("[emitter] stopped", { id: emitterId });
|
|
3036
|
+
};
|
|
3037
|
+
}
|
|
3038
|
+
|
|
3039
|
+
// src/runtime/streams-reorg-poll.ts
|
|
3040
|
+
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
3041
|
+
import { IndexHttpClient as IndexHttpClient2 } from "@secondlayer/shared/index-http";
|
|
3042
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
3043
|
+
var POLL_MS = Number(process.env.SUBGRAPH_REORG_POLL_MS) || 15000;
|
|
3044
|
+
var STARTUP_MARGIN_MS = 60 * 60 * 1000;
|
|
3045
|
+
async function pollReorgsOnce(http, cursor, handleReorg, loadDef) {
|
|
3046
|
+
const { reorgs, next_since } = await http.listReorgs(cursor);
|
|
3047
|
+
const sorted = [...reorgs].sort((a, b) => a.fork_point_height - b.fork_point_height);
|
|
3048
|
+
for (const r of sorted) {
|
|
3049
|
+
logger11.info("Streams reorg — rewinding subgraphs", {
|
|
3050
|
+
forkPointHeight: r.fork_point_height
|
|
3051
|
+
});
|
|
3052
|
+
await handleReorg(r.fork_point_height, loadDef);
|
|
3053
|
+
}
|
|
3054
|
+
return next_since ?? cursor;
|
|
3055
|
+
}
|
|
3056
|
+
function startStreamsReorgPoll(handleReorg, loadDef) {
|
|
3057
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
3058
|
+
const http = new IndexHttpClient2({
|
|
3059
|
+
indexBaseUrl: baseUrl,
|
|
3060
|
+
streamsBaseUrl: baseUrl,
|
|
3061
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
3062
|
+
});
|
|
3063
|
+
let since = new Date(Date.now() - STARTUP_MARGIN_MS).toISOString();
|
|
3064
|
+
let running = true;
|
|
3065
|
+
let timer;
|
|
3066
|
+
const tick = async () => {
|
|
3067
|
+
if (!running)
|
|
3068
|
+
return;
|
|
3069
|
+
try {
|
|
3070
|
+
since = await pollReorgsOnce(http, since, handleReorg, loadDef);
|
|
3071
|
+
} catch (err) {
|
|
3072
|
+
logger11.error("Streams reorg poll failed", {
|
|
3073
|
+
error: getErrorMessage4(err)
|
|
3074
|
+
});
|
|
3075
|
+
}
|
|
3076
|
+
if (running)
|
|
3077
|
+
timer = setTimeout(tick, POLL_MS);
|
|
3078
|
+
};
|
|
3079
|
+
timer = setTimeout(tick, POLL_MS);
|
|
3080
|
+
logger11.info("Streams reorg poll started", { pollMs: POLL_MS });
|
|
3081
|
+
return () => {
|
|
3082
|
+
running = false;
|
|
3083
|
+
if (timer)
|
|
3084
|
+
clearTimeout(timer);
|
|
2825
3085
|
};
|
|
2826
3086
|
}
|
|
2827
3087
|
|
|
@@ -2844,11 +3104,11 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
2844
3104
|
const def = await loadSubgraphDefinition(sg);
|
|
2845
3105
|
await catchUpSubgraph(def, sg.name);
|
|
2846
3106
|
} catch (err) {
|
|
2847
|
-
const msg =
|
|
3107
|
+
const msg = getErrorMessage5(err);
|
|
2848
3108
|
if (isHandlerNotFoundError(err)) {
|
|
2849
3109
|
await updateSubgraphStatus3(db, sg.name, "error");
|
|
2850
3110
|
}
|
|
2851
|
-
|
|
3111
|
+
logger12.error("Subgraph catch-up failed", {
|
|
2852
3112
|
subgraph: sg.name,
|
|
2853
3113
|
error: msg
|
|
2854
3114
|
});
|
|
@@ -2894,7 +3154,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
2894
3154
|
definitionCache.set(sg.name, def);
|
|
2895
3155
|
if (prevVersion && prevVersion !== sg.version) {
|
|
2896
3156
|
invalidateSubgraphRoute(sg.name);
|
|
2897
|
-
|
|
3157
|
+
logger12.info("Subgraph handler reloaded", {
|
|
2898
3158
|
subgraph: sg.name,
|
|
2899
3159
|
from: prevVersion,
|
|
2900
3160
|
to: sg.version
|
|
@@ -2928,7 +3188,7 @@ async function synthesizeLegacyReindexOperations() {
|
|
|
2928
3188
|
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
2929
3189
|
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
2930
3190
|
});
|
|
2931
|
-
|
|
3191
|
+
logger12.info("Queued legacy reindex resume operation", {
|
|
2932
3192
|
subgraph: sg.name
|
|
2933
3193
|
});
|
|
2934
3194
|
} catch (err) {
|
|
@@ -2984,7 +3244,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
2984
3244
|
const activeRuns = new Map;
|
|
2985
3245
|
let running = true;
|
|
2986
3246
|
let draining = false;
|
|
2987
|
-
|
|
3247
|
+
logger12.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
2988
3248
|
const startOperation = (operation) => {
|
|
2989
3249
|
const controller = new AbortController;
|
|
2990
3250
|
active.set(operation.id, controller);
|
|
@@ -2992,9 +3252,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
2992
3252
|
if (!running)
|
|
2993
3253
|
return;
|
|
2994
3254
|
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
2995
|
-
|
|
3255
|
+
logger12.warn("Subgraph operation heartbeat failed", {
|
|
2996
3256
|
operationId: operation.id,
|
|
2997
|
-
error:
|
|
3257
|
+
error: getErrorMessage5(err)
|
|
2998
3258
|
});
|
|
2999
3259
|
});
|
|
3000
3260
|
}, HEARTBEAT_INTERVAL_MS);
|
|
@@ -3004,9 +3264,9 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3004
3264
|
controller.abort("user-cancelled");
|
|
3005
3265
|
}
|
|
3006
3266
|
}).catch((err) => {
|
|
3007
|
-
|
|
3267
|
+
logger12.warn("Subgraph operation cancel poll failed", {
|
|
3008
3268
|
operationId: operation.id,
|
|
3009
|
-
error:
|
|
3269
|
+
error: getErrorMessage5(err)
|
|
3010
3270
|
});
|
|
3011
3271
|
});
|
|
3012
3272
|
}, CANCEL_POLL_INTERVAL_MS);
|
|
@@ -3021,14 +3281,14 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3021
3281
|
const reason = String(controller.signal.reason ?? "");
|
|
3022
3282
|
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
3023
3283
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3024
|
-
|
|
3284
|
+
logger12.info("Subgraph operation cancelled", {
|
|
3025
3285
|
operationId: operation.id,
|
|
3026
3286
|
subgraph: operation.subgraph_name
|
|
3027
3287
|
});
|
|
3028
3288
|
return;
|
|
3029
3289
|
}
|
|
3030
3290
|
if (controller.signal.aborted) {
|
|
3031
|
-
|
|
3291
|
+
logger12.info("Subgraph operation interrupted", {
|
|
3032
3292
|
operationId: operation.id,
|
|
3033
3293
|
subgraph: operation.subgraph_name,
|
|
3034
3294
|
reason
|
|
@@ -3036,7 +3296,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3036
3296
|
return;
|
|
3037
3297
|
}
|
|
3038
3298
|
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3039
|
-
|
|
3299
|
+
logger12.info("Subgraph operation completed", {
|
|
3040
3300
|
operationId: operation.id,
|
|
3041
3301
|
subgraph: operation.subgraph_name,
|
|
3042
3302
|
processed
|
|
@@ -3044,7 +3304,7 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3044
3304
|
} catch (err) {
|
|
3045
3305
|
const reason = String(controller.signal.reason ?? "");
|
|
3046
3306
|
if (controller.signal.aborted && reason === "shutdown") {
|
|
3047
|
-
|
|
3307
|
+
logger12.info("Subgraph operation interrupted by shutdown", {
|
|
3048
3308
|
operationId: operation.id,
|
|
3049
3309
|
subgraph: operation.subgraph_name
|
|
3050
3310
|
});
|
|
@@ -3054,11 +3314,11 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3054
3314
|
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
3055
3315
|
return;
|
|
3056
3316
|
}
|
|
3057
|
-
await failSubgraphOperation(db, operation.id, lockedBy,
|
|
3058
|
-
|
|
3317
|
+
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage5(err), processed);
|
|
3318
|
+
logger12.error("Subgraph operation failed", {
|
|
3059
3319
|
operationId: operation.id,
|
|
3060
3320
|
subgraph: operation.subgraph_name,
|
|
3061
|
-
error:
|
|
3321
|
+
error: getErrorMessage5(err)
|
|
3062
3322
|
});
|
|
3063
3323
|
} finally {
|
|
3064
3324
|
clearInterval(heartbeat);
|
|
@@ -3102,13 +3362,13 @@ async function startSubgraphOperationRunner(opts) {
|
|
|
3102
3362
|
controller.abort("shutdown");
|
|
3103
3363
|
}
|
|
3104
3364
|
await Promise.allSettled(activeRuns.values());
|
|
3105
|
-
|
|
3365
|
+
logger12.info("Subgraph operation runner stopped");
|
|
3106
3366
|
};
|
|
3107
3367
|
}
|
|
3108
3368
|
async function startSubgraphProcessor(opts) {
|
|
3109
3369
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
3110
3370
|
let running = true;
|
|
3111
|
-
|
|
3371
|
+
logger12.info("Starting subgraph processor", { concurrency });
|
|
3112
3372
|
const stopOperations = await startSubgraphOperationRunner({
|
|
3113
3373
|
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
3114
3374
|
});
|
|
@@ -3133,8 +3393,8 @@ async function startSubgraphProcessor(opts) {
|
|
|
3133
3393
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
3134
3394
|
}
|
|
3135
3395
|
} catch (err) {
|
|
3136
|
-
|
|
3137
|
-
error:
|
|
3396
|
+
logger12.error("Subgraph reorg handling failed", {
|
|
3397
|
+
error: getErrorMessage5(err)
|
|
3138
3398
|
});
|
|
3139
3399
|
}
|
|
3140
3400
|
}, { connectionString: sourceListenerUrl() });
|
|
@@ -3146,22 +3406,24 @@ async function startSubgraphProcessor(opts) {
|
|
|
3146
3406
|
cleanupCaches(subgraphs);
|
|
3147
3407
|
await catchUpAll(subgraphs, db, concurrency);
|
|
3148
3408
|
}, POLL_INTERVAL_MS);
|
|
3409
|
+
const stopStreamsReorgPoll = process.env.SUBGRAPH_SOURCE === "streams-index" ? startStreamsReorgPoll(handleSubgraphReorg, loadSubgraphDefinition) : undefined;
|
|
3149
3410
|
const stopEmitter = await startEmitter();
|
|
3150
|
-
|
|
3411
|
+
logger12.info("Subgraph processor ready");
|
|
3151
3412
|
return async () => {
|
|
3152
3413
|
running = false;
|
|
3153
3414
|
clearInterval(pollInterval);
|
|
3154
3415
|
await stopListening();
|
|
3155
3416
|
await stopReorgListening();
|
|
3417
|
+
stopStreamsReorgPoll?.();
|
|
3156
3418
|
await stopOperations();
|
|
3157
3419
|
await stopEmitter();
|
|
3158
|
-
|
|
3420
|
+
logger12.info("Subgraph processor stopped");
|
|
3159
3421
|
};
|
|
3160
3422
|
}
|
|
3161
3423
|
|
|
3162
3424
|
// src/service.ts
|
|
3163
3425
|
import { getDb } from "@secondlayer/shared/db";
|
|
3164
|
-
import { logger as
|
|
3426
|
+
import { logger as logger13 } from "@secondlayer/shared/logger";
|
|
3165
3427
|
import { sql as sql5 } from "kysely";
|
|
3166
3428
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
3167
3429
|
var SERVICE_NAME = "subgraph-processor";
|
|
@@ -3169,7 +3431,7 @@ async function writeHeartbeat() {
|
|
|
3169
3431
|
try {
|
|
3170
3432
|
await getDb().insertInto("service_heartbeats").values({ name: SERVICE_NAME }).onConflict((oc) => oc.column("name").doUpdateSet({ updated_at: sql5`now()` })).execute();
|
|
3171
3433
|
} catch (err) {
|
|
3172
|
-
|
|
3434
|
+
logger13.warn("subgraph-processor heartbeat write failed", {
|
|
3173
3435
|
error: err instanceof Error ? err.message : String(err)
|
|
3174
3436
|
});
|
|
3175
3437
|
}
|
|
@@ -3180,7 +3442,7 @@ var processor = await startSubgraphProcessor({
|
|
|
3180
3442
|
await writeHeartbeat();
|
|
3181
3443
|
var heartbeatInterval = setInterval(writeHeartbeat, HEARTBEAT_INTERVAL_MS2);
|
|
3182
3444
|
var shutdown = async () => {
|
|
3183
|
-
|
|
3445
|
+
logger13.info("Shutting down subgraph processor...");
|
|
3184
3446
|
clearInterval(heartbeatInterval);
|
|
3185
3447
|
await processor();
|
|
3186
3448
|
process.exit(0);
|
|
@@ -3188,5 +3450,5 @@ var shutdown = async () => {
|
|
|
3188
3450
|
process.on("SIGINT", shutdown);
|
|
3189
3451
|
process.on("SIGTERM", shutdown);
|
|
3190
3452
|
|
|
3191
|
-
//# debugId=
|
|
3453
|
+
//# debugId=518DC73E15D2D69264756E2164756E21
|
|
3192
3454
|
//# sourceMappingURL=service.js.map
|