@secondlayer/subgraphs 3.5.0 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +327 -113
- package/dist/src/index.js.map +12 -10
- package/dist/src/runtime/block-processor.js +292 -30
- package/dist/src/runtime/block-processor.js.map +8 -5
- package/dist/src/runtime/catchup.js +306 -91
- package/dist/src/runtime/catchup.js.map +9 -7
- package/dist/src/runtime/processor.js +433 -171
- package/dist/src/runtime/processor.js.map +14 -11
- package/dist/src/runtime/reindex.js +325 -111
- package/dist/src/runtime/reindex.js.map +10 -8
- package/dist/src/runtime/reorg.js +298 -36
- package/dist/src/runtime/reorg.js.map +8 -5
- package/dist/src/runtime/runner.d.ts +6 -1
- package/dist/src/runtime/runner.js +10 -7
- package/dist/src/runtime/runner.js.map +3 -3
- package/dist/src/schema/index.js.map +1 -1
- package/dist/src/service.js +436 -174
- package/dist/src/service.js.map +13 -10
- package/package.json +2 -2
|
@@ -518,21 +518,21 @@ function buildEventPayload(filter, tx, event) {
|
|
|
518
518
|
return {
|
|
519
519
|
sender: decoded.sender,
|
|
520
520
|
recipient: decoded.recipient,
|
|
521
|
-
tokenId: decoded.value,
|
|
521
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
522
522
|
assetIdentifier: decoded.asset_identifier,
|
|
523
523
|
tx: txMeta
|
|
524
524
|
};
|
|
525
525
|
case "nft_mint":
|
|
526
526
|
return {
|
|
527
527
|
recipient: decoded.recipient,
|
|
528
|
-
tokenId: decoded.value,
|
|
528
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
529
529
|
assetIdentifier: decoded.asset_identifier,
|
|
530
530
|
tx: txMeta
|
|
531
531
|
};
|
|
532
532
|
case "nft_burn":
|
|
533
533
|
return {
|
|
534
534
|
sender: decoded.sender,
|
|
535
|
-
tokenId: decoded.value,
|
|
535
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
536
536
|
assetIdentifier: decoded.asset_identifier,
|
|
537
537
|
tx: txMeta
|
|
538
538
|
};
|
|
@@ -564,8 +564,8 @@ function buildEventPayload(filter, tx, event) {
|
|
|
564
564
|
tx: txMeta
|
|
565
565
|
};
|
|
566
566
|
case "print_event": {
|
|
567
|
-
const
|
|
568
|
-
const rawValue =
|
|
567
|
+
const rawHex = event.data?.raw_value;
|
|
568
|
+
const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
|
|
569
569
|
const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
|
|
570
570
|
const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
|
|
571
571
|
const { topic: _, ...rest } = clarityObj ?? {};
|
|
@@ -578,9 +578,11 @@ function buildEventPayload(filter, tx, event) {
|
|
|
578
578
|
};
|
|
579
579
|
}
|
|
580
580
|
case "contract_call": {
|
|
581
|
+
const ccRawHex = event.data?.raw_value;
|
|
582
|
+
const normalized = typeof ccRawHex === "string" && ccRawHex.startsWith("0x") ? { ...decoded, value: decodeClarityValue(ccRawHex) } : decoded;
|
|
581
583
|
const input = buildContractCallInput(filter, tx);
|
|
582
584
|
return {
|
|
583
|
-
...
|
|
585
|
+
...normalized,
|
|
584
586
|
type: "contract_call",
|
|
585
587
|
_eventType: event.type,
|
|
586
588
|
contractId: tx.contract_id ?? "",
|
|
@@ -949,10 +951,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
|
|
|
949
951
|
}
|
|
950
952
|
|
|
951
953
|
// src/runtime/block-processor.ts
|
|
952
|
-
import {
|
|
953
|
-
getSourceDb,
|
|
954
|
-
getTargetDb
|
|
955
|
-
} from "@secondlayer/shared/db";
|
|
954
|
+
import { getTargetDb } from "@secondlayer/shared/db";
|
|
956
955
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
957
956
|
import {
|
|
958
957
|
isByoSubgraph,
|
|
@@ -960,15 +959,286 @@ import {
|
|
|
960
959
|
resolveSubgraphDb,
|
|
961
960
|
updateSubgraphStatus
|
|
962
961
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
963
|
-
import { logger as
|
|
962
|
+
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
964
963
|
import { sql as sql3 } from "kysely";
|
|
965
964
|
|
|
966
965
|
// src/schema/utils.ts
|
|
967
966
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
968
967
|
|
|
968
|
+
// src/runtime/block-source.ts
|
|
969
|
+
import { getSourceDb } from "@secondlayer/shared/db";
|
|
970
|
+
import { IndexHttpClient } from "@secondlayer/shared/index-http";
|
|
971
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
972
|
+
|
|
973
|
+
// src/runtime/batch-loader.ts
|
|
974
|
+
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
975
|
+
const [blocks, txs, events] = await Promise.all([
|
|
976
|
+
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
977
|
+
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
978
|
+
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
979
|
+
]);
|
|
980
|
+
const txsByHeight = new Map;
|
|
981
|
+
for (const tx of txs) {
|
|
982
|
+
const h = Number(tx.block_height);
|
|
983
|
+
const list = txsByHeight.get(h) ?? [];
|
|
984
|
+
list.push(tx);
|
|
985
|
+
txsByHeight.set(h, list);
|
|
986
|
+
}
|
|
987
|
+
const eventsByHeight = new Map;
|
|
988
|
+
for (const evt of events) {
|
|
989
|
+
const h = Number(evt.block_height);
|
|
990
|
+
const list = eventsByHeight.get(h) ?? [];
|
|
991
|
+
list.push(evt);
|
|
992
|
+
eventsByHeight.set(h, list);
|
|
993
|
+
}
|
|
994
|
+
const result = new Map;
|
|
995
|
+
for (const block of blocks) {
|
|
996
|
+
const h = Number(block.height);
|
|
997
|
+
result.set(h, {
|
|
998
|
+
block,
|
|
999
|
+
txs: txsByHeight.get(h) ?? [],
|
|
1000
|
+
events: eventsByHeight.get(h) ?? []
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
return result;
|
|
1004
|
+
}
|
|
1005
|
+
function avgEventsPerBlock(batch) {
|
|
1006
|
+
if (batch.size === 0)
|
|
1007
|
+
return 0;
|
|
1008
|
+
let totalEvents = 0;
|
|
1009
|
+
for (const data of batch.values()) {
|
|
1010
|
+
totalEvents += data.events.length;
|
|
1011
|
+
}
|
|
1012
|
+
return totalEvents / batch.size;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
// src/runtime/reconstruct.ts
|
|
1016
|
+
function isoToUnixSeconds(iso) {
|
|
1017
|
+
if (!iso)
|
|
1018
|
+
return 0;
|
|
1019
|
+
return Math.floor(new Date(iso).getTime() / 1000);
|
|
1020
|
+
}
|
|
1021
|
+
function reconstructBlock(b) {
|
|
1022
|
+
return {
|
|
1023
|
+
height: b.block_height,
|
|
1024
|
+
hash: b.block_hash,
|
|
1025
|
+
parent_hash: b.parent_hash,
|
|
1026
|
+
burn_block_height: b.burn_block_height,
|
|
1027
|
+
burn_block_hash: b.burn_block_hash,
|
|
1028
|
+
timestamp: isoToUnixSeconds(b.block_time),
|
|
1029
|
+
canonical: true,
|
|
1030
|
+
created_at: new Date(0)
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
function reconstructTransaction(t) {
|
|
1034
|
+
return {
|
|
1035
|
+
tx_id: t.tx_id,
|
|
1036
|
+
block_height: t.block_height,
|
|
1037
|
+
tx_index: t.tx_index,
|
|
1038
|
+
type: t.tx_type,
|
|
1039
|
+
sender: t.sender,
|
|
1040
|
+
status: t.status,
|
|
1041
|
+
contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
|
|
1042
|
+
function_name: t.contract_call?.function_name ?? null,
|
|
1043
|
+
function_args: t.contract_call?.function_args_hex ?? [],
|
|
1044
|
+
raw_result: t.contract_call?.result_hex ?? null,
|
|
1045
|
+
raw_tx: "",
|
|
1046
|
+
created_at: new Date(0)
|
|
1047
|
+
};
|
|
1048
|
+
}
|
|
1049
|
+
function reconstructEvent(e) {
|
|
1050
|
+
const base = {
|
|
1051
|
+
id: `${e.tx_id}#${e.event_index}`,
|
|
1052
|
+
tx_id: e.tx_id,
|
|
1053
|
+
block_height: e.block_height,
|
|
1054
|
+
event_index: e.event_index,
|
|
1055
|
+
created_at: new Date(0)
|
|
1056
|
+
};
|
|
1057
|
+
switch (e.event_type) {
|
|
1058
|
+
case "ft_transfer":
|
|
1059
|
+
case "ft_mint":
|
|
1060
|
+
case "ft_burn":
|
|
1061
|
+
return {
|
|
1062
|
+
...base,
|
|
1063
|
+
type: `${e.event_type}_event`,
|
|
1064
|
+
data: {
|
|
1065
|
+
asset_identifier: e.asset_identifier,
|
|
1066
|
+
sender: e.sender,
|
|
1067
|
+
recipient: e.recipient,
|
|
1068
|
+
amount: e.amount
|
|
1069
|
+
}
|
|
1070
|
+
};
|
|
1071
|
+
case "nft_transfer":
|
|
1072
|
+
case "nft_mint":
|
|
1073
|
+
case "nft_burn":
|
|
1074
|
+
return {
|
|
1075
|
+
...base,
|
|
1076
|
+
type: `${e.event_type}_event`,
|
|
1077
|
+
data: {
|
|
1078
|
+
asset_identifier: e.asset_identifier,
|
|
1079
|
+
sender: e.sender,
|
|
1080
|
+
recipient: e.recipient,
|
|
1081
|
+
raw_value: e.value
|
|
1082
|
+
}
|
|
1083
|
+
};
|
|
1084
|
+
case "stx_transfer":
|
|
1085
|
+
case "stx_mint":
|
|
1086
|
+
case "stx_burn":
|
|
1087
|
+
return {
|
|
1088
|
+
...base,
|
|
1089
|
+
type: `${e.event_type}_event`,
|
|
1090
|
+
data: {
|
|
1091
|
+
sender: e.sender,
|
|
1092
|
+
recipient: e.recipient,
|
|
1093
|
+
amount: e.amount,
|
|
1094
|
+
...e.event_type === "stx_transfer" ? { memo: e.memo ?? "" } : {}
|
|
1095
|
+
}
|
|
1096
|
+
};
|
|
1097
|
+
case "stx_lock":
|
|
1098
|
+
return {
|
|
1099
|
+
...base,
|
|
1100
|
+
type: "stx_lock_event",
|
|
1101
|
+
data: {
|
|
1102
|
+
locked_address: e.sender,
|
|
1103
|
+
locked_amount: e.amount,
|
|
1104
|
+
unlock_height: e.payload.unlock_height
|
|
1105
|
+
}
|
|
1106
|
+
};
|
|
1107
|
+
case "print":
|
|
1108
|
+
return {
|
|
1109
|
+
...base,
|
|
1110
|
+
type: "contract_event",
|
|
1111
|
+
data: {
|
|
1112
|
+
topic: e.payload.topic,
|
|
1113
|
+
contract_identifier: e.contract_id,
|
|
1114
|
+
value: e.payload.value,
|
|
1115
|
+
raw_value: e.payload.raw_value
|
|
1116
|
+
}
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// src/runtime/block-source.ts
|
|
1122
|
+
class PostgresBlockSource {
|
|
1123
|
+
async getTip() {
|
|
1124
|
+
const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1125
|
+
return progress ? Number(progress.highest_seen_block) : 0;
|
|
1126
|
+
}
|
|
1127
|
+
loadBlockRange(fromHeight, toHeight) {
|
|
1128
|
+
return loadBlockRange(getSourceDb(), fromHeight, toHeight);
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
var EVENT_FILTER_TO_INDEX_TYPE = {
|
|
1132
|
+
stx_transfer: "stx_transfer",
|
|
1133
|
+
stx_mint: "stx_mint",
|
|
1134
|
+
stx_burn: "stx_burn",
|
|
1135
|
+
stx_lock: "stx_lock",
|
|
1136
|
+
ft_transfer: "ft_transfer",
|
|
1137
|
+
ft_mint: "ft_mint",
|
|
1138
|
+
ft_burn: "ft_burn",
|
|
1139
|
+
nft_transfer: "nft_transfer",
|
|
1140
|
+
nft_mint: "nft_mint",
|
|
1141
|
+
nft_burn: "nft_burn",
|
|
1142
|
+
print_event: "print"
|
|
1143
|
+
};
|
|
1144
|
+
var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
1145
|
+
var ALL_INDEX_EVENT_TYPES = [
|
|
1146
|
+
...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
|
|
1147
|
+
];
|
|
1148
|
+
function sourceFilters(subgraph) {
|
|
1149
|
+
const sources = subgraph.sources;
|
|
1150
|
+
return Array.isArray(sources) ? sources : Object.values(sources);
|
|
1151
|
+
}
|
|
1152
|
+
function referencedIndexEventTypes(subgraph) {
|
|
1153
|
+
const filters = sourceFilters(subgraph);
|
|
1154
|
+
if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
|
|
1155
|
+
return ALL_INDEX_EVENT_TYPES;
|
|
1156
|
+
}
|
|
1157
|
+
const types = new Set;
|
|
1158
|
+
for (const f of filters) {
|
|
1159
|
+
const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
|
|
1160
|
+
if (t)
|
|
1161
|
+
types.add(t);
|
|
1162
|
+
}
|
|
1163
|
+
return [...types];
|
|
1164
|
+
}
|
|
1165
|
+
function isStreamsIndexEligible(subgraph) {
|
|
1166
|
+
if (Array.isArray(subgraph.sources))
|
|
1167
|
+
return false;
|
|
1168
|
+
const filters = sourceFilters(subgraph);
|
|
1169
|
+
if (filters.length === 0)
|
|
1170
|
+
return false;
|
|
1171
|
+
for (const f of filters) {
|
|
1172
|
+
const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
|
|
1173
|
+
if (!known)
|
|
1174
|
+
return false;
|
|
1175
|
+
}
|
|
1176
|
+
return true;
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
class PublicApiBlockSource {
|
|
1180
|
+
http;
|
|
1181
|
+
eventTypes;
|
|
1182
|
+
constructor(http, eventTypes) {
|
|
1183
|
+
this.http = http;
|
|
1184
|
+
this.eventTypes = eventTypes;
|
|
1185
|
+
}
|
|
1186
|
+
getTip() {
|
|
1187
|
+
return this.http.getIndexTip();
|
|
1188
|
+
}
|
|
1189
|
+
async loadBlockRange(fromHeight, toHeight) {
|
|
1190
|
+
const [blocks, txs, eventLists] = await Promise.all([
|
|
1191
|
+
this.http.walkBlocks(fromHeight, toHeight),
|
|
1192
|
+
this.http.walkTransactions(fromHeight, toHeight),
|
|
1193
|
+
Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
|
|
1194
|
+
]);
|
|
1195
|
+
const map = new Map;
|
|
1196
|
+
for (const b of blocks) {
|
|
1197
|
+
map.set(b.block_height, {
|
|
1198
|
+
block: reconstructBlock(b),
|
|
1199
|
+
txs: [],
|
|
1200
|
+
events: []
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
for (const t of txs) {
|
|
1204
|
+
map.get(t.block_height)?.txs.push(reconstructTransaction(t));
|
|
1205
|
+
}
|
|
1206
|
+
for (const list of eventLists) {
|
|
1207
|
+
for (const e of list) {
|
|
1208
|
+
map.get(e.block_height)?.events.push(reconstructEvent(e));
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
for (const bd of map.values()) {
|
|
1212
|
+
bd.txs.sort((a, b) => a.tx_index - b.tx_index);
|
|
1213
|
+
bd.events.sort((a, b) => a.event_index - b.event_index);
|
|
1214
|
+
}
|
|
1215
|
+
return map;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
var postgresBlockSource = new PostgresBlockSource;
|
|
1219
|
+
function buildHttpClient() {
|
|
1220
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
1221
|
+
return new IndexHttpClient({
|
|
1222
|
+
indexBaseUrl: baseUrl,
|
|
1223
|
+
streamsBaseUrl: baseUrl,
|
|
1224
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
function resolveBlockSource(subgraph) {
|
|
1228
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
|
|
1229
|
+
return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
|
|
1230
|
+
}
|
|
1231
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
|
|
1232
|
+
logger3.debug("Subgraph not streams-index eligible, using DB tap", {
|
|
1233
|
+
subgraph: subgraph.name
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
return postgresBlockSource;
|
|
1237
|
+
}
|
|
1238
|
+
|
|
969
1239
|
// src/runtime/outbox-emit.ts
|
|
970
1240
|
import { createHash } from "node:crypto";
|
|
971
|
-
import { logger as
|
|
1241
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
972
1242
|
var loggedKillSwitch = false;
|
|
973
1243
|
function isEmitOutboxEnabled() {
|
|
974
1244
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
@@ -987,7 +1257,7 @@ function stableStringify(obj) {
|
|
|
987
1257
|
async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
988
1258
|
if (!isEmitOutboxEnabled()) {
|
|
989
1259
|
if (!loggedKillSwitch) {
|
|
990
|
-
|
|
1260
|
+
logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
991
1261
|
loggedKillSwitch = true;
|
|
992
1262
|
}
|
|
993
1263
|
return 0;
|
|
@@ -1228,7 +1498,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
|
|
|
1228
1498
|
return resolved;
|
|
1229
1499
|
}
|
|
1230
1500
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
1231
|
-
const sourceDb = getSourceDb();
|
|
1232
1501
|
const targetDb = getTargetDb();
|
|
1233
1502
|
const blockStart = performance.now();
|
|
1234
1503
|
const result = {
|
|
@@ -1246,25 +1515,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1246
1515
|
txs = opts.preloaded.txs;
|
|
1247
1516
|
evts = opts.preloaded.events;
|
|
1248
1517
|
} else {
|
|
1249
|
-
|
|
1250
|
-
if (!
|
|
1251
|
-
|
|
1252
|
-
subgraph: subgraphName,
|
|
1253
|
-
blockHeight
|
|
1254
|
-
});
|
|
1255
|
-
result.skipped = true;
|
|
1256
|
-
return result;
|
|
1257
|
-
}
|
|
1258
|
-
if (!block.canonical) {
|
|
1259
|
-
logger4.debug("Skipping non-canonical block", {
|
|
1518
|
+
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
1519
|
+
if (!data) {
|
|
1520
|
+
logger5.debug("Block not found or non-canonical for subgraph processing", {
|
|
1260
1521
|
subgraph: subgraphName,
|
|
1261
1522
|
blockHeight
|
|
1262
1523
|
});
|
|
1263
1524
|
result.skipped = true;
|
|
1264
1525
|
return result;
|
|
1265
1526
|
}
|
|
1266
|
-
|
|
1267
|
-
|
|
1527
|
+
block = data.block;
|
|
1528
|
+
txs = data.txs;
|
|
1529
|
+
evts = data.events;
|
|
1268
1530
|
}
|
|
1269
1531
|
const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
|
|
1270
1532
|
const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
|
|
@@ -1355,7 +1617,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1355
1617
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
1356
1618
|
const count = Number(rows[0]?.count ?? 0);
|
|
1357
1619
|
if (count >= 1e7) {
|
|
1358
|
-
|
|
1620
|
+
logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
1359
1621
|
subgraph: subgraphName,
|
|
1360
1622
|
table,
|
|
1361
1623
|
count
|
|
@@ -1363,7 +1625,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1363
1625
|
}
|
|
1364
1626
|
}
|
|
1365
1627
|
} catch (err) {
|
|
1366
|
-
|
|
1628
|
+
logger5.debug("Row count sample failed", {
|
|
1367
1629
|
subgraph: subgraphName,
|
|
1368
1630
|
error: err instanceof Error ? err.message : String(err)
|
|
1369
1631
|
});
|
|
@@ -1437,7 +1699,7 @@ class StatsAccumulator {
|
|
|
1437
1699
|
|
|
1438
1700
|
// src/runtime/reindex.ts
|
|
1439
1701
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
1440
|
-
import { getRawClient,
|
|
1702
|
+
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
1441
1703
|
import {
|
|
1442
1704
|
recordGapBatch,
|
|
1443
1705
|
resolveGaps
|
|
@@ -1446,7 +1708,7 @@ import {
|
|
|
1446
1708
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1447
1709
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
1448
1710
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1449
|
-
import { logger as
|
|
1711
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
1450
1712
|
|
|
1451
1713
|
// src/schema/generator.ts
|
|
1452
1714
|
import { createHash as createHash2 } from "node:crypto";
|
|
@@ -1539,48 +1801,6 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
1539
1801
|
return { statements, hash };
|
|
1540
1802
|
}
|
|
1541
1803
|
|
|
1542
|
-
// src/runtime/batch-loader.ts
|
|
1543
|
-
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1544
|
-
const [blocks, txs, events] = await Promise.all([
|
|
1545
|
-
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1546
|
-
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1547
|
-
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1548
|
-
]);
|
|
1549
|
-
const txsByHeight = new Map;
|
|
1550
|
-
for (const tx of txs) {
|
|
1551
|
-
const h = Number(tx.block_height);
|
|
1552
|
-
const list = txsByHeight.get(h) ?? [];
|
|
1553
|
-
list.push(tx);
|
|
1554
|
-
txsByHeight.set(h, list);
|
|
1555
|
-
}
|
|
1556
|
-
const eventsByHeight = new Map;
|
|
1557
|
-
for (const evt of events) {
|
|
1558
|
-
const h = Number(evt.block_height);
|
|
1559
|
-
const list = eventsByHeight.get(h) ?? [];
|
|
1560
|
-
list.push(evt);
|
|
1561
|
-
eventsByHeight.set(h, list);
|
|
1562
|
-
}
|
|
1563
|
-
const result = new Map;
|
|
1564
|
-
for (const block of blocks) {
|
|
1565
|
-
const h = Number(block.height);
|
|
1566
|
-
result.set(h, {
|
|
1567
|
-
block,
|
|
1568
|
-
txs: txsByHeight.get(h) ?? [],
|
|
1569
|
-
events: eventsByHeight.get(h) ?? []
|
|
1570
|
-
});
|
|
1571
|
-
}
|
|
1572
|
-
return result;
|
|
1573
|
-
}
|
|
1574
|
-
function avgEventsPerBlock(batch) {
|
|
1575
|
-
if (batch.size === 0)
|
|
1576
|
-
return 0;
|
|
1577
|
-
let totalEvents = 0;
|
|
1578
|
-
for (const data of batch.values()) {
|
|
1579
|
-
totalEvents += data.events.length;
|
|
1580
|
-
}
|
|
1581
|
-
return totalEvents / batch.size;
|
|
1582
|
-
}
|
|
1583
|
-
|
|
1584
1804
|
// src/runtime/reindex.ts
|
|
1585
1805
|
var LOG_INTERVAL = 1000;
|
|
1586
1806
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
@@ -1641,7 +1861,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
1641
1861
|
return ranges;
|
|
1642
1862
|
}
|
|
1643
1863
|
async function processBlockRange(def, opts) {
|
|
1644
|
-
const
|
|
1864
|
+
const source = resolveBlockSource(def);
|
|
1645
1865
|
const targetDb = getTargetDb2();
|
|
1646
1866
|
const subgraphName = def.name;
|
|
1647
1867
|
const { fromBlock, toBlock, status } = opts;
|
|
@@ -1671,11 +1891,11 @@ async function processBlockRange(def, opts) {
|
|
|
1671
1891
|
lastHealthFlushAt = Date.now();
|
|
1672
1892
|
};
|
|
1673
1893
|
let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
1674
|
-
let nextBatchPromise = loadBlockRange(
|
|
1894
|
+
let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
|
|
1675
1895
|
while (currentHeight <= toBlock) {
|
|
1676
1896
|
if (opts.signal?.aborted) {
|
|
1677
1897
|
aborted = true;
|
|
1678
|
-
|
|
1898
|
+
logger6.info("Block processing aborted", {
|
|
1679
1899
|
subgraph: subgraphName,
|
|
1680
1900
|
currentBlock: currentHeight,
|
|
1681
1901
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -1687,7 +1907,7 @@ async function processBlockRange(def, opts) {
|
|
|
1687
1907
|
const nextStart = batchEnd + 1;
|
|
1688
1908
|
if (nextStart <= toBlock) {
|
|
1689
1909
|
nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
1690
|
-
nextBatchPromise = loadBlockRange(
|
|
1910
|
+
nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
|
|
1691
1911
|
}
|
|
1692
1912
|
const batchFailedBlocks = [];
|
|
1693
1913
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -1705,7 +1925,7 @@ async function processBlockRange(def, opts) {
|
|
|
1705
1925
|
});
|
|
1706
1926
|
} catch (err) {
|
|
1707
1927
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1708
|
-
|
|
1928
|
+
logger6.error("Block processing error", {
|
|
1709
1929
|
subgraph: subgraphName,
|
|
1710
1930
|
blockHeight: height,
|
|
1711
1931
|
error: errorMsg
|
|
@@ -1739,7 +1959,7 @@ async function processBlockRange(def, opts) {
|
|
|
1739
1959
|
lastProgressFlushAt = now;
|
|
1740
1960
|
}
|
|
1741
1961
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1742
|
-
|
|
1962
|
+
logger6.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1743
1963
|
subgraph: subgraphName,
|
|
1744
1964
|
processed: blocksProcessed,
|
|
1745
1965
|
total: totalBlocks,
|
|
@@ -1758,7 +1978,7 @@ async function processBlockRange(def, opts) {
|
|
|
1758
1978
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1759
1979
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1760
1980
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1761
|
-
|
|
1981
|
+
logger6.warn("Failed to record subgraph gaps", {
|
|
1762
1982
|
subgraph: subgraphName,
|
|
1763
1983
|
error: err instanceof Error ? err.message : String(err)
|
|
1764
1984
|
});
|
|
@@ -1775,32 +1995,26 @@ async function processBlockRange(def, opts) {
|
|
|
1775
1995
|
await flushHealth();
|
|
1776
1996
|
return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
|
|
1777
1997
|
}
|
|
1778
|
-
async function resolveBlockRange(
|
|
1998
|
+
async function resolveBlockRange(source, def, opts) {
|
|
1779
1999
|
const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
|
|
1780
|
-
|
|
1781
|
-
if (opts?.toBlock != null) {
|
|
1782
|
-
toBlock = opts.toBlock;
|
|
1783
|
-
} else {
|
|
1784
|
-
const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1785
|
-
toBlock = progress?.highest_seen_block ?? 0;
|
|
1786
|
-
}
|
|
2000
|
+
const toBlock = opts?.toBlock != null ? opts.toBlock : await source.getTip();
|
|
1787
2001
|
return { fromBlock, toBlock };
|
|
1788
2002
|
}
|
|
1789
2003
|
async function clearReindexMetadata(db, subgraphName) {
|
|
1790
2004
|
await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
|
|
1791
2005
|
}
|
|
1792
2006
|
async function reindexSubgraph(def, opts) {
|
|
1793
|
-
const
|
|
2007
|
+
const source = resolveBlockSource(def);
|
|
1794
2008
|
const targetDb = getTargetDb2();
|
|
1795
2009
|
const client = getRawClient("target");
|
|
1796
2010
|
const subgraphName = def.name;
|
|
1797
2011
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1798
2012
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1799
|
-
|
|
2013
|
+
logger6.info("Reindex starting", { subgraph: subgraphName });
|
|
1800
2014
|
try {
|
|
1801
|
-
const { fromBlock, toBlock } = await resolveBlockRange(
|
|
2015
|
+
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
1802
2016
|
if (fromBlock > toBlock) {
|
|
1803
|
-
|
|
2017
|
+
logger6.info("No blocks to reindex", {
|
|
1804
2018
|
subgraph: subgraphName,
|
|
1805
2019
|
fromBlock,
|
|
1806
2020
|
toBlock
|
|
@@ -1813,7 +2027,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1813
2027
|
for (const stmt of statements) {
|
|
1814
2028
|
await client.unsafe(stmt);
|
|
1815
2029
|
}
|
|
1816
|
-
|
|
2030
|
+
logger6.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1817
2031
|
await targetDb.updateTable("subgraphs").set({
|
|
1818
2032
|
last_processed_block: initialReindexProgressBlock(fromBlock),
|
|
1819
2033
|
reindex_from_block: fromBlock,
|
|
@@ -1824,7 +2038,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1824
2038
|
last_error_at: null,
|
|
1825
2039
|
updated_at: new Date
|
|
1826
2040
|
}).where("name", "=", subgraphName).execute();
|
|
1827
|
-
|
|
2041
|
+
logger6.info("Reindexing blocks", {
|
|
1828
2042
|
subgraph: subgraphName,
|
|
1829
2043
|
fromBlock,
|
|
1830
2044
|
toBlock,
|
|
@@ -1845,9 +2059,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
1845
2059
|
if (reason === "user-cancelled") {
|
|
1846
2060
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1847
2061
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1848
|
-
|
|
2062
|
+
logger6.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1849
2063
|
} else {
|
|
1850
|
-
|
|
2064
|
+
logger6.info("Reindex interrupted by shutdown, will resume", {
|
|
1851
2065
|
subgraph: subgraphName
|
|
1852
2066
|
});
|
|
1853
2067
|
}
|
|
@@ -1855,7 +2069,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1855
2069
|
}
|
|
1856
2070
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1857
2071
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1858
|
-
|
|
2072
|
+
logger6.info("Reindex complete", {
|
|
1859
2073
|
subgraph: subgraphName,
|
|
1860
2074
|
blocks: result.blocksProcessed,
|
|
1861
2075
|
events: result.totalEventsProcessed,
|
|
@@ -1863,7 +2077,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1863
2077
|
});
|
|
1864
2078
|
return { processed: result.blocksProcessed };
|
|
1865
2079
|
} catch (err) {
|
|
1866
|
-
|
|
2080
|
+
logger6.error("Reindex failed", {
|
|
1867
2081
|
subgraph: subgraphName,
|
|
1868
2082
|
error: getErrorMessage2(err)
|
|
1869
2083
|
});
|
|
@@ -1885,7 +2099,7 @@ async function resumeReindex(def, opts) {
|
|
|
1885
2099
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
1886
2100
|
const toBlock = Number(row.reindex_to_block);
|
|
1887
2101
|
if (fromBlock == null) {
|
|
1888
|
-
|
|
2102
|
+
logger6.info("No reindex metadata, starting fresh reindex", {
|
|
1889
2103
|
subgraph: subgraphName
|
|
1890
2104
|
});
|
|
1891
2105
|
return reindexSubgraph(def, {
|
|
@@ -1894,12 +2108,12 @@ async function resumeReindex(def, opts) {
|
|
|
1894
2108
|
});
|
|
1895
2109
|
}
|
|
1896
2110
|
if (fromBlock > toBlock) {
|
|
1897
|
-
|
|
2111
|
+
logger6.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1898
2112
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1899
2113
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1900
2114
|
return { processed: 0 };
|
|
1901
2115
|
}
|
|
1902
|
-
|
|
2116
|
+
logger6.info("Resuming reindex", {
|
|
1903
2117
|
subgraph: subgraphName,
|
|
1904
2118
|
fromBlock,
|
|
1905
2119
|
toBlock,
|
|
@@ -1920,9 +2134,9 @@ async function resumeReindex(def, opts) {
|
|
|
1920
2134
|
if (reason === "user-cancelled") {
|
|
1921
2135
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1922
2136
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1923
|
-
|
|
2137
|
+
logger6.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
1924
2138
|
} else {
|
|
1925
|
-
|
|
2139
|
+
logger6.info("Resume interrupted by shutdown, will resume again", {
|
|
1926
2140
|
subgraph: subgraphName
|
|
1927
2141
|
});
|
|
1928
2142
|
}
|
|
@@ -1930,13 +2144,13 @@ async function resumeReindex(def, opts) {
|
|
|
1930
2144
|
}
|
|
1931
2145
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1932
2146
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1933
|
-
|
|
2147
|
+
logger6.info("Resumed reindex complete", {
|
|
1934
2148
|
subgraph: subgraphName,
|
|
1935
2149
|
blocks: result.blocksProcessed
|
|
1936
2150
|
});
|
|
1937
2151
|
return { processed: result.blocksProcessed };
|
|
1938
2152
|
} catch (err) {
|
|
1939
|
-
|
|
2153
|
+
logger6.error("Resumed reindex failed", {
|
|
1940
2154
|
subgraph: subgraphName,
|
|
1941
2155
|
error: getErrorMessage2(err)
|
|
1942
2156
|
});
|
|
@@ -1947,7 +2161,7 @@ async function resumeReindex(def, opts) {
|
|
|
1947
2161
|
async function backfillSubgraph(def, opts) {
|
|
1948
2162
|
const targetDb = getTargetDb2();
|
|
1949
2163
|
const subgraphName = def.name;
|
|
1950
|
-
|
|
2164
|
+
logger6.info("Backfill starting", {
|
|
1951
2165
|
subgraph: subgraphName,
|
|
1952
2166
|
from: opts.fromBlock,
|
|
1953
2167
|
to: opts.toBlock
|
|
@@ -1964,17 +2178,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
1964
2178
|
signal: opts.signal
|
|
1965
2179
|
});
|
|
1966
2180
|
if (result.aborted) {
|
|
1967
|
-
|
|
2181
|
+
logger6.info("Backfill aborted", { subgraph: subgraphName });
|
|
1968
2182
|
return { processed: result.blocksProcessed };
|
|
1969
2183
|
}
|
|
1970
2184
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
1971
2185
|
if (resolved > 0) {
|
|
1972
|
-
|
|
2186
|
+
logger6.info("Resolved subgraph gaps via backfill", {
|
|
1973
2187
|
subgraph: subgraphName,
|
|
1974
2188
|
resolved
|
|
1975
2189
|
});
|
|
1976
2190
|
}
|
|
1977
|
-
|
|
2191
|
+
logger6.info("Backfill complete", {
|
|
1978
2192
|
subgraph: subgraphName,
|
|
1979
2193
|
blocks: result.blocksProcessed,
|
|
1980
2194
|
events: result.totalEventsProcessed,
|
|
@@ -1982,7 +2196,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
1982
2196
|
});
|
|
1983
2197
|
return { processed: result.blocksProcessed };
|
|
1984
2198
|
} catch (err) {
|
|
1985
|
-
|
|
2199
|
+
logger6.error("Backfill failed", {
|
|
1986
2200
|
subgraph: subgraphName,
|
|
1987
2201
|
error: getErrorMessage2(err)
|
|
1988
2202
|
});
|
|
@@ -1998,5 +2212,5 @@ export {
|
|
|
1998
2212
|
backfillSubgraph
|
|
1999
2213
|
};
|
|
2000
2214
|
|
|
2001
|
-
//# debugId=
|
|
2215
|
+
//# debugId=37BEA2EC8642CAEB64756E2164756E21
|
|
2002
2216
|
//# sourceMappingURL=reindex.js.map
|