@secondlayer/subgraphs 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +16 -1
- package/dist/src/index.js +325 -112
- package/dist/src/index.js.map +12 -10
- package/dist/src/runtime/block-processor.js +290 -29
- package/dist/src/runtime/block-processor.js.map +8 -5
- package/dist/src/runtime/catchup.js +304 -90
- package/dist/src/runtime/catchup.js.map +9 -7
- package/dist/src/runtime/processor.js +431 -170
- package/dist/src/runtime/processor.js.map +14 -11
- package/dist/src/runtime/reindex.js +323 -110
- package/dist/src/runtime/reindex.js.map +10 -8
- package/dist/src/runtime/reorg.js +296 -35
- package/dist/src/runtime/reorg.js.map +8 -5
- package/dist/src/runtime/runner.d.ts +6 -1
- package/dist/src/runtime/runner.js +7 -6
- package/dist/src/runtime/runner.js.map +3 -3
- package/dist/src/schema/index.js.map +1 -1
- package/dist/src/service.js +434 -173
- package/dist/src/service.js.map +13 -10
- package/package.json +2 -2
package/dist/src/index.js
CHANGED
|
@@ -594,21 +594,21 @@ function buildEventPayload(filter, tx, event) {
|
|
|
594
594
|
return {
|
|
595
595
|
sender: decoded.sender,
|
|
596
596
|
recipient: decoded.recipient,
|
|
597
|
-
tokenId: decoded.value,
|
|
597
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
598
598
|
assetIdentifier: decoded.asset_identifier,
|
|
599
599
|
tx: txMeta
|
|
600
600
|
};
|
|
601
601
|
case "nft_mint":
|
|
602
602
|
return {
|
|
603
603
|
recipient: decoded.recipient,
|
|
604
|
-
tokenId: decoded.value,
|
|
604
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
605
605
|
assetIdentifier: decoded.asset_identifier,
|
|
606
606
|
tx: txMeta
|
|
607
607
|
};
|
|
608
608
|
case "nft_burn":
|
|
609
609
|
return {
|
|
610
610
|
sender: decoded.sender,
|
|
611
|
-
tokenId: decoded.value,
|
|
611
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
612
612
|
assetIdentifier: decoded.asset_identifier,
|
|
613
613
|
tx: txMeta
|
|
614
614
|
};
|
|
@@ -640,8 +640,8 @@ function buildEventPayload(filter, tx, event) {
|
|
|
640
640
|
tx: txMeta
|
|
641
641
|
};
|
|
642
642
|
case "print_event": {
|
|
643
|
-
const
|
|
644
|
-
const rawValue =
|
|
643
|
+
const rawHex = event.data?.raw_value;
|
|
644
|
+
const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
|
|
645
645
|
const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
|
|
646
646
|
const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
|
|
647
647
|
const { topic: _, ...rest } = clarityObj ?? {};
|
|
@@ -1025,10 +1025,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
|
|
|
1025
1025
|
}
|
|
1026
1026
|
|
|
1027
1027
|
// src/runtime/block-processor.ts
|
|
1028
|
-
import {
|
|
1029
|
-
getSourceDb,
|
|
1030
|
-
getTargetDb
|
|
1031
|
-
} from "@secondlayer/shared/db";
|
|
1028
|
+
import { getTargetDb } from "@secondlayer/shared/db";
|
|
1032
1029
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
1033
1030
|
import {
|
|
1034
1031
|
isByoSubgraph,
|
|
@@ -1036,15 +1033,287 @@ import {
|
|
|
1036
1033
|
resolveSubgraphDb,
|
|
1037
1034
|
updateSubgraphStatus
|
|
1038
1035
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1039
|
-
import { logger as
|
|
1036
|
+
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
1040
1037
|
import { sql as sql3 } from "kysely";
|
|
1041
1038
|
|
|
1042
1039
|
// src/schema/utils.ts
|
|
1043
1040
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
1044
1041
|
|
|
1042
|
+
// src/runtime/block-source.ts
|
|
1043
|
+
import { getSourceDb } from "@secondlayer/shared/db";
|
|
1044
|
+
import { IndexHttpClient } from "@secondlayer/shared/index-http";
|
|
1045
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
1046
|
+
|
|
1047
|
+
// src/runtime/batch-loader.ts
|
|
1048
|
+
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1049
|
+
const [blocks, txs, events] = await Promise.all([
|
|
1050
|
+
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1051
|
+
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1052
|
+
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1053
|
+
]);
|
|
1054
|
+
const txsByHeight = new Map;
|
|
1055
|
+
for (const tx of txs) {
|
|
1056
|
+
const h = Number(tx.block_height);
|
|
1057
|
+
const list = txsByHeight.get(h) ?? [];
|
|
1058
|
+
list.push(tx);
|
|
1059
|
+
txsByHeight.set(h, list);
|
|
1060
|
+
}
|
|
1061
|
+
const eventsByHeight = new Map;
|
|
1062
|
+
for (const evt of events) {
|
|
1063
|
+
const h = Number(evt.block_height);
|
|
1064
|
+
const list = eventsByHeight.get(h) ?? [];
|
|
1065
|
+
list.push(evt);
|
|
1066
|
+
eventsByHeight.set(h, list);
|
|
1067
|
+
}
|
|
1068
|
+
const result = new Map;
|
|
1069
|
+
for (const block of blocks) {
|
|
1070
|
+
const h = Number(block.height);
|
|
1071
|
+
result.set(h, {
|
|
1072
|
+
block,
|
|
1073
|
+
txs: txsByHeight.get(h) ?? [],
|
|
1074
|
+
events: eventsByHeight.get(h) ?? []
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
return result;
|
|
1078
|
+
}
|
|
1079
|
+
function avgEventsPerBlock(batch) {
|
|
1080
|
+
if (batch.size === 0)
|
|
1081
|
+
return 0;
|
|
1082
|
+
let totalEvents = 0;
|
|
1083
|
+
for (const data of batch.values()) {
|
|
1084
|
+
totalEvents += data.events.length;
|
|
1085
|
+
}
|
|
1086
|
+
return totalEvents / batch.size;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// src/runtime/reconstruct.ts
|
|
1090
|
+
function isoToUnixSeconds(iso) {
|
|
1091
|
+
if (!iso)
|
|
1092
|
+
return 0;
|
|
1093
|
+
return Math.floor(new Date(iso).getTime() / 1000);
|
|
1094
|
+
}
|
|
1095
|
+
function reconstructBlock(b) {
|
|
1096
|
+
return {
|
|
1097
|
+
height: b.block_height,
|
|
1098
|
+
hash: b.block_hash,
|
|
1099
|
+
parent_hash: b.parent_hash,
|
|
1100
|
+
burn_block_height: b.burn_block_height,
|
|
1101
|
+
burn_block_hash: b.burn_block_hash,
|
|
1102
|
+
timestamp: isoToUnixSeconds(b.block_time),
|
|
1103
|
+
canonical: true,
|
|
1104
|
+
created_at: new Date(0)
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
function reconstructTransaction(t) {
|
|
1108
|
+
return {
|
|
1109
|
+
tx_id: t.tx_id,
|
|
1110
|
+
block_height: t.block_height,
|
|
1111
|
+
tx_index: t.tx_index,
|
|
1112
|
+
type: t.tx_type,
|
|
1113
|
+
sender: t.sender,
|
|
1114
|
+
status: t.status,
|
|
1115
|
+
contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
|
|
1116
|
+
function_name: t.contract_call?.function_name ?? null,
|
|
1117
|
+
function_args: t.contract_call?.function_args_hex ?? [],
|
|
1118
|
+
raw_result: t.contract_call?.result_hex ?? null,
|
|
1119
|
+
raw_tx: "",
|
|
1120
|
+
created_at: new Date(0)
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
function reconstructEvent(e) {
|
|
1124
|
+
const base = {
|
|
1125
|
+
id: `${e.tx_id}#${e.event_index}`,
|
|
1126
|
+
tx_id: e.tx_id,
|
|
1127
|
+
block_height: e.block_height,
|
|
1128
|
+
event_index: e.event_index,
|
|
1129
|
+
created_at: new Date(0)
|
|
1130
|
+
};
|
|
1131
|
+
switch (e.event_type) {
|
|
1132
|
+
case "ft_transfer":
|
|
1133
|
+
case "ft_mint":
|
|
1134
|
+
case "ft_burn":
|
|
1135
|
+
return {
|
|
1136
|
+
...base,
|
|
1137
|
+
type: `${e.event_type}_event`,
|
|
1138
|
+
data: {
|
|
1139
|
+
asset_identifier: e.asset_identifier,
|
|
1140
|
+
sender: e.sender,
|
|
1141
|
+
recipient: e.recipient,
|
|
1142
|
+
amount: e.amount
|
|
1143
|
+
}
|
|
1144
|
+
};
|
|
1145
|
+
case "nft_transfer":
|
|
1146
|
+
case "nft_mint":
|
|
1147
|
+
case "nft_burn":
|
|
1148
|
+
return {
|
|
1149
|
+
...base,
|
|
1150
|
+
type: `${e.event_type}_event`,
|
|
1151
|
+
data: {
|
|
1152
|
+
asset_identifier: e.asset_identifier,
|
|
1153
|
+
sender: e.sender,
|
|
1154
|
+
recipient: e.recipient,
|
|
1155
|
+
raw_value: e.value
|
|
1156
|
+
}
|
|
1157
|
+
};
|
|
1158
|
+
case "stx_transfer":
|
|
1159
|
+
case "stx_mint":
|
|
1160
|
+
case "stx_burn":
|
|
1161
|
+
return {
|
|
1162
|
+
...base,
|
|
1163
|
+
type: `${e.event_type}_event`,
|
|
1164
|
+
data: {
|
|
1165
|
+
sender: e.sender,
|
|
1166
|
+
recipient: e.recipient,
|
|
1167
|
+
amount: e.amount,
|
|
1168
|
+
..."memo" in e ? { memo: e.memo ?? undefined } : {}
|
|
1169
|
+
}
|
|
1170
|
+
};
|
|
1171
|
+
case "stx_lock":
|
|
1172
|
+
return {
|
|
1173
|
+
...base,
|
|
1174
|
+
type: "stx_lock_event",
|
|
1175
|
+
data: {
|
|
1176
|
+
locked_address: e.sender,
|
|
1177
|
+
locked_amount: e.amount,
|
|
1178
|
+
unlock_height: e.payload.unlock_height
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
case "print":
|
|
1182
|
+
return {
|
|
1183
|
+
...base,
|
|
1184
|
+
type: "contract_event",
|
|
1185
|
+
data: {
|
|
1186
|
+
topic: e.payload.topic,
|
|
1187
|
+
contract_id: e.contract_id,
|
|
1188
|
+
contract_identifier: e.contract_id,
|
|
1189
|
+
value: e.payload.value,
|
|
1190
|
+
raw_value: e.payload.raw_value
|
|
1191
|
+
}
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
// src/runtime/block-source.ts
|
|
1197
|
+
class PostgresBlockSource {
|
|
1198
|
+
async getTip() {
|
|
1199
|
+
const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1200
|
+
return progress ? Number(progress.highest_seen_block) : 0;
|
|
1201
|
+
}
|
|
1202
|
+
loadBlockRange(fromHeight, toHeight) {
|
|
1203
|
+
return loadBlockRange(getSourceDb(), fromHeight, toHeight);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
var EVENT_FILTER_TO_INDEX_TYPE = {
|
|
1207
|
+
stx_transfer: "stx_transfer",
|
|
1208
|
+
stx_mint: "stx_mint",
|
|
1209
|
+
stx_burn: "stx_burn",
|
|
1210
|
+
stx_lock: "stx_lock",
|
|
1211
|
+
ft_transfer: "ft_transfer",
|
|
1212
|
+
ft_mint: "ft_mint",
|
|
1213
|
+
ft_burn: "ft_burn",
|
|
1214
|
+
nft_transfer: "nft_transfer",
|
|
1215
|
+
nft_mint: "nft_mint",
|
|
1216
|
+
nft_burn: "nft_burn",
|
|
1217
|
+
print_event: "print"
|
|
1218
|
+
};
|
|
1219
|
+
var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
1220
|
+
var ALL_INDEX_EVENT_TYPES = [
|
|
1221
|
+
...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
|
|
1222
|
+
];
|
|
1223
|
+
function sourceFilters(subgraph) {
|
|
1224
|
+
const sources = subgraph.sources;
|
|
1225
|
+
return Array.isArray(sources) ? sources : Object.values(sources);
|
|
1226
|
+
}
|
|
1227
|
+
function referencedIndexEventTypes(subgraph) {
|
|
1228
|
+
const filters = sourceFilters(subgraph);
|
|
1229
|
+
if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
|
|
1230
|
+
return ALL_INDEX_EVENT_TYPES;
|
|
1231
|
+
}
|
|
1232
|
+
const types = new Set;
|
|
1233
|
+
for (const f of filters) {
|
|
1234
|
+
const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
|
|
1235
|
+
if (t)
|
|
1236
|
+
types.add(t);
|
|
1237
|
+
}
|
|
1238
|
+
return [...types];
|
|
1239
|
+
}
|
|
1240
|
+
function isStreamsIndexEligible(subgraph) {
|
|
1241
|
+
if (Array.isArray(subgraph.sources))
|
|
1242
|
+
return false;
|
|
1243
|
+
const filters = sourceFilters(subgraph);
|
|
1244
|
+
if (filters.length === 0)
|
|
1245
|
+
return false;
|
|
1246
|
+
for (const f of filters) {
|
|
1247
|
+
const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
|
|
1248
|
+
if (!known)
|
|
1249
|
+
return false;
|
|
1250
|
+
}
|
|
1251
|
+
return true;
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
class PublicApiBlockSource {
|
|
1255
|
+
http;
|
|
1256
|
+
eventTypes;
|
|
1257
|
+
constructor(http, eventTypes) {
|
|
1258
|
+
this.http = http;
|
|
1259
|
+
this.eventTypes = eventTypes;
|
|
1260
|
+
}
|
|
1261
|
+
getTip() {
|
|
1262
|
+
return this.http.getIndexTip();
|
|
1263
|
+
}
|
|
1264
|
+
async loadBlockRange(fromHeight, toHeight) {
|
|
1265
|
+
const [blocks, txs, eventLists] = await Promise.all([
|
|
1266
|
+
this.http.walkBlocks(fromHeight, toHeight),
|
|
1267
|
+
this.http.walkTransactions(fromHeight, toHeight),
|
|
1268
|
+
Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
|
|
1269
|
+
]);
|
|
1270
|
+
const map = new Map;
|
|
1271
|
+
for (const b of blocks) {
|
|
1272
|
+
map.set(b.block_height, {
|
|
1273
|
+
block: reconstructBlock(b),
|
|
1274
|
+
txs: [],
|
|
1275
|
+
events: []
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
for (const t of txs) {
|
|
1279
|
+
map.get(t.block_height)?.txs.push(reconstructTransaction(t));
|
|
1280
|
+
}
|
|
1281
|
+
for (const list of eventLists) {
|
|
1282
|
+
for (const e of list) {
|
|
1283
|
+
map.get(e.block_height)?.events.push(reconstructEvent(e));
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
for (const bd of map.values()) {
|
|
1287
|
+
bd.txs.sort((a, b) => a.tx_index - b.tx_index);
|
|
1288
|
+
bd.events.sort((a, b) => a.event_index - b.event_index);
|
|
1289
|
+
}
|
|
1290
|
+
return map;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
var postgresBlockSource = new PostgresBlockSource;
|
|
1294
|
+
function buildHttpClient() {
|
|
1295
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
1296
|
+
return new IndexHttpClient({
|
|
1297
|
+
indexBaseUrl: baseUrl,
|
|
1298
|
+
streamsBaseUrl: baseUrl,
|
|
1299
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
function resolveBlockSource(subgraph) {
|
|
1303
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
|
|
1304
|
+
return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
|
|
1305
|
+
}
|
|
1306
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
|
|
1307
|
+
logger3.debug("Subgraph not streams-index eligible, using DB tap", {
|
|
1308
|
+
subgraph: subgraph.name
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
return postgresBlockSource;
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1045
1314
|
// src/runtime/outbox-emit.ts
|
|
1046
1315
|
import { createHash } from "node:crypto";
|
|
1047
|
-
import { logger as
|
|
1316
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
1048
1317
|
var loggedKillSwitch = false;
|
|
1049
1318
|
function isEmitOutboxEnabled() {
|
|
1050
1319
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
@@ -1063,7 +1332,7 @@ function stableStringify(obj) {
|
|
|
1063
1332
|
async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
1064
1333
|
if (!isEmitOutboxEnabled()) {
|
|
1065
1334
|
if (!loggedKillSwitch) {
|
|
1066
|
-
|
|
1335
|
+
logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
1067
1336
|
loggedKillSwitch = true;
|
|
1068
1337
|
}
|
|
1069
1338
|
return 0;
|
|
@@ -1304,7 +1573,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
|
|
|
1304
1573
|
return resolved;
|
|
1305
1574
|
}
|
|
1306
1575
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
1307
|
-
const sourceDb = getSourceDb();
|
|
1308
1576
|
const targetDb = getTargetDb();
|
|
1309
1577
|
const blockStart = performance.now();
|
|
1310
1578
|
const result = {
|
|
@@ -1322,25 +1590,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1322
1590
|
txs = opts.preloaded.txs;
|
|
1323
1591
|
evts = opts.preloaded.events;
|
|
1324
1592
|
} else {
|
|
1325
|
-
|
|
1326
|
-
if (!
|
|
1327
|
-
|
|
1328
|
-
subgraph: subgraphName,
|
|
1329
|
-
blockHeight
|
|
1330
|
-
});
|
|
1331
|
-
result.skipped = true;
|
|
1332
|
-
return result;
|
|
1333
|
-
}
|
|
1334
|
-
if (!block.canonical) {
|
|
1335
|
-
logger4.debug("Skipping non-canonical block", {
|
|
1593
|
+
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
1594
|
+
if (!data) {
|
|
1595
|
+
logger5.debug("Block not found or non-canonical for subgraph processing", {
|
|
1336
1596
|
subgraph: subgraphName,
|
|
1337
1597
|
blockHeight
|
|
1338
1598
|
});
|
|
1339
1599
|
result.skipped = true;
|
|
1340
1600
|
return result;
|
|
1341
1601
|
}
|
|
1342
|
-
|
|
1343
|
-
|
|
1602
|
+
block = data.block;
|
|
1603
|
+
txs = data.txs;
|
|
1604
|
+
evts = data.events;
|
|
1344
1605
|
}
|
|
1345
1606
|
const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
|
|
1346
1607
|
const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
|
|
@@ -1431,7 +1692,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1431
1692
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
1432
1693
|
const count = Number(rows[0]?.count ?? 0);
|
|
1433
1694
|
if (count >= 1e7) {
|
|
1434
|
-
|
|
1695
|
+
logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
1435
1696
|
subgraph: subgraphName,
|
|
1436
1697
|
table,
|
|
1437
1698
|
count
|
|
@@ -1439,7 +1700,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1439
1700
|
}
|
|
1440
1701
|
}
|
|
1441
1702
|
} catch (err) {
|
|
1442
|
-
|
|
1703
|
+
logger5.debug("Row count sample failed", {
|
|
1443
1704
|
subgraph: subgraphName,
|
|
1444
1705
|
error: err instanceof Error ? err.message : String(err)
|
|
1445
1706
|
});
|
|
@@ -1513,7 +1774,7 @@ class StatsAccumulator {
|
|
|
1513
1774
|
|
|
1514
1775
|
// src/runtime/reindex.ts
|
|
1515
1776
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
1516
|
-
import { getRawClient,
|
|
1777
|
+
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
1517
1778
|
import {
|
|
1518
1779
|
recordGapBatch,
|
|
1519
1780
|
resolveGaps
|
|
@@ -1522,7 +1783,7 @@ import {
|
|
|
1522
1783
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1523
1784
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
1524
1785
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1525
|
-
import { logger as
|
|
1786
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
1526
1787
|
|
|
1527
1788
|
// src/schema/generator.ts
|
|
1528
1789
|
import { createHash as createHash2 } from "node:crypto";
|
|
@@ -1615,48 +1876,6 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
1615
1876
|
return { statements, hash };
|
|
1616
1877
|
}
|
|
1617
1878
|
|
|
1618
|
-
// src/runtime/batch-loader.ts
|
|
1619
|
-
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1620
|
-
const [blocks, txs, events] = await Promise.all([
|
|
1621
|
-
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1622
|
-
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1623
|
-
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1624
|
-
]);
|
|
1625
|
-
const txsByHeight = new Map;
|
|
1626
|
-
for (const tx of txs) {
|
|
1627
|
-
const h = Number(tx.block_height);
|
|
1628
|
-
const list = txsByHeight.get(h) ?? [];
|
|
1629
|
-
list.push(tx);
|
|
1630
|
-
txsByHeight.set(h, list);
|
|
1631
|
-
}
|
|
1632
|
-
const eventsByHeight = new Map;
|
|
1633
|
-
for (const evt of events) {
|
|
1634
|
-
const h = Number(evt.block_height);
|
|
1635
|
-
const list = eventsByHeight.get(h) ?? [];
|
|
1636
|
-
list.push(evt);
|
|
1637
|
-
eventsByHeight.set(h, list);
|
|
1638
|
-
}
|
|
1639
|
-
const result = new Map;
|
|
1640
|
-
for (const block of blocks) {
|
|
1641
|
-
const h = Number(block.height);
|
|
1642
|
-
result.set(h, {
|
|
1643
|
-
block,
|
|
1644
|
-
txs: txsByHeight.get(h) ?? [],
|
|
1645
|
-
events: eventsByHeight.get(h) ?? []
|
|
1646
|
-
});
|
|
1647
|
-
}
|
|
1648
|
-
return result;
|
|
1649
|
-
}
|
|
1650
|
-
function avgEventsPerBlock(batch) {
|
|
1651
|
-
if (batch.size === 0)
|
|
1652
|
-
return 0;
|
|
1653
|
-
let totalEvents = 0;
|
|
1654
|
-
for (const data of batch.values()) {
|
|
1655
|
-
totalEvents += data.events.length;
|
|
1656
|
-
}
|
|
1657
|
-
return totalEvents / batch.size;
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
1879
|
// src/runtime/reindex.ts
|
|
1661
1880
|
var LOG_INTERVAL = 1000;
|
|
1662
1881
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
@@ -1717,7 +1936,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
1717
1936
|
return ranges;
|
|
1718
1937
|
}
|
|
1719
1938
|
async function processBlockRange(def, opts) {
|
|
1720
|
-
const
|
|
1939
|
+
const source = resolveBlockSource(def);
|
|
1721
1940
|
const targetDb = getTargetDb2();
|
|
1722
1941
|
const subgraphName = def.name;
|
|
1723
1942
|
const { fromBlock, toBlock, status } = opts;
|
|
@@ -1747,11 +1966,11 @@ async function processBlockRange(def, opts) {
|
|
|
1747
1966
|
lastHealthFlushAt = Date.now();
|
|
1748
1967
|
};
|
|
1749
1968
|
let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
1750
|
-
let nextBatchPromise = loadBlockRange(
|
|
1969
|
+
let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
|
|
1751
1970
|
while (currentHeight <= toBlock) {
|
|
1752
1971
|
if (opts.signal?.aborted) {
|
|
1753
1972
|
aborted = true;
|
|
1754
|
-
|
|
1973
|
+
logger6.info("Block processing aborted", {
|
|
1755
1974
|
subgraph: subgraphName,
|
|
1756
1975
|
currentBlock: currentHeight,
|
|
1757
1976
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -1763,7 +1982,7 @@ async function processBlockRange(def, opts) {
|
|
|
1763
1982
|
const nextStart = batchEnd + 1;
|
|
1764
1983
|
if (nextStart <= toBlock) {
|
|
1765
1984
|
nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
1766
|
-
nextBatchPromise = loadBlockRange(
|
|
1985
|
+
nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
|
|
1767
1986
|
}
|
|
1768
1987
|
const batchFailedBlocks = [];
|
|
1769
1988
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -1781,7 +2000,7 @@ async function processBlockRange(def, opts) {
|
|
|
1781
2000
|
});
|
|
1782
2001
|
} catch (err) {
|
|
1783
2002
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1784
|
-
|
|
2003
|
+
logger6.error("Block processing error", {
|
|
1785
2004
|
subgraph: subgraphName,
|
|
1786
2005
|
blockHeight: height,
|
|
1787
2006
|
error: errorMsg
|
|
@@ -1815,7 +2034,7 @@ async function processBlockRange(def, opts) {
|
|
|
1815
2034
|
lastProgressFlushAt = now;
|
|
1816
2035
|
}
|
|
1817
2036
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1818
|
-
|
|
2037
|
+
logger6.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1819
2038
|
subgraph: subgraphName,
|
|
1820
2039
|
processed: blocksProcessed,
|
|
1821
2040
|
total: totalBlocks,
|
|
@@ -1834,7 +2053,7 @@ async function processBlockRange(def, opts) {
|
|
|
1834
2053
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1835
2054
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1836
2055
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1837
|
-
|
|
2056
|
+
logger6.warn("Failed to record subgraph gaps", {
|
|
1838
2057
|
subgraph: subgraphName,
|
|
1839
2058
|
error: err instanceof Error ? err.message : String(err)
|
|
1840
2059
|
});
|
|
@@ -1851,32 +2070,26 @@ async function processBlockRange(def, opts) {
|
|
|
1851
2070
|
await flushHealth();
|
|
1852
2071
|
return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
|
|
1853
2072
|
}
|
|
1854
|
-
async function resolveBlockRange(
|
|
2073
|
+
async function resolveBlockRange(source, def, opts) {
|
|
1855
2074
|
const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
|
|
1856
|
-
|
|
1857
|
-
if (opts?.toBlock != null) {
|
|
1858
|
-
toBlock = opts.toBlock;
|
|
1859
|
-
} else {
|
|
1860
|
-
const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1861
|
-
toBlock = progress?.highest_seen_block ?? 0;
|
|
1862
|
-
}
|
|
2075
|
+
const toBlock = opts?.toBlock != null ? opts.toBlock : await source.getTip();
|
|
1863
2076
|
return { fromBlock, toBlock };
|
|
1864
2077
|
}
|
|
1865
2078
|
async function clearReindexMetadata(db, subgraphName) {
|
|
1866
2079
|
await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
|
|
1867
2080
|
}
|
|
1868
2081
|
async function reindexSubgraph(def, opts) {
|
|
1869
|
-
const
|
|
2082
|
+
const source = resolveBlockSource(def);
|
|
1870
2083
|
const targetDb = getTargetDb2();
|
|
1871
2084
|
const client = getRawClient("target");
|
|
1872
2085
|
const subgraphName = def.name;
|
|
1873
2086
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1874
2087
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1875
|
-
|
|
2088
|
+
logger6.info("Reindex starting", { subgraph: subgraphName });
|
|
1876
2089
|
try {
|
|
1877
|
-
const { fromBlock, toBlock } = await resolveBlockRange(
|
|
2090
|
+
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
1878
2091
|
if (fromBlock > toBlock) {
|
|
1879
|
-
|
|
2092
|
+
logger6.info("No blocks to reindex", {
|
|
1880
2093
|
subgraph: subgraphName,
|
|
1881
2094
|
fromBlock,
|
|
1882
2095
|
toBlock
|
|
@@ -1889,7 +2102,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1889
2102
|
for (const stmt of statements) {
|
|
1890
2103
|
await client.unsafe(stmt);
|
|
1891
2104
|
}
|
|
1892
|
-
|
|
2105
|
+
logger6.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1893
2106
|
await targetDb.updateTable("subgraphs").set({
|
|
1894
2107
|
last_processed_block: initialReindexProgressBlock(fromBlock),
|
|
1895
2108
|
reindex_from_block: fromBlock,
|
|
@@ -1900,7 +2113,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1900
2113
|
last_error_at: null,
|
|
1901
2114
|
updated_at: new Date
|
|
1902
2115
|
}).where("name", "=", subgraphName).execute();
|
|
1903
|
-
|
|
2116
|
+
logger6.info("Reindexing blocks", {
|
|
1904
2117
|
subgraph: subgraphName,
|
|
1905
2118
|
fromBlock,
|
|
1906
2119
|
toBlock,
|
|
@@ -1921,9 +2134,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
1921
2134
|
if (reason === "user-cancelled") {
|
|
1922
2135
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1923
2136
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1924
|
-
|
|
2137
|
+
logger6.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1925
2138
|
} else {
|
|
1926
|
-
|
|
2139
|
+
logger6.info("Reindex interrupted by shutdown, will resume", {
|
|
1927
2140
|
subgraph: subgraphName
|
|
1928
2141
|
});
|
|
1929
2142
|
}
|
|
@@ -1931,7 +2144,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1931
2144
|
}
|
|
1932
2145
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1933
2146
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1934
|
-
|
|
2147
|
+
logger6.info("Reindex complete", {
|
|
1935
2148
|
subgraph: subgraphName,
|
|
1936
2149
|
blocks: result.blocksProcessed,
|
|
1937
2150
|
events: result.totalEventsProcessed,
|
|
@@ -1939,7 +2152,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1939
2152
|
});
|
|
1940
2153
|
return { processed: result.blocksProcessed };
|
|
1941
2154
|
} catch (err) {
|
|
1942
|
-
|
|
2155
|
+
logger6.error("Reindex failed", {
|
|
1943
2156
|
subgraph: subgraphName,
|
|
1944
2157
|
error: getErrorMessage2(err)
|
|
1945
2158
|
});
|
|
@@ -1961,7 +2174,7 @@ async function resumeReindex(def, opts) {
|
|
|
1961
2174
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
1962
2175
|
const toBlock = Number(row.reindex_to_block);
|
|
1963
2176
|
if (fromBlock == null) {
|
|
1964
|
-
|
|
2177
|
+
logger6.info("No reindex metadata, starting fresh reindex", {
|
|
1965
2178
|
subgraph: subgraphName
|
|
1966
2179
|
});
|
|
1967
2180
|
return reindexSubgraph(def, {
|
|
@@ -1970,12 +2183,12 @@ async function resumeReindex(def, opts) {
|
|
|
1970
2183
|
});
|
|
1971
2184
|
}
|
|
1972
2185
|
if (fromBlock > toBlock) {
|
|
1973
|
-
|
|
2186
|
+
logger6.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1974
2187
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1975
2188
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1976
2189
|
return { processed: 0 };
|
|
1977
2190
|
}
|
|
1978
|
-
|
|
2191
|
+
logger6.info("Resuming reindex", {
|
|
1979
2192
|
subgraph: subgraphName,
|
|
1980
2193
|
fromBlock,
|
|
1981
2194
|
toBlock,
|
|
@@ -1996,9 +2209,9 @@ async function resumeReindex(def, opts) {
|
|
|
1996
2209
|
if (reason === "user-cancelled") {
|
|
1997
2210
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1998
2211
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1999
|
-
|
|
2212
|
+
logger6.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
2000
2213
|
} else {
|
|
2001
|
-
|
|
2214
|
+
logger6.info("Resume interrupted by shutdown, will resume again", {
|
|
2002
2215
|
subgraph: subgraphName
|
|
2003
2216
|
});
|
|
2004
2217
|
}
|
|
@@ -2006,13 +2219,13 @@ async function resumeReindex(def, opts) {
|
|
|
2006
2219
|
}
|
|
2007
2220
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2008
2221
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2009
|
-
|
|
2222
|
+
logger6.info("Resumed reindex complete", {
|
|
2010
2223
|
subgraph: subgraphName,
|
|
2011
2224
|
blocks: result.blocksProcessed
|
|
2012
2225
|
});
|
|
2013
2226
|
return { processed: result.blocksProcessed };
|
|
2014
2227
|
} catch (err) {
|
|
2015
|
-
|
|
2228
|
+
logger6.error("Resumed reindex failed", {
|
|
2016
2229
|
subgraph: subgraphName,
|
|
2017
2230
|
error: getErrorMessage2(err)
|
|
2018
2231
|
});
|
|
@@ -2023,7 +2236,7 @@ async function resumeReindex(def, opts) {
|
|
|
2023
2236
|
async function backfillSubgraph(def, opts) {
|
|
2024
2237
|
const targetDb = getTargetDb2();
|
|
2025
2238
|
const subgraphName = def.name;
|
|
2026
|
-
|
|
2239
|
+
logger6.info("Backfill starting", {
|
|
2027
2240
|
subgraph: subgraphName,
|
|
2028
2241
|
from: opts.fromBlock,
|
|
2029
2242
|
to: opts.toBlock
|
|
@@ -2040,17 +2253,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
2040
2253
|
signal: opts.signal
|
|
2041
2254
|
});
|
|
2042
2255
|
if (result.aborted) {
|
|
2043
|
-
|
|
2256
|
+
logger6.info("Backfill aborted", { subgraph: subgraphName });
|
|
2044
2257
|
return { processed: result.blocksProcessed };
|
|
2045
2258
|
}
|
|
2046
2259
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
2047
2260
|
if (resolved > 0) {
|
|
2048
|
-
|
|
2261
|
+
logger6.info("Resolved subgraph gaps via backfill", {
|
|
2049
2262
|
subgraph: subgraphName,
|
|
2050
2263
|
resolved
|
|
2051
2264
|
});
|
|
2052
2265
|
}
|
|
2053
|
-
|
|
2266
|
+
logger6.info("Backfill complete", {
|
|
2054
2267
|
subgraph: subgraphName,
|
|
2055
2268
|
blocks: result.blocksProcessed,
|
|
2056
2269
|
events: result.totalEventsProcessed,
|
|
@@ -2058,7 +2271,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
2058
2271
|
});
|
|
2059
2272
|
return { processed: result.blocksProcessed };
|
|
2060
2273
|
} catch (err) {
|
|
2061
|
-
|
|
2274
|
+
logger6.error("Backfill failed", {
|
|
2062
2275
|
subgraph: subgraphName,
|
|
2063
2276
|
error: getErrorMessage2(err)
|
|
2064
2277
|
});
|
|
@@ -2169,7 +2382,7 @@ function generatePrismaSchema(def, opts = {}) {
|
|
|
2169
2382
|
const schemaName = opts.schemaName ?? pgSchemaName(def.name);
|
|
2170
2383
|
const env = opts.datasourceEnv ?? "DATABASE_URL";
|
|
2171
2384
|
const header = [
|
|
2172
|
-
`// Generated by \`sl
|
|
2385
|
+
`// Generated by \`sl subgraphs codegen --target prisma\` from subgraph "${def.name}". Do not edit by hand.`,
|
|
2173
2386
|
"",
|
|
2174
2387
|
"datasource db {",
|
|
2175
2388
|
' provider = "postgresql"',
|
|
@@ -2329,7 +2542,7 @@ function generateDrizzleSchema(def, opts = {}) {
|
|
|
2329
2542
|
const relationDecls = renderRelations(def.schema);
|
|
2330
2543
|
const typeExports = Object.keys(def.schema).map((name) => `export type ${pascalCase2(name)} = typeof ${snakeToCamel2(name)}.$inferSelect;`);
|
|
2331
2544
|
return [
|
|
2332
|
-
`// Generated by \`sl
|
|
2545
|
+
`// Generated by \`sl subgraphs codegen --target drizzle\` from subgraph "${def.name}". Do not edit by hand.`,
|
|
2333
2546
|
imports,
|
|
2334
2547
|
"",
|
|
2335
2548
|
`export const sg = pgSchema("${schemaName}");`,
|
|
@@ -2625,5 +2838,5 @@ export {
|
|
|
2625
2838
|
backfillSubgraph
|
|
2626
2839
|
};
|
|
2627
2840
|
|
|
2628
|
-
//# debugId=
|
|
2841
|
+
//# debugId=4E10631FF8DB802F64756E2164756E21
|
|
2629
2842
|
//# sourceMappingURL=index.js.map
|