@secondlayer/subgraphs 3.5.0 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +327 -113
- package/dist/src/index.js.map +12 -10
- package/dist/src/runtime/block-processor.js +292 -30
- package/dist/src/runtime/block-processor.js.map +8 -5
- package/dist/src/runtime/catchup.js +306 -91
- package/dist/src/runtime/catchup.js.map +9 -7
- package/dist/src/runtime/processor.js +433 -171
- package/dist/src/runtime/processor.js.map +14 -11
- package/dist/src/runtime/reindex.js +325 -111
- package/dist/src/runtime/reindex.js.map +10 -8
- package/dist/src/runtime/reorg.js +298 -36
- package/dist/src/runtime/reorg.js.map +8 -5
- package/dist/src/runtime/runner.d.ts +6 -1
- package/dist/src/runtime/runner.js +10 -7
- package/dist/src/runtime/runner.js.map +3 -3
- package/dist/src/schema/index.js.map +1 -1
- package/dist/src/service.js +436 -174
- package/dist/src/service.js.map +13 -10
- package/package.json +2 -2
package/dist/src/index.js
CHANGED
|
@@ -594,21 +594,21 @@ function buildEventPayload(filter, tx, event) {
|
|
|
594
594
|
return {
|
|
595
595
|
sender: decoded.sender,
|
|
596
596
|
recipient: decoded.recipient,
|
|
597
|
-
tokenId: decoded.value,
|
|
597
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
598
598
|
assetIdentifier: decoded.asset_identifier,
|
|
599
599
|
tx: txMeta
|
|
600
600
|
};
|
|
601
601
|
case "nft_mint":
|
|
602
602
|
return {
|
|
603
603
|
recipient: decoded.recipient,
|
|
604
|
-
tokenId: decoded.value,
|
|
604
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
605
605
|
assetIdentifier: decoded.asset_identifier,
|
|
606
606
|
tx: txMeta
|
|
607
607
|
};
|
|
608
608
|
case "nft_burn":
|
|
609
609
|
return {
|
|
610
610
|
sender: decoded.sender,
|
|
611
|
-
tokenId: decoded.value,
|
|
611
|
+
tokenId: decoded.raw_value ?? decoded.value,
|
|
612
612
|
assetIdentifier: decoded.asset_identifier,
|
|
613
613
|
tx: txMeta
|
|
614
614
|
};
|
|
@@ -640,8 +640,8 @@ function buildEventPayload(filter, tx, event) {
|
|
|
640
640
|
tx: txMeta
|
|
641
641
|
};
|
|
642
642
|
case "print_event": {
|
|
643
|
-
const
|
|
644
|
-
const rawValue =
|
|
643
|
+
const rawHex = event.data?.raw_value;
|
|
644
|
+
const rawValue = typeof rawHex === "string" && rawHex.startsWith("0x") ? decodeClarityValue(rawHex) : decoded.value;
|
|
645
645
|
const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
|
|
646
646
|
const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
|
|
647
647
|
const { topic: _, ...rest } = clarityObj ?? {};
|
|
@@ -654,9 +654,11 @@ function buildEventPayload(filter, tx, event) {
|
|
|
654
654
|
};
|
|
655
655
|
}
|
|
656
656
|
case "contract_call": {
|
|
657
|
+
const ccRawHex = event.data?.raw_value;
|
|
658
|
+
const normalized = typeof ccRawHex === "string" && ccRawHex.startsWith("0x") ? { ...decoded, value: decodeClarityValue(ccRawHex) } : decoded;
|
|
657
659
|
const input = buildContractCallInput(filter, tx);
|
|
658
660
|
return {
|
|
659
|
-
...
|
|
661
|
+
...normalized,
|
|
660
662
|
type: "contract_call",
|
|
661
663
|
_eventType: event.type,
|
|
662
664
|
contractId: tx.contract_id ?? "",
|
|
@@ -1025,10 +1027,7 @@ function matchSources(sources, transactions, events, traitContracts = new Map) {
|
|
|
1025
1027
|
}
|
|
1026
1028
|
|
|
1027
1029
|
// src/runtime/block-processor.ts
|
|
1028
|
-
import {
|
|
1029
|
-
getSourceDb,
|
|
1030
|
-
getTargetDb
|
|
1031
|
-
} from "@secondlayer/shared/db";
|
|
1030
|
+
import { getTargetDb } from "@secondlayer/shared/db";
|
|
1032
1031
|
import { resolveTraitContractIds } from "@secondlayer/shared/db/queries/contracts";
|
|
1033
1032
|
import {
|
|
1034
1033
|
isByoSubgraph,
|
|
@@ -1036,15 +1035,286 @@ import {
|
|
|
1036
1035
|
resolveSubgraphDb,
|
|
1037
1036
|
updateSubgraphStatus
|
|
1038
1037
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1039
|
-
import { logger as
|
|
1038
|
+
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
1040
1039
|
import { sql as sql3 } from "kysely";
|
|
1041
1040
|
|
|
1042
1041
|
// src/schema/utils.ts
|
|
1043
1042
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
1044
1043
|
|
|
1044
|
+
// src/runtime/block-source.ts
|
|
1045
|
+
import { getSourceDb } from "@secondlayer/shared/db";
|
|
1046
|
+
import { IndexHttpClient } from "@secondlayer/shared/index-http";
|
|
1047
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
1048
|
+
|
|
1049
|
+
// src/runtime/batch-loader.ts
|
|
1050
|
+
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1051
|
+
const [blocks, txs, events] = await Promise.all([
|
|
1052
|
+
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1053
|
+
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1054
|
+
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1055
|
+
]);
|
|
1056
|
+
const txsByHeight = new Map;
|
|
1057
|
+
for (const tx of txs) {
|
|
1058
|
+
const h = Number(tx.block_height);
|
|
1059
|
+
const list = txsByHeight.get(h) ?? [];
|
|
1060
|
+
list.push(tx);
|
|
1061
|
+
txsByHeight.set(h, list);
|
|
1062
|
+
}
|
|
1063
|
+
const eventsByHeight = new Map;
|
|
1064
|
+
for (const evt of events) {
|
|
1065
|
+
const h = Number(evt.block_height);
|
|
1066
|
+
const list = eventsByHeight.get(h) ?? [];
|
|
1067
|
+
list.push(evt);
|
|
1068
|
+
eventsByHeight.set(h, list);
|
|
1069
|
+
}
|
|
1070
|
+
const result = new Map;
|
|
1071
|
+
for (const block of blocks) {
|
|
1072
|
+
const h = Number(block.height);
|
|
1073
|
+
result.set(h, {
|
|
1074
|
+
block,
|
|
1075
|
+
txs: txsByHeight.get(h) ?? [],
|
|
1076
|
+
events: eventsByHeight.get(h) ?? []
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
return result;
|
|
1080
|
+
}
|
|
1081
|
+
function avgEventsPerBlock(batch) {
|
|
1082
|
+
if (batch.size === 0)
|
|
1083
|
+
return 0;
|
|
1084
|
+
let totalEvents = 0;
|
|
1085
|
+
for (const data of batch.values()) {
|
|
1086
|
+
totalEvents += data.events.length;
|
|
1087
|
+
}
|
|
1088
|
+
return totalEvents / batch.size;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
// src/runtime/reconstruct.ts
|
|
1092
|
+
function isoToUnixSeconds(iso) {
|
|
1093
|
+
if (!iso)
|
|
1094
|
+
return 0;
|
|
1095
|
+
return Math.floor(new Date(iso).getTime() / 1000);
|
|
1096
|
+
}
|
|
1097
|
+
function reconstructBlock(b) {
|
|
1098
|
+
return {
|
|
1099
|
+
height: b.block_height,
|
|
1100
|
+
hash: b.block_hash,
|
|
1101
|
+
parent_hash: b.parent_hash,
|
|
1102
|
+
burn_block_height: b.burn_block_height,
|
|
1103
|
+
burn_block_hash: b.burn_block_hash,
|
|
1104
|
+
timestamp: isoToUnixSeconds(b.block_time),
|
|
1105
|
+
canonical: true,
|
|
1106
|
+
created_at: new Date(0)
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
function reconstructTransaction(t) {
|
|
1110
|
+
return {
|
|
1111
|
+
tx_id: t.tx_id,
|
|
1112
|
+
block_height: t.block_height,
|
|
1113
|
+
tx_index: t.tx_index,
|
|
1114
|
+
type: t.tx_type,
|
|
1115
|
+
sender: t.sender,
|
|
1116
|
+
status: t.status,
|
|
1117
|
+
contract_id: t.contract_call?.contract_id ?? t.smart_contract?.contract_id ?? null,
|
|
1118
|
+
function_name: t.contract_call?.function_name ?? null,
|
|
1119
|
+
function_args: t.contract_call?.function_args_hex ?? [],
|
|
1120
|
+
raw_result: t.contract_call?.result_hex ?? null,
|
|
1121
|
+
raw_tx: "",
|
|
1122
|
+
created_at: new Date(0)
|
|
1123
|
+
};
|
|
1124
|
+
}
|
|
1125
|
+
function reconstructEvent(e) {
|
|
1126
|
+
const base = {
|
|
1127
|
+
id: `${e.tx_id}#${e.event_index}`,
|
|
1128
|
+
tx_id: e.tx_id,
|
|
1129
|
+
block_height: e.block_height,
|
|
1130
|
+
event_index: e.event_index,
|
|
1131
|
+
created_at: new Date(0)
|
|
1132
|
+
};
|
|
1133
|
+
switch (e.event_type) {
|
|
1134
|
+
case "ft_transfer":
|
|
1135
|
+
case "ft_mint":
|
|
1136
|
+
case "ft_burn":
|
|
1137
|
+
return {
|
|
1138
|
+
...base,
|
|
1139
|
+
type: `${e.event_type}_event`,
|
|
1140
|
+
data: {
|
|
1141
|
+
asset_identifier: e.asset_identifier,
|
|
1142
|
+
sender: e.sender,
|
|
1143
|
+
recipient: e.recipient,
|
|
1144
|
+
amount: e.amount
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
case "nft_transfer":
|
|
1148
|
+
case "nft_mint":
|
|
1149
|
+
case "nft_burn":
|
|
1150
|
+
return {
|
|
1151
|
+
...base,
|
|
1152
|
+
type: `${e.event_type}_event`,
|
|
1153
|
+
data: {
|
|
1154
|
+
asset_identifier: e.asset_identifier,
|
|
1155
|
+
sender: e.sender,
|
|
1156
|
+
recipient: e.recipient,
|
|
1157
|
+
raw_value: e.value
|
|
1158
|
+
}
|
|
1159
|
+
};
|
|
1160
|
+
case "stx_transfer":
|
|
1161
|
+
case "stx_mint":
|
|
1162
|
+
case "stx_burn":
|
|
1163
|
+
return {
|
|
1164
|
+
...base,
|
|
1165
|
+
type: `${e.event_type}_event`,
|
|
1166
|
+
data: {
|
|
1167
|
+
sender: e.sender,
|
|
1168
|
+
recipient: e.recipient,
|
|
1169
|
+
amount: e.amount,
|
|
1170
|
+
...e.event_type === "stx_transfer" ? { memo: e.memo ?? "" } : {}
|
|
1171
|
+
}
|
|
1172
|
+
};
|
|
1173
|
+
case "stx_lock":
|
|
1174
|
+
return {
|
|
1175
|
+
...base,
|
|
1176
|
+
type: "stx_lock_event",
|
|
1177
|
+
data: {
|
|
1178
|
+
locked_address: e.sender,
|
|
1179
|
+
locked_amount: e.amount,
|
|
1180
|
+
unlock_height: e.payload.unlock_height
|
|
1181
|
+
}
|
|
1182
|
+
};
|
|
1183
|
+
case "print":
|
|
1184
|
+
return {
|
|
1185
|
+
...base,
|
|
1186
|
+
type: "contract_event",
|
|
1187
|
+
data: {
|
|
1188
|
+
topic: e.payload.topic,
|
|
1189
|
+
contract_identifier: e.contract_id,
|
|
1190
|
+
value: e.payload.value,
|
|
1191
|
+
raw_value: e.payload.raw_value
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
// src/runtime/block-source.ts
|
|
1198
|
+
class PostgresBlockSource {
|
|
1199
|
+
async getTip() {
|
|
1200
|
+
const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1201
|
+
return progress ? Number(progress.highest_seen_block) : 0;
|
|
1202
|
+
}
|
|
1203
|
+
loadBlockRange(fromHeight, toHeight) {
|
|
1204
|
+
return loadBlockRange(getSourceDb(), fromHeight, toHeight);
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
var EVENT_FILTER_TO_INDEX_TYPE = {
|
|
1208
|
+
stx_transfer: "stx_transfer",
|
|
1209
|
+
stx_mint: "stx_mint",
|
|
1210
|
+
stx_burn: "stx_burn",
|
|
1211
|
+
stx_lock: "stx_lock",
|
|
1212
|
+
ft_transfer: "ft_transfer",
|
|
1213
|
+
ft_mint: "ft_mint",
|
|
1214
|
+
ft_burn: "ft_burn",
|
|
1215
|
+
nft_transfer: "nft_transfer",
|
|
1216
|
+
nft_mint: "nft_mint",
|
|
1217
|
+
nft_burn: "nft_burn",
|
|
1218
|
+
print_event: "print"
|
|
1219
|
+
};
|
|
1220
|
+
var TX_SOURCE_TYPES = new Set(["contract_call", "contract_deploy"]);
|
|
1221
|
+
var ALL_INDEX_EVENT_TYPES = [
|
|
1222
|
+
...new Set(Object.values(EVENT_FILTER_TO_INDEX_TYPE))
|
|
1223
|
+
];
|
|
1224
|
+
function sourceFilters(subgraph) {
|
|
1225
|
+
const sources = subgraph.sources;
|
|
1226
|
+
return Array.isArray(sources) ? sources : Object.values(sources);
|
|
1227
|
+
}
|
|
1228
|
+
function referencedIndexEventTypes(subgraph) {
|
|
1229
|
+
const filters = sourceFilters(subgraph);
|
|
1230
|
+
if (filters.some((f) => TX_SOURCE_TYPES.has(f.type))) {
|
|
1231
|
+
return ALL_INDEX_EVENT_TYPES;
|
|
1232
|
+
}
|
|
1233
|
+
const types = new Set;
|
|
1234
|
+
for (const f of filters) {
|
|
1235
|
+
const t = EVENT_FILTER_TO_INDEX_TYPE[f.type];
|
|
1236
|
+
if (t)
|
|
1237
|
+
types.add(t);
|
|
1238
|
+
}
|
|
1239
|
+
return [...types];
|
|
1240
|
+
}
|
|
1241
|
+
function isStreamsIndexEligible(subgraph) {
|
|
1242
|
+
if (Array.isArray(subgraph.sources))
|
|
1243
|
+
return false;
|
|
1244
|
+
const filters = sourceFilters(subgraph);
|
|
1245
|
+
if (filters.length === 0)
|
|
1246
|
+
return false;
|
|
1247
|
+
for (const f of filters) {
|
|
1248
|
+
const known = EVENT_FILTER_TO_INDEX_TYPE[f.type] || TX_SOURCE_TYPES.has(f.type);
|
|
1249
|
+
if (!known)
|
|
1250
|
+
return false;
|
|
1251
|
+
}
|
|
1252
|
+
return true;
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
class PublicApiBlockSource {
|
|
1256
|
+
http;
|
|
1257
|
+
eventTypes;
|
|
1258
|
+
constructor(http, eventTypes) {
|
|
1259
|
+
this.http = http;
|
|
1260
|
+
this.eventTypes = eventTypes;
|
|
1261
|
+
}
|
|
1262
|
+
getTip() {
|
|
1263
|
+
return this.http.getIndexTip();
|
|
1264
|
+
}
|
|
1265
|
+
async loadBlockRange(fromHeight, toHeight) {
|
|
1266
|
+
const [blocks, txs, eventLists] = await Promise.all([
|
|
1267
|
+
this.http.walkBlocks(fromHeight, toHeight),
|
|
1268
|
+
this.http.walkTransactions(fromHeight, toHeight),
|
|
1269
|
+
Promise.all(this.eventTypes.map((t) => this.http.walkEvents(t, fromHeight, toHeight)))
|
|
1270
|
+
]);
|
|
1271
|
+
const map = new Map;
|
|
1272
|
+
for (const b of blocks) {
|
|
1273
|
+
map.set(b.block_height, {
|
|
1274
|
+
block: reconstructBlock(b),
|
|
1275
|
+
txs: [],
|
|
1276
|
+
events: []
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
for (const t of txs) {
|
|
1280
|
+
map.get(t.block_height)?.txs.push(reconstructTransaction(t));
|
|
1281
|
+
}
|
|
1282
|
+
for (const list of eventLists) {
|
|
1283
|
+
for (const e of list) {
|
|
1284
|
+
map.get(e.block_height)?.events.push(reconstructEvent(e));
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
for (const bd of map.values()) {
|
|
1288
|
+
bd.txs.sort((a, b) => a.tx_index - b.tx_index);
|
|
1289
|
+
bd.events.sort((a, b) => a.event_index - b.event_index);
|
|
1290
|
+
}
|
|
1291
|
+
return map;
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
var postgresBlockSource = new PostgresBlockSource;
|
|
1295
|
+
function buildHttpClient() {
|
|
1296
|
+
const baseUrl = process.env.SUBGRAPH_INDEX_API_URL ?? process.env.STREAMS_API_URL ?? "http://api:3800";
|
|
1297
|
+
return new IndexHttpClient({
|
|
1298
|
+
indexBaseUrl: baseUrl,
|
|
1299
|
+
streamsBaseUrl: baseUrl,
|
|
1300
|
+
streamsApiKey: process.env.STREAMS_INTERNAL_API_KEY ?? "sk-sl_streams_l2_internal"
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1303
|
+
function resolveBlockSource(subgraph) {
|
|
1304
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
|
|
1305
|
+
return new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph));
|
|
1306
|
+
}
|
|
1307
|
+
if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
|
|
1308
|
+
logger3.debug("Subgraph not streams-index eligible, using DB tap", {
|
|
1309
|
+
subgraph: subgraph.name
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1312
|
+
return postgresBlockSource;
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1045
1315
|
// src/runtime/outbox-emit.ts
|
|
1046
1316
|
import { createHash } from "node:crypto";
|
|
1047
|
-
import { logger as
|
|
1317
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
1048
1318
|
var loggedKillSwitch = false;
|
|
1049
1319
|
function isEmitOutboxEnabled() {
|
|
1050
1320
|
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
@@ -1063,7 +1333,7 @@ function stableStringify(obj) {
|
|
|
1063
1333
|
async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
1064
1334
|
if (!isEmitOutboxEnabled()) {
|
|
1065
1335
|
if (!loggedKillSwitch) {
|
|
1066
|
-
|
|
1336
|
+
logger4.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
1067
1337
|
loggedKillSwitch = true;
|
|
1068
1338
|
}
|
|
1069
1339
|
return 0;
|
|
@@ -1304,7 +1574,6 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
|
|
|
1304
1574
|
return resolved;
|
|
1305
1575
|
}
|
|
1306
1576
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
1307
|
-
const sourceDb = getSourceDb();
|
|
1308
1577
|
const targetDb = getTargetDb();
|
|
1309
1578
|
const blockStart = performance.now();
|
|
1310
1579
|
const result = {
|
|
@@ -1322,25 +1591,18 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1322
1591
|
txs = opts.preloaded.txs;
|
|
1323
1592
|
evts = opts.preloaded.events;
|
|
1324
1593
|
} else {
|
|
1325
|
-
|
|
1326
|
-
if (!
|
|
1327
|
-
|
|
1328
|
-
subgraph: subgraphName,
|
|
1329
|
-
blockHeight
|
|
1330
|
-
});
|
|
1331
|
-
result.skipped = true;
|
|
1332
|
-
return result;
|
|
1333
|
-
}
|
|
1334
|
-
if (!block.canonical) {
|
|
1335
|
-
logger4.debug("Skipping non-canonical block", {
|
|
1594
|
+
const data = (await resolveBlockSource(subgraph).loadBlockRange(blockHeight, blockHeight)).get(blockHeight);
|
|
1595
|
+
if (!data) {
|
|
1596
|
+
logger5.debug("Block not found or non-canonical for subgraph processing", {
|
|
1336
1597
|
subgraph: subgraphName,
|
|
1337
1598
|
blockHeight
|
|
1338
1599
|
});
|
|
1339
1600
|
result.skipped = true;
|
|
1340
1601
|
return result;
|
|
1341
1602
|
}
|
|
1342
|
-
|
|
1343
|
-
|
|
1603
|
+
block = data.block;
|
|
1604
|
+
txs = data.txs;
|
|
1605
|
+
evts = data.events;
|
|
1344
1606
|
}
|
|
1345
1607
|
const traitContracts = await resolveTraitContracts(subgraph, blockHeight, targetDb);
|
|
1346
1608
|
const matched = matchSources(subgraph.sources, txs, evts, traitContracts);
|
|
@@ -1431,7 +1693,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1431
1693
|
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(route.dataDb);
|
|
1432
1694
|
const count = Number(rows[0]?.count ?? 0);
|
|
1433
1695
|
if (count >= 1e7) {
|
|
1434
|
-
|
|
1696
|
+
logger5.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
1435
1697
|
subgraph: subgraphName,
|
|
1436
1698
|
table,
|
|
1437
1699
|
count
|
|
@@ -1439,7 +1701,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1439
1701
|
}
|
|
1440
1702
|
}
|
|
1441
1703
|
} catch (err) {
|
|
1442
|
-
|
|
1704
|
+
logger5.debug("Row count sample failed", {
|
|
1443
1705
|
subgraph: subgraphName,
|
|
1444
1706
|
error: err instanceof Error ? err.message : String(err)
|
|
1445
1707
|
});
|
|
@@ -1513,7 +1775,7 @@ class StatsAccumulator {
|
|
|
1513
1775
|
|
|
1514
1776
|
// src/runtime/reindex.ts
|
|
1515
1777
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
1516
|
-
import { getRawClient,
|
|
1778
|
+
import { getRawClient, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
1517
1779
|
import {
|
|
1518
1780
|
recordGapBatch,
|
|
1519
1781
|
resolveGaps
|
|
@@ -1522,7 +1784,7 @@ import {
|
|
|
1522
1784
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1523
1785
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
1524
1786
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1525
|
-
import { logger as
|
|
1787
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
1526
1788
|
|
|
1527
1789
|
// src/schema/generator.ts
|
|
1528
1790
|
import { createHash as createHash2 } from "node:crypto";
|
|
@@ -1615,48 +1877,6 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
1615
1877
|
return { statements, hash };
|
|
1616
1878
|
}
|
|
1617
1879
|
|
|
1618
|
-
// src/runtime/batch-loader.ts
|
|
1619
|
-
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1620
|
-
const [blocks, txs, events] = await Promise.all([
|
|
1621
|
-
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
1622
|
-
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
1623
|
-
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
1624
|
-
]);
|
|
1625
|
-
const txsByHeight = new Map;
|
|
1626
|
-
for (const tx of txs) {
|
|
1627
|
-
const h = Number(tx.block_height);
|
|
1628
|
-
const list = txsByHeight.get(h) ?? [];
|
|
1629
|
-
list.push(tx);
|
|
1630
|
-
txsByHeight.set(h, list);
|
|
1631
|
-
}
|
|
1632
|
-
const eventsByHeight = new Map;
|
|
1633
|
-
for (const evt of events) {
|
|
1634
|
-
const h = Number(evt.block_height);
|
|
1635
|
-
const list = eventsByHeight.get(h) ?? [];
|
|
1636
|
-
list.push(evt);
|
|
1637
|
-
eventsByHeight.set(h, list);
|
|
1638
|
-
}
|
|
1639
|
-
const result = new Map;
|
|
1640
|
-
for (const block of blocks) {
|
|
1641
|
-
const h = Number(block.height);
|
|
1642
|
-
result.set(h, {
|
|
1643
|
-
block,
|
|
1644
|
-
txs: txsByHeight.get(h) ?? [],
|
|
1645
|
-
events: eventsByHeight.get(h) ?? []
|
|
1646
|
-
});
|
|
1647
|
-
}
|
|
1648
|
-
return result;
|
|
1649
|
-
}
|
|
1650
|
-
function avgEventsPerBlock(batch) {
|
|
1651
|
-
if (batch.size === 0)
|
|
1652
|
-
return 0;
|
|
1653
|
-
let totalEvents = 0;
|
|
1654
|
-
for (const data of batch.values()) {
|
|
1655
|
-
totalEvents += data.events.length;
|
|
1656
|
-
}
|
|
1657
|
-
return totalEvents / batch.size;
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
1880
|
// src/runtime/reindex.ts
|
|
1661
1881
|
var LOG_INTERVAL = 1000;
|
|
1662
1882
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
@@ -1717,7 +1937,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
1717
1937
|
return ranges;
|
|
1718
1938
|
}
|
|
1719
1939
|
async function processBlockRange(def, opts) {
|
|
1720
|
-
const
|
|
1940
|
+
const source = resolveBlockSource(def);
|
|
1721
1941
|
const targetDb = getTargetDb2();
|
|
1722
1942
|
const subgraphName = def.name;
|
|
1723
1943
|
const { fromBlock, toBlock, status } = opts;
|
|
@@ -1747,11 +1967,11 @@ async function processBlockRange(def, opts) {
|
|
|
1747
1967
|
lastHealthFlushAt = Date.now();
|
|
1748
1968
|
};
|
|
1749
1969
|
let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
1750
|
-
let nextBatchPromise = loadBlockRange(
|
|
1970
|
+
let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
|
|
1751
1971
|
while (currentHeight <= toBlock) {
|
|
1752
1972
|
if (opts.signal?.aborted) {
|
|
1753
1973
|
aborted = true;
|
|
1754
|
-
|
|
1974
|
+
logger6.info("Block processing aborted", {
|
|
1755
1975
|
subgraph: subgraphName,
|
|
1756
1976
|
currentBlock: currentHeight,
|
|
1757
1977
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -1763,7 +1983,7 @@ async function processBlockRange(def, opts) {
|
|
|
1763
1983
|
const nextStart = batchEnd + 1;
|
|
1764
1984
|
if (nextStart <= toBlock) {
|
|
1765
1985
|
nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
1766
|
-
nextBatchPromise = loadBlockRange(
|
|
1986
|
+
nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
|
|
1767
1987
|
}
|
|
1768
1988
|
const batchFailedBlocks = [];
|
|
1769
1989
|
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
@@ -1781,7 +2001,7 @@ async function processBlockRange(def, opts) {
|
|
|
1781
2001
|
});
|
|
1782
2002
|
} catch (err) {
|
|
1783
2003
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1784
|
-
|
|
2004
|
+
logger6.error("Block processing error", {
|
|
1785
2005
|
subgraph: subgraphName,
|
|
1786
2006
|
blockHeight: height,
|
|
1787
2007
|
error: errorMsg
|
|
@@ -1815,7 +2035,7 @@ async function processBlockRange(def, opts) {
|
|
|
1815
2035
|
lastProgressFlushAt = now;
|
|
1816
2036
|
}
|
|
1817
2037
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1818
|
-
|
|
2038
|
+
logger6.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1819
2039
|
subgraph: subgraphName,
|
|
1820
2040
|
processed: blocksProcessed,
|
|
1821
2041
|
total: totalBlocks,
|
|
@@ -1834,7 +2054,7 @@ async function processBlockRange(def, opts) {
|
|
|
1834
2054
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1835
2055
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1836
2056
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1837
|
-
|
|
2057
|
+
logger6.warn("Failed to record subgraph gaps", {
|
|
1838
2058
|
subgraph: subgraphName,
|
|
1839
2059
|
error: err instanceof Error ? err.message : String(err)
|
|
1840
2060
|
});
|
|
@@ -1851,32 +2071,26 @@ async function processBlockRange(def, opts) {
|
|
|
1851
2071
|
await flushHealth();
|
|
1852
2072
|
return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
|
|
1853
2073
|
}
|
|
1854
|
-
async function resolveBlockRange(
|
|
2074
|
+
async function resolveBlockRange(source, def, opts) {
|
|
1855
2075
|
const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
|
|
1856
|
-
|
|
1857
|
-
if (opts?.toBlock != null) {
|
|
1858
|
-
toBlock = opts.toBlock;
|
|
1859
|
-
} else {
|
|
1860
|
-
const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1861
|
-
toBlock = progress?.highest_seen_block ?? 0;
|
|
1862
|
-
}
|
|
2076
|
+
const toBlock = opts?.toBlock != null ? opts.toBlock : await source.getTip();
|
|
1863
2077
|
return { fromBlock, toBlock };
|
|
1864
2078
|
}
|
|
1865
2079
|
async function clearReindexMetadata(db, subgraphName) {
|
|
1866
2080
|
await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
|
|
1867
2081
|
}
|
|
1868
2082
|
async function reindexSubgraph(def, opts) {
|
|
1869
|
-
const
|
|
2083
|
+
const source = resolveBlockSource(def);
|
|
1870
2084
|
const targetDb = getTargetDb2();
|
|
1871
2085
|
const client = getRawClient("target");
|
|
1872
2086
|
const subgraphName = def.name;
|
|
1873
2087
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1874
2088
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1875
|
-
|
|
2089
|
+
logger6.info("Reindex starting", { subgraph: subgraphName });
|
|
1876
2090
|
try {
|
|
1877
|
-
const { fromBlock, toBlock } = await resolveBlockRange(
|
|
2091
|
+
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
1878
2092
|
if (fromBlock > toBlock) {
|
|
1879
|
-
|
|
2093
|
+
logger6.info("No blocks to reindex", {
|
|
1880
2094
|
subgraph: subgraphName,
|
|
1881
2095
|
fromBlock,
|
|
1882
2096
|
toBlock
|
|
@@ -1889,7 +2103,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1889
2103
|
for (const stmt of statements) {
|
|
1890
2104
|
await client.unsafe(stmt);
|
|
1891
2105
|
}
|
|
1892
|
-
|
|
2106
|
+
logger6.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1893
2107
|
await targetDb.updateTable("subgraphs").set({
|
|
1894
2108
|
last_processed_block: initialReindexProgressBlock(fromBlock),
|
|
1895
2109
|
reindex_from_block: fromBlock,
|
|
@@ -1900,7 +2114,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1900
2114
|
last_error_at: null,
|
|
1901
2115
|
updated_at: new Date
|
|
1902
2116
|
}).where("name", "=", subgraphName).execute();
|
|
1903
|
-
|
|
2117
|
+
logger6.info("Reindexing blocks", {
|
|
1904
2118
|
subgraph: subgraphName,
|
|
1905
2119
|
fromBlock,
|
|
1906
2120
|
toBlock,
|
|
@@ -1921,9 +2135,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
1921
2135
|
if (reason === "user-cancelled") {
|
|
1922
2136
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1923
2137
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1924
|
-
|
|
2138
|
+
logger6.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1925
2139
|
} else {
|
|
1926
|
-
|
|
2140
|
+
logger6.info("Reindex interrupted by shutdown, will resume", {
|
|
1927
2141
|
subgraph: subgraphName
|
|
1928
2142
|
});
|
|
1929
2143
|
}
|
|
@@ -1931,7 +2145,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1931
2145
|
}
|
|
1932
2146
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1933
2147
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1934
|
-
|
|
2148
|
+
logger6.info("Reindex complete", {
|
|
1935
2149
|
subgraph: subgraphName,
|
|
1936
2150
|
blocks: result.blocksProcessed,
|
|
1937
2151
|
events: result.totalEventsProcessed,
|
|
@@ -1939,7 +2153,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1939
2153
|
});
|
|
1940
2154
|
return { processed: result.blocksProcessed };
|
|
1941
2155
|
} catch (err) {
|
|
1942
|
-
|
|
2156
|
+
logger6.error("Reindex failed", {
|
|
1943
2157
|
subgraph: subgraphName,
|
|
1944
2158
|
error: getErrorMessage2(err)
|
|
1945
2159
|
});
|
|
@@ -1961,7 +2175,7 @@ async function resumeReindex(def, opts) {
|
|
|
1961
2175
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
1962
2176
|
const toBlock = Number(row.reindex_to_block);
|
|
1963
2177
|
if (fromBlock == null) {
|
|
1964
|
-
|
|
2178
|
+
logger6.info("No reindex metadata, starting fresh reindex", {
|
|
1965
2179
|
subgraph: subgraphName
|
|
1966
2180
|
});
|
|
1967
2181
|
return reindexSubgraph(def, {
|
|
@@ -1970,12 +2184,12 @@ async function resumeReindex(def, opts) {
|
|
|
1970
2184
|
});
|
|
1971
2185
|
}
|
|
1972
2186
|
if (fromBlock > toBlock) {
|
|
1973
|
-
|
|
2187
|
+
logger6.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1974
2188
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1975
2189
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1976
2190
|
return { processed: 0 };
|
|
1977
2191
|
}
|
|
1978
|
-
|
|
2192
|
+
logger6.info("Resuming reindex", {
|
|
1979
2193
|
subgraph: subgraphName,
|
|
1980
2194
|
fromBlock,
|
|
1981
2195
|
toBlock,
|
|
@@ -1996,9 +2210,9 @@ async function resumeReindex(def, opts) {
|
|
|
1996
2210
|
if (reason === "user-cancelled") {
|
|
1997
2211
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1998
2212
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1999
|
-
|
|
2213
|
+
logger6.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
2000
2214
|
} else {
|
|
2001
|
-
|
|
2215
|
+
logger6.info("Resume interrupted by shutdown, will resume again", {
|
|
2002
2216
|
subgraph: subgraphName
|
|
2003
2217
|
});
|
|
2004
2218
|
}
|
|
@@ -2006,13 +2220,13 @@ async function resumeReindex(def, opts) {
|
|
|
2006
2220
|
}
|
|
2007
2221
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
2008
2222
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
2009
|
-
|
|
2223
|
+
logger6.info("Resumed reindex complete", {
|
|
2010
2224
|
subgraph: subgraphName,
|
|
2011
2225
|
blocks: result.blocksProcessed
|
|
2012
2226
|
});
|
|
2013
2227
|
return { processed: result.blocksProcessed };
|
|
2014
2228
|
} catch (err) {
|
|
2015
|
-
|
|
2229
|
+
logger6.error("Resumed reindex failed", {
|
|
2016
2230
|
subgraph: subgraphName,
|
|
2017
2231
|
error: getErrorMessage2(err)
|
|
2018
2232
|
});
|
|
@@ -2023,7 +2237,7 @@ async function resumeReindex(def, opts) {
|
|
|
2023
2237
|
async function backfillSubgraph(def, opts) {
|
|
2024
2238
|
const targetDb = getTargetDb2();
|
|
2025
2239
|
const subgraphName = def.name;
|
|
2026
|
-
|
|
2240
|
+
logger6.info("Backfill starting", {
|
|
2027
2241
|
subgraph: subgraphName,
|
|
2028
2242
|
from: opts.fromBlock,
|
|
2029
2243
|
to: opts.toBlock
|
|
@@ -2040,17 +2254,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
2040
2254
|
signal: opts.signal
|
|
2041
2255
|
});
|
|
2042
2256
|
if (result.aborted) {
|
|
2043
|
-
|
|
2257
|
+
logger6.info("Backfill aborted", { subgraph: subgraphName });
|
|
2044
2258
|
return { processed: result.blocksProcessed };
|
|
2045
2259
|
}
|
|
2046
2260
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
2047
2261
|
if (resolved > 0) {
|
|
2048
|
-
|
|
2262
|
+
logger6.info("Resolved subgraph gaps via backfill", {
|
|
2049
2263
|
subgraph: subgraphName,
|
|
2050
2264
|
resolved
|
|
2051
2265
|
});
|
|
2052
2266
|
}
|
|
2053
|
-
|
|
2267
|
+
logger6.info("Backfill complete", {
|
|
2054
2268
|
subgraph: subgraphName,
|
|
2055
2269
|
blocks: result.blocksProcessed,
|
|
2056
2270
|
events: result.totalEventsProcessed,
|
|
@@ -2058,7 +2272,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
2058
2272
|
});
|
|
2059
2273
|
return { processed: result.blocksProcessed };
|
|
2060
2274
|
} catch (err) {
|
|
2061
|
-
|
|
2275
|
+
logger6.error("Backfill failed", {
|
|
2062
2276
|
subgraph: subgraphName,
|
|
2063
2277
|
error: getErrorMessage2(err)
|
|
2064
2278
|
});
|
|
@@ -2169,7 +2383,7 @@ function generatePrismaSchema(def, opts = {}) {
|
|
|
2169
2383
|
const schemaName = opts.schemaName ?? pgSchemaName(def.name);
|
|
2170
2384
|
const env = opts.datasourceEnv ?? "DATABASE_URL";
|
|
2171
2385
|
const header = [
|
|
2172
|
-
`// Generated by \`sl
|
|
2386
|
+
`// Generated by \`sl subgraphs codegen --target prisma\` from subgraph "${def.name}". Do not edit by hand.`,
|
|
2173
2387
|
"",
|
|
2174
2388
|
"datasource db {",
|
|
2175
2389
|
' provider = "postgresql"',
|
|
@@ -2329,7 +2543,7 @@ function generateDrizzleSchema(def, opts = {}) {
|
|
|
2329
2543
|
const relationDecls = renderRelations(def.schema);
|
|
2330
2544
|
const typeExports = Object.keys(def.schema).map((name) => `export type ${pascalCase2(name)} = typeof ${snakeToCamel2(name)}.$inferSelect;`);
|
|
2331
2545
|
return [
|
|
2332
|
-
`// Generated by \`sl
|
|
2546
|
+
`// Generated by \`sl subgraphs codegen --target drizzle\` from subgraph "${def.name}". Do not edit by hand.`,
|
|
2333
2547
|
imports,
|
|
2334
2548
|
"",
|
|
2335
2549
|
`export const sg = pgSchema("${schemaName}");`,
|
|
@@ -2625,5 +2839,5 @@ export {
|
|
|
2625
2839
|
backfillSubgraph
|
|
2626
2840
|
};
|
|
2627
2841
|
|
|
2628
|
-
//# debugId=
|
|
2842
|
+
//# debugId=393F56BE14245E2264756E2164756E21
|
|
2629
2843
|
//# sourceMappingURL=index.js.map
|