@secondlayer/subgraphs 1.0.0-alpha.0 → 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +261 -49
- package/dist/src/index.js.map +8 -5
- package/dist/src/runtime/block-processor.js +222 -10
- package/dist/src/runtime/block-processor.js.map +9 -6
- package/dist/src/runtime/catchup.js +229 -17
- package/dist/src/runtime/catchup.js.map +8 -5
- package/dist/src/runtime/context.d.ts +22 -2
- package/dist/src/runtime/context.js +24 -3
- package/dist/src/runtime/context.js.map +3 -3
- package/dist/src/runtime/processor.js +524 -37
- package/dist/src/runtime/processor.js.map +11 -6
- package/dist/src/runtime/reindex.js +247 -35
- package/dist/src/runtime/reindex.js.map +8 -5
- package/dist/src/runtime/reorg.js +227 -15
- package/dist/src/runtime/reorg.js.map +8 -5
- package/dist/src/runtime/runner.d.ts +21 -1
- package/dist/src/service.js +526 -39
- package/dist/src/service.js.map +11 -6
- package/package.json +2 -2
package/dist/src/index.js
CHANGED
|
@@ -237,7 +237,7 @@ class SubgraphContext {
|
|
|
237
237
|
}
|
|
238
238
|
async flush() {
|
|
239
239
|
if (this.ops.length === 0)
|
|
240
|
-
return 0;
|
|
240
|
+
return { count: 0, writes: [] };
|
|
241
241
|
const opsToFlush = [...this.ops];
|
|
242
242
|
this.ops.length = 0;
|
|
243
243
|
const statements = this.buildStatements(opsToFlush);
|
|
@@ -252,7 +252,21 @@ class SubgraphContext {
|
|
|
252
252
|
}
|
|
253
253
|
});
|
|
254
254
|
}
|
|
255
|
-
|
|
255
|
+
const writes = opsToFlush.map((op, rowIndex) => {
|
|
256
|
+
const blockHeight = op.data._block_height ?? this.block.height;
|
|
257
|
+
const txId = op.data._tx_id ?? this._tx.txId;
|
|
258
|
+
const baseRow = op.kind === "update" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
|
|
259
|
+
delete baseRow._upsert_keys;
|
|
260
|
+
delete baseRow._upsert_fallback_keys;
|
|
261
|
+
delete baseRow._upsert_fallback_set;
|
|
262
|
+
return {
|
|
263
|
+
op: op.kind,
|
|
264
|
+
table: op.table,
|
|
265
|
+
row: jsonSafe(baseRow),
|
|
266
|
+
pk: { blockHeight, txId, rowIndex }
|
|
267
|
+
};
|
|
268
|
+
});
|
|
269
|
+
return { count: opsToFlush.length, writes };
|
|
256
270
|
}
|
|
257
271
|
prepareInsert(op) {
|
|
258
272
|
const upsertKeys = op.data._upsert_keys;
|
|
@@ -345,6 +359,13 @@ class SubgraphContext {
|
|
|
345
359
|
}
|
|
346
360
|
}
|
|
347
361
|
}
|
|
362
|
+
function jsonSafe(row) {
|
|
363
|
+
const out = {};
|
|
364
|
+
for (const [k, v] of Object.entries(row)) {
|
|
365
|
+
out[k] = typeof v === "bigint" ? v.toString() : v;
|
|
366
|
+
}
|
|
367
|
+
return out;
|
|
368
|
+
}
|
|
348
369
|
function escapeLiteral(value) {
|
|
349
370
|
if (value === null || value === undefined)
|
|
350
371
|
return "NULL";
|
|
@@ -926,12 +947,200 @@ import {
|
|
|
926
947
|
recordSubgraphProcessed,
|
|
927
948
|
updateSubgraphStatus
|
|
928
949
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
929
|
-
import { logger as
|
|
930
|
-
import { sql as
|
|
950
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
951
|
+
import { sql as sql3 } from "kysely";
|
|
931
952
|
|
|
932
953
|
// src/schema/utils.ts
|
|
933
954
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
934
955
|
|
|
956
|
+
// src/runtime/outbox-emit.ts
|
|
957
|
+
import { createHash } from "node:crypto";
|
|
958
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
959
|
+
var loggedKillSwitch = false;
|
|
960
|
+
function isEmitOutboxEnabled() {
|
|
961
|
+
return process.env.SECONDLAYER_EMIT_OUTBOX !== "false";
|
|
962
|
+
}
|
|
963
|
+
function dedupKey(subgraphName, tableName, blockHeight, txId, rowIndex, row) {
|
|
964
|
+
const canonical = `${subgraphName}:${tableName}:${blockHeight}:${txId}:${rowIndex}:${stableStringify(row)}`;
|
|
965
|
+
return createHash("sha256").update(canonical).digest("hex").slice(0, 32);
|
|
966
|
+
}
|
|
967
|
+
function stableStringify(obj) {
|
|
968
|
+
const keys = Object.keys(obj).sort();
|
|
969
|
+
return JSON.stringify(keys.reduce((acc, k) => {
|
|
970
|
+
acc[k] = obj[k];
|
|
971
|
+
return acc;
|
|
972
|
+
}, {}));
|
|
973
|
+
}
|
|
974
|
+
async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
975
|
+
if (!isEmitOutboxEnabled()) {
|
|
976
|
+
if (!loggedKillSwitch) {
|
|
977
|
+
logger3.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
978
|
+
loggedKillSwitch = true;
|
|
979
|
+
}
|
|
980
|
+
return 0;
|
|
981
|
+
}
|
|
982
|
+
loggedKillSwitch = false;
|
|
983
|
+
if (manifest.count === 0 || matcher.size() === 0)
|
|
984
|
+
return 0;
|
|
985
|
+
const rows = [];
|
|
986
|
+
for (const write of manifest.writes) {
|
|
987
|
+
if (write.op !== "insert")
|
|
988
|
+
continue;
|
|
989
|
+
const subs = matcher.match(subgraphName, write.table, write.row);
|
|
990
|
+
if (subs.length === 0)
|
|
991
|
+
continue;
|
|
992
|
+
const eventType = `${subgraphName}.${write.table}.created`;
|
|
993
|
+
for (const s of subs) {
|
|
994
|
+
rows.push({
|
|
995
|
+
subscription_id: s.id,
|
|
996
|
+
subgraph_name: subgraphName,
|
|
997
|
+
table_name: write.table,
|
|
998
|
+
block_height: blockHeight,
|
|
999
|
+
tx_id: write.pk.txId || null,
|
|
1000
|
+
row_pk: write.pk,
|
|
1001
|
+
event_type: eventType,
|
|
1002
|
+
payload: write.row,
|
|
1003
|
+
dedup_key: dedupKey(subgraphName, write.table, write.pk.blockHeight, write.pk.txId, write.pk.rowIndex, write.row)
|
|
1004
|
+
});
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
if (rows.length === 0)
|
|
1008
|
+
return 0;
|
|
1009
|
+
await tx.insertInto("subscription_outbox").values(rows).onConflict((oc) => oc.columns(["subscription_id", "dedup_key"]).doNothing()).execute();
|
|
1010
|
+
return rows.length;
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
// src/runtime/subscription-state.ts
|
|
1014
|
+
import { listSubscriptions } from "@secondlayer/shared/db/queries/subscriptions";
|
|
1015
|
+
import { sql as sql2 } from "kysely";
|
|
1016
|
+
|
|
1017
|
+
// src/runtime/emitter-matcher.ts
|
|
1018
|
+
function isPrimitive(v) {
|
|
1019
|
+
const t = typeof v;
|
|
1020
|
+
return t === "string" || t === "number" || t === "boolean";
|
|
1021
|
+
}
|
|
1022
|
+
function coerceNumeric(v) {
|
|
1023
|
+
if (typeof v === "number" && Number.isFinite(v))
|
|
1024
|
+
return v;
|
|
1025
|
+
if (typeof v === "bigint")
|
|
1026
|
+
return Number(v);
|
|
1027
|
+
if (typeof v === "string" && v !== "" && !Number.isNaN(Number(v))) {
|
|
1028
|
+
return Number(v);
|
|
1029
|
+
}
|
|
1030
|
+
return null;
|
|
1031
|
+
}
|
|
1032
|
+
function matchClause(rowValue, clause) {
|
|
1033
|
+
if (isPrimitive(clause)) {
|
|
1034
|
+
if (typeof clause === "number" || typeof rowValue === "number") {
|
|
1035
|
+
const a = coerceNumeric(rowValue);
|
|
1036
|
+
const b = coerceNumeric(clause);
|
|
1037
|
+
if (a !== null && b !== null)
|
|
1038
|
+
return a === b;
|
|
1039
|
+
}
|
|
1040
|
+
return rowValue === clause || String(rowValue) === String(clause);
|
|
1041
|
+
}
|
|
1042
|
+
if (clause === null || typeof clause !== "object" || Array.isArray(clause)) {
|
|
1043
|
+
return false;
|
|
1044
|
+
}
|
|
1045
|
+
const keys = Object.keys(clause);
|
|
1046
|
+
if (keys.length !== 1)
|
|
1047
|
+
return false;
|
|
1048
|
+
const op = keys[0];
|
|
1049
|
+
const c = clause;
|
|
1050
|
+
switch (op) {
|
|
1051
|
+
case "eq":
|
|
1052
|
+
return matchClause(rowValue, c.eq);
|
|
1053
|
+
case "neq":
|
|
1054
|
+
return !matchClause(rowValue, c.neq);
|
|
1055
|
+
case "gt":
|
|
1056
|
+
case "gte":
|
|
1057
|
+
case "lt":
|
|
1058
|
+
case "lte": {
|
|
1059
|
+
const a = coerceNumeric(rowValue);
|
|
1060
|
+
const b = coerceNumeric(c[op]);
|
|
1061
|
+
if (a === null || b === null)
|
|
1062
|
+
return false;
|
|
1063
|
+
if (op === "gt")
|
|
1064
|
+
return a > b;
|
|
1065
|
+
if (op === "gte")
|
|
1066
|
+
return a >= b;
|
|
1067
|
+
if (op === "lt")
|
|
1068
|
+
return a < b;
|
|
1069
|
+
return a <= b;
|
|
1070
|
+
}
|
|
1071
|
+
case "in": {
|
|
1072
|
+
const list = c.in;
|
|
1073
|
+
if (!Array.isArray(list))
|
|
1074
|
+
return false;
|
|
1075
|
+
return list.some((item) => isPrimitive(item) && matchClause(rowValue, item));
|
|
1076
|
+
}
|
|
1077
|
+
default:
|
|
1078
|
+
return false;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function matchesFilter(filter, row) {
|
|
1082
|
+
if (!filter || Object.keys(filter).length === 0)
|
|
1083
|
+
return true;
|
|
1084
|
+
for (const [col, clause] of Object.entries(filter)) {
|
|
1085
|
+
if (!matchClause(row[col], clause))
|
|
1086
|
+
return false;
|
|
1087
|
+
}
|
|
1088
|
+
return true;
|
|
1089
|
+
}
|
|
1090
|
+
function key(subgraphName, tableName) {
|
|
1091
|
+
return `${subgraphName}\x00${tableName}`;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
class SubscriptionMatcher {
|
|
1095
|
+
byKey = new Map;
|
|
1096
|
+
byId = new Map;
|
|
1097
|
+
setAll(subs) {
|
|
1098
|
+
this.byKey.clear();
|
|
1099
|
+
this.byId.clear();
|
|
1100
|
+
for (const sub of subs) {
|
|
1101
|
+
if (sub.status !== "active")
|
|
1102
|
+
continue;
|
|
1103
|
+
this.byId.set(sub.id, sub);
|
|
1104
|
+
const k = key(sub.subgraph_name, sub.table_name);
|
|
1105
|
+
const arr = this.byKey.get(k);
|
|
1106
|
+
if (arr)
|
|
1107
|
+
arr.push(sub);
|
|
1108
|
+
else
|
|
1109
|
+
this.byKey.set(k, [sub]);
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
match(subgraphName, tableName, row) {
|
|
1113
|
+
const bucket = this.byKey.get(key(subgraphName, tableName));
|
|
1114
|
+
if (!bucket)
|
|
1115
|
+
return [];
|
|
1116
|
+
const hits = [];
|
|
1117
|
+
for (const sub of bucket) {
|
|
1118
|
+
if (matchesFilter(sub.filter, row))
|
|
1119
|
+
hits.push(sub);
|
|
1120
|
+
}
|
|
1121
|
+
return hits;
|
|
1122
|
+
}
|
|
1123
|
+
has(subgraphName, tableName) {
|
|
1124
|
+
return this.byKey.has(key(subgraphName, tableName));
|
|
1125
|
+
}
|
|
1126
|
+
size() {
|
|
1127
|
+
return this.byId.size;
|
|
1128
|
+
}
|
|
1129
|
+
get(id) {
|
|
1130
|
+
return this.byId.get(id);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
// src/runtime/subscription-state.ts
|
|
1135
|
+
var matcher = new SubscriptionMatcher;
|
|
1136
|
+
async function refreshMatcher(db) {
|
|
1137
|
+
const rows = await sql2`
|
|
1138
|
+
SELECT * FROM subscriptions WHERE status = 'active'
|
|
1139
|
+
`.execute(db);
|
|
1140
|
+
matcher.setAll(rows.rows);
|
|
1141
|
+
return matcher.size();
|
|
1142
|
+
}
|
|
1143
|
+
|
|
935
1144
|
// src/runtime/block-processor.ts
|
|
936
1145
|
var schemaNameCache = new Map;
|
|
937
1146
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
@@ -955,7 +1164,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
955
1164
|
} else {
|
|
956
1165
|
block = await sourceDb.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
|
|
957
1166
|
if (!block) {
|
|
958
|
-
|
|
1167
|
+
logger4.warn("Block not found for subgraph processing", {
|
|
959
1168
|
subgraph: subgraphName,
|
|
960
1169
|
blockHeight
|
|
961
1170
|
});
|
|
@@ -963,7 +1172,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
963
1172
|
return result;
|
|
964
1173
|
}
|
|
965
1174
|
if (!block.canonical) {
|
|
966
|
-
|
|
1175
|
+
logger4.debug("Skipping non-canonical block", {
|
|
967
1176
|
subgraph: subgraphName,
|
|
968
1177
|
blockHeight
|
|
969
1178
|
});
|
|
@@ -1010,7 +1219,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1010
1219
|
result.errors = runResult.errors;
|
|
1011
1220
|
if (ctx.pendingOps > 0) {
|
|
1012
1221
|
const flushStart = performance.now();
|
|
1013
|
-
await ctx.flush();
|
|
1222
|
+
const manifest = await ctx.flush();
|
|
1223
|
+
if (manifest.count > 0) {
|
|
1224
|
+
await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
|
|
1225
|
+
}
|
|
1014
1226
|
flushMs = performance.now() - flushStart;
|
|
1015
1227
|
}
|
|
1016
1228
|
if (!opts?.skipProgressUpdate) {
|
|
@@ -1032,10 +1244,10 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
1032
1244
|
try {
|
|
1033
1245
|
const tables = Object.keys(subgraph.schema);
|
|
1034
1246
|
for (const table of tables) {
|
|
1035
|
-
const { rows } = await
|
|
1247
|
+
const { rows } = await sql3.raw(`SELECT n_live_tup AS count FROM pg_stat_user_tables WHERE schemaname = '${schemaName}' AND relname = '${table}'`).execute(targetDb);
|
|
1036
1248
|
const count = Number(rows[0]?.count ?? 0);
|
|
1037
1249
|
if (count >= 1e7) {
|
|
1038
|
-
|
|
1250
|
+
logger4.warn("Subgraph table exceeds 10M rows (estimate)", {
|
|
1039
1251
|
subgraph: subgraphName,
|
|
1040
1252
|
table,
|
|
1041
1253
|
count
|
|
@@ -1121,7 +1333,7 @@ import {
|
|
|
1121
1333
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1122
1334
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
1123
1335
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1124
|
-
import { logger as
|
|
1336
|
+
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
1125
1337
|
|
|
1126
1338
|
// src/schema/generator.ts
|
|
1127
1339
|
var TYPE_MAP = {
|
|
@@ -1294,7 +1506,7 @@ async function processBlockRange(def, opts) {
|
|
|
1294
1506
|
while (currentHeight <= toBlock) {
|
|
1295
1507
|
if (opts.signal?.aborted) {
|
|
1296
1508
|
aborted = true;
|
|
1297
|
-
|
|
1509
|
+
logger5.info("Block processing aborted", {
|
|
1298
1510
|
subgraph: subgraphName,
|
|
1299
1511
|
currentBlock: currentHeight,
|
|
1300
1512
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -1324,7 +1536,7 @@ async function processBlockRange(def, opts) {
|
|
|
1324
1536
|
});
|
|
1325
1537
|
} catch (err) {
|
|
1326
1538
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1327
|
-
|
|
1539
|
+
logger5.error("Block processing error", {
|
|
1328
1540
|
subgraph: subgraphName,
|
|
1329
1541
|
blockHeight: height,
|
|
1330
1542
|
error: errorMsg
|
|
@@ -1349,7 +1561,7 @@ async function processBlockRange(def, opts) {
|
|
|
1349
1561
|
await updateSubgraphStatus2(targetDb, subgraphName, status, height);
|
|
1350
1562
|
}
|
|
1351
1563
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1352
|
-
|
|
1564
|
+
logger5.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1353
1565
|
subgraph: subgraphName,
|
|
1354
1566
|
processed: blocksProcessed,
|
|
1355
1567
|
total: totalBlocks,
|
|
@@ -1361,7 +1573,7 @@ async function processBlockRange(def, opts) {
|
|
|
1361
1573
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1362
1574
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1363
1575
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1364
|
-
|
|
1576
|
+
logger5.warn("Failed to record subgraph gaps", {
|
|
1365
1577
|
subgraph: subgraphName,
|
|
1366
1578
|
error: err instanceof Error ? err.message : String(err)
|
|
1367
1579
|
});
|
|
@@ -1398,11 +1610,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
1398
1610
|
const subgraphName = def.name;
|
|
1399
1611
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1400
1612
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1401
|
-
|
|
1613
|
+
logger5.info("Reindex starting", { subgraph: subgraphName });
|
|
1402
1614
|
try {
|
|
1403
1615
|
const { fromBlock, toBlock } = await resolveBlockRange(sourceDb, def, opts);
|
|
1404
1616
|
if (fromBlock > toBlock) {
|
|
1405
|
-
|
|
1617
|
+
logger5.info("No blocks to reindex", {
|
|
1406
1618
|
subgraph: subgraphName,
|
|
1407
1619
|
fromBlock,
|
|
1408
1620
|
toBlock
|
|
@@ -1416,8 +1628,8 @@ async function reindexSubgraph(def, opts) {
|
|
|
1416
1628
|
for (const stmt of statements) {
|
|
1417
1629
|
await client.unsafe(stmt);
|
|
1418
1630
|
}
|
|
1419
|
-
|
|
1420
|
-
|
|
1631
|
+
logger5.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1632
|
+
logger5.info("Reindexing blocks", {
|
|
1421
1633
|
subgraph: subgraphName,
|
|
1422
1634
|
fromBlock,
|
|
1423
1635
|
toBlock,
|
|
@@ -1438,9 +1650,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
1438
1650
|
if (reason === "user-cancelled") {
|
|
1439
1651
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1440
1652
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1441
|
-
|
|
1653
|
+
logger5.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1442
1654
|
} else {
|
|
1443
|
-
|
|
1655
|
+
logger5.info("Reindex interrupted by shutdown, will resume", {
|
|
1444
1656
|
subgraph: subgraphName
|
|
1445
1657
|
});
|
|
1446
1658
|
}
|
|
@@ -1452,7 +1664,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1452
1664
|
}
|
|
1453
1665
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1454
1666
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1455
|
-
|
|
1667
|
+
logger5.info("Reindex complete", {
|
|
1456
1668
|
subgraph: subgraphName,
|
|
1457
1669
|
blocks: result.blocksProcessed,
|
|
1458
1670
|
events: result.totalEventsProcessed,
|
|
@@ -1460,7 +1672,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
1460
1672
|
});
|
|
1461
1673
|
return { processed: result.blocksProcessed };
|
|
1462
1674
|
} catch (err) {
|
|
1463
|
-
|
|
1675
|
+
logger5.error("Reindex failed", {
|
|
1464
1676
|
subgraph: subgraphName,
|
|
1465
1677
|
error: getErrorMessage2(err)
|
|
1466
1678
|
});
|
|
@@ -1480,7 +1692,7 @@ async function resumeReindex(def, opts) {
|
|
|
1480
1692
|
if (!row)
|
|
1481
1693
|
throw new Error(`Subgraph "${subgraphName}" not found`);
|
|
1482
1694
|
if (row.reindex_from_block == null || row.reindex_to_block == null) {
|
|
1483
|
-
|
|
1695
|
+
logger5.info("No reindex metadata, starting fresh reindex", {
|
|
1484
1696
|
subgraph: subgraphName
|
|
1485
1697
|
});
|
|
1486
1698
|
return reindexSubgraph(def, {
|
|
@@ -1491,12 +1703,12 @@ async function resumeReindex(def, opts) {
|
|
|
1491
1703
|
const fromBlock = Math.max(row.last_processed_block + 1, row.reindex_from_block);
|
|
1492
1704
|
const toBlock = row.reindex_to_block;
|
|
1493
1705
|
if (fromBlock > toBlock) {
|
|
1494
|
-
|
|
1706
|
+
logger5.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1495
1707
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1496
1708
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1497
1709
|
return { processed: 0 };
|
|
1498
1710
|
}
|
|
1499
|
-
|
|
1711
|
+
logger5.info("Resuming reindex", {
|
|
1500
1712
|
subgraph: subgraphName,
|
|
1501
1713
|
fromBlock,
|
|
1502
1714
|
toBlock,
|
|
@@ -1517,9 +1729,9 @@ async function resumeReindex(def, opts) {
|
|
|
1517
1729
|
if (reason === "user-cancelled") {
|
|
1518
1730
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1519
1731
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1520
|
-
|
|
1732
|
+
logger5.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
1521
1733
|
} else {
|
|
1522
|
-
|
|
1734
|
+
logger5.info("Resume interrupted by shutdown, will resume again", {
|
|
1523
1735
|
subgraph: subgraphName
|
|
1524
1736
|
});
|
|
1525
1737
|
}
|
|
@@ -1531,13 +1743,13 @@ async function resumeReindex(def, opts) {
|
|
|
1531
1743
|
}
|
|
1532
1744
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1533
1745
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
1534
|
-
|
|
1746
|
+
logger5.info("Resumed reindex complete", {
|
|
1535
1747
|
subgraph: subgraphName,
|
|
1536
1748
|
blocks: result.blocksProcessed
|
|
1537
1749
|
});
|
|
1538
1750
|
return { processed: result.blocksProcessed };
|
|
1539
1751
|
} catch (err) {
|
|
1540
|
-
|
|
1752
|
+
logger5.error("Resumed reindex failed", {
|
|
1541
1753
|
subgraph: subgraphName,
|
|
1542
1754
|
error: getErrorMessage2(err)
|
|
1543
1755
|
});
|
|
@@ -1548,7 +1760,7 @@ async function resumeReindex(def, opts) {
|
|
|
1548
1760
|
async function backfillSubgraph(def, opts) {
|
|
1549
1761
|
const targetDb = getTargetDb2();
|
|
1550
1762
|
const subgraphName = def.name;
|
|
1551
|
-
|
|
1763
|
+
logger5.info("Backfill starting", {
|
|
1552
1764
|
subgraph: subgraphName,
|
|
1553
1765
|
from: opts.fromBlock,
|
|
1554
1766
|
to: opts.toBlock
|
|
@@ -1565,12 +1777,12 @@ async function backfillSubgraph(def, opts) {
|
|
|
1565
1777
|
signal: opts.signal
|
|
1566
1778
|
});
|
|
1567
1779
|
if (result.aborted) {
|
|
1568
|
-
|
|
1780
|
+
logger5.info("Backfill aborted", { subgraph: subgraphName });
|
|
1569
1781
|
return { processed: result.blocksProcessed };
|
|
1570
1782
|
}
|
|
1571
1783
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
1572
1784
|
if (resolved > 0) {
|
|
1573
|
-
|
|
1785
|
+
logger5.info("Resolved subgraph gaps via backfill", {
|
|
1574
1786
|
subgraph: subgraphName,
|
|
1575
1787
|
resolved
|
|
1576
1788
|
});
|
|
@@ -1579,7 +1791,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
1579
1791
|
if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
|
|
1580
1792
|
await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
|
|
1581
1793
|
}
|
|
1582
|
-
|
|
1794
|
+
logger5.info("Backfill complete", {
|
|
1583
1795
|
subgraph: subgraphName,
|
|
1584
1796
|
blocks: result.blocksProcessed,
|
|
1585
1797
|
events: result.totalEventsProcessed,
|
|
@@ -1587,7 +1799,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
1587
1799
|
});
|
|
1588
1800
|
return { processed: result.blocksProcessed };
|
|
1589
1801
|
} catch (err) {
|
|
1590
|
-
|
|
1802
|
+
logger5.error("Backfill failed", {
|
|
1591
1803
|
subgraph: subgraphName,
|
|
1592
1804
|
error: getErrorMessage2(err)
|
|
1593
1805
|
});
|
|
@@ -1600,7 +1812,7 @@ function defineSubgraph(def) {
|
|
|
1600
1812
|
return def;
|
|
1601
1813
|
}
|
|
1602
1814
|
// src/schema/deployer.ts
|
|
1603
|
-
import { sql as
|
|
1815
|
+
import { sql as sql4 } from "kysely";
|
|
1604
1816
|
function toJsonSafe(obj) {
|
|
1605
1817
|
return JSON.parse(JSON.stringify(obj, (_key, value) => typeof value === "bigint" ? value.toString() : value));
|
|
1606
1818
|
}
|
|
@@ -1690,9 +1902,9 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
1690
1902
|
};
|
|
1691
1903
|
}
|
|
1692
1904
|
if (existing.schema_hash === hash && opts?.forceReindex) {
|
|
1693
|
-
await
|
|
1905
|
+
await sql4.raw(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`).execute(db);
|
|
1694
1906
|
for (const stmt of statements) {
|
|
1695
|
-
await
|
|
1907
|
+
await sql4.raw(stmt).execute(db);
|
|
1696
1908
|
}
|
|
1697
1909
|
const sg2 = await registerSubgraph(db, regData);
|
|
1698
1910
|
return { action: "reindexed", subgraphId: sg2.id, version: newVersion };
|
|
@@ -1701,9 +1913,9 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
1701
1913
|
const diff = diffSchema(existing.definition.schema, def.schema);
|
|
1702
1914
|
const { breaking, reasons } = hasBreakingChanges(diff);
|
|
1703
1915
|
if (breaking || opts?.forceReindex) {
|
|
1704
|
-
await
|
|
1916
|
+
await sql4.raw(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`).execute(db);
|
|
1705
1917
|
for (const stmt of statements) {
|
|
1706
|
-
await
|
|
1918
|
+
await sql4.raw(stmt).execute(db);
|
|
1707
1919
|
}
|
|
1708
1920
|
const sg3 = await registerSubgraph(db, regData);
|
|
1709
1921
|
const deployDiff2 = {
|
|
@@ -1737,18 +1949,18 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
1737
1949
|
continue;
|
|
1738
1950
|
colDefs.push(`${colName} ${sqlType}${nullable}`);
|
|
1739
1951
|
}
|
|
1740
|
-
await
|
|
1952
|
+
await sql4.raw(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
|
|
1741
1953
|
${colDefs.join(`,
|
|
1742
1954
|
`)}
|
|
1743
1955
|
)`).execute(db);
|
|
1744
|
-
await
|
|
1745
|
-
await
|
|
1956
|
+
await sql4.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`).execute(db);
|
|
1957
|
+
await sql4.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`).execute(db);
|
|
1746
1958
|
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1747
1959
|
if (col.indexed) {
|
|
1748
|
-
await
|
|
1960
|
+
await sql4.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`).execute(db);
|
|
1749
1961
|
}
|
|
1750
1962
|
if (col.search) {
|
|
1751
|
-
await
|
|
1963
|
+
await sql4.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`).execute(db);
|
|
1752
1964
|
}
|
|
1753
1965
|
}
|
|
1754
1966
|
}
|
|
@@ -1767,12 +1979,12 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
1767
1979
|
if (!sqlType)
|
|
1768
1980
|
continue;
|
|
1769
1981
|
const nullable = col.nullable ? "" : ` NOT NULL DEFAULT ${getDefault(col.type)}`;
|
|
1770
|
-
await
|
|
1982
|
+
await sql4.raw(`ALTER TABLE ${qualifiedName} ADD COLUMN ${colName} ${sqlType}${nullable}`).execute(db);
|
|
1771
1983
|
if (col.indexed) {
|
|
1772
|
-
await
|
|
1984
|
+
await sql4.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`).execute(db);
|
|
1773
1985
|
}
|
|
1774
1986
|
if (col.search) {
|
|
1775
|
-
await
|
|
1987
|
+
await sql4.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`).execute(db);
|
|
1776
1988
|
}
|
|
1777
1989
|
}
|
|
1778
1990
|
}
|
|
@@ -1797,7 +2009,7 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
1797
2009
|
}
|
|
1798
2010
|
}
|
|
1799
2011
|
for (const stmt of statements) {
|
|
1800
|
-
await
|
|
2012
|
+
await sql4.raw(stmt).execute(db);
|
|
1801
2013
|
}
|
|
1802
2014
|
const sg = await registerSubgraph(db, regData);
|
|
1803
2015
|
return { action: "created", subgraphId: sg.id, version: newVersion };
|
|
@@ -1832,5 +2044,5 @@ export {
|
|
|
1832
2044
|
backfillSubgraph
|
|
1833
2045
|
};
|
|
1834
2046
|
|
|
1835
|
-
//# debugId=
|
|
2047
|
+
//# debugId=C453EF737BBEFDDD64756E2164756E21
|
|
1836
2048
|
//# sourceMappingURL=index.js.map
|