@secondlayer/subgraphs 3.26.0 → 3.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +367 -76
- package/dist/src/index.js.map +7 -5
- package/dist/src/runtime/block-processor.js +29 -9
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.d.ts +34 -1
- package/dist/src/runtime/catchup.js +42 -11
- package/dist/src/runtime/catchup.js.map +4 -4
- package/dist/src/runtime/context.d.ts +20 -0
- package/dist/src/runtime/context.js +29 -9
- package/dist/src/runtime/context.js.map +3 -3
- package/dist/src/runtime/processor.js +344 -313
- package/dist/src/runtime/processor.js.map +7 -7
- package/dist/src/runtime/reindex.js +342 -51
- package/dist/src/runtime/reindex.js.map +7 -5
- package/dist/src/runtime/reorg.js +42 -13
- package/dist/src/runtime/reorg.js.map +4 -4
- package/dist/src/runtime/runner.d.ts +20 -0
- package/dist/src/service.js +346 -315
- package/dist/src/service.js.map +7 -7
- package/dist/src/testing/index.js +29 -9
- package/dist/src/testing/index.js.map +3 -3
- package/package.json +1 -1
|
@@ -605,14 +605,10 @@ class SubgraphContext {
|
|
|
605
605
|
this.ops.length = 0;
|
|
606
606
|
const statements = this.buildStatements(opsToFlush);
|
|
607
607
|
if ("isTransaction" in this.db) {
|
|
608
|
-
|
|
609
|
-
await sql.raw(stmt).execute(this.db);
|
|
610
|
-
}
|
|
608
|
+
await this.execStatements(this.db, statements);
|
|
611
609
|
} else {
|
|
612
610
|
await this.db.transaction().execute(async (tx) => {
|
|
613
|
-
|
|
614
|
-
await sql.raw(stmt).execute(tx);
|
|
615
|
-
}
|
|
611
|
+
await this.execStatements(tx, statements);
|
|
616
612
|
});
|
|
617
613
|
}
|
|
618
614
|
const writes = opsToFlush.map((op, rowIndex) => {
|
|
@@ -631,6 +627,25 @@ class SubgraphContext {
|
|
|
631
627
|
});
|
|
632
628
|
return { count: opsToFlush.length, writes };
|
|
633
629
|
}
|
|
630
|
+
async execStatements(executor, statements) {
|
|
631
|
+
const affected = (r) => r.numAffectedRows ?? 0n;
|
|
632
|
+
for (const stmt of statements) {
|
|
633
|
+
if (typeof stmt === "string") {
|
|
634
|
+
await sql.raw(stmt).execute(executor);
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
const updated = await sql.raw(stmt.update).execute(executor);
|
|
638
|
+
if (affected(updated) > 0n)
|
|
639
|
+
continue;
|
|
640
|
+
const inserted = await sql.raw(stmt.insert).execute(executor);
|
|
641
|
+
if (affected(inserted) > 0n)
|
|
642
|
+
continue;
|
|
643
|
+
const retried = await sql.raw(stmt.update).execute(executor);
|
|
644
|
+
if (affected(retried) === 0n) {
|
|
645
|
+
throw new Error(`increment could not be applied to ${this.pgSchemaName}.${stmt.table} where ${stmt.keyPred}: row neither updated nor inserted`);
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
}
|
|
634
649
|
prepareInsert(op) {
|
|
635
650
|
const upsertKeys = op.data._upsert_keys;
|
|
636
651
|
const data = { ...op.data };
|
|
@@ -710,7 +725,6 @@ class SubgraphContext {
|
|
|
710
725
|
for (const r of batch.rows.values()) {
|
|
711
726
|
const keyPred = batch.keyCols.map((k) => `"${k}" = ${escapeLiteral(r.keys[k])}`).join(" AND ");
|
|
712
727
|
const updSet = batch.deltaCols.map((c) => `"${c}" = COALESCE("${c}", 0) + (${String(r.deltas[c] ?? 0n)})`).join(", ");
|
|
713
|
-
statements.push(`UPDATE ${qualifiedTable} SET ${updSet} WHERE ${keyPred}`);
|
|
714
728
|
const insertVals = [
|
|
715
729
|
...batch.keyCols.map((k) => escapeLiteral(r.keys[k])),
|
|
716
730
|
...batch.deltaCols.map((c) => String(r.deltas[c] ?? 0n)),
|
|
@@ -718,7 +732,13 @@ class SubgraphContext {
|
|
|
718
732
|
escapeLiteral(r.meta.txId),
|
|
719
733
|
"NOW()"
|
|
720
734
|
];
|
|
721
|
-
statements.push(
|
|
735
|
+
statements.push({
|
|
736
|
+
kind: "increment",
|
|
737
|
+
table: batch.table,
|
|
738
|
+
keyPred,
|
|
739
|
+
update: `UPDATE ${qualifiedTable} SET ${updSet} WHERE ${keyPred}`,
|
|
740
|
+
insert: `INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) ` + `SELECT ${insertVals.join(", ")} ` + `WHERE NOT EXISTS (SELECT 1 FROM ${qualifiedTable} WHERE ${keyPred})`
|
|
741
|
+
});
|
|
722
742
|
}
|
|
723
743
|
incBatch = null;
|
|
724
744
|
incBatchKey = "";
|
|
@@ -3024,9 +3044,278 @@ class StatsAccumulator {
|
|
|
3024
3044
|
}
|
|
3025
3045
|
}
|
|
3026
3046
|
|
|
3047
|
+
// src/runtime/catchup.ts
|
|
3048
|
+
import { getTargetDb as getTargetDb2, sql as sql4 } from "@secondlayer/shared/db";
|
|
3049
|
+
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
3050
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
3051
|
+
|
|
3052
|
+
// src/runtime/catchup-leader.ts
|
|
3053
|
+
import {
|
|
3054
|
+
SUBGRAPH_CATCHUP_LOCK_KEY,
|
|
3055
|
+
createPostgresLeaderBackend,
|
|
3056
|
+
withLeaderLock
|
|
3057
|
+
} from "@secondlayer/shared/leader";
|
|
3058
|
+
import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
3059
|
+
var catchUpLeader = false;
|
|
3060
|
+
function isCatchUpLeader() {
|
|
3061
|
+
return catchUpLeader;
|
|
3062
|
+
}
|
|
3063
|
+
function startCatchUpLeader(opts = {}) {
|
|
3064
|
+
return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
|
|
3065
|
+
catchUpLeader = true;
|
|
3066
|
+
await opts.onAcquire?.();
|
|
3067
|
+
return () => {
|
|
3068
|
+
catchUpLeader = false;
|
|
3069
|
+
};
|
|
3070
|
+
}, {
|
|
3071
|
+
pollMs: opts.pollMs,
|
|
3072
|
+
heartbeatMs: opts.heartbeatMs,
|
|
3073
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
|
|
3074
|
+
});
|
|
3075
|
+
}
|
|
3076
|
+
|
|
3077
|
+
// src/runtime/catchup.ts
|
|
3078
|
+
var LOG_INTERVAL = 1000;
|
|
3079
|
+
var STANDARD_CATCHUP_BATCH_CONFIG = {
|
|
3080
|
+
defaultBatchSize: 500,
|
|
3081
|
+
minBatchSize: 100,
|
|
3082
|
+
maxBatchSize: 1000,
|
|
3083
|
+
prefetch: true
|
|
3084
|
+
};
|
|
3085
|
+
var catchingUp = new Set;
|
|
3086
|
+
var subgraphBlockLockHeld = new Set;
|
|
3087
|
+
var subgraphBlockLockWaiters = new Map;
|
|
3088
|
+
var reorgEpoch = new Map;
|
|
3089
|
+
async function acquireSubgraphBlockLock(name) {
|
|
3090
|
+
if (!subgraphBlockLockHeld.has(name)) {
|
|
3091
|
+
subgraphBlockLockHeld.add(name);
|
|
3092
|
+
return () => releaseSubgraphBlockLock(name);
|
|
3093
|
+
}
|
|
3094
|
+
return new Promise((resolve) => {
|
|
3095
|
+
const waiters = subgraphBlockLockWaiters.get(name) ?? [];
|
|
3096
|
+
waiters.push(() => resolve(() => releaseSubgraphBlockLock(name)));
|
|
3097
|
+
subgraphBlockLockWaiters.set(name, waiters);
|
|
3098
|
+
});
|
|
3099
|
+
}
|
|
3100
|
+
function releaseSubgraphBlockLock(name) {
|
|
3101
|
+
const waiters = subgraphBlockLockWaiters.get(name);
|
|
3102
|
+
if (waiters && waiters.length > 0) {
|
|
3103
|
+
const next = waiters.shift();
|
|
3104
|
+
if (waiters.length === 0)
|
|
3105
|
+
subgraphBlockLockWaiters.delete(name);
|
|
3106
|
+
next();
|
|
3107
|
+
return;
|
|
3108
|
+
}
|
|
3109
|
+
subgraphBlockLockHeld.delete(name);
|
|
3110
|
+
}
|
|
3111
|
+
async function withSubgraphBlockLock(name, fn) {
|
|
3112
|
+
const release = await acquireSubgraphBlockLock(name);
|
|
3113
|
+
try {
|
|
3114
|
+
return await fn();
|
|
3115
|
+
} finally {
|
|
3116
|
+
release();
|
|
3117
|
+
}
|
|
3118
|
+
}
|
|
3119
|
+
async function withSubgraphBlockAdvisoryLock(name, fn) {
|
|
3120
|
+
return getTargetDb2().transaction().execute(async (lockTx) => {
|
|
3121
|
+
await sql4`SELECT pg_advisory_xact_lock(hashtext(${`subgraph-block:${name}`}))`.execute(lockTx);
|
|
3122
|
+
return fn();
|
|
3123
|
+
});
|
|
3124
|
+
}
|
|
3125
|
+
async function withSubgraphBlockWriteLock(name, fn) {
|
|
3126
|
+
return withSubgraphBlockLock(name, () => withSubgraphBlockAdvisoryLock(name, fn));
|
|
3127
|
+
}
|
|
3128
|
+
function bumpReorgEpoch(name) {
|
|
3129
|
+
reorgEpoch.set(name, (reorgEpoch.get(name) ?? 0) + 1);
|
|
3130
|
+
}
|
|
3131
|
+
function getReorgEpoch(name) {
|
|
3132
|
+
return reorgEpoch.get(name) ?? 0;
|
|
3133
|
+
}
|
|
3134
|
+
|
|
3135
|
+
class ReorgEpochChangedError extends Error {
|
|
3136
|
+
constructor(name) {
|
|
3137
|
+
super(`reorg epoch changed for ${name} mid catch-up`);
|
|
3138
|
+
this.name = "ReorgEpochChangedError";
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
function parsePositiveInt(value) {
|
|
3142
|
+
if (value == null || value.trim() === "")
|
|
3143
|
+
return;
|
|
3144
|
+
const parsed = Number.parseInt(value, 10);
|
|
3145
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
3146
|
+
}
|
|
3147
|
+
function parseBoolean(value) {
|
|
3148
|
+
if (value == null || value.trim() === "")
|
|
3149
|
+
return;
|
|
3150
|
+
const normalized = value.trim().toLowerCase();
|
|
3151
|
+
if (normalized === "true")
|
|
3152
|
+
return true;
|
|
3153
|
+
if (normalized === "false")
|
|
3154
|
+
return false;
|
|
3155
|
+
return;
|
|
3156
|
+
}
|
|
3157
|
+
function resolveCatchupBatchConfig(env = process.env) {
|
|
3158
|
+
const base = STANDARD_CATCHUP_BATCH_CONFIG;
|
|
3159
|
+
const minBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MIN_BATCH_SIZE) ?? base.minBatchSize;
|
|
3160
|
+
const maxBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MAX_BATCH_SIZE) ?? base.maxBatchSize;
|
|
3161
|
+
const defaultBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_BATCH_SIZE) ?? base.defaultBatchSize;
|
|
3162
|
+
return {
|
|
3163
|
+
minBatchSize,
|
|
3164
|
+
maxBatchSize,
|
|
3165
|
+
defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize),
|
|
3166
|
+
prefetch: parseBoolean(env.SUBGRAPH_CATCHUP_PREFETCH) ?? base.prefetch
|
|
3167
|
+
};
|
|
3168
|
+
}
|
|
3169
|
+
function adjustBatchSize(current, avgEvents, config) {
|
|
3170
|
+
if (avgEvents > 50)
|
|
3171
|
+
return Math.max(Math.round(current * 0.5), config.minBatchSize);
|
|
3172
|
+
if (avgEvents < 10)
|
|
3173
|
+
return Math.min(Math.round(current * 1.5), config.maxBatchSize);
|
|
3174
|
+
return current;
|
|
3175
|
+
}
|
|
3176
|
+
async function catchUpSubgraph(subgraph, subgraphName) {
|
|
3177
|
+
if (catchingUp.has(subgraphName))
|
|
3178
|
+
return 0;
|
|
3179
|
+
catchingUp.add(subgraphName);
|
|
3180
|
+
try {
|
|
3181
|
+
const source = resolveBlockSource(subgraph);
|
|
3182
|
+
const targetDb = getTargetDb2();
|
|
3183
|
+
const subgraphRow = await getSubgraph(targetDb, subgraphName);
|
|
3184
|
+
if (!subgraphRow)
|
|
3185
|
+
return 0;
|
|
3186
|
+
const lastProcessedBlock = Number(subgraphRow.last_processed_block);
|
|
3187
|
+
const chainTip = await source.getTip();
|
|
3188
|
+
if (chainTip <= 0 || lastProcessedBlock >= chainTip)
|
|
3189
|
+
return 0;
|
|
3190
|
+
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
3191
|
+
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
3192
|
+
const totalBlocks = chainTip - lastProcessedBlock;
|
|
3193
|
+
logger8.info("Subgraph catch-up starting", {
|
|
3194
|
+
subgraph: subgraphName,
|
|
3195
|
+
from: startBlock,
|
|
3196
|
+
to: chainTip,
|
|
3197
|
+
blocks: totalBlocks
|
|
3198
|
+
});
|
|
3199
|
+
const stats = new StatsAccumulator(subgraphName, true);
|
|
3200
|
+
let processed = 0;
|
|
3201
|
+
const batchConfig = resolveCatchupBatchConfig();
|
|
3202
|
+
let batchSize = batchConfig.defaultBatchSize;
|
|
3203
|
+
let currentHeight = startBlock;
|
|
3204
|
+
const walkEpoch = getReorgEpoch(subgraphName);
|
|
3205
|
+
let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
3206
|
+
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
3207
|
+
while (currentHeight <= chainTip) {
|
|
3208
|
+
if (!isCatchUpLeader()) {
|
|
3209
|
+
logger8.info("Lost catch-up leadership, stopping catch-up", {
|
|
3210
|
+
subgraph: subgraphName,
|
|
3211
|
+
processed
|
|
3212
|
+
});
|
|
3213
|
+
break;
|
|
3214
|
+
}
|
|
3215
|
+
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
3216
|
+
if (!currentRow || currentRow.status !== "active") {
|
|
3217
|
+
logger8.info("Subgraph status changed, stopping catch-up", {
|
|
3218
|
+
subgraph: subgraphName,
|
|
3219
|
+
status: currentRow?.status ?? "deleted"
|
|
3220
|
+
});
|
|
3221
|
+
break;
|
|
3222
|
+
}
|
|
3223
|
+
let batchEnd;
|
|
3224
|
+
let batch;
|
|
3225
|
+
if (nextBatchPromise) {
|
|
3226
|
+
batch = await nextBatchPromise;
|
|
3227
|
+
batchEnd = prefetchedBatchEnd;
|
|
3228
|
+
const nextStart = batchEnd + 1;
|
|
3229
|
+
if (nextStart <= chainTip) {
|
|
3230
|
+
prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
|
|
3231
|
+
nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
|
|
3232
|
+
} else {
|
|
3233
|
+
nextBatchPromise = undefined;
|
|
3234
|
+
}
|
|
3235
|
+
} else {
|
|
3236
|
+
batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
3237
|
+
batch = await source.loadBlockRange(currentHeight, batchEnd);
|
|
3238
|
+
}
|
|
3239
|
+
let stopCatchup = false;
|
|
3240
|
+
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
3241
|
+
let blockData = batch.get(height);
|
|
3242
|
+
if (!blockData) {
|
|
3243
|
+
blockData = (await source.loadBlockRange(height, height)).get(height);
|
|
3244
|
+
}
|
|
3245
|
+
if (!blockData) {
|
|
3246
|
+
logger8.warn("Block missing during catch-up, deferring to next tick", {
|
|
3247
|
+
subgraph: subgraphName,
|
|
3248
|
+
blockHeight: height
|
|
3249
|
+
});
|
|
3250
|
+
stopCatchup = true;
|
|
3251
|
+
break;
|
|
3252
|
+
}
|
|
3253
|
+
const preloaded = blockData;
|
|
3254
|
+
let result;
|
|
3255
|
+
try {
|
|
3256
|
+
result = await withSubgraphBlockWriteLock(subgraphName, async () => {
|
|
3257
|
+
if (getReorgEpoch(subgraphName) !== walkEpoch) {
|
|
3258
|
+
throw new ReorgEpochChangedError(subgraphName);
|
|
3259
|
+
}
|
|
3260
|
+
return processBlockWithRetry(subgraph, subgraphName, height, {
|
|
3261
|
+
preloaded
|
|
3262
|
+
});
|
|
3263
|
+
});
|
|
3264
|
+
} catch (err) {
|
|
3265
|
+
if (err instanceof ReorgEpochChangedError) {
|
|
3266
|
+
logger8.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
|
|
3267
|
+
stopCatchup = true;
|
|
3268
|
+
break;
|
|
3269
|
+
}
|
|
3270
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
3271
|
+
logger8.error("Block processing failed persistently during catch-up", {
|
|
3272
|
+
subgraph: subgraphName,
|
|
3273
|
+
blockHeight: height,
|
|
3274
|
+
error: errorMsg
|
|
3275
|
+
});
|
|
3276
|
+
const { updateSubgraphStatus: updateSubgraphStatus2, recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
3277
|
+
await recordSubgraphProcessed2(targetDb, subgraphName, 0, 1, `catch-up halted at block ${height}: ${errorMsg}`).catch(() => {});
|
|
3278
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "error").catch(() => {});
|
|
3279
|
+
stopCatchup = true;
|
|
3280
|
+
break;
|
|
3281
|
+
}
|
|
3282
|
+
processed++;
|
|
3283
|
+
if (result.timing) {
|
|
3284
|
+
stats.record(result.timing, result.processed);
|
|
3285
|
+
if (stats.shouldFlush()) {
|
|
3286
|
+
await stats.flush(targetDb);
|
|
3287
|
+
}
|
|
3288
|
+
}
|
|
3289
|
+
if (processed % LOG_INTERVAL === 0) {
|
|
3290
|
+
logger8.info("Subgraph catch-up progress", {
|
|
3291
|
+
subgraph: subgraphName,
|
|
3292
|
+
processed,
|
|
3293
|
+
total: totalBlocks,
|
|
3294
|
+
currentBlock: height,
|
|
3295
|
+
pct: Math.round(processed / totalBlocks * 100)
|
|
3296
|
+
});
|
|
3297
|
+
}
|
|
3298
|
+
}
|
|
3299
|
+
if (stopCatchup)
|
|
3300
|
+
break;
|
|
3301
|
+
const avg = avgEventsPerBlock(batch);
|
|
3302
|
+
batchSize = adjustBatchSize(batchSize, avg, batchConfig);
|
|
3303
|
+
currentHeight = batchEnd + 1;
|
|
3304
|
+
}
|
|
3305
|
+
await stats.flush(targetDb);
|
|
3306
|
+
logger8.info("Subgraph catch-up complete", {
|
|
3307
|
+
subgraph: subgraphName,
|
|
3308
|
+
processed
|
|
3309
|
+
});
|
|
3310
|
+
return processed;
|
|
3311
|
+
} finally {
|
|
3312
|
+
catchingUp.delete(subgraphName);
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3027
3316
|
// src/runtime/reindex.ts
|
|
3028
3317
|
import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
|
|
3029
|
-
import { getRawClient, getTargetDb as
|
|
3318
|
+
import { getRawClient, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3030
3319
|
import {
|
|
3031
3320
|
recordGapBatch,
|
|
3032
3321
|
resolveGaps
|
|
@@ -3039,11 +3328,11 @@ import {
|
|
|
3039
3328
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
3040
3329
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
3041
3330
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3042
|
-
import { logger as
|
|
3331
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
3043
3332
|
|
|
3044
3333
|
// src/runtime/reindex-notify.ts
|
|
3045
3334
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
3046
|
-
import { logger as
|
|
3335
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
3047
3336
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
3048
3337
|
try {
|
|
3049
3338
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -3054,7 +3343,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
3054
3343
|
return;
|
|
3055
3344
|
const resendKey = process.env.RESEND_API_KEY;
|
|
3056
3345
|
if (!resendKey) {
|
|
3057
|
-
|
|
3346
|
+
logger9.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
3058
3347
|
subgraph: subgraphName
|
|
3059
3348
|
});
|
|
3060
3349
|
return;
|
|
@@ -3076,14 +3365,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
3076
3365
|
});
|
|
3077
3366
|
if (!res.ok) {
|
|
3078
3367
|
const text = await res.text().catch(() => "");
|
|
3079
|
-
|
|
3368
|
+
logger9.warn("reindex-complete email failed", {
|
|
3080
3369
|
subgraph: subgraphName,
|
|
3081
3370
|
status: res.status,
|
|
3082
3371
|
body: text.slice(0, 200)
|
|
3083
3372
|
});
|
|
3084
3373
|
}
|
|
3085
3374
|
} catch (err) {
|
|
3086
|
-
|
|
3375
|
+
logger9.warn("reindex-complete email threw", {
|
|
3087
3376
|
subgraph: subgraphName,
|
|
3088
3377
|
error: getErrorMessage2(err)
|
|
3089
3378
|
});
|
|
@@ -3091,7 +3380,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
3091
3380
|
}
|
|
3092
3381
|
|
|
3093
3382
|
// src/runtime/reindex.ts
|
|
3094
|
-
var
|
|
3383
|
+
var LOG_INTERVAL2 = 1000;
|
|
3095
3384
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
3096
3385
|
var PROGRESS_FLUSH_INTERVAL_MS = 5000;
|
|
3097
3386
|
var EMPTY_BATCH_HALT_THRESHOLD = 3;
|
|
@@ -3111,7 +3400,7 @@ function resolveReindexResumeBlock(row) {
|
|
|
3111
3400
|
const reindexFromBlock = Number(row.reindex_from_block);
|
|
3112
3401
|
return Math.max(lastProcessedBlock + 1, reindexFromBlock);
|
|
3113
3402
|
}
|
|
3114
|
-
function
|
|
3403
|
+
function parsePositiveInt2(value) {
|
|
3115
3404
|
if (value == null || value.trim() === "")
|
|
3116
3405
|
return;
|
|
3117
3406
|
const parsed = Number.parseInt(value, 10);
|
|
@@ -3119,9 +3408,9 @@ function parsePositiveInt(value) {
|
|
|
3119
3408
|
}
|
|
3120
3409
|
function resolveReindexBatchConfig(env = process.env) {
|
|
3121
3410
|
const base = STANDARD_REINDEX_BATCH_CONFIG;
|
|
3122
|
-
const minBatchSize =
|
|
3123
|
-
const maxBatchSize =
|
|
3124
|
-
const defaultBatchSize =
|
|
3411
|
+
const minBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_MIN_BATCH_SIZE) ?? base.minBatchSize;
|
|
3412
|
+
const maxBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_MAX_BATCH_SIZE) ?? base.maxBatchSize;
|
|
3413
|
+
const defaultBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_BATCH_SIZE) ?? base.defaultBatchSize;
|
|
3125
3414
|
return {
|
|
3126
3415
|
minBatchSize,
|
|
3127
3416
|
maxBatchSize,
|
|
@@ -3152,7 +3441,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
3152
3441
|
}
|
|
3153
3442
|
async function processBlockRange(def, opts) {
|
|
3154
3443
|
const source = resolveBlockSource(def);
|
|
3155
|
-
const targetDb =
|
|
3444
|
+
const targetDb = getTargetDb3();
|
|
3156
3445
|
const subgraphName = def.name;
|
|
3157
3446
|
const { fromBlock, toBlock, status } = opts;
|
|
3158
3447
|
const totalBlocks = toBlock - fromBlock + 1;
|
|
@@ -3195,7 +3484,7 @@ async function processBlockRange(def, opts) {
|
|
|
3195
3484
|
while (currentHeight <= toBlock) {
|
|
3196
3485
|
if (opts.signal?.aborted) {
|
|
3197
3486
|
aborted = true;
|
|
3198
|
-
|
|
3487
|
+
logger10.info("Block processing aborted", {
|
|
3199
3488
|
subgraph: subgraphName,
|
|
3200
3489
|
currentBlock: currentHeight,
|
|
3201
3490
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -3237,14 +3526,16 @@ async function processBlockRange(def, opts) {
|
|
|
3237
3526
|
}
|
|
3238
3527
|
let result;
|
|
3239
3528
|
try {
|
|
3240
|
-
|
|
3529
|
+
const mayWrite = blockData.txs.length > 0 || blockData.events.length > 0;
|
|
3530
|
+
const runBlock = () => processBlockWithRetry(def, subgraphName, height, {
|
|
3241
3531
|
skipProgressUpdate: true,
|
|
3242
3532
|
atomicProgress,
|
|
3243
3533
|
preloaded: blockData
|
|
3244
3534
|
});
|
|
3535
|
+
result = mayWrite ? await withSubgraphBlockWriteLock(subgraphName, runBlock) : await runBlock();
|
|
3245
3536
|
} catch (err) {
|
|
3246
3537
|
const errorMsg = getErrorMessage3(err);
|
|
3247
|
-
|
|
3538
|
+
logger10.error("Block processing failed persistently", {
|
|
3248
3539
|
subgraph: subgraphName,
|
|
3249
3540
|
blockHeight: height,
|
|
3250
3541
|
error: errorMsg
|
|
@@ -3289,8 +3580,8 @@ async function processBlockRange(def, opts) {
|
|
|
3289
3580
|
}
|
|
3290
3581
|
lastProgressFlushAt = now;
|
|
3291
3582
|
}
|
|
3292
|
-
if (blocksProcessed %
|
|
3293
|
-
|
|
3583
|
+
if (blocksProcessed % LOG_INTERVAL2 === 0) {
|
|
3584
|
+
logger10.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
3294
3585
|
subgraph: subgraphName,
|
|
3295
3586
|
processed: blocksProcessed,
|
|
3296
3587
|
total: totalBlocks,
|
|
@@ -3310,7 +3601,7 @@ async function processBlockRange(def, opts) {
|
|
|
3310
3601
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
3311
3602
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
3312
3603
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
3313
|
-
|
|
3604
|
+
logger10.warn("Failed to record subgraph gaps", {
|
|
3314
3605
|
subgraph: subgraphName,
|
|
3315
3606
|
error: err instanceof Error ? err.message : String(err)
|
|
3316
3607
|
});
|
|
@@ -3327,7 +3618,7 @@ async function processBlockRange(def, opts) {
|
|
|
3327
3618
|
} else {
|
|
3328
3619
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
3329
3620
|
}
|
|
3330
|
-
|
|
3621
|
+
logger10.info("Sparse skip", {
|
|
3331
3622
|
subgraph: subgraphName,
|
|
3332
3623
|
from: batchEnd + 1,
|
|
3333
3624
|
to: jumpTo - 1,
|
|
@@ -3362,16 +3653,16 @@ async function clearReindexMetadata(db, subgraphName) {
|
|
|
3362
3653
|
}
|
|
3363
3654
|
async function reindexSubgraph(def, opts) {
|
|
3364
3655
|
const source = resolveBlockSource(def);
|
|
3365
|
-
const targetDb =
|
|
3656
|
+
const targetDb = getTargetDb3();
|
|
3366
3657
|
const client = getRawClient("target");
|
|
3367
3658
|
const subgraphName = def.name;
|
|
3368
3659
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3369
3660
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3370
|
-
|
|
3661
|
+
logger10.info("Reindex starting", { subgraph: subgraphName });
|
|
3371
3662
|
try {
|
|
3372
3663
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3373
3664
|
if (fromBlock > toBlock) {
|
|
3374
|
-
|
|
3665
|
+
logger10.info("No blocks to reindex", {
|
|
3375
3666
|
subgraph: subgraphName,
|
|
3376
3667
|
fromBlock,
|
|
3377
3668
|
toBlock
|
|
@@ -3385,7 +3676,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3385
3676
|
await client.unsafe(stmt);
|
|
3386
3677
|
}
|
|
3387
3678
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3388
|
-
|
|
3679
|
+
logger10.info("Schema recreated for reindex", {
|
|
3389
3680
|
subgraph: subgraphName,
|
|
3390
3681
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3391
3682
|
});
|
|
@@ -3399,7 +3690,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3399
3690
|
last_error_at: null,
|
|
3400
3691
|
updated_at: new Date
|
|
3401
3692
|
}).where("name", "=", subgraphName).execute();
|
|
3402
|
-
|
|
3693
|
+
logger10.info("Reindexing blocks", {
|
|
3403
3694
|
subgraph: subgraphName,
|
|
3404
3695
|
fromBlock,
|
|
3405
3696
|
toBlock,
|
|
@@ -3421,9 +3712,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3421
3712
|
if (reason === "user-cancelled") {
|
|
3422
3713
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3423
3714
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3424
|
-
|
|
3715
|
+
logger10.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3425
3716
|
} else {
|
|
3426
|
-
|
|
3717
|
+
logger10.info("Reindex interrupted by shutdown, will resume", {
|
|
3427
3718
|
subgraph: subgraphName
|
|
3428
3719
|
});
|
|
3429
3720
|
}
|
|
@@ -3431,7 +3722,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3431
3722
|
}
|
|
3432
3723
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3433
3724
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3434
|
-
|
|
3725
|
+
logger10.info("Reindex complete", {
|
|
3435
3726
|
subgraph: subgraphName,
|
|
3436
3727
|
blocks: result.blocksProcessed,
|
|
3437
3728
|
events: result.totalEventsProcessed,
|
|
@@ -3444,7 +3735,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3444
3735
|
});
|
|
3445
3736
|
return { processed: result.blocksProcessed };
|
|
3446
3737
|
} catch (err) {
|
|
3447
|
-
|
|
3738
|
+
logger10.error("Reindex failed", {
|
|
3448
3739
|
subgraph: subgraphName,
|
|
3449
3740
|
error: getErrorMessage3(err)
|
|
3450
3741
|
});
|
|
@@ -3453,7 +3744,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3453
3744
|
}
|
|
3454
3745
|
}
|
|
3455
3746
|
async function resumeReindex(def, opts) {
|
|
3456
|
-
const targetDb =
|
|
3747
|
+
const targetDb = getTargetDb3();
|
|
3457
3748
|
const subgraphName = def.name;
|
|
3458
3749
|
const row = await targetDb.selectFrom("subgraphs").select([
|
|
3459
3750
|
"id",
|
|
@@ -3466,7 +3757,7 @@ async function resumeReindex(def, opts) {
|
|
|
3466
3757
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3467
3758
|
const toBlock = Number(row.reindex_to_block);
|
|
3468
3759
|
if (fromBlock == null) {
|
|
3469
|
-
|
|
3760
|
+
logger10.info("No reindex metadata, starting fresh reindex", {
|
|
3470
3761
|
subgraph: subgraphName
|
|
3471
3762
|
});
|
|
3472
3763
|
return reindexSubgraph(def, {
|
|
@@ -3475,12 +3766,12 @@ async function resumeReindex(def, opts) {
|
|
|
3475
3766
|
});
|
|
3476
3767
|
}
|
|
3477
3768
|
if (fromBlock > toBlock) {
|
|
3478
|
-
|
|
3769
|
+
logger10.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3479
3770
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3480
3771
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3481
3772
|
return { processed: 0 };
|
|
3482
3773
|
}
|
|
3483
|
-
|
|
3774
|
+
logger10.info("Resuming reindex", {
|
|
3484
3775
|
subgraph: subgraphName,
|
|
3485
3776
|
fromBlock,
|
|
3486
3777
|
toBlock,
|
|
@@ -3502,9 +3793,9 @@ async function resumeReindex(def, opts) {
|
|
|
3502
3793
|
if (reason === "user-cancelled") {
|
|
3503
3794
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3504
3795
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3505
|
-
|
|
3796
|
+
logger10.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3506
3797
|
} else {
|
|
3507
|
-
|
|
3798
|
+
logger10.info("Resume interrupted by shutdown, will resume again", {
|
|
3508
3799
|
subgraph: subgraphName
|
|
3509
3800
|
});
|
|
3510
3801
|
}
|
|
@@ -3512,13 +3803,13 @@ async function resumeReindex(def, opts) {
|
|
|
3512
3803
|
}
|
|
3513
3804
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3514
3805
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3515
|
-
|
|
3806
|
+
logger10.info("Resumed reindex complete", {
|
|
3516
3807
|
subgraph: subgraphName,
|
|
3517
3808
|
blocks: result.blocksProcessed
|
|
3518
3809
|
});
|
|
3519
3810
|
return { processed: result.blocksProcessed };
|
|
3520
3811
|
} catch (err) {
|
|
3521
|
-
|
|
3812
|
+
logger10.error("Resumed reindex failed", {
|
|
3522
3813
|
subgraph: subgraphName,
|
|
3523
3814
|
error: getErrorMessage3(err)
|
|
3524
3815
|
});
|
|
@@ -3527,9 +3818,9 @@ async function resumeReindex(def, opts) {
|
|
|
3527
3818
|
}
|
|
3528
3819
|
}
|
|
3529
3820
|
async function backfillSubgraph(def, opts) {
|
|
3530
|
-
const targetDb =
|
|
3821
|
+
const targetDb = getTargetDb3();
|
|
3531
3822
|
const subgraphName = def.name;
|
|
3532
|
-
|
|
3823
|
+
logger10.info("Backfill starting", {
|
|
3533
3824
|
subgraph: subgraphName,
|
|
3534
3825
|
from: opts.fromBlock,
|
|
3535
3826
|
to: opts.toBlock
|
|
@@ -3547,17 +3838,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3547
3838
|
signal: opts.signal
|
|
3548
3839
|
});
|
|
3549
3840
|
if (result.aborted) {
|
|
3550
|
-
|
|
3841
|
+
logger10.info("Backfill aborted", { subgraph: subgraphName });
|
|
3551
3842
|
return { processed: result.blocksProcessed };
|
|
3552
3843
|
}
|
|
3553
3844
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3554
3845
|
if (resolved > 0) {
|
|
3555
|
-
|
|
3846
|
+
logger10.info("Resolved subgraph gaps via backfill", {
|
|
3556
3847
|
subgraph: subgraphName,
|
|
3557
3848
|
resolved
|
|
3558
3849
|
});
|
|
3559
3850
|
}
|
|
3560
|
-
|
|
3851
|
+
logger10.info("Backfill complete", {
|
|
3561
3852
|
subgraph: subgraphName,
|
|
3562
3853
|
blocks: result.blocksProcessed,
|
|
3563
3854
|
events: result.totalEventsProcessed,
|
|
@@ -3565,7 +3856,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3565
3856
|
});
|
|
3566
3857
|
return { processed: result.blocksProcessed };
|
|
3567
3858
|
} catch (err) {
|
|
3568
|
-
|
|
3859
|
+
logger10.error("Backfill failed", {
|
|
3569
3860
|
subgraph: subgraphName,
|
|
3570
3861
|
error: getErrorMessage3(err)
|
|
3571
3862
|
});
|
|
@@ -3582,5 +3873,5 @@ export {
|
|
|
3582
3873
|
backfillSubgraph
|
|
3583
3874
|
};
|
|
3584
3875
|
|
|
3585
|
-
//# debugId=
|
|
3876
|
+
//# debugId=1CF5DC6B2286F56764756E2164756E21
|
|
3586
3877
|
//# sourceMappingURL=reindex.js.map
|