@secondlayer/subgraphs 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +16 -4
- package/dist/src/index.js +259 -95
- package/dist/src/index.js.map +8 -7
- package/dist/src/runtime/block-processor.d.ts +9 -1
- package/dist/src/runtime/block-processor.js +89 -47
- package/dist/src/runtime/block-processor.js.map +4 -4
- package/dist/src/runtime/catchup.d.ts +2 -2
- package/dist/src/runtime/catchup.js +194 -63
- package/dist/src/runtime/catchup.js.map +7 -6
- package/dist/src/runtime/context.d.ts +3 -1
- package/dist/src/runtime/context.js +63 -33
- package/dist/src/runtime/context.js.map +3 -3
- package/dist/src/runtime/processor.js +194 -63
- package/dist/src/runtime/processor.js.map +7 -6
- package/dist/src/runtime/reindex.d.ts +13 -1
- package/dist/src/runtime/reindex.js +258 -94
- package/dist/src/runtime/reindex.js.map +8 -7
- package/dist/src/runtime/reorg.js +89 -47
- package/dist/src/runtime/reorg.js.map +4 -4
- package/dist/src/runtime/runner.d.ts +3 -1
- package/dist/src/schema/index.js +2 -2
- package/dist/src/schema/index.js.map +3 -3
- package/dist/src/service.js +194 -63
- package/dist/src/service.js.map +7 -6
- package/dist/src/validate.js +2 -2
- package/dist/src/validate.js.map +3 -3
- package/package.json +2 -2
package/dist/src/index.d.ts
CHANGED
|
@@ -94,6 +94,18 @@ declare function reindexSubgraph(def: SubgraphDefinition, opts?: ReindexOptions)
|
|
|
94
94
|
processed: number
|
|
95
95
|
}>;
|
|
96
96
|
/**
|
|
97
|
+
* Backfill a subgraph by re-processing a block range WITHOUT dropping the schema.
|
|
98
|
+
* Uses upserts so existing data is updated, not duplicated. Safe to run while
|
|
99
|
+
* the subgraph is actively syncing.
|
|
100
|
+
*/
|
|
101
|
+
declare function backfillSubgraph(def: SubgraphDefinition, opts: {
|
|
102
|
+
fromBlock: number
|
|
103
|
+
toBlock: number
|
|
104
|
+
schemaName?: string
|
|
105
|
+
}): Promise<{
|
|
106
|
+
processed: number
|
|
107
|
+
}>;
|
|
108
|
+
/**
|
|
97
109
|
* Identity function that preserves schema literal types for type inference.
|
|
98
110
|
*
|
|
99
111
|
* The generic `S` is narrowed to the exact schema shape so that column type
|
|
@@ -162,13 +174,13 @@ declare function deploySchema(db: AnyDb, def: SubgraphDefinition, handlerPath: s
|
|
|
162
174
|
subgraphId: string
|
|
163
175
|
}>;
|
|
164
176
|
/** Maps a ColumnType string literal to its TypeScript equivalent */
|
|
165
|
-
type ColumnToTS<T extends string> = T extends "uint" | "int" ?
|
|
177
|
+
type ColumnToTS<T extends string> = T extends "uint" | "int" ? bigint : T extends "text" | "principal" | "timestamp" ? string : T extends "boolean" ? boolean : T extends "jsonb" ? Record<string, unknown> : unknown;
|
|
166
178
|
/** Infer TS type for a single column definition, respecting nullable */
|
|
167
179
|
type InferColumnType<C extends SubgraphColumn> = C["type"] extends ColumnType ? C["nullable"] extends true ? ColumnToTS<C["type"]> | null : ColumnToTS<C["type"]> : unknown;
|
|
168
180
|
/** Shape of system columns on every returned row (camelCase, underscore-prefixed) */
|
|
169
181
|
interface SystemRow {
|
|
170
182
|
_id: string;
|
|
171
|
-
_blockHeight:
|
|
183
|
+
_blockHeight: bigint;
|
|
172
184
|
_txId: string;
|
|
173
185
|
_createdAt: string;
|
|
174
186
|
}
|
|
@@ -190,7 +202,7 @@ type WhereInput<TRow> = { [K in keyof TRow]? : TRow[K] | ComparisonFilter<TRow[K
|
|
|
190
202
|
* Both `_blockHeight` and `blockHeight` are valid — serializer handles mapping.
|
|
191
203
|
*/
|
|
192
204
|
type SystemWhereAliases = {
|
|
193
|
-
blockHeight?:
|
|
205
|
+
blockHeight?: bigint | ComparisonFilter<bigint>
|
|
194
206
|
txId?: string | ComparisonFilter<string>
|
|
195
207
|
createdAt?: string | ComparisonFilter<string>
|
|
196
208
|
id?: string | ComparisonFilter<string>
|
|
@@ -216,4 +228,4 @@ interface SubgraphTableClient<TRow> {
|
|
|
216
228
|
type InferSubgraphClient<T> = T extends {
|
|
217
229
|
schema: infer S
|
|
218
230
|
} ? { [K in keyof S] : S[K] extends SubgraphTable ? SubgraphTableClient<InferTableRow<S[K]>> : never } : never;
|
|
219
|
-
export { validateSubgraphDefinition, sourceKey, reindexSubgraph, pgSchemaName, generateSubgraphSQL, diffSchema, deploySchema, defineSubgraph, WhereInput, TableDiff, SystemRow, SubgraphTableClient, SubgraphTable, SubgraphSource, SubgraphSchema, SubgraphHandler, SubgraphDefinition, SubgraphContext, SubgraphColumn, ReindexOptions, InferTableRow, InferSubgraphClient, InferColumnType, GeneratedSQL, FindManyOptions, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff };
|
|
231
|
+
export { validateSubgraphDefinition, sourceKey, reindexSubgraph, pgSchemaName, generateSubgraphSQL, diffSchema, deploySchema, defineSubgraph, backfillSubgraph, WhereInput, TableDiff, SystemRow, SubgraphTableClient, SubgraphTable, SubgraphSource, SubgraphSchema, SubgraphHandler, SubgraphDefinition, SubgraphContext, SubgraphColumn, ReindexOptions, InferTableRow, InferSubgraphClient, InferColumnType, GeneratedSQL, FindManyOptions, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff };
|
package/dist/src/index.js
CHANGED
|
@@ -53,7 +53,7 @@ var SubgraphDefinitionSchema = z.object({
|
|
|
53
53
|
description: z.string().optional(),
|
|
54
54
|
sources: z.array(SubgraphSourceSchema).min(1, "Must have at least one source"),
|
|
55
55
|
schema: SubgraphSchemaSchema,
|
|
56
|
-
handlers: z.record(z.string(), z.
|
|
56
|
+
handlers: z.record(z.string(), z.any())
|
|
57
57
|
});
|
|
58
58
|
function validateSubgraphDefinition(def) {
|
|
59
59
|
return SubgraphDefinitionSchema.parse(def);
|
|
@@ -166,53 +166,83 @@ class SubgraphContext {
|
|
|
166
166
|
}
|
|
167
167
|
return opsToFlush.length;
|
|
168
168
|
}
|
|
169
|
+
prepareInsert(op) {
|
|
170
|
+
const upsertKeys = op.data._upsert_keys;
|
|
171
|
+
const data = { ...op.data };
|
|
172
|
+
delete data._upsert_keys;
|
|
173
|
+
delete data._upsert_fallback_keys;
|
|
174
|
+
delete data._upsert_fallback_set;
|
|
175
|
+
data._block_height = this.block.height;
|
|
176
|
+
data._tx_id = this._tx.txId;
|
|
177
|
+
data._created_at = "NOW()";
|
|
178
|
+
const cols = Object.keys(data);
|
|
179
|
+
cols.forEach(validateColumnName);
|
|
180
|
+
const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
|
|
181
|
+
const batchKey = `${op.table}:${[...cols].sort().join(",")}:${upsertKeys ? [...upsertKeys].sort().join(",") : ""}`;
|
|
182
|
+
return { data, cols, vals, upsertKeys, batchKey };
|
|
183
|
+
}
|
|
169
184
|
buildStatements(ops) {
|
|
170
185
|
const statements = [];
|
|
186
|
+
let currentBatch = null;
|
|
187
|
+
let currentBatchKey = "";
|
|
188
|
+
const flushInsertBatch = () => {
|
|
189
|
+
if (!currentBatch)
|
|
190
|
+
return;
|
|
191
|
+
const qualifiedTable = `"${this.pgSchemaName}"."${currentBatch.table}"`;
|
|
192
|
+
const colList = currentBatch.cols.map((c) => `"${c}"`).join(", ");
|
|
193
|
+
let rows = currentBatch.rows;
|
|
194
|
+
if (currentBatch.upsertKeys && currentBatch.upsertKeys.length > 0) {
|
|
195
|
+
const keyIndices = currentBatch.upsertKeys.map((k) => currentBatch.cols.indexOf(k));
|
|
196
|
+
const seen = new Map;
|
|
197
|
+
for (let i = 0;i < rows.length; i++) {
|
|
198
|
+
const key = keyIndices.map((ki) => rows[i][ki]).join("\x00");
|
|
199
|
+
seen.set(key, i);
|
|
200
|
+
}
|
|
201
|
+
if (seen.size < rows.length) {
|
|
202
|
+
rows = Array.from(seen.values()).map((i) => rows[i]);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
const valuesList = rows.map((r) => `(${r.join(", ")})`).join(", ");
|
|
206
|
+
let stmt = `INSERT INTO ${qualifiedTable} (${colList}) VALUES ${valuesList}`;
|
|
207
|
+
if (currentBatch.upsertKeys && currentBatch.upsertKeys.length > 0) {
|
|
208
|
+
const updateCols = currentBatch.cols.filter((c) => !currentBatch.upsertKeys.includes(c) && !c.startsWith("_"));
|
|
209
|
+
if (updateCols.length > 0) {
|
|
210
|
+
const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
|
|
211
|
+
stmt += ` ON CONFLICT (${currentBatch.upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
|
|
212
|
+
} else {
|
|
213
|
+
stmt += ` ON CONFLICT (${currentBatch.upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
statements.push(stmt);
|
|
217
|
+
currentBatch = null;
|
|
218
|
+
currentBatchKey = "";
|
|
219
|
+
};
|
|
171
220
|
for (const op of ops) {
|
|
172
221
|
const qualifiedTable = `"${this.pgSchemaName}"."${op.table}"`;
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
delete data._upsert_fallback_set;
|
|
182
|
-
data._block_height = this.block.height;
|
|
183
|
-
data._tx_id = this._tx.txId;
|
|
184
|
-
data._created_at = "NOW()";
|
|
185
|
-
const cols = Object.keys(data);
|
|
186
|
-
cols.forEach(validateColumnName);
|
|
187
|
-
const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
|
|
188
|
-
let stmt = `INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) VALUES (${vals.join(", ")})`;
|
|
189
|
-
if (upsertKeys && upsertKeys.length > 0) {
|
|
190
|
-
const updateCols = cols.filter((c) => !upsertKeys.includes(c) && !c.startsWith("_"));
|
|
191
|
-
if (updateCols.length > 0) {
|
|
192
|
-
const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
|
|
193
|
-
stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
|
|
194
|
-
} else {
|
|
195
|
-
stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
|
|
196
|
-
}
|
|
197
|
-
} else if (fallbackKeys && fallbackSet) {}
|
|
198
|
-
statements.push(stmt);
|
|
199
|
-
break;
|
|
222
|
+
if (op.kind === "insert") {
|
|
223
|
+
const { cols, vals, upsertKeys, batchKey } = this.prepareInsert(op);
|
|
224
|
+
if (batchKey === currentBatchKey && currentBatch) {
|
|
225
|
+
currentBatch.rows.push(vals);
|
|
226
|
+
} else {
|
|
227
|
+
flushInsertBatch();
|
|
228
|
+
currentBatch = { table: op.table, cols, rows: [vals], upsertKeys };
|
|
229
|
+
currentBatchKey = batchKey;
|
|
200
230
|
}
|
|
201
|
-
|
|
231
|
+
} else {
|
|
232
|
+
flushInsertBatch();
|
|
233
|
+
if (op.kind === "update") {
|
|
202
234
|
const setEntries = Object.entries(op.set);
|
|
203
235
|
setEntries.forEach(([k]) => validateColumnName(k));
|
|
204
236
|
const setClauses = setEntries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
|
|
205
237
|
const { clause } = buildWhereClause(op.data);
|
|
206
238
|
statements.push(`UPDATE ${qualifiedTable} SET ${setClauses.join(", ")} WHERE ${clause}`);
|
|
207
|
-
|
|
208
|
-
}
|
|
209
|
-
case "delete": {
|
|
239
|
+
} else if (op.kind === "delete") {
|
|
210
240
|
const { clause } = buildWhereClause(op.data);
|
|
211
241
|
statements.push(`DELETE FROM ${qualifiedTable} WHERE ${clause}`);
|
|
212
|
-
break;
|
|
213
242
|
}
|
|
214
243
|
}
|
|
215
244
|
}
|
|
245
|
+
flushInsertBatch();
|
|
216
246
|
return statements;
|
|
217
247
|
}
|
|
218
248
|
validateTable(table) {
|
|
@@ -462,6 +492,7 @@ import { updateSubgraphStatus, recordSubgraphProcessed } from "@secondlayer/shar
|
|
|
462
492
|
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
463
493
|
|
|
464
494
|
// src/runtime/block-processor.ts
|
|
495
|
+
var schemaNameCache = new Map;
|
|
465
496
|
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
466
497
|
const db = getDb();
|
|
467
498
|
const blockStart = performance.now();
|
|
@@ -472,19 +503,26 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
472
503
|
errors: 0,
|
|
473
504
|
skipped: false
|
|
474
505
|
};
|
|
475
|
-
|
|
476
|
-
if (
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
506
|
+
let block, txs, evts;
|
|
507
|
+
if (opts?.preloaded) {
|
|
508
|
+
block = opts.preloaded.block;
|
|
509
|
+
txs = opts.preloaded.txs;
|
|
510
|
+
evts = opts.preloaded.events;
|
|
511
|
+
} else {
|
|
512
|
+
block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
|
|
513
|
+
if (!block) {
|
|
514
|
+
logger3.warn("Block not found for subgraph processing", { subgraph: subgraphName, blockHeight });
|
|
515
|
+
result.skipped = true;
|
|
516
|
+
return result;
|
|
517
|
+
}
|
|
518
|
+
if (!block.canonical) {
|
|
519
|
+
logger3.debug("Skipping non-canonical block", { subgraph: subgraphName, blockHeight });
|
|
520
|
+
result.skipped = true;
|
|
521
|
+
return result;
|
|
522
|
+
}
|
|
523
|
+
txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
|
|
524
|
+
evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
|
|
485
525
|
}
|
|
486
|
-
const txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
|
|
487
|
-
const evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
|
|
488
526
|
const matched = matchSources(subgraph.sources, txs, evts);
|
|
489
527
|
result.matched = matched.length;
|
|
490
528
|
if (matched.length === 0) {
|
|
@@ -493,8 +531,12 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
493
531
|
}
|
|
494
532
|
return result;
|
|
495
533
|
}
|
|
496
|
-
|
|
497
|
-
|
|
534
|
+
let schemaName = schemaNameCache.get(subgraphName);
|
|
535
|
+
if (!schemaName) {
|
|
536
|
+
const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
|
|
537
|
+
schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
|
|
538
|
+
schemaNameCache.set(subgraphName, schemaName);
|
|
539
|
+
}
|
|
498
540
|
const blockMeta = {
|
|
499
541
|
height: block.height,
|
|
500
542
|
hash: block.hash,
|
|
@@ -709,45 +751,96 @@ function generateSubgraphSQL(def, schemaNameOverride) {
|
|
|
709
751
|
return { statements, hash };
|
|
710
752
|
}
|
|
711
753
|
|
|
754
|
+
// src/runtime/batch-loader.ts
|
|
755
|
+
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
756
|
+
const [blocks, txs, events] = await Promise.all([
|
|
757
|
+
db.selectFrom("blocks").selectAll().where("height", ">=", fromHeight).where("height", "<=", toHeight).where("canonical", "=", true).execute(),
|
|
758
|
+
db.selectFrom("transactions").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute(),
|
|
759
|
+
db.selectFrom("events").selectAll().where("block_height", ">=", fromHeight).where("block_height", "<=", toHeight).execute()
|
|
760
|
+
]);
|
|
761
|
+
const txsByHeight = new Map;
|
|
762
|
+
for (const tx of txs) {
|
|
763
|
+
const h = Number(tx.block_height);
|
|
764
|
+
const list = txsByHeight.get(h) ?? [];
|
|
765
|
+
list.push(tx);
|
|
766
|
+
txsByHeight.set(h, list);
|
|
767
|
+
}
|
|
768
|
+
const eventsByHeight = new Map;
|
|
769
|
+
for (const evt of events) {
|
|
770
|
+
const h = Number(evt.block_height);
|
|
771
|
+
const list = eventsByHeight.get(h) ?? [];
|
|
772
|
+
list.push(evt);
|
|
773
|
+
eventsByHeight.set(h, list);
|
|
774
|
+
}
|
|
775
|
+
const result = new Map;
|
|
776
|
+
for (const block of blocks) {
|
|
777
|
+
const h = Number(block.height);
|
|
778
|
+
result.set(h, {
|
|
779
|
+
block,
|
|
780
|
+
txs: txsByHeight.get(h) ?? [],
|
|
781
|
+
events: eventsByHeight.get(h) ?? []
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
return result;
|
|
785
|
+
}
|
|
786
|
+
function avgEventsPerBlock(batch) {
|
|
787
|
+
if (batch.size === 0)
|
|
788
|
+
return 0;
|
|
789
|
+
let totalEvents = 0;
|
|
790
|
+
for (const data of batch.values()) {
|
|
791
|
+
totalEvents += data.events.length;
|
|
792
|
+
}
|
|
793
|
+
return totalEvents / batch.size;
|
|
794
|
+
}
|
|
795
|
+
|
|
712
796
|
// src/runtime/reindex.ts
|
|
713
797
|
var LOG_INTERVAL = 1000;
|
|
714
|
-
|
|
798
|
+
var DEFAULT_BATCH_SIZE = 500;
|
|
799
|
+
var MIN_BATCH_SIZE = 100;
|
|
800
|
+
var MAX_BATCH_SIZE = 1000;
|
|
801
|
+
async function processBlockRange(def, opts) {
|
|
715
802
|
const db = getDb2();
|
|
716
|
-
const client = getRawClient();
|
|
717
803
|
const subgraphName = def.name;
|
|
718
|
-
const
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
804
|
+
const { fromBlock, toBlock, status } = opts;
|
|
805
|
+
const totalBlocks = toBlock - fromBlock + 1;
|
|
806
|
+
const stats = new StatsAccumulator(subgraphName, opts.apiKeyId, opts.isCatchup);
|
|
807
|
+
let blocksProcessed = 0;
|
|
808
|
+
let totalEventsProcessed = 0;
|
|
809
|
+
let totalErrors = 0;
|
|
810
|
+
let batchSize = DEFAULT_BATCH_SIZE;
|
|
811
|
+
let currentHeight = fromBlock;
|
|
812
|
+
let nextBatchPromise = loadBlockRange(db, currentHeight, Math.min(currentHeight + batchSize - 1, toBlock));
|
|
813
|
+
while (currentHeight <= toBlock) {
|
|
814
|
+
const batch = await nextBatchPromise;
|
|
815
|
+
const batchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
816
|
+
const nextStart = batchEnd + 1;
|
|
817
|
+
if (nextStart <= toBlock) {
|
|
818
|
+
const nextEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
819
|
+
nextBatchPromise = loadBlockRange(db, nextStart, nextEnd);
|
|
726
820
|
}
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
const result = await processBlock(def, subgraphName, height, { skipProgressUpdate: true });
|
|
821
|
+
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
822
|
+
const blockData = batch.get(height);
|
|
823
|
+
if (!blockData) {
|
|
824
|
+
blocksProcessed++;
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
let result;
|
|
828
|
+
try {
|
|
829
|
+
result = await processBlock(def, subgraphName, height, {
|
|
830
|
+
skipProgressUpdate: true,
|
|
831
|
+
preloaded: blockData
|
|
832
|
+
});
|
|
833
|
+
} catch (err) {
|
|
834
|
+
logger4.error("Block processing error", {
|
|
835
|
+
subgraph: subgraphName,
|
|
836
|
+
blockHeight: height,
|
|
837
|
+
error: err instanceof Error ? err.message : String(err)
|
|
838
|
+
});
|
|
839
|
+
await updateSubgraphStatus2(db, subgraphName, status, height).catch(() => {});
|
|
840
|
+
blocksProcessed++;
|
|
841
|
+
totalErrors++;
|
|
842
|
+
continue;
|
|
843
|
+
}
|
|
751
844
|
blocksProcessed++;
|
|
752
845
|
totalEventsProcessed += result.processed;
|
|
753
846
|
totalErrors += result.errors;
|
|
@@ -757,11 +850,11 @@ async function reindexSubgraph(def, opts) {
|
|
|
757
850
|
await stats.flush(db);
|
|
758
851
|
}
|
|
759
852
|
}
|
|
760
|
-
if (blocksProcessed %
|
|
761
|
-
await updateSubgraphStatus2(db, subgraphName,
|
|
853
|
+
if (blocksProcessed % 100 === 0) {
|
|
854
|
+
await updateSubgraphStatus2(db, subgraphName, status, height);
|
|
762
855
|
}
|
|
763
856
|
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
764
|
-
logger4.info("Reindex
|
|
857
|
+
logger4.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
765
858
|
subgraph: subgraphName,
|
|
766
859
|
processed: blocksProcessed,
|
|
767
860
|
total: totalBlocks,
|
|
@@ -770,23 +863,93 @@ async function reindexSubgraph(def, opts) {
|
|
|
770
863
|
});
|
|
771
864
|
}
|
|
772
865
|
}
|
|
773
|
-
|
|
866
|
+
const avg = avgEventsPerBlock(batch);
|
|
867
|
+
if (avg > 50)
|
|
868
|
+
batchSize = Math.max(Math.round(batchSize * 0.5), MIN_BATCH_SIZE);
|
|
869
|
+
else if (avg < 10)
|
|
870
|
+
batchSize = Math.min(Math.round(batchSize * 1.5), MAX_BATCH_SIZE);
|
|
871
|
+
currentHeight = batchEnd + 1;
|
|
872
|
+
}
|
|
873
|
+
await stats.flush(db);
|
|
874
|
+
return { blocksProcessed, totalEventsProcessed, totalErrors };
|
|
875
|
+
}
|
|
876
|
+
async function resolveBlockRange(db, opts) {
|
|
877
|
+
const fromBlock = opts?.fromBlock ?? 1;
|
|
878
|
+
let toBlock;
|
|
879
|
+
if (opts?.toBlock != null) {
|
|
880
|
+
toBlock = opts.toBlock;
|
|
881
|
+
} else {
|
|
882
|
+
const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
883
|
+
toBlock = progress?.last_indexed_block ?? progress?.last_contiguous_block ?? 0;
|
|
884
|
+
}
|
|
885
|
+
return { fromBlock, toBlock };
|
|
886
|
+
}
|
|
887
|
+
async function reindexSubgraph(def, opts) {
|
|
888
|
+
const db = getDb2();
|
|
889
|
+
const client = getRawClient();
|
|
890
|
+
const subgraphName = def.name;
|
|
891
|
+
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
892
|
+
await updateSubgraphStatus2(db, subgraphName, "reindexing");
|
|
893
|
+
logger4.info("Reindex starting", { subgraph: subgraphName });
|
|
894
|
+
try {
|
|
895
|
+
await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
|
|
896
|
+
const { statements } = generateSubgraphSQL(def, schemaName);
|
|
897
|
+
for (const stmt of statements) {
|
|
898
|
+
await client.unsafe(stmt);
|
|
899
|
+
}
|
|
900
|
+
logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
901
|
+
const { fromBlock, toBlock } = await resolveBlockRange(db, opts);
|
|
902
|
+
if (fromBlock > toBlock) {
|
|
903
|
+
logger4.info("No blocks to reindex", { subgraph: subgraphName, fromBlock, toBlock });
|
|
904
|
+
await updateSubgraphStatus2(db, subgraphName, "active", 0);
|
|
905
|
+
return { processed: 0 };
|
|
906
|
+
}
|
|
907
|
+
logger4.info("Reindexing blocks", { subgraph: subgraphName, fromBlock, toBlock, totalBlocks: toBlock - fromBlock + 1 });
|
|
908
|
+
const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
|
|
909
|
+
const result = await processBlockRange(def, {
|
|
910
|
+
fromBlock,
|
|
911
|
+
toBlock,
|
|
912
|
+
status: "reindexing",
|
|
913
|
+
isCatchup: false,
|
|
914
|
+
apiKeyId: subgraphRow?.api_key_id ?? null
|
|
915
|
+
});
|
|
774
916
|
const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
775
|
-
if (totalEventsProcessed > 0 || totalErrors > 0) {
|
|
776
|
-
await recordSubgraphProcessed2(db, subgraphName, totalEventsProcessed, totalErrors, totalErrors > 0 ? `${totalErrors} error(s) during reindex` : undefined);
|
|
917
|
+
if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
|
|
918
|
+
await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
|
|
777
919
|
}
|
|
778
920
|
await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
|
|
779
|
-
logger4.info("Reindex complete", { subgraph: subgraphName, blocks: blocksProcessed, events: totalEventsProcessed, errors: totalErrors });
|
|
780
|
-
return { processed: blocksProcessed };
|
|
921
|
+
logger4.info("Reindex complete", { subgraph: subgraphName, blocks: result.blocksProcessed, events: result.totalEventsProcessed, errors: result.totalErrors });
|
|
922
|
+
return { processed: result.blocksProcessed };
|
|
781
923
|
} catch (err) {
|
|
782
|
-
logger4.error("Reindex failed", {
|
|
783
|
-
subgraph: subgraphName,
|
|
784
|
-
error: getErrorMessage2(err)
|
|
785
|
-
});
|
|
924
|
+
logger4.error("Reindex failed", { subgraph: subgraphName, error: getErrorMessage2(err) });
|
|
786
925
|
await updateSubgraphStatus2(db, subgraphName, "error");
|
|
787
926
|
throw err;
|
|
788
927
|
}
|
|
789
928
|
}
|
|
929
|
+
async function backfillSubgraph(def, opts) {
|
|
930
|
+
const db = getDb2();
|
|
931
|
+
const subgraphName = def.name;
|
|
932
|
+
logger4.info("Backfill starting", { subgraph: subgraphName, from: opts.fromBlock, to: opts.toBlock });
|
|
933
|
+
try {
|
|
934
|
+
const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
|
|
935
|
+
const result = await processBlockRange(def, {
|
|
936
|
+
fromBlock: opts.fromBlock,
|
|
937
|
+
toBlock: opts.toBlock,
|
|
938
|
+
status: "active",
|
|
939
|
+
isCatchup: false,
|
|
940
|
+
apiKeyId: subgraphRow?.api_key_id ?? null
|
|
941
|
+
});
|
|
942
|
+
const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
943
|
+
if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
|
|
944
|
+
await recordSubgraphProcessed2(db, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
|
|
945
|
+
}
|
|
946
|
+
logger4.info("Backfill complete", { subgraph: subgraphName, blocks: result.blocksProcessed, events: result.totalEventsProcessed, errors: result.totalErrors });
|
|
947
|
+
return { processed: result.blocksProcessed };
|
|
948
|
+
} catch (err) {
|
|
949
|
+
logger4.error("Backfill failed", { subgraph: subgraphName, error: getErrorMessage2(err) });
|
|
950
|
+
throw err;
|
|
951
|
+
}
|
|
952
|
+
}
|
|
790
953
|
// src/define.ts
|
|
791
954
|
function defineSubgraph(def) {
|
|
792
955
|
return def;
|
|
@@ -961,8 +1124,9 @@ export {
|
|
|
961
1124
|
generateSubgraphSQL,
|
|
962
1125
|
diffSchema,
|
|
963
1126
|
deploySchema,
|
|
964
|
-
defineSubgraph
|
|
1127
|
+
defineSubgraph,
|
|
1128
|
+
backfillSubgraph
|
|
965
1129
|
};
|
|
966
1130
|
|
|
967
|
-
//# debugId=
|
|
1131
|
+
//# debugId=FAC592B26F4F381E64756E2164756E21
|
|
968
1132
|
//# sourceMappingURL=index.js.map
|