@secondlayer/subgraphs 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +219 -0
- package/dist/src/index.js +956 -0
- package/dist/src/index.js.map +22 -0
- package/dist/src/runtime/block-processor.d.ts +100 -0
- package/dist/src/runtime/block-processor.js +501 -0
- package/dist/src/runtime/block-processor.js.map +16 -0
- package/dist/src/runtime/catchup.d.ts +78 -0
- package/dist/src/runtime/catchup.js +630 -0
- package/dist/src/runtime/catchup.js.map +18 -0
- package/dist/src/runtime/clarity.d.ts +15 -0
- package/dist/src/runtime/clarity.js +41 -0
- package/dist/src/runtime/clarity.js.map +10 -0
- package/dist/src/runtime/context.d.ts +69 -0
- package/dist/src/runtime/context.js +177 -0
- package/dist/src/runtime/context.js.map +10 -0
- package/dist/src/runtime/processor.d.ts +8 -0
- package/dist/src/runtime/processor.js +770 -0
- package/dist/src/runtime/processor.js.map +20 -0
- package/dist/src/runtime/reindex.d.ts +84 -0
- package/dist/src/runtime/reindex.js +738 -0
- package/dist/src/runtime/reindex.js.map +19 -0
- package/dist/src/runtime/reorg.d.ts +78 -0
- package/dist/src/runtime/reorg.js +536 -0
- package/dist/src/runtime/reorg.js.map +17 -0
- package/dist/src/runtime/runner.d.ts +147 -0
- package/dist/src/runtime/runner.js +130 -0
- package/dist/src/runtime/runner.js.map +11 -0
- package/dist/src/runtime/source-matcher.d.ts +53 -0
- package/dist/src/runtime/source-matcher.js +111 -0
- package/dist/src/runtime/source-matcher.js.map +11 -0
- package/dist/src/runtime/stats.d.ts +24 -0
- package/dist/src/runtime/stats.js +75 -0
- package/dist/src/runtime/stats.js.map +10 -0
- package/dist/src/schema/index.d.ts +117 -0
- package/dist/src/schema/index.js +305 -0
- package/dist/src/schema/index.js.map +13 -0
- package/dist/src/service.d.ts +0 -0
- package/dist/src/service.js +780 -0
- package/dist/src/service.js.map +21 -0
- package/dist/src/types.d.ts +80 -0
- package/dist/src/types.js +22 -0
- package/dist/src/types.js.map +10 -0
- package/dist/src/validate.d.ts +84 -0
- package/dist/src/validate.js +59 -0
- package/dist/src/validate.js.map +10 -0
- package/package.json +49 -0
|
@@ -0,0 +1,738 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
3
|
+
|
|
4
|
+
// src/types.ts
|
|
5
|
+
function sourceKey(source) {
|
|
6
|
+
if (source.contract) {
|
|
7
|
+
if (source.function)
|
|
8
|
+
return `${source.contract}::${source.function}`;
|
|
9
|
+
if (source.event)
|
|
10
|
+
return `${source.contract}::${source.event}`;
|
|
11
|
+
return source.contract;
|
|
12
|
+
}
|
|
13
|
+
if (source.type)
|
|
14
|
+
return source.type;
|
|
15
|
+
return "*";
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// src/runtime/context.ts
|
|
19
|
+
import { sql } from "kysely";
|
|
20
|
+
import { logger } from "@secondlayer/shared/logger";
|
|
21
|
+
function validateColumnName(name) {
|
|
22
|
+
if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
|
|
23
|
+
throw new Error(`Invalid column name: ${name}`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
class SubgraphContext {
|
|
28
|
+
block;
|
|
29
|
+
_tx;
|
|
30
|
+
db;
|
|
31
|
+
pgSchemaName;
|
|
32
|
+
subgraphSchema;
|
|
33
|
+
ops = [];
|
|
34
|
+
constructor(db, pgSchemaName, subgraphSchema, block, tx) {
|
|
35
|
+
this.db = db;
|
|
36
|
+
this.pgSchemaName = pgSchemaName;
|
|
37
|
+
this.subgraphSchema = subgraphSchema;
|
|
38
|
+
this.block = block;
|
|
39
|
+
this._tx = tx;
|
|
40
|
+
}
|
|
41
|
+
get tx() {
|
|
42
|
+
return this._tx;
|
|
43
|
+
}
|
|
44
|
+
setTx(tx) {
|
|
45
|
+
this._tx = tx;
|
|
46
|
+
}
|
|
47
|
+
insert(table, row) {
|
|
48
|
+
this.validateTable(table);
|
|
49
|
+
this.ops.push({ kind: "insert", table, data: row });
|
|
50
|
+
}
|
|
51
|
+
update(table, where, set) {
|
|
52
|
+
this.validateTable(table);
|
|
53
|
+
this.ops.push({ kind: "update", table, data: where, set });
|
|
54
|
+
}
|
|
55
|
+
upsert(table, key, row) {
|
|
56
|
+
this.validateTable(table);
|
|
57
|
+
const tableDef = this.subgraphSchema[table];
|
|
58
|
+
const keyColumns = Object.keys(key);
|
|
59
|
+
const hasUniqueConstraint = tableDef.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
|
|
60
|
+
if (hasUniqueConstraint) {
|
|
61
|
+
this.ops.push({ kind: "insert", table, data: { ...key, ...row, _upsert_keys: keyColumns } });
|
|
62
|
+
} else {
|
|
63
|
+
logger.warn("upsert called without matching uniqueKeys constraint, using fallback", {
|
|
64
|
+
table,
|
|
65
|
+
keys: keyColumns
|
|
66
|
+
});
|
|
67
|
+
this.ops.push({ kind: "insert", table, data: { ...key, ...row, _upsert_fallback_keys: keyColumns, _upsert_fallback_set: row } });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
delete(table, where) {
|
|
71
|
+
this.validateTable(table);
|
|
72
|
+
this.ops.push({ kind: "delete", table, data: where });
|
|
73
|
+
}
|
|
74
|
+
async findOne(table, where) {
|
|
75
|
+
this.validateTable(table);
|
|
76
|
+
const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
|
|
77
|
+
const { clause } = buildWhereClause(where);
|
|
78
|
+
const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause} LIMIT 1`;
|
|
79
|
+
const { rows } = await sql.raw(query).execute(this.db);
|
|
80
|
+
return rows[0] ?? null;
|
|
81
|
+
}
|
|
82
|
+
async findMany(table, where) {
|
|
83
|
+
this.validateTable(table);
|
|
84
|
+
const qualifiedTable = `"${this.pgSchemaName}"."${table}"`;
|
|
85
|
+
const { clause } = buildWhereClause(where);
|
|
86
|
+
const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause}`;
|
|
87
|
+
const { rows } = await sql.raw(query).execute(this.db);
|
|
88
|
+
return rows;
|
|
89
|
+
}
|
|
90
|
+
get pendingOps() {
|
|
91
|
+
return this.ops.length;
|
|
92
|
+
}
|
|
93
|
+
async flush() {
|
|
94
|
+
if (this.ops.length === 0)
|
|
95
|
+
return 0;
|
|
96
|
+
const opsToFlush = [...this.ops];
|
|
97
|
+
this.ops.length = 0;
|
|
98
|
+
const statements = this.buildStatements(opsToFlush);
|
|
99
|
+
if ("isTransaction" in this.db) {
|
|
100
|
+
for (const stmt of statements) {
|
|
101
|
+
await sql.raw(stmt).execute(this.db);
|
|
102
|
+
}
|
|
103
|
+
} else {
|
|
104
|
+
await this.db.transaction().execute(async (tx) => {
|
|
105
|
+
for (const stmt of statements) {
|
|
106
|
+
await sql.raw(stmt).execute(tx);
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
return opsToFlush.length;
|
|
111
|
+
}
|
|
112
|
+
buildStatements(ops) {
|
|
113
|
+
const statements = [];
|
|
114
|
+
for (const op of ops) {
|
|
115
|
+
const qualifiedTable = `"${this.pgSchemaName}"."${op.table}"`;
|
|
116
|
+
switch (op.kind) {
|
|
117
|
+
case "insert": {
|
|
118
|
+
const upsertKeys = op.data._upsert_keys;
|
|
119
|
+
const fallbackKeys = op.data._upsert_fallback_keys;
|
|
120
|
+
const fallbackSet = op.data._upsert_fallback_set;
|
|
121
|
+
const data = { ...op.data };
|
|
122
|
+
delete data._upsert_keys;
|
|
123
|
+
delete data._upsert_fallback_keys;
|
|
124
|
+
delete data._upsert_fallback_set;
|
|
125
|
+
data._block_height = this.block.height;
|
|
126
|
+
data._tx_id = this._tx.txId;
|
|
127
|
+
data._created_at = "NOW()";
|
|
128
|
+
const cols = Object.keys(data);
|
|
129
|
+
cols.forEach(validateColumnName);
|
|
130
|
+
const vals = cols.map((c) => data[c] === "NOW()" ? "NOW()" : escapeLiteral(data[c]));
|
|
131
|
+
let stmt = `INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) VALUES (${vals.join(", ")})`;
|
|
132
|
+
if (upsertKeys && upsertKeys.length > 0) {
|
|
133
|
+
const updateCols = cols.filter((c) => !upsertKeys.includes(c) && !c.startsWith("_"));
|
|
134
|
+
if (updateCols.length > 0) {
|
|
135
|
+
const setClauses = updateCols.map((c) => `"${c}" = EXCLUDED."${c}"`);
|
|
136
|
+
stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`;
|
|
137
|
+
} else {
|
|
138
|
+
stmt += ` ON CONFLICT (${upsertKeys.map((k) => `"${k}"`).join(", ")}) DO NOTHING`;
|
|
139
|
+
}
|
|
140
|
+
} else if (fallbackKeys && fallbackSet) {}
|
|
141
|
+
statements.push(stmt);
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
case "update": {
|
|
145
|
+
const setEntries = Object.entries(op.set);
|
|
146
|
+
setEntries.forEach(([k]) => validateColumnName(k));
|
|
147
|
+
const setClauses = setEntries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
|
|
148
|
+
const { clause } = buildWhereClause(op.data);
|
|
149
|
+
statements.push(`UPDATE ${qualifiedTable} SET ${setClauses.join(", ")} WHERE ${clause}`);
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
case "delete": {
|
|
153
|
+
const { clause } = buildWhereClause(op.data);
|
|
154
|
+
statements.push(`DELETE FROM ${qualifiedTable} WHERE ${clause}`);
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return statements;
|
|
160
|
+
}
|
|
161
|
+
validateTable(table) {
|
|
162
|
+
if (!this.subgraphSchema[table]) {
|
|
163
|
+
throw new Error(`Table "${table}" not found in subgraph schema. Available: [${Object.keys(this.subgraphSchema).join(", ")}]`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
function escapeLiteral(value) {
|
|
168
|
+
if (value === null || value === undefined)
|
|
169
|
+
return "NULL";
|
|
170
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
171
|
+
return String(value);
|
|
172
|
+
if (typeof value === "boolean")
|
|
173
|
+
return value ? "TRUE" : "FALSE";
|
|
174
|
+
if (typeof value === "object")
|
|
175
|
+
return `'${JSON.stringify(value).replace(/'/g, "''")}'::jsonb`;
|
|
176
|
+
return `'${String(value).replace(/'/g, "''")}'`;
|
|
177
|
+
}
|
|
178
|
+
function buildWhereClause(where) {
|
|
179
|
+
const entries = Object.entries(where);
|
|
180
|
+
if (entries.length === 0)
|
|
181
|
+
return { clause: "TRUE", values: [] };
|
|
182
|
+
entries.forEach(([k]) => validateColumnName(k));
|
|
183
|
+
const parts = entries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
|
|
184
|
+
return { clause: parts.join(" AND "), values: [] };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// src/runtime/source-matcher.ts
|
|
188
|
+
function matchPattern(value, pattern) {
|
|
189
|
+
if (!pattern.includes("*"))
|
|
190
|
+
return value === pattern;
|
|
191
|
+
const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
192
|
+
return new RegExp(`^${regex}$`).test(value);
|
|
193
|
+
}
|
|
194
|
+
function matchSource(source, transactions, eventsByTx) {
|
|
195
|
+
const results = [];
|
|
196
|
+
const key = sourceKey(source);
|
|
197
|
+
for (const tx of transactions) {
|
|
198
|
+
if (source.type) {
|
|
199
|
+
if (!matchPattern(tx.type, source.type))
|
|
200
|
+
continue;
|
|
201
|
+
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
202
|
+
let matchedEvents = txEvents;
|
|
203
|
+
if (source.minAmount !== undefined) {
|
|
204
|
+
const amountEvents = matchedEvents.filter((e) => {
|
|
205
|
+
const data = e.data;
|
|
206
|
+
const rawAmount = data?.amount;
|
|
207
|
+
if (rawAmount === undefined)
|
|
208
|
+
return false;
|
|
209
|
+
const amount = BigInt(rawAmount);
|
|
210
|
+
return amount >= source.minAmount;
|
|
211
|
+
});
|
|
212
|
+
if (amountEvents.length === 0)
|
|
213
|
+
continue;
|
|
214
|
+
matchedEvents = amountEvents;
|
|
215
|
+
}
|
|
216
|
+
results.push({ tx, events: matchedEvents, sourceKey: key });
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
if (source.contract) {
|
|
220
|
+
const txContractMatch = tx.contract_id && matchPattern(tx.contract_id, source.contract);
|
|
221
|
+
if (source.function && tx.function_name) {
|
|
222
|
+
if (!matchPattern(tx.function_name, source.function))
|
|
223
|
+
continue;
|
|
224
|
+
} else if (source.function && !tx.function_name) {
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
228
|
+
let matchedEvents = txEvents;
|
|
229
|
+
if (!txContractMatch) {
|
|
230
|
+
matchedEvents = txEvents.filter((e) => {
|
|
231
|
+
const data = e.data;
|
|
232
|
+
const contractIdentifier = data?.contract_identifier;
|
|
233
|
+
return contractIdentifier && matchPattern(contractIdentifier, source.contract);
|
|
234
|
+
});
|
|
235
|
+
if (matchedEvents.length === 0)
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (source.event) {
|
|
239
|
+
matchedEvents = matchedEvents.filter((e) => {
|
|
240
|
+
if (matchPattern(e.type, source.event))
|
|
241
|
+
return true;
|
|
242
|
+
const data = e.data;
|
|
243
|
+
const topic = data?.topic;
|
|
244
|
+
return topic ? matchPattern(topic, source.event) : false;
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
if (txContractMatch || matchedEvents.length > 0) {
|
|
248
|
+
results.push({ tx, events: matchedEvents, sourceKey: key });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return results;
|
|
253
|
+
}
|
|
254
|
+
function matchSources(sources, transactions, events) {
|
|
255
|
+
const eventsByTx = new Map;
|
|
256
|
+
for (const event of events) {
|
|
257
|
+
const list = eventsByTx.get(event.tx_id) ?? [];
|
|
258
|
+
list.push(event);
|
|
259
|
+
eventsByTx.set(event.tx_id, list);
|
|
260
|
+
}
|
|
261
|
+
const seen = new Set;
|
|
262
|
+
const results = [];
|
|
263
|
+
for (const source of sources) {
|
|
264
|
+
const matches = matchSource(source, transactions, eventsByTx);
|
|
265
|
+
for (const match of matches) {
|
|
266
|
+
const dedupeKey = `${match.tx.tx_id}:${match.sourceKey}`;
|
|
267
|
+
if (!seen.has(dedupeKey)) {
|
|
268
|
+
seen.add(dedupeKey);
|
|
269
|
+
results.push(match);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return results;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// src/runtime/clarity.ts
|
|
277
|
+
import { cvToJSON, deserializeCV } from "@secondlayer/stacks/clarity";
|
|
278
|
+
function decodeClarityValue(hex) {
|
|
279
|
+
try {
|
|
280
|
+
const cleanHex = hex.startsWith("0x") ? hex.slice(2) : hex;
|
|
281
|
+
const cv = deserializeCV(cleanHex);
|
|
282
|
+
return cvToJSON(cv);
|
|
283
|
+
} catch {
|
|
284
|
+
return hex;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function decodeEventData(data) {
|
|
288
|
+
if (typeof data === "string" && data.startsWith("0x") && data.length > 10) {
|
|
289
|
+
return decodeClarityValue(data);
|
|
290
|
+
}
|
|
291
|
+
if (Array.isArray(data)) {
|
|
292
|
+
return data.map(decodeEventData);
|
|
293
|
+
}
|
|
294
|
+
if (typeof data === "object" && data !== null) {
|
|
295
|
+
const decoded = {};
|
|
296
|
+
for (const [key, value] of Object.entries(data)) {
|
|
297
|
+
decoded[key] = decodeEventData(value);
|
|
298
|
+
}
|
|
299
|
+
return decoded;
|
|
300
|
+
}
|
|
301
|
+
return data;
|
|
302
|
+
}
|
|
303
|
+
function decodeFunctionArgs(args) {
|
|
304
|
+
return args.map(decodeClarityValue);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// src/runtime/runner.ts
|
|
308
|
+
import { logger as logger2 } from "@secondlayer/shared/logger";
|
|
309
|
+
import { getErrorMessage } from "@secondlayer/shared";
|
|
310
|
+
var DEFAULT_ERROR_THRESHOLD = 50;
|
|
311
|
+
function resolveHandler(handlers, key) {
|
|
312
|
+
return handlers[key] ?? handlers["*"] ?? null;
|
|
313
|
+
}
|
|
314
|
+
async function runHandlers(subgraph, matched, ctx, opts) {
|
|
315
|
+
let processed = 0;
|
|
316
|
+
let errors = 0;
|
|
317
|
+
const threshold = opts?.errorThreshold ?? DEFAULT_ERROR_THRESHOLD;
|
|
318
|
+
for (const { tx, events, sourceKey: sourceKey2 } of matched) {
|
|
319
|
+
const handler = resolveHandler(subgraph.handlers, sourceKey2);
|
|
320
|
+
if (!handler) {
|
|
321
|
+
logger2.warn("No handler found for source key", { subgraph: subgraph.name, sourceKey: sourceKey2, txId: tx.tx_id });
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
ctx.setTx({
|
|
325
|
+
txId: tx.tx_id,
|
|
326
|
+
sender: tx.sender,
|
|
327
|
+
type: tx.type,
|
|
328
|
+
status: tx.status
|
|
329
|
+
});
|
|
330
|
+
if (events.length === 0) {
|
|
331
|
+
try {
|
|
332
|
+
const txPayload = {
|
|
333
|
+
tx: {
|
|
334
|
+
txId: tx.tx_id,
|
|
335
|
+
sender: tx.sender,
|
|
336
|
+
type: tx.type,
|
|
337
|
+
status: tx.status,
|
|
338
|
+
contractId: tx.contract_id,
|
|
339
|
+
functionName: tx.function_name
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
await handler(txPayload, ctx);
|
|
343
|
+
processed++;
|
|
344
|
+
} catch (err) {
|
|
345
|
+
errors++;
|
|
346
|
+
logger2.error("Subgraph handler error", {
|
|
347
|
+
subgraph: subgraph.name,
|
|
348
|
+
sourceKey: sourceKey2,
|
|
349
|
+
txId: tx.tx_id,
|
|
350
|
+
error: getErrorMessage(err)
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
for (const event of events) {
|
|
356
|
+
if (errors >= threshold) {
|
|
357
|
+
logger2.error("Subgraph error threshold reached, skipping remaining events", {
|
|
358
|
+
subgraph: subgraph.name,
|
|
359
|
+
errors,
|
|
360
|
+
threshold
|
|
361
|
+
});
|
|
362
|
+
return { processed, errors };
|
|
363
|
+
}
|
|
364
|
+
try {
|
|
365
|
+
const decoded = decodeEventData(event.data);
|
|
366
|
+
const eventPayload = {
|
|
367
|
+
...decoded,
|
|
368
|
+
_eventId: event.id,
|
|
369
|
+
_eventType: event.type,
|
|
370
|
+
_eventIndex: event.event_index,
|
|
371
|
+
tx: {
|
|
372
|
+
txId: tx.tx_id,
|
|
373
|
+
sender: tx.sender,
|
|
374
|
+
type: tx.type,
|
|
375
|
+
status: tx.status,
|
|
376
|
+
contractId: tx.contract_id,
|
|
377
|
+
functionName: tx.function_name
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
await handler(eventPayload, ctx);
|
|
381
|
+
processed++;
|
|
382
|
+
} catch (err) {
|
|
383
|
+
errors++;
|
|
384
|
+
logger2.error("Subgraph handler error", {
|
|
385
|
+
subgraph: subgraph.name,
|
|
386
|
+
sourceKey: sourceKey2,
|
|
387
|
+
txId: tx.tx_id,
|
|
388
|
+
eventId: event.id,
|
|
389
|
+
eventType: event.type,
|
|
390
|
+
error: getErrorMessage(err)
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
return { processed, errors };
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// src/runtime/block-processor.ts
|
|
399
|
+
import { sql as sql2 } from "kysely";
|
|
400
|
+
import { getDb } from "@secondlayer/shared/db";
|
|
401
|
+
import { logger as logger3 } from "@secondlayer/shared/logger";
|
|
402
|
+
import { updateSubgraphStatus, recordSubgraphProcessed } from "@secondlayer/shared/db/queries/subgraphs";
|
|
403
|
+
|
|
404
|
+
// src/schema/utils.ts
|
|
405
|
+
import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
|
|
406
|
+
|
|
407
|
+
// src/runtime/block-processor.ts
|
|
408
|
+
async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
409
|
+
const db = getDb();
|
|
410
|
+
const blockStart = performance.now();
|
|
411
|
+
const result = {
|
|
412
|
+
blockHeight,
|
|
413
|
+
matched: 0,
|
|
414
|
+
processed: 0,
|
|
415
|
+
errors: 0,
|
|
416
|
+
skipped: false
|
|
417
|
+
};
|
|
418
|
+
const block = await db.selectFrom("blocks").selectAll().where("height", "=", blockHeight).executeTakeFirst();
|
|
419
|
+
if (!block) {
|
|
420
|
+
logger3.warn("Block not found for subgraph processing", { subgraph: subgraphName, blockHeight });
|
|
421
|
+
result.skipped = true;
|
|
422
|
+
return result;
|
|
423
|
+
}
|
|
424
|
+
if (!block.canonical) {
|
|
425
|
+
logger3.debug("Skipping non-canonical block", { subgraph: subgraphName, blockHeight });
|
|
426
|
+
result.skipped = true;
|
|
427
|
+
return result;
|
|
428
|
+
}
|
|
429
|
+
const txs = await db.selectFrom("transactions").selectAll().where("block_height", "=", blockHeight).execute();
|
|
430
|
+
const evts = await db.selectFrom("events").selectAll().where("block_height", "=", blockHeight).execute();
|
|
431
|
+
const matched = matchSources(subgraph.sources, txs, evts);
|
|
432
|
+
result.matched = matched.length;
|
|
433
|
+
if (matched.length === 0) {
|
|
434
|
+
if (!opts?.skipProgressUpdate) {
|
|
435
|
+
await updateSubgraphStatus(db, subgraphName, "active", blockHeight);
|
|
436
|
+
}
|
|
437
|
+
return result;
|
|
438
|
+
}
|
|
439
|
+
const subgraphRecord = await db.selectFrom("subgraphs").select("schema_name").where("name", "=", subgraphName).executeTakeFirst();
|
|
440
|
+
const schemaName = subgraphRecord?.schema_name ?? pgSchemaName(subgraphName);
|
|
441
|
+
const blockMeta = {
|
|
442
|
+
height: block.height,
|
|
443
|
+
hash: block.hash,
|
|
444
|
+
timestamp: block.timestamp,
|
|
445
|
+
burnBlockHeight: block.burn_block_height
|
|
446
|
+
};
|
|
447
|
+
const initialTx = {
|
|
448
|
+
txId: "",
|
|
449
|
+
sender: "",
|
|
450
|
+
type: "",
|
|
451
|
+
status: ""
|
|
452
|
+
};
|
|
453
|
+
let handlerMs = 0;
|
|
454
|
+
let flushMs = 0;
|
|
455
|
+
await db.transaction().execute(async (tx) => {
|
|
456
|
+
const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx);
|
|
457
|
+
const handlerStart = performance.now();
|
|
458
|
+
const runResult = await runHandlers(subgraph, matched, ctx);
|
|
459
|
+
handlerMs = performance.now() - handlerStart;
|
|
460
|
+
result.processed = runResult.processed;
|
|
461
|
+
result.errors = runResult.errors;
|
|
462
|
+
if (ctx.pendingOps > 0) {
|
|
463
|
+
const flushStart = performance.now();
|
|
464
|
+
await ctx.flush();
|
|
465
|
+
flushMs = performance.now() - flushStart;
|
|
466
|
+
}
|
|
467
|
+
if (!opts?.skipProgressUpdate) {
|
|
468
|
+
const status = runResult.errors > 0 && runResult.processed === 0 ? "error" : "active";
|
|
469
|
+
await updateSubgraphStatus(tx, subgraphName, status, blockHeight);
|
|
470
|
+
}
|
|
471
|
+
if (!opts?.skipProgressUpdate && (runResult.processed > 0 || runResult.errors > 0)) {
|
|
472
|
+
const lastError = runResult.errors > 0 ? `${runResult.errors} error(s) at block ${blockHeight}` : undefined;
|
|
473
|
+
await recordSubgraphProcessed(tx, subgraphName, runResult.processed, runResult.errors, lastError);
|
|
474
|
+
}
|
|
475
|
+
});
|
|
476
|
+
const totalMs = performance.now() - blockStart;
|
|
477
|
+
result.timing = {
|
|
478
|
+
totalMs: Math.round(totalMs),
|
|
479
|
+
handlerMs: Math.round(handlerMs),
|
|
480
|
+
flushMs: Math.round(flushMs)
|
|
481
|
+
};
|
|
482
|
+
if (blockHeight % 1000 === 0) {
|
|
483
|
+
try {
|
|
484
|
+
const tables = Object.keys(subgraph.schema);
|
|
485
|
+
for (const table of tables) {
|
|
486
|
+
const { rows } = await sql2.raw(`SELECT COUNT(*) as count FROM "${schemaName}"."${table}"`).execute(db);
|
|
487
|
+
const count = Number(rows[0].count);
|
|
488
|
+
if (count >= 1e7) {
|
|
489
|
+
logger3.warn("Subgraph table exceeds 10M rows", { subgraph: subgraphName, table, count });
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
} catch {}
|
|
493
|
+
}
|
|
494
|
+
return result;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// src/runtime/stats.ts
|
|
498
|
+
var FLUSH_INTERVAL_BLOCKS = 100;
|
|
499
|
+
var FLUSH_INTERVAL_MS = 60000;
|
|
500
|
+
|
|
501
|
+
class StatsAccumulator {
|
|
502
|
+
subgraphName;
|
|
503
|
+
apiKeyId;
|
|
504
|
+
isCatchup;
|
|
505
|
+
current;
|
|
506
|
+
lastFlush = Date.now();
|
|
507
|
+
constructor(subgraphName, apiKeyId, isCatchup) {
|
|
508
|
+
this.subgraphName = subgraphName;
|
|
509
|
+
this.apiKeyId = apiKeyId;
|
|
510
|
+
this.isCatchup = isCatchup;
|
|
511
|
+
this.current = this.newEntry();
|
|
512
|
+
}
|
|
513
|
+
record(timing, opsCount) {
|
|
514
|
+
this.current.blocksProcessed++;
|
|
515
|
+
this.current.totalTimeMs += timing.totalMs;
|
|
516
|
+
this.current.handlerTimeMs += timing.handlerMs;
|
|
517
|
+
this.current.flushTimeMs += timing.flushMs;
|
|
518
|
+
this.current.maxBlockTimeMs = Math.max(this.current.maxBlockTimeMs, timing.totalMs);
|
|
519
|
+
this.current.maxHandlerTimeMs = Math.max(this.current.maxHandlerTimeMs, timing.handlerMs);
|
|
520
|
+
this.current.totalOps += opsCount;
|
|
521
|
+
}
|
|
522
|
+
shouldFlush() {
|
|
523
|
+
return this.current.blocksProcessed >= FLUSH_INTERVAL_BLOCKS || Date.now() - this.lastFlush >= FLUSH_INTERVAL_MS;
|
|
524
|
+
}
|
|
525
|
+
async flush(db) {
|
|
526
|
+
if (this.current.blocksProcessed === 0)
|
|
527
|
+
return;
|
|
528
|
+
const entry = this.current;
|
|
529
|
+
this.current = this.newEntry();
|
|
530
|
+
this.lastFlush = Date.now();
|
|
531
|
+
const avgOpsPerBlock = entry.blocksProcessed > 0 ? entry.totalOps / entry.blocksProcessed : 0;
|
|
532
|
+
await db.insertInto("subgraph_processing_stats").values({
|
|
533
|
+
subgraph_name: entry.subgraphName,
|
|
534
|
+
api_key_id: entry.apiKeyId,
|
|
535
|
+
bucket_start: entry.bucketStart,
|
|
536
|
+
bucket_end: new Date,
|
|
537
|
+
blocks_processed: entry.blocksProcessed,
|
|
538
|
+
total_time_ms: Math.round(entry.totalTimeMs),
|
|
539
|
+
handler_time_ms: Math.round(entry.handlerTimeMs),
|
|
540
|
+
flush_time_ms: Math.round(entry.flushTimeMs),
|
|
541
|
+
max_block_time_ms: Math.round(entry.maxBlockTimeMs),
|
|
542
|
+
max_handler_time_ms: Math.round(entry.maxHandlerTimeMs),
|
|
543
|
+
avg_ops_per_block: parseFloat(avgOpsPerBlock.toFixed(2)),
|
|
544
|
+
is_catchup: entry.isCatchup
|
|
545
|
+
}).execute();
|
|
546
|
+
}
|
|
547
|
+
newEntry() {
|
|
548
|
+
return {
|
|
549
|
+
subgraphName: this.subgraphName,
|
|
550
|
+
apiKeyId: this.apiKeyId,
|
|
551
|
+
bucketStart: new Date,
|
|
552
|
+
blocksProcessed: 0,
|
|
553
|
+
totalTimeMs: 0,
|
|
554
|
+
handlerTimeMs: 0,
|
|
555
|
+
flushTimeMs: 0,
|
|
556
|
+
maxBlockTimeMs: 0,
|
|
557
|
+
maxHandlerTimeMs: 0,
|
|
558
|
+
totalOps: 0,
|
|
559
|
+
isCatchup: this.isCatchup
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// src/runtime/reindex.ts
|
|
565
|
+
import { getDb as getDb2, getRawClient } from "@secondlayer/shared/db";
|
|
566
|
+
import { updateSubgraphStatus as updateSubgraphStatus2 } from "@secondlayer/shared/db/queries/subgraphs";
|
|
567
|
+
import { logger as logger4 } from "@secondlayer/shared/logger";
|
|
568
|
+
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
569
|
+
|
|
570
|
+
// src/schema/generator.ts
|
|
571
|
+
var TYPE_MAP = {
|
|
572
|
+
text: "TEXT",
|
|
573
|
+
uint: "BIGINT",
|
|
574
|
+
int: "INTEGER",
|
|
575
|
+
principal: "TEXT",
|
|
576
|
+
boolean: "BOOLEAN",
|
|
577
|
+
timestamp: "TIMESTAMPTZ",
|
|
578
|
+
jsonb: "JSONB"
|
|
579
|
+
};
|
|
580
|
+
function escapeLiteralDefault(value) {
|
|
581
|
+
if (value === null || value === undefined)
|
|
582
|
+
return "NULL";
|
|
583
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
584
|
+
return String(value);
|
|
585
|
+
if (typeof value === "boolean")
|
|
586
|
+
return value ? "TRUE" : "FALSE";
|
|
587
|
+
return `'${String(value).replace(/'/g, "''")}'`;
|
|
588
|
+
}
|
|
589
|
+
function generateSubgraphSQL(def, schemaNameOverride) {
|
|
590
|
+
const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
|
|
591
|
+
const statements = [];
|
|
592
|
+
const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
|
|
593
|
+
if (needsTrgm) {
|
|
594
|
+
statements.push(`CREATE EXTENSION IF NOT EXISTS pg_trgm`);
|
|
595
|
+
}
|
|
596
|
+
statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
|
|
597
|
+
for (const [tableName, tableDef] of Object.entries(def.schema)) {
|
|
598
|
+
const qualifiedName = `${schemaName}.${tableName}`;
|
|
599
|
+
const columnDefs = [
|
|
600
|
+
`_id BIGSERIAL PRIMARY KEY`,
|
|
601
|
+
`_block_height BIGINT NOT NULL`,
|
|
602
|
+
`_tx_id TEXT NOT NULL`,
|
|
603
|
+
`_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()`
|
|
604
|
+
];
|
|
605
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
606
|
+
const sqlType = TYPE_MAP[col.type];
|
|
607
|
+
const nullable = col.nullable ? "" : " NOT NULL";
|
|
608
|
+
let colDef = `${colName} ${sqlType}${nullable}`;
|
|
609
|
+
if (col.default !== undefined) {
|
|
610
|
+
colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
|
|
611
|
+
}
|
|
612
|
+
columnDefs.push(colDef);
|
|
613
|
+
}
|
|
614
|
+
statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
|
|
615
|
+
${columnDefs.join(`,
|
|
616
|
+
`)}
|
|
617
|
+
)`);
|
|
618
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
|
|
619
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
|
|
620
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
621
|
+
if (col.indexed) {
|
|
622
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
626
|
+
if (col.search) {
|
|
627
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
if (tableDef.indexes) {
|
|
631
|
+
for (let i = 0;i < tableDef.indexes.length; i++) {
|
|
632
|
+
const cols = tableDef.indexes[i];
|
|
633
|
+
const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
|
|
634
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
if (tableDef.uniqueKeys) {
|
|
638
|
+
for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
|
|
639
|
+
const cols = tableDef.uniqueKeys[i];
|
|
640
|
+
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
|
|
641
|
+
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
const hashInput = JSON.stringify({
|
|
646
|
+
name: def.name,
|
|
647
|
+
version: def.version,
|
|
648
|
+
schema: def.schema,
|
|
649
|
+
sources: def.sources
|
|
650
|
+
}, (_key, value) => typeof value === "bigint" ? value.toString() : value);
|
|
651
|
+
const hash = String(Bun.hash(hashInput));
|
|
652
|
+
return { statements, hash };
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// src/runtime/reindex.ts
|
|
656
|
+
var LOG_INTERVAL = 1000;
|
|
657
|
+
async function reindexSubgraph(def, opts) {
|
|
658
|
+
const db = getDb2();
|
|
659
|
+
const client = getRawClient();
|
|
660
|
+
const subgraphName = def.name;
|
|
661
|
+
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
662
|
+
await updateSubgraphStatus2(db, subgraphName, "reindexing");
|
|
663
|
+
logger4.info("Reindex starting", { subgraph: subgraphName });
|
|
664
|
+
try {
|
|
665
|
+
await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
|
|
666
|
+
const { statements } = generateSubgraphSQL(def, schemaName);
|
|
667
|
+
for (const stmt of statements) {
|
|
668
|
+
await client.unsafe(stmt);
|
|
669
|
+
}
|
|
670
|
+
logger4.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
671
|
+
const fromBlock = opts?.fromBlock ?? 1;
|
|
672
|
+
let toBlock;
|
|
673
|
+
if (opts?.toBlock != null) {
|
|
674
|
+
toBlock = opts.toBlock;
|
|
675
|
+
} else {
|
|
676
|
+
const progress = await db.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
677
|
+
toBlock = progress?.last_indexed_block ?? progress?.last_contiguous_block ?? 0;
|
|
678
|
+
}
|
|
679
|
+
if (fromBlock > toBlock) {
|
|
680
|
+
logger4.info("No blocks to reindex", { subgraph: subgraphName, fromBlock, toBlock });
|
|
681
|
+
await updateSubgraphStatus2(db, subgraphName, "active", 0);
|
|
682
|
+
return { processed: 0 };
|
|
683
|
+
}
|
|
684
|
+
const totalBlocks = toBlock - fromBlock + 1;
|
|
685
|
+
logger4.info("Reindexing blocks", { subgraph: subgraphName, fromBlock, toBlock, totalBlocks });
|
|
686
|
+
const subgraphRow = await db.selectFrom("subgraphs").select("api_key_id").where("name", "=", subgraphName).executeTakeFirst();
|
|
687
|
+
const stats = new StatsAccumulator(subgraphName, subgraphRow?.api_key_id ?? null, false);
|
|
688
|
+
let blocksProcessed = 0;
|
|
689
|
+
let totalEventsProcessed = 0;
|
|
690
|
+
let totalErrors = 0;
|
|
691
|
+
const PROGRESS_INTERVAL = 100;
|
|
692
|
+
for (let height = fromBlock;height <= toBlock; height++) {
|
|
693
|
+
const result = await processBlock(def, subgraphName, height, { skipProgressUpdate: true });
|
|
694
|
+
blocksProcessed++;
|
|
695
|
+
totalEventsProcessed += result.processed;
|
|
696
|
+
totalErrors += result.errors;
|
|
697
|
+
if (result.timing) {
|
|
698
|
+
stats.record(result.timing, result.processed);
|
|
699
|
+
if (stats.shouldFlush()) {
|
|
700
|
+
await stats.flush(db);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
if (blocksProcessed % PROGRESS_INTERVAL === 0) {
|
|
704
|
+
await updateSubgraphStatus2(db, subgraphName, "reindexing", height);
|
|
705
|
+
}
|
|
706
|
+
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
707
|
+
logger4.info("Reindex progress", {
|
|
708
|
+
subgraph: subgraphName,
|
|
709
|
+
processed: blocksProcessed,
|
|
710
|
+
total: totalBlocks,
|
|
711
|
+
currentBlock: height,
|
|
712
|
+
pct: Math.round(blocksProcessed / totalBlocks * 100)
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
await stats.flush(db);
|
|
717
|
+
const { recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
718
|
+
if (totalEventsProcessed > 0 || totalErrors > 0) {
|
|
719
|
+
await recordSubgraphProcessed2(db, subgraphName, totalEventsProcessed, totalErrors, totalErrors > 0 ? `${totalErrors} error(s) during reindex` : undefined);
|
|
720
|
+
}
|
|
721
|
+
await updateSubgraphStatus2(db, subgraphName, "active", toBlock);
|
|
722
|
+
logger4.info("Reindex complete", { subgraph: subgraphName, blocks: blocksProcessed, events: totalEventsProcessed, errors: totalErrors });
|
|
723
|
+
return { processed: blocksProcessed };
|
|
724
|
+
} catch (err) {
|
|
725
|
+
logger4.error("Reindex failed", {
|
|
726
|
+
subgraph: subgraphName,
|
|
727
|
+
error: getErrorMessage2(err)
|
|
728
|
+
});
|
|
729
|
+
await updateSubgraphStatus2(db, subgraphName, "error");
|
|
730
|
+
throw err;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
export {
|
|
734
|
+
reindexSubgraph
|
|
735
|
+
};
|
|
736
|
+
|
|
737
|
+
//# debugId=1C1BF85F61D4F46664756E2164756E21
|
|
738
|
+
//# sourceMappingURL=reindex.js.map
|