@secondlayer/subgraphs 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/src/index.js +55 -13
- package/dist/src/index.js.map +5 -5
- package/dist/src/runtime/block-processor.js +4 -3
- package/dist/src/runtime/block-processor.js.map +4 -4
- package/dist/src/runtime/catchup.js +4 -3
- package/dist/src/runtime/catchup.js.map +4 -4
- package/dist/src/runtime/processor.d.ts +4 -1
- package/dist/src/runtime/processor.js +766 -67
- package/dist/src/runtime/processor.js.map +9 -7
- package/dist/src/runtime/reindex.d.ts +20 -1
- package/dist/src/runtime/reindex.js +58 -13
- package/dist/src/runtime/reindex.js.map +5 -5
- package/dist/src/runtime/reorg.js +4 -3
- package/dist/src/runtime/reorg.js.map +4 -4
- package/dist/src/runtime/runner.js +3 -2
- package/dist/src/runtime/runner.js.map +3 -3
- package/dist/src/runtime/source-matcher.js +2 -2
- package/dist/src/runtime/source-matcher.js.map +3 -3
- package/dist/src/service.js +766 -68
- package/dist/src/service.js.map +9 -7
- package/package.json +2 -2
package/dist/src/service.js
CHANGED
|
@@ -513,7 +513,8 @@ function buildEventPayload(filter, tx, event) {
|
|
|
513
513
|
tx: txMeta
|
|
514
514
|
};
|
|
515
515
|
case "print_event": {
|
|
516
|
-
const
|
|
516
|
+
const decodedRawValue = decoded.raw_value;
|
|
517
|
+
const rawValue = decodedRawValue && typeof decodedRawValue === "object" && !Array.isArray(decodedRawValue) ? decodedRawValue : decoded.value;
|
|
517
518
|
const clarityObj = rawValue && typeof rawValue === "object" && !Array.isArray(rawValue) ? rawValue : null;
|
|
518
519
|
const topic = clarityObj?.topic ? String(clarityObj.topic) : decoded.topic ?? "";
|
|
519
520
|
const { topic: _, ...rest } = clarityObj ?? {};
|
|
@@ -824,7 +825,7 @@ function matchFilter(filter, transactions, eventsByTx) {
|
|
|
824
825
|
for (const tx of transactions) {
|
|
825
826
|
const txEvents = eventsByTx.get(tx.tx_id) ?? [];
|
|
826
827
|
const matched = txEvents.filter((e) => {
|
|
827
|
-
if (e.type !== "smart_contract_event")
|
|
828
|
+
if (e.type !== "smart_contract_event" && e.type !== "contract_event")
|
|
828
829
|
return false;
|
|
829
830
|
const data = e.data;
|
|
830
831
|
if (!data)
|
|
@@ -1298,14 +1299,103 @@ class StatsAccumulator {
|
|
|
1298
1299
|
}
|
|
1299
1300
|
}
|
|
1300
1301
|
|
|
1301
|
-
// src/runtime/
|
|
1302
|
-
import {
|
|
1302
|
+
// src/runtime/reindex.ts
|
|
1303
|
+
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
1304
|
+
import { getRawClient, getSourceDb as getSourceDb2, getTargetDb as getTargetDb2 } from "@secondlayer/shared/db";
|
|
1303
1305
|
import {
|
|
1304
|
-
recordGapBatch
|
|
1306
|
+
recordGapBatch,
|
|
1307
|
+
resolveGaps
|
|
1305
1308
|
} from "@secondlayer/shared/db/queries/subgraph-gaps";
|
|
1306
|
-
import {
|
|
1309
|
+
import {
|
|
1310
|
+
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
1311
|
+
updateSubgraphStatus as updateSubgraphStatus2
|
|
1312
|
+
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1307
1313
|
import { logger as logger5 } from "@secondlayer/shared/logger";
|
|
1308
1314
|
|
|
1315
|
+
// src/schema/generator.ts
|
|
1316
|
+
var TYPE_MAP = {
|
|
1317
|
+
text: "TEXT",
|
|
1318
|
+
uint: "NUMERIC",
|
|
1319
|
+
int: "NUMERIC",
|
|
1320
|
+
principal: "TEXT",
|
|
1321
|
+
boolean: "BOOLEAN",
|
|
1322
|
+
timestamp: "TIMESTAMPTZ",
|
|
1323
|
+
jsonb: "JSONB"
|
|
1324
|
+
};
|
|
1325
|
+
function escapeLiteralDefault(value) {
|
|
1326
|
+
if (value === null || value === undefined)
|
|
1327
|
+
return "NULL";
|
|
1328
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
1329
|
+
return String(value);
|
|
1330
|
+
if (typeof value === "boolean")
|
|
1331
|
+
return value ? "TRUE" : "FALSE";
|
|
1332
|
+
return `'${String(value).replace(/'/g, "''")}'`;
|
|
1333
|
+
}
|
|
1334
|
+
function generateSubgraphSQL(def, schemaNameOverride) {
|
|
1335
|
+
const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
|
|
1336
|
+
const statements = [];
|
|
1337
|
+
const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
|
|
1338
|
+
if (needsTrgm) {
|
|
1339
|
+
statements.push(`CREATE EXTENSION IF NOT EXISTS pg_trgm`);
|
|
1340
|
+
}
|
|
1341
|
+
statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
|
|
1342
|
+
for (const [tableName, tableDef] of Object.entries(def.schema)) {
|
|
1343
|
+
const qualifiedName = `${schemaName}.${tableName}`;
|
|
1344
|
+
const columnDefs = [
|
|
1345
|
+
`_id BIGSERIAL PRIMARY KEY`,
|
|
1346
|
+
`_block_height BIGINT NOT NULL`,
|
|
1347
|
+
`_tx_id TEXT NOT NULL`,
|
|
1348
|
+
`_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()`
|
|
1349
|
+
];
|
|
1350
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1351
|
+
const sqlType = TYPE_MAP[col.type];
|
|
1352
|
+
const nullable = col.nullable ? "" : " NOT NULL";
|
|
1353
|
+
let colDef = `${colName} ${sqlType}${nullable}`;
|
|
1354
|
+
if (col.default !== undefined) {
|
|
1355
|
+
colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
|
|
1356
|
+
}
|
|
1357
|
+
columnDefs.push(colDef);
|
|
1358
|
+
}
|
|
1359
|
+
statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
|
|
1360
|
+
${columnDefs.join(`,
|
|
1361
|
+
`)}
|
|
1362
|
+
)`);
|
|
1363
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
|
|
1364
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
|
|
1365
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1366
|
+
if (col.indexed) {
|
|
1367
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
for (const [colName, col] of Object.entries(tableDef.columns)) {
|
|
1371
|
+
if (col.search) {
|
|
1372
|
+
statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
if (tableDef.indexes) {
|
|
1376
|
+
for (let i = 0;i < tableDef.indexes.length; i++) {
|
|
1377
|
+
const cols = tableDef.indexes[i];
|
|
1378
|
+
const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
|
|
1379
|
+
statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
if (tableDef.uniqueKeys) {
|
|
1383
|
+
for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
|
|
1384
|
+
const cols = tableDef.uniqueKeys[i];
|
|
1385
|
+
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
|
|
1386
|
+
statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
const hashInput = JSON.stringify({
|
|
1391
|
+
name: def.name,
|
|
1392
|
+
schema: def.schema,
|
|
1393
|
+
sources: def.sources
|
|
1394
|
+
}, (_key, value) => typeof value === "bigint" ? value.toString() : value);
|
|
1395
|
+
const hash = String(Bun.hash(hashInput));
|
|
1396
|
+
return { statements, hash };
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1309
1399
|
// src/runtime/batch-loader.ts
|
|
1310
1400
|
async function loadBlockRange(db, fromHeight, toHeight) {
|
|
1311
1401
|
const [blocks, txs, events] = await Promise.all([
|
|
@@ -1348,8 +1438,401 @@ function avgEventsPerBlock(batch) {
|
|
|
1348
1438
|
return totalEvents / batch.size;
|
|
1349
1439
|
}
|
|
1350
1440
|
|
|
1351
|
-
// src/runtime/
|
|
1441
|
+
// src/runtime/reindex.ts
|
|
1352
1442
|
var LOG_INTERVAL = 1000;
|
|
1443
|
+
var HOBBY_REINDEX_BATCH_CONFIG = {
|
|
1444
|
+
defaultBatchSize: 50,
|
|
1445
|
+
minBatchSize: 25,
|
|
1446
|
+
maxBatchSize: 100
|
|
1447
|
+
};
|
|
1448
|
+
var STANDARD_REINDEX_BATCH_CONFIG = {
|
|
1449
|
+
defaultBatchSize: 500,
|
|
1450
|
+
minBatchSize: 100,
|
|
1451
|
+
maxBatchSize: 1000
|
|
1452
|
+
};
|
|
1453
|
+
function initialReindexProgressBlock(fromBlock) {
|
|
1454
|
+
return Math.max(0, fromBlock - 1);
|
|
1455
|
+
}
|
|
1456
|
+
function resolveReindexResumeBlock(row) {
|
|
1457
|
+
if (row.reindex_from_block == null || row.reindex_to_block == null) {
|
|
1458
|
+
return null;
|
|
1459
|
+
}
|
|
1460
|
+
const lastProcessedBlock = Number(row.last_processed_block);
|
|
1461
|
+
const reindexFromBlock = Number(row.reindex_from_block);
|
|
1462
|
+
return Math.max(lastProcessedBlock + 1, reindexFromBlock);
|
|
1463
|
+
}
|
|
1464
|
+
function parsePositiveInt(value) {
|
|
1465
|
+
if (value == null || value.trim() === "")
|
|
1466
|
+
return;
|
|
1467
|
+
const parsed = Number.parseInt(value, 10);
|
|
1468
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
1469
|
+
}
|
|
1470
|
+
function resolveReindexBatchConfig(env = process.env) {
|
|
1471
|
+
const base = env.TENANT_PLAN?.trim().toLowerCase() === "hobby" ? HOBBY_REINDEX_BATCH_CONFIG : STANDARD_REINDEX_BATCH_CONFIG;
|
|
1472
|
+
const minBatchSize = parsePositiveInt(env.SUBGRAPH_REINDEX_MIN_BATCH_SIZE) ?? base.minBatchSize;
|
|
1473
|
+
const maxBatchSize = parsePositiveInt(env.SUBGRAPH_REINDEX_MAX_BATCH_SIZE) ?? base.maxBatchSize;
|
|
1474
|
+
const defaultBatchSize = parsePositiveInt(env.SUBGRAPH_REINDEX_BATCH_SIZE) ?? base.defaultBatchSize;
|
|
1475
|
+
return {
|
|
1476
|
+
minBatchSize,
|
|
1477
|
+
maxBatchSize,
|
|
1478
|
+
defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize)
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
function coalesceFailedBlocks(blocks) {
|
|
1482
|
+
if (blocks.length === 0)
|
|
1483
|
+
return [];
|
|
1484
|
+
blocks.sort((a, b) => a.height - b.height);
|
|
1485
|
+
const ranges = [];
|
|
1486
|
+
let start = blocks[0].height;
|
|
1487
|
+
let end = blocks[0].height;
|
|
1488
|
+
let reason = blocks[0].reason;
|
|
1489
|
+
for (let i = 1;i < blocks.length; i++) {
|
|
1490
|
+
const b = blocks[i];
|
|
1491
|
+
if (b.height === end + 1 && b.reason === reason) {
|
|
1492
|
+
end = b.height;
|
|
1493
|
+
} else {
|
|
1494
|
+
ranges.push({ start, end, reason });
|
|
1495
|
+
start = b.height;
|
|
1496
|
+
end = b.height;
|
|
1497
|
+
reason = b.reason;
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
ranges.push({ start, end, reason });
|
|
1501
|
+
return ranges;
|
|
1502
|
+
}
|
|
1503
|
+
async function processBlockRange(def, opts) {
|
|
1504
|
+
const sourceDb = getSourceDb2();
|
|
1505
|
+
const targetDb = getTargetDb2();
|
|
1506
|
+
const subgraphName = def.name;
|
|
1507
|
+
const { fromBlock, toBlock, status } = opts;
|
|
1508
|
+
const totalBlocks = toBlock - fromBlock + 1;
|
|
1509
|
+
const stats = new StatsAccumulator(subgraphName, opts.isCatchup);
|
|
1510
|
+
let blocksProcessed = 0;
|
|
1511
|
+
let totalEventsProcessed = 0;
|
|
1512
|
+
let totalErrors = 0;
|
|
1513
|
+
const batchConfig = resolveReindexBatchConfig();
|
|
1514
|
+
let batchSize = batchConfig.defaultBatchSize;
|
|
1515
|
+
let currentHeight = fromBlock;
|
|
1516
|
+
let aborted = false;
|
|
1517
|
+
let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
|
|
1518
|
+
let nextBatchPromise = loadBlockRange(sourceDb, currentHeight, nextBatchEnd);
|
|
1519
|
+
while (currentHeight <= toBlock) {
|
|
1520
|
+
if (opts.signal?.aborted) {
|
|
1521
|
+
aborted = true;
|
|
1522
|
+
logger5.info("Block processing aborted", {
|
|
1523
|
+
subgraph: subgraphName,
|
|
1524
|
+
currentBlock: currentHeight,
|
|
1525
|
+
reason: String(opts.signal.reason ?? "unknown")
|
|
1526
|
+
});
|
|
1527
|
+
break;
|
|
1528
|
+
}
|
|
1529
|
+
const batch = await nextBatchPromise;
|
|
1530
|
+
const batchEnd = nextBatchEnd;
|
|
1531
|
+
const nextStart = batchEnd + 1;
|
|
1532
|
+
if (nextStart <= toBlock) {
|
|
1533
|
+
nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
|
|
1534
|
+
nextBatchPromise = loadBlockRange(sourceDb, nextStart, nextBatchEnd);
|
|
1535
|
+
}
|
|
1536
|
+
const batchFailedBlocks = [];
|
|
1537
|
+
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
1538
|
+
const blockData = batch.get(height);
|
|
1539
|
+
if (!blockData) {
|
|
1540
|
+
batchFailedBlocks.push({ height, reason: "block_missing" });
|
|
1541
|
+
blocksProcessed++;
|
|
1542
|
+
continue;
|
|
1543
|
+
}
|
|
1544
|
+
let result;
|
|
1545
|
+
try {
|
|
1546
|
+
result = await processBlock(def, subgraphName, height, {
|
|
1547
|
+
skipProgressUpdate: true,
|
|
1548
|
+
preloaded: blockData
|
|
1549
|
+
});
|
|
1550
|
+
} catch (err) {
|
|
1551
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
1552
|
+
logger5.error("Block processing error", {
|
|
1553
|
+
subgraph: subgraphName,
|
|
1554
|
+
blockHeight: height,
|
|
1555
|
+
error: errorMsg
|
|
1556
|
+
});
|
|
1557
|
+
batchFailedBlocks.push({ height, reason: "processing_error" });
|
|
1558
|
+
await updateSubgraphStatus2(targetDb, subgraphName, status, height).catch(() => {});
|
|
1559
|
+
await recordSubgraphProcessed2(targetDb, subgraphName, 0, 1, errorMsg).catch(() => {});
|
|
1560
|
+
blocksProcessed++;
|
|
1561
|
+
totalErrors++;
|
|
1562
|
+
continue;
|
|
1563
|
+
}
|
|
1564
|
+
blocksProcessed++;
|
|
1565
|
+
totalEventsProcessed += result.processed;
|
|
1566
|
+
totalErrors += result.errors;
|
|
1567
|
+
if (result.timing) {
|
|
1568
|
+
stats.record(result.timing, result.processed);
|
|
1569
|
+
if (stats.shouldFlush()) {
|
|
1570
|
+
await stats.flush(targetDb);
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
if (blocksProcessed % 100 === 0) {
|
|
1574
|
+
await updateSubgraphStatus2(targetDb, subgraphName, status, height);
|
|
1575
|
+
}
|
|
1576
|
+
if (blocksProcessed % LOG_INTERVAL === 0) {
|
|
1577
|
+
logger5.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
1578
|
+
subgraph: subgraphName,
|
|
1579
|
+
processed: blocksProcessed,
|
|
1580
|
+
total: totalBlocks,
|
|
1581
|
+
currentBlock: height,
|
|
1582
|
+
pct: Math.round(blocksProcessed / totalBlocks * 100)
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
1587
|
+
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
1588
|
+
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
1589
|
+
logger5.warn("Failed to record subgraph gaps", {
|
|
1590
|
+
subgraph: subgraphName,
|
|
1591
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1592
|
+
});
|
|
1593
|
+
});
|
|
1594
|
+
}
|
|
1595
|
+
const avg = avgEventsPerBlock(batch);
|
|
1596
|
+
if (avg > 50)
|
|
1597
|
+
batchSize = Math.max(Math.round(batchSize * 0.5), batchConfig.minBatchSize);
|
|
1598
|
+
else if (avg < 10)
|
|
1599
|
+
batchSize = Math.min(Math.round(batchSize * 1.5), batchConfig.maxBatchSize);
|
|
1600
|
+
currentHeight = batchEnd + 1;
|
|
1601
|
+
}
|
|
1602
|
+
await stats.flush(targetDb);
|
|
1603
|
+
return { blocksProcessed, totalEventsProcessed, totalErrors, aborted };
|
|
1604
|
+
}
|
|
1605
|
+
async function resolveBlockRange(sourceDb, def, opts) {
|
|
1606
|
+
const fromBlock = opts?.fromBlock ?? def.startBlock ?? 1;
|
|
1607
|
+
let toBlock;
|
|
1608
|
+
if (opts?.toBlock != null) {
|
|
1609
|
+
toBlock = opts.toBlock;
|
|
1610
|
+
} else {
|
|
1611
|
+
const progress = await sourceDb.selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
|
|
1612
|
+
toBlock = progress?.highest_seen_block ?? 0;
|
|
1613
|
+
}
|
|
1614
|
+
return { fromBlock, toBlock };
|
|
1615
|
+
}
|
|
1616
|
+
async function clearReindexMetadata(db, subgraphName) {
|
|
1617
|
+
await db.updateTable("subgraphs").set({ reindex_from_block: null, reindex_to_block: null }).where("name", "=", subgraphName).execute();
|
|
1618
|
+
}
|
|
1619
|
+
async function reindexSubgraph(def, opts) {
|
|
1620
|
+
const sourceDb = getSourceDb2();
|
|
1621
|
+
const targetDb = getTargetDb2();
|
|
1622
|
+
const client = getRawClient("target");
|
|
1623
|
+
const subgraphName = def.name;
|
|
1624
|
+
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
1625
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
1626
|
+
logger5.info("Reindex starting", { subgraph: subgraphName });
|
|
1627
|
+
try {
|
|
1628
|
+
const { fromBlock, toBlock } = await resolveBlockRange(sourceDb, def, opts);
|
|
1629
|
+
if (fromBlock > toBlock) {
|
|
1630
|
+
logger5.info("No blocks to reindex", {
|
|
1631
|
+
subgraph: subgraphName,
|
|
1632
|
+
fromBlock,
|
|
1633
|
+
toBlock
|
|
1634
|
+
});
|
|
1635
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "active", 0);
|
|
1636
|
+
return { processed: 0 };
|
|
1637
|
+
}
|
|
1638
|
+
await client.unsafe(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`);
|
|
1639
|
+
const { statements } = generateSubgraphSQL(def, schemaName);
|
|
1640
|
+
for (const stmt of statements) {
|
|
1641
|
+
await client.unsafe(stmt);
|
|
1642
|
+
}
|
|
1643
|
+
logger5.info("Schema recreated for reindex", { subgraph: subgraphName });
|
|
1644
|
+
await targetDb.updateTable("subgraphs").set({
|
|
1645
|
+
last_processed_block: initialReindexProgressBlock(fromBlock),
|
|
1646
|
+
reindex_from_block: fromBlock,
|
|
1647
|
+
reindex_to_block: toBlock,
|
|
1648
|
+
updated_at: new Date
|
|
1649
|
+
}).where("name", "=", subgraphName).execute();
|
|
1650
|
+
logger5.info("Reindexing blocks", {
|
|
1651
|
+
subgraph: subgraphName,
|
|
1652
|
+
fromBlock,
|
|
1653
|
+
toBlock,
|
|
1654
|
+
totalBlocks: toBlock - fromBlock + 1
|
|
1655
|
+
});
|
|
1656
|
+
const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
1657
|
+
const result = await processBlockRange(def, {
|
|
1658
|
+
fromBlock,
|
|
1659
|
+
toBlock,
|
|
1660
|
+
status: "reindexing",
|
|
1661
|
+
isCatchup: false,
|
|
1662
|
+
apiKeyId: null,
|
|
1663
|
+
subgraphId: subgraphRow?.id,
|
|
1664
|
+
signal: opts?.signal
|
|
1665
|
+
});
|
|
1666
|
+
if (result.aborted) {
|
|
1667
|
+
const reason = String(opts?.signal?.reason ?? "unknown");
|
|
1668
|
+
if (reason === "user-cancelled") {
|
|
1669
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1670
|
+
await clearReindexMetadata(targetDb, subgraphName);
|
|
1671
|
+
logger5.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
1672
|
+
} else {
|
|
1673
|
+
logger5.info("Reindex interrupted by shutdown, will resume", {
|
|
1674
|
+
subgraph: subgraphName
|
|
1675
|
+
});
|
|
1676
|
+
}
|
|
1677
|
+
return { processed: result.blocksProcessed };
|
|
1678
|
+
}
|
|
1679
|
+
const { recordSubgraphProcessed: recordSubgraphProcessed3 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
1680
|
+
if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
|
|
1681
|
+
await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during reindex` : undefined);
|
|
1682
|
+
}
|
|
1683
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1684
|
+
await clearReindexMetadata(targetDb, subgraphName);
|
|
1685
|
+
logger5.info("Reindex complete", {
|
|
1686
|
+
subgraph: subgraphName,
|
|
1687
|
+
blocks: result.blocksProcessed,
|
|
1688
|
+
events: result.totalEventsProcessed,
|
|
1689
|
+
errors: result.totalErrors
|
|
1690
|
+
});
|
|
1691
|
+
return { processed: result.blocksProcessed };
|
|
1692
|
+
} catch (err) {
|
|
1693
|
+
logger5.error("Reindex failed", {
|
|
1694
|
+
subgraph: subgraphName,
|
|
1695
|
+
error: getErrorMessage2(err)
|
|
1696
|
+
});
|
|
1697
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "error");
|
|
1698
|
+
throw err;
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
async function resumeReindex(def, opts) {
|
|
1702
|
+
const targetDb = getTargetDb2();
|
|
1703
|
+
const subgraphName = def.name;
|
|
1704
|
+
const row = await targetDb.selectFrom("subgraphs").select([
|
|
1705
|
+
"id",
|
|
1706
|
+
"last_processed_block",
|
|
1707
|
+
"reindex_from_block",
|
|
1708
|
+
"reindex_to_block"
|
|
1709
|
+
]).where("name", "=", subgraphName).executeTakeFirst();
|
|
1710
|
+
if (!row)
|
|
1711
|
+
throw new Error(`Subgraph "${subgraphName}" not found`);
|
|
1712
|
+
const fromBlock = resolveReindexResumeBlock(row);
|
|
1713
|
+
const toBlock = Number(row.reindex_to_block);
|
|
1714
|
+
if (fromBlock == null) {
|
|
1715
|
+
logger5.info("No reindex metadata, starting fresh reindex", {
|
|
1716
|
+
subgraph: subgraphName
|
|
1717
|
+
});
|
|
1718
|
+
return reindexSubgraph(def, {
|
|
1719
|
+
schemaName: opts.schemaName,
|
|
1720
|
+
signal: opts.signal
|
|
1721
|
+
});
|
|
1722
|
+
}
|
|
1723
|
+
if (fromBlock > toBlock) {
|
|
1724
|
+
logger5.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
1725
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1726
|
+
await clearReindexMetadata(targetDb, subgraphName);
|
|
1727
|
+
return { processed: 0 };
|
|
1728
|
+
}
|
|
1729
|
+
logger5.info("Resuming reindex", {
|
|
1730
|
+
subgraph: subgraphName,
|
|
1731
|
+
fromBlock,
|
|
1732
|
+
toBlock,
|
|
1733
|
+
remaining: toBlock - fromBlock + 1
|
|
1734
|
+
});
|
|
1735
|
+
try {
|
|
1736
|
+
const result = await processBlockRange(def, {
|
|
1737
|
+
fromBlock,
|
|
1738
|
+
toBlock,
|
|
1739
|
+
status: "reindexing",
|
|
1740
|
+
isCatchup: false,
|
|
1741
|
+
apiKeyId: null,
|
|
1742
|
+
subgraphId: row.id,
|
|
1743
|
+
signal: opts.signal
|
|
1744
|
+
});
|
|
1745
|
+
if (result.aborted) {
|
|
1746
|
+
const reason = String(opts.signal?.reason ?? "unknown");
|
|
1747
|
+
if (reason === "user-cancelled") {
|
|
1748
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
1749
|
+
await clearReindexMetadata(targetDb, subgraphName);
|
|
1750
|
+
logger5.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
1751
|
+
} else {
|
|
1752
|
+
logger5.info("Resume interrupted by shutdown, will resume again", {
|
|
1753
|
+
subgraph: subgraphName
|
|
1754
|
+
});
|
|
1755
|
+
}
|
|
1756
|
+
return { processed: result.blocksProcessed };
|
|
1757
|
+
}
|
|
1758
|
+
const { recordSubgraphProcessed: recordSubgraphProcessed3 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
1759
|
+
if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
|
|
1760
|
+
await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during resumed reindex` : undefined);
|
|
1761
|
+
}
|
|
1762
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
1763
|
+
await clearReindexMetadata(targetDb, subgraphName);
|
|
1764
|
+
logger5.info("Resumed reindex complete", {
|
|
1765
|
+
subgraph: subgraphName,
|
|
1766
|
+
blocks: result.blocksProcessed
|
|
1767
|
+
});
|
|
1768
|
+
return { processed: result.blocksProcessed };
|
|
1769
|
+
} catch (err) {
|
|
1770
|
+
logger5.error("Resumed reindex failed", {
|
|
1771
|
+
subgraph: subgraphName,
|
|
1772
|
+
error: getErrorMessage2(err)
|
|
1773
|
+
});
|
|
1774
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "error");
|
|
1775
|
+
throw err;
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
async function backfillSubgraph(def, opts) {
|
|
1779
|
+
const targetDb = getTargetDb2();
|
|
1780
|
+
const subgraphName = def.name;
|
|
1781
|
+
logger5.info("Backfill starting", {
|
|
1782
|
+
subgraph: subgraphName,
|
|
1783
|
+
from: opts.fromBlock,
|
|
1784
|
+
to: opts.toBlock
|
|
1785
|
+
});
|
|
1786
|
+
try {
|
|
1787
|
+
const subgraphRow = await targetDb.selectFrom("subgraphs").select(["id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
1788
|
+
const result = await processBlockRange(def, {
|
|
1789
|
+
fromBlock: opts.fromBlock,
|
|
1790
|
+
toBlock: opts.toBlock,
|
|
1791
|
+
status: "active",
|
|
1792
|
+
isCatchup: false,
|
|
1793
|
+
apiKeyId: null,
|
|
1794
|
+
subgraphId: subgraphRow?.id,
|
|
1795
|
+
signal: opts.signal
|
|
1796
|
+
});
|
|
1797
|
+
if (result.aborted) {
|
|
1798
|
+
logger5.info("Backfill aborted", { subgraph: subgraphName });
|
|
1799
|
+
return { processed: result.blocksProcessed };
|
|
1800
|
+
}
|
|
1801
|
+
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
1802
|
+
if (resolved > 0) {
|
|
1803
|
+
logger5.info("Resolved subgraph gaps via backfill", {
|
|
1804
|
+
subgraph: subgraphName,
|
|
1805
|
+
resolved
|
|
1806
|
+
});
|
|
1807
|
+
}
|
|
1808
|
+
const { recordSubgraphProcessed: recordSubgraphProcessed3 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
1809
|
+
if (result.totalEventsProcessed > 0 || result.totalErrors > 0) {
|
|
1810
|
+
await recordSubgraphProcessed3(targetDb, subgraphName, result.totalEventsProcessed, result.totalErrors, result.totalErrors > 0 ? `${result.totalErrors} error(s) during backfill` : undefined);
|
|
1811
|
+
}
|
|
1812
|
+
logger5.info("Backfill complete", {
|
|
1813
|
+
subgraph: subgraphName,
|
|
1814
|
+
blocks: result.blocksProcessed,
|
|
1815
|
+
events: result.totalEventsProcessed,
|
|
1816
|
+
errors: result.totalErrors
|
|
1817
|
+
});
|
|
1818
|
+
return { processed: result.blocksProcessed };
|
|
1819
|
+
} catch (err) {
|
|
1820
|
+
logger5.error("Backfill failed", {
|
|
1821
|
+
subgraph: subgraphName,
|
|
1822
|
+
error: getErrorMessage2(err)
|
|
1823
|
+
});
|
|
1824
|
+
throw err;
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
// src/runtime/catchup.ts
|
|
1829
|
+
import { getSourceDb as getSourceDb3, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
1830
|
+
import {
|
|
1831
|
+
recordGapBatch as recordGapBatch2
|
|
1832
|
+
} from "@secondlayer/shared/db/queries/subgraph-gaps";
|
|
1833
|
+
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
1834
|
+
import { logger as logger6 } from "@secondlayer/shared/logger";
|
|
1835
|
+
var LOG_INTERVAL2 = 1000;
|
|
1353
1836
|
var DEFAULT_BATCH_SIZE = 500;
|
|
1354
1837
|
var MIN_BATCH_SIZE = 100;
|
|
1355
1838
|
var MAX_BATCH_SIZE = 1000;
|
|
@@ -1388,8 +1871,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1388
1871
|
return 0;
|
|
1389
1872
|
catchingUp.add(subgraphName);
|
|
1390
1873
|
try {
|
|
1391
|
-
const sourceDb =
|
|
1392
|
-
const targetDb =
|
|
1874
|
+
const sourceDb = getSourceDb3();
|
|
1875
|
+
const targetDb = getTargetDb3();
|
|
1393
1876
|
const subgraphRow = await getSubgraph(targetDb, subgraphName);
|
|
1394
1877
|
if (!subgraphRow)
|
|
1395
1878
|
return 0;
|
|
@@ -1403,7 +1886,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1403
1886
|
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
1404
1887
|
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
1405
1888
|
const totalBlocks = chainTip - lastProcessedBlock;
|
|
1406
|
-
|
|
1889
|
+
logger6.info("Subgraph catch-up starting", {
|
|
1407
1890
|
subgraph: subgraphName,
|
|
1408
1891
|
from: startBlock,
|
|
1409
1892
|
to: chainTip,
|
|
@@ -1418,7 +1901,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1418
1901
|
while (currentHeight <= chainTip) {
|
|
1419
1902
|
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
1420
1903
|
if (!currentRow || currentRow.status !== "active") {
|
|
1421
|
-
|
|
1904
|
+
logger6.info("Subgraph status changed, stopping catch-up", {
|
|
1422
1905
|
subgraph: subgraphName,
|
|
1423
1906
|
status: currentRow?.status ?? "deleted"
|
|
1424
1907
|
});
|
|
@@ -1445,14 +1928,14 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1445
1928
|
preloaded: blockData
|
|
1446
1929
|
});
|
|
1447
1930
|
} catch (err) {
|
|
1448
|
-
|
|
1931
|
+
logger6.error("Block processing error during catch-up", {
|
|
1449
1932
|
subgraph: subgraphName,
|
|
1450
1933
|
blockHeight: height,
|
|
1451
1934
|
error: err instanceof Error ? err.message : String(err)
|
|
1452
1935
|
});
|
|
1453
1936
|
batchFailedBlocks.push({ height, reason: "processing_error" });
|
|
1454
|
-
const { updateSubgraphStatus:
|
|
1455
|
-
await
|
|
1937
|
+
const { updateSubgraphStatus: updateSubgraphStatus3 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
1938
|
+
await updateSubgraphStatus3(targetDb, subgraphName, "active", height).catch(() => {});
|
|
1456
1939
|
processed++;
|
|
1457
1940
|
continue;
|
|
1458
1941
|
}
|
|
@@ -1463,8 +1946,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1463
1946
|
await stats.flush(targetDb);
|
|
1464
1947
|
}
|
|
1465
1948
|
}
|
|
1466
|
-
if (processed %
|
|
1467
|
-
|
|
1949
|
+
if (processed % LOG_INTERVAL2 === 0) {
|
|
1950
|
+
logger6.info("Subgraph catch-up progress", {
|
|
1468
1951
|
subgraph: subgraphName,
|
|
1469
1952
|
processed,
|
|
1470
1953
|
total: totalBlocks,
|
|
@@ -1475,8 +1958,8 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1475
1958
|
}
|
|
1476
1959
|
if (batchFailedBlocks.length > 0) {
|
|
1477
1960
|
const gaps = coalesceGaps(batchFailedBlocks);
|
|
1478
|
-
await
|
|
1479
|
-
|
|
1961
|
+
await recordGapBatch2(targetDb, subgraphRow.id, subgraphName, gaps).catch((err) => {
|
|
1962
|
+
logger6.warn("Failed to record subgraph gaps", {
|
|
1480
1963
|
subgraph: subgraphName,
|
|
1481
1964
|
error: err instanceof Error ? err.message : String(err)
|
|
1482
1965
|
});
|
|
@@ -1487,7 +1970,7 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1487
1970
|
currentHeight = batchEnd + 1;
|
|
1488
1971
|
}
|
|
1489
1972
|
await stats.flush(targetDb);
|
|
1490
|
-
|
|
1973
|
+
logger6.info("Subgraph catch-up complete", {
|
|
1491
1974
|
subgraph: subgraphName,
|
|
1492
1975
|
processed
|
|
1493
1976
|
});
|
|
@@ -1498,17 +1981,17 @@ async function catchUpSubgraph(subgraph, subgraphName) {
|
|
|
1498
1981
|
}
|
|
1499
1982
|
|
|
1500
1983
|
// src/runtime/reorg.ts
|
|
1501
|
-
import { getErrorMessage as
|
|
1502
|
-
import { getRawClient, getTargetDb as
|
|
1984
|
+
import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
|
|
1985
|
+
import { getRawClient as getRawClient2, getTargetDb as getTargetDb4 } from "@secondlayer/shared/db";
|
|
1503
1986
|
import { listSubgraphs } from "@secondlayer/shared/db/queries/subgraphs";
|
|
1504
|
-
import { logger as
|
|
1987
|
+
import { logger as logger7 } from "@secondlayer/shared/logger";
|
|
1505
1988
|
async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
1506
|
-
const targetDb =
|
|
1507
|
-
const client =
|
|
1989
|
+
const targetDb = getTargetDb4();
|
|
1990
|
+
const client = getRawClient2("target");
|
|
1508
1991
|
const activeSubgraphs = (await listSubgraphs(targetDb)).filter((v) => v.status === "active");
|
|
1509
1992
|
if (activeSubgraphs.length === 0)
|
|
1510
1993
|
return;
|
|
1511
|
-
|
|
1994
|
+
logger7.info("Propagating reorg to subgraphs", {
|
|
1512
1995
|
blockHeight,
|
|
1513
1996
|
subgraphCount: activeSubgraphs.length
|
|
1514
1997
|
});
|
|
@@ -1521,51 +2004,65 @@ async function handleSubgraphReorg(blockHeight, loadSubgraphDef) {
|
|
|
1521
2004
|
for (const tableName of Object.keys(schema)) {
|
|
1522
2005
|
await client.unsafe(`DELETE FROM "${schemaName}"."${tableName}" WHERE "_block_height" = $1`, [blockHeight]);
|
|
1523
2006
|
}
|
|
1524
|
-
|
|
2007
|
+
logger7.info("Subgraph reorg cleanup done", {
|
|
1525
2008
|
subgraph: sg.name,
|
|
1526
2009
|
blockHeight
|
|
1527
2010
|
});
|
|
1528
2011
|
const def = await loadSubgraphDef(sg);
|
|
1529
2012
|
await processBlock(def, sg.name, blockHeight);
|
|
1530
|
-
|
|
2013
|
+
logger7.info("Subgraph reorg reprocessed", {
|
|
1531
2014
|
subgraph: sg.name,
|
|
1532
2015
|
blockHeight
|
|
1533
2016
|
});
|
|
1534
2017
|
} catch (err) {
|
|
1535
|
-
|
|
2018
|
+
logger7.error("Subgraph reorg handling failed", {
|
|
1536
2019
|
subgraph: sg.name,
|
|
1537
2020
|
blockHeight,
|
|
1538
|
-
error:
|
|
2021
|
+
error: getErrorMessage3(err)
|
|
1539
2022
|
});
|
|
1540
2023
|
}
|
|
1541
2024
|
}
|
|
1542
2025
|
}
|
|
1543
2026
|
|
|
1544
2027
|
// src/runtime/processor.ts
|
|
2028
|
+
import { randomUUID } from "node:crypto";
|
|
2029
|
+
import { hostname } from "node:os";
|
|
1545
2030
|
import { resolve } from "node:path";
|
|
1546
2031
|
import { pathToFileURL } from "node:url";
|
|
1547
|
-
import { getErrorMessage as
|
|
1548
|
-
import { getTargetDb as
|
|
2032
|
+
import { getErrorMessage as getErrorMessage4 } from "@secondlayer/shared";
|
|
2033
|
+
import { getTargetDb as getTargetDb6 } from "@secondlayer/shared/db";
|
|
2034
|
+
import {
|
|
2035
|
+
cancelSubgraphOperation,
|
|
2036
|
+
claimSubgraphOperation,
|
|
2037
|
+
completeSubgraphOperation,
|
|
2038
|
+
createSubgraphOperation,
|
|
2039
|
+
failSubgraphOperation,
|
|
2040
|
+
findActiveSubgraphOperation,
|
|
2041
|
+
getSubgraphOperation,
|
|
2042
|
+
heartbeatSubgraphOperation,
|
|
2043
|
+
isActiveSubgraphOperationConflict
|
|
2044
|
+
} from "@secondlayer/shared/db/queries/subgraph-operations";
|
|
1549
2045
|
import {
|
|
1550
2046
|
listSubgraphs as listSubgraphs2,
|
|
1551
|
-
|
|
2047
|
+
pgSchemaName as pgSchemaName2,
|
|
2048
|
+
updateSubgraphStatus as updateSubgraphStatus3
|
|
1552
2049
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
1553
|
-
import { logger as
|
|
2050
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
1554
2051
|
import { listen as listen2 } from "@secondlayer/shared/queue/listener";
|
|
1555
2052
|
|
|
1556
2053
|
// src/runtime/emitter.ts
|
|
1557
2054
|
import {
|
|
1558
|
-
getTargetDb as
|
|
2055
|
+
getTargetDb as getTargetDb5
|
|
1559
2056
|
} from "@secondlayer/shared/db";
|
|
1560
2057
|
import {
|
|
1561
2058
|
getSubscriptionSigningSecret
|
|
1562
2059
|
} from "@secondlayer/shared/db/queries/subscriptions";
|
|
1563
|
-
import { logger as
|
|
2060
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
1564
2061
|
import { listen } from "@secondlayer/shared/queue/listener";
|
|
1565
2062
|
import { sql as sql4 } from "kysely";
|
|
1566
2063
|
|
|
1567
2064
|
// src/runtime/formats/index.ts
|
|
1568
|
-
import { logger as
|
|
2065
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
1569
2066
|
|
|
1570
2067
|
// src/runtime/formats/cloudevents.ts
|
|
1571
2068
|
function buildCloudEvents(outboxRow, _sub) {
|
|
@@ -1712,7 +2209,7 @@ function buildForFormat(outboxRow, sub, signingSecret) {
|
|
|
1712
2209
|
case "standard-webhooks":
|
|
1713
2210
|
return buildStandardWebhooks(outboxRow, signingSecret);
|
|
1714
2211
|
default:
|
|
1715
|
-
|
|
2212
|
+
logger8.warn("Unknown subscription format, falling back to standard-webhooks", {
|
|
1716
2213
|
format: sub.format,
|
|
1717
2214
|
subscriptionId: sub.id
|
|
1718
2215
|
});
|
|
@@ -1795,7 +2292,7 @@ async function dispatchOne(db, outboxRow, sub) {
|
|
|
1795
2292
|
let responseHeaders = {};
|
|
1796
2293
|
if (isPrivateEgress(sub.url) && !allowPrivateEgress()) {
|
|
1797
2294
|
error = "refused private egress (set SECONDLAYER_ALLOW_PRIVATE_EGRESS=true to allow)";
|
|
1798
|
-
|
|
2295
|
+
logger9.warn("[emitter] refused private egress", {
|
|
1799
2296
|
subscription: sub.name,
|
|
1800
2297
|
url: sub.url
|
|
1801
2298
|
});
|
|
@@ -1891,7 +2388,7 @@ async function settleFailed(db, outboxRow, sub, errText) {
|
|
|
1891
2388
|
circuit_opened_at: new Date,
|
|
1892
2389
|
updated_at: new Date
|
|
1893
2390
|
}).where("id", "=", sub.id).execute();
|
|
1894
|
-
|
|
2391
|
+
logger9.warn("Subscription circuit tripped — paused after consecutive failures", {
|
|
1895
2392
|
subscription: sub.name,
|
|
1896
2393
|
failures: newFailures
|
|
1897
2394
|
});
|
|
@@ -1979,7 +2476,7 @@ async function drainForSub(db, state, sub, rows) {
|
|
|
1979
2476
|
await settleFailed(db, row, sub, err);
|
|
1980
2477
|
}
|
|
1981
2478
|
} catch (err) {
|
|
1982
|
-
|
|
2479
|
+
logger9.error("Emitter dispatch crashed", {
|
|
1983
2480
|
outboxId: row.id,
|
|
1984
2481
|
error: err instanceof Error ? err.message : String(err)
|
|
1985
2482
|
});
|
|
@@ -2008,7 +2505,7 @@ async function runRetention(db) {
|
|
|
2008
2505
|
}
|
|
2009
2506
|
async function startEmitter(opts) {
|
|
2010
2507
|
const emitterId = `emitter-${Math.random().toString(36).slice(2, 10)}`;
|
|
2011
|
-
const db =
|
|
2508
|
+
const db = getTargetDb5();
|
|
2012
2509
|
const state = {
|
|
2013
2510
|
running: true,
|
|
2014
2511
|
inFlightBySub: new Map,
|
|
@@ -2016,7 +2513,7 @@ async function startEmitter(opts) {
|
|
|
2016
2513
|
};
|
|
2017
2514
|
const pollIntervalMs = opts?.pollIntervalMs ?? 120000;
|
|
2018
2515
|
const retentionIntervalMs = opts?.retentionIntervalMs ?? 60 * 60000;
|
|
2019
|
-
|
|
2516
|
+
logger9.info("[emitter] started", { id: emitterId });
|
|
2020
2517
|
const MATCHER_BOOT_ATTEMPTS = 5;
|
|
2021
2518
|
let lastErr = null;
|
|
2022
2519
|
for (let i = 0;i < MATCHER_BOOT_ATTEMPTS; i++) {
|
|
@@ -2027,7 +2524,7 @@ async function startEmitter(opts) {
|
|
|
2027
2524
|
} catch (err) {
|
|
2028
2525
|
lastErr = err;
|
|
2029
2526
|
const delayMs = 500 * 2 ** i;
|
|
2030
|
-
|
|
2527
|
+
logger9.warn("[emitter] matcher refresh failed, retrying", {
|
|
2031
2528
|
attempt: i + 1,
|
|
2032
2529
|
delayMs,
|
|
2033
2530
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -2041,21 +2538,21 @@ async function startEmitter(opts) {
|
|
|
2041
2538
|
const stopNew = await listen("subscriptions:new_outbox", () => {
|
|
2042
2539
|
if (!state.running)
|
|
2043
2540
|
return;
|
|
2044
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
2541
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger9.error("[emitter] claim failed", {
|
|
2045
2542
|
error: err instanceof Error ? err.message : String(err)
|
|
2046
2543
|
}));
|
|
2047
2544
|
});
|
|
2048
2545
|
const stopChanged = await listen("subscriptions:changed", () => {
|
|
2049
2546
|
if (!state.running)
|
|
2050
2547
|
return;
|
|
2051
|
-
refreshMatcher(db).catch((err) =>
|
|
2548
|
+
refreshMatcher(db).catch((err) => logger9.error("[emitter] matcher refresh failed", {
|
|
2052
2549
|
error: err instanceof Error ? err.message : String(err)
|
|
2053
2550
|
}));
|
|
2054
2551
|
});
|
|
2055
2552
|
const poll = setInterval(() => {
|
|
2056
2553
|
if (!state.running)
|
|
2057
2554
|
return;
|
|
2058
|
-
claimAndDrain(db, state, emitterId).catch((err) =>
|
|
2555
|
+
claimAndDrain(db, state, emitterId).catch((err) => logger9.error("[emitter] poll claim failed", {
|
|
2059
2556
|
error: err instanceof Error ? err.message : String(err)
|
|
2060
2557
|
}));
|
|
2061
2558
|
}, pollIntervalMs);
|
|
@@ -2063,7 +2560,7 @@ async function startEmitter(opts) {
|
|
|
2063
2560
|
const retention = setInterval(() => {
|
|
2064
2561
|
if (!state.running)
|
|
2065
2562
|
return;
|
|
2066
|
-
runRetention(db).catch((err) =>
|
|
2563
|
+
runRetention(db).catch((err) => logger9.error("[emitter] retention failed", {
|
|
2067
2564
|
error: err instanceof Error ? err.message : String(err)
|
|
2068
2565
|
}));
|
|
2069
2566
|
}, retentionIntervalMs);
|
|
@@ -2073,20 +2570,27 @@ async function startEmitter(opts) {
|
|
|
2073
2570
|
clearInterval(retention);
|
|
2074
2571
|
await stopNew();
|
|
2075
2572
|
await stopChanged();
|
|
2076
|
-
|
|
2573
|
+
logger9.info("[emitter] stopped", { id: emitterId });
|
|
2077
2574
|
};
|
|
2078
2575
|
}
|
|
2079
2576
|
|
|
2080
2577
|
// src/runtime/processor.ts
|
|
2081
2578
|
var CHANNEL_NEW_BLOCK = "indexer:new_block";
|
|
2579
|
+
var CHANNEL_SUBGRAPH_OPERATIONS = "subgraph_operations:new";
|
|
2082
2580
|
var DEFAULT_CONCURRENCY = 5;
|
|
2581
|
+
var DEFAULT_OPERATION_CONCURRENCY = 1;
|
|
2083
2582
|
var POLL_INTERVAL_MS = 5000;
|
|
2583
|
+
var HEARTBEAT_INTERVAL_MS = 15000;
|
|
2584
|
+
var CANCEL_POLL_INTERVAL_MS = 1000;
|
|
2084
2585
|
function handlerImportUrl(handlerPath, cacheBust = Date.now()) {
|
|
2085
2586
|
return `${pathToFileURL(resolve(handlerPath)).href}?t=${cacheBust}`;
|
|
2086
2587
|
}
|
|
2087
2588
|
function sourceListenerUrl() {
|
|
2088
2589
|
return process.env.SOURCE_DATABASE_URL ?? process.env.DATABASE_URL;
|
|
2089
2590
|
}
|
|
2591
|
+
function targetListenerUrl() {
|
|
2592
|
+
return process.env.TARGET_DATABASE_URL ?? process.env.DATABASE_URL;
|
|
2593
|
+
}
|
|
2090
2594
|
function isHandlerNotFoundError(err) {
|
|
2091
2595
|
if (!(err instanceof Error))
|
|
2092
2596
|
return false;
|
|
@@ -2114,7 +2618,7 @@ async function loadSubgraphDefinition(sg) {
|
|
|
2114
2618
|
knownVersions.set(sg.name, sg.version);
|
|
2115
2619
|
definitionCache.set(sg.name, def);
|
|
2116
2620
|
if (prevVersion && prevVersion !== sg.version) {
|
|
2117
|
-
|
|
2621
|
+
logger10.info("Subgraph handler reloaded", {
|
|
2118
2622
|
subgraph: sg.name,
|
|
2119
2623
|
from: prevVersion,
|
|
2120
2624
|
to: sg.version
|
|
@@ -2131,22 +2635,215 @@ function cleanupCaches(active) {
|
|
|
2131
2635
|
}
|
|
2132
2636
|
}
|
|
2133
2637
|
}
|
|
2638
|
+
async function synthesizeLegacyReindexOperations() {
|
|
2639
|
+
const db = getTargetDb6();
|
|
2640
|
+
const stale = (await listSubgraphs2(db)).filter((sg) => sg.status === "reindexing");
|
|
2641
|
+
for (const sg of stale) {
|
|
2642
|
+
const active = await findActiveSubgraphOperation(db, sg.id);
|
|
2643
|
+
if (active)
|
|
2644
|
+
continue;
|
|
2645
|
+
try {
|
|
2646
|
+
await createSubgraphOperation(db, {
|
|
2647
|
+
subgraphId: sg.id,
|
|
2648
|
+
subgraphName: sg.name,
|
|
2649
|
+
accountId: sg.account_id,
|
|
2650
|
+
kind: "reindex",
|
|
2651
|
+
fromBlock: sg.reindex_from_block == null ? undefined : Number(sg.reindex_from_block),
|
|
2652
|
+
toBlock: sg.reindex_to_block == null ? undefined : Number(sg.reindex_to_block)
|
|
2653
|
+
});
|
|
2654
|
+
logger10.info("Queued legacy reindex resume operation", {
|
|
2655
|
+
subgraph: sg.name
|
|
2656
|
+
});
|
|
2657
|
+
} catch (err) {
|
|
2658
|
+
if (isActiveSubgraphOperationConflict(err))
|
|
2659
|
+
continue;
|
|
2660
|
+
throw err;
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
async function runSubgraphOperation(operation, signal) {
|
|
2665
|
+
if (operation.cancel_requested) {
|
|
2666
|
+
return 0;
|
|
2667
|
+
}
|
|
2668
|
+
const db = getTargetDb6();
|
|
2669
|
+
const subgraph = await db.selectFrom("subgraphs").selectAll().where("id", "=", operation.subgraph_id).executeTakeFirst();
|
|
2670
|
+
if (!subgraph)
|
|
2671
|
+
throw new Error(`Subgraph not found: ${operation.subgraph_id}`);
|
|
2672
|
+
const def = await loadSubgraphDefinition(subgraph);
|
|
2673
|
+
const schemaName = subgraph.schema_name ?? pgSchemaName2(subgraph.name);
|
|
2674
|
+
if (operation.kind === "backfill") {
|
|
2675
|
+
if (operation.from_block == null || operation.to_block == null) {
|
|
2676
|
+
throw new Error("Backfill operation is missing from_block or to_block");
|
|
2677
|
+
}
|
|
2678
|
+
const result2 = await backfillSubgraph(def, {
|
|
2679
|
+
fromBlock: Number(operation.from_block),
|
|
2680
|
+
toBlock: Number(operation.to_block),
|
|
2681
|
+
schemaName,
|
|
2682
|
+
signal
|
|
2683
|
+
});
|
|
2684
|
+
return result2.processed;
|
|
2685
|
+
}
|
|
2686
|
+
const hasResumeMetadata = subgraph.status === "reindexing" && subgraph.reindex_from_block != null && subgraph.reindex_to_block != null;
|
|
2687
|
+
if (hasResumeMetadata) {
|
|
2688
|
+
const result2 = await resumeReindex(def, { schemaName, signal });
|
|
2689
|
+
return result2.processed;
|
|
2690
|
+
}
|
|
2691
|
+
const result = await reindexSubgraph(def, {
|
|
2692
|
+
fromBlock: operation.from_block == null ? undefined : Number(operation.from_block),
|
|
2693
|
+
toBlock: operation.to_block == null ? undefined : Number(operation.to_block),
|
|
2694
|
+
schemaName,
|
|
2695
|
+
signal
|
|
2696
|
+
});
|
|
2697
|
+
return result.processed;
|
|
2698
|
+
}
|
|
2699
|
+
async function startSubgraphOperationRunner(opts) {
|
|
2700
|
+
const concurrency = opts?.concurrency ?? DEFAULT_OPERATION_CONCURRENCY;
|
|
2701
|
+
const db = getTargetDb6();
|
|
2702
|
+
const lockedBy = `${hostname()}:${process.pid}:${randomUUID()}`;
|
|
2703
|
+
const active = new Map;
|
|
2704
|
+
const activeRuns = new Map;
|
|
2705
|
+
let running = true;
|
|
2706
|
+
let draining = false;
|
|
2707
|
+
logger10.info("Starting subgraph operation runner", { concurrency, lockedBy });
|
|
2708
|
+
const startOperation = (operation) => {
|
|
2709
|
+
const controller = new AbortController;
|
|
2710
|
+
active.set(operation.id, controller);
|
|
2711
|
+
const heartbeat = setInterval(() => {
|
|
2712
|
+
if (!running)
|
|
2713
|
+
return;
|
|
2714
|
+
heartbeatSubgraphOperation(db, operation.id, lockedBy).catch((err) => {
|
|
2715
|
+
logger10.warn("Subgraph operation heartbeat failed", {
|
|
2716
|
+
operationId: operation.id,
|
|
2717
|
+
error: getErrorMessage4(err)
|
|
2718
|
+
});
|
|
2719
|
+
});
|
|
2720
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
2721
|
+
const cancelPoll = setInterval(() => {
|
|
2722
|
+
getSubgraphOperation(db, operation.id).then((row) => {
|
|
2723
|
+
if (row?.cancel_requested && !controller.signal.aborted) {
|
|
2724
|
+
controller.abort("user-cancelled");
|
|
2725
|
+
}
|
|
2726
|
+
}).catch((err) => {
|
|
2727
|
+
logger10.warn("Subgraph operation cancel poll failed", {
|
|
2728
|
+
operationId: operation.id,
|
|
2729
|
+
error: getErrorMessage4(err)
|
|
2730
|
+
});
|
|
2731
|
+
});
|
|
2732
|
+
}, CANCEL_POLL_INTERVAL_MS);
|
|
2733
|
+
const run = (async () => {
|
|
2734
|
+
let processed = 0;
|
|
2735
|
+
try {
|
|
2736
|
+
if (operation.cancel_requested) {
|
|
2737
|
+
controller.abort("user-cancelled");
|
|
2738
|
+
} else {
|
|
2739
|
+
processed = await runSubgraphOperation(operation, controller.signal);
|
|
2740
|
+
}
|
|
2741
|
+
const reason = String(controller.signal.reason ?? "");
|
|
2742
|
+
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
2743
|
+
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
2744
|
+
logger10.info("Subgraph operation cancelled", {
|
|
2745
|
+
operationId: operation.id,
|
|
2746
|
+
subgraph: operation.subgraph_name
|
|
2747
|
+
});
|
|
2748
|
+
return;
|
|
2749
|
+
}
|
|
2750
|
+
if (controller.signal.aborted) {
|
|
2751
|
+
logger10.info("Subgraph operation interrupted", {
|
|
2752
|
+
operationId: operation.id,
|
|
2753
|
+
subgraph: operation.subgraph_name,
|
|
2754
|
+
reason
|
|
2755
|
+
});
|
|
2756
|
+
return;
|
|
2757
|
+
}
|
|
2758
|
+
await completeSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
2759
|
+
logger10.info("Subgraph operation completed", {
|
|
2760
|
+
operationId: operation.id,
|
|
2761
|
+
subgraph: operation.subgraph_name,
|
|
2762
|
+
processed
|
|
2763
|
+
});
|
|
2764
|
+
} catch (err) {
|
|
2765
|
+
const reason = String(controller.signal.reason ?? "");
|
|
2766
|
+
if (controller.signal.aborted && reason === "shutdown") {
|
|
2767
|
+
logger10.info("Subgraph operation interrupted by shutdown", {
|
|
2768
|
+
operationId: operation.id,
|
|
2769
|
+
subgraph: operation.subgraph_name
|
|
2770
|
+
});
|
|
2771
|
+
return;
|
|
2772
|
+
}
|
|
2773
|
+
if (controller.signal.aborted && reason === "user-cancelled") {
|
|
2774
|
+
await cancelSubgraphOperation(db, operation.id, lockedBy, processed);
|
|
2775
|
+
return;
|
|
2776
|
+
}
|
|
2777
|
+
await failSubgraphOperation(db, operation.id, lockedBy, getErrorMessage4(err), processed);
|
|
2778
|
+
logger10.error("Subgraph operation failed", {
|
|
2779
|
+
operationId: operation.id,
|
|
2780
|
+
subgraph: operation.subgraph_name,
|
|
2781
|
+
error: getErrorMessage4(err)
|
|
2782
|
+
});
|
|
2783
|
+
} finally {
|
|
2784
|
+
clearInterval(heartbeat);
|
|
2785
|
+
clearInterval(cancelPoll);
|
|
2786
|
+
active.delete(operation.id);
|
|
2787
|
+
activeRuns.delete(operation.id);
|
|
2788
|
+
if (running)
|
|
2789
|
+
drain();
|
|
2790
|
+
}
|
|
2791
|
+
})();
|
|
2792
|
+
activeRuns.set(operation.id, run);
|
|
2793
|
+
};
|
|
2794
|
+
const drain = async () => {
|
|
2795
|
+
if (!running || draining)
|
|
2796
|
+
return;
|
|
2797
|
+
draining = true;
|
|
2798
|
+
try {
|
|
2799
|
+
while (running && active.size < concurrency) {
|
|
2800
|
+
const operation = await claimSubgraphOperation(db, lockedBy);
|
|
2801
|
+
if (!operation)
|
|
2802
|
+
break;
|
|
2803
|
+
startOperation(operation);
|
|
2804
|
+
}
|
|
2805
|
+
} finally {
|
|
2806
|
+
draining = false;
|
|
2807
|
+
}
|
|
2808
|
+
};
|
|
2809
|
+
await synthesizeLegacyReindexOperations();
|
|
2810
|
+
await drain();
|
|
2811
|
+
const stopListening = await listen2(CHANNEL_SUBGRAPH_OPERATIONS, () => {
|
|
2812
|
+
drain();
|
|
2813
|
+
}, { connectionString: targetListenerUrl() });
|
|
2814
|
+
const pollInterval = setInterval(() => {
|
|
2815
|
+
drain();
|
|
2816
|
+
}, POLL_INTERVAL_MS);
|
|
2817
|
+
return async () => {
|
|
2818
|
+
running = false;
|
|
2819
|
+
clearInterval(pollInterval);
|
|
2820
|
+
await stopListening();
|
|
2821
|
+
for (const controller of active.values()) {
|
|
2822
|
+
controller.abort("shutdown");
|
|
2823
|
+
}
|
|
2824
|
+
await Promise.allSettled(activeRuns.values());
|
|
2825
|
+
logger10.info("Subgraph operation runner stopped");
|
|
2826
|
+
};
|
|
2827
|
+
}
|
|
2134
2828
|
async function startSubgraphProcessor(opts) {
|
|
2135
2829
|
const concurrency = opts?.concurrency ?? DEFAULT_CONCURRENCY;
|
|
2136
2830
|
let running = true;
|
|
2137
|
-
|
|
2138
|
-
const
|
|
2831
|
+
logger10.info("Starting subgraph processor", { concurrency });
|
|
2832
|
+
const stopOperations = await startSubgraphOperationRunner({
|
|
2833
|
+
concurrency: Number.parseInt(process.env.SUBGRAPH_OPERATION_CONCURRENCY ?? String(DEFAULT_OPERATION_CONCURRENCY))
|
|
2834
|
+
});
|
|
2835
|
+
const targetDb = getTargetDb6();
|
|
2139
2836
|
const activeSubgraphs = (await listSubgraphs2(targetDb)).filter((v) => v.status === "active");
|
|
2140
2837
|
for (const sg of activeSubgraphs) {
|
|
2141
2838
|
try {
|
|
2142
2839
|
const def = await loadSubgraphDefinition(sg);
|
|
2143
2840
|
await catchUpSubgraph(def, sg.name);
|
|
2144
2841
|
} catch (err) {
|
|
2145
|
-
const msg =
|
|
2842
|
+
const msg = getErrorMessage4(err);
|
|
2146
2843
|
if (isHandlerNotFoundError(err)) {
|
|
2147
|
-
await
|
|
2844
|
+
await updateSubgraphStatus3(targetDb, sg.name, "error");
|
|
2148
2845
|
}
|
|
2149
|
-
|
|
2846
|
+
logger10.error("Subgraph catch-up failed on startup", {
|
|
2150
2847
|
subgraph: sg.name,
|
|
2151
2848
|
error: msg
|
|
2152
2849
|
});
|
|
@@ -2155,7 +2852,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
2155
2852
|
const stopListening = await listen2(CHANNEL_NEW_BLOCK, async () => {
|
|
2156
2853
|
if (!running)
|
|
2157
2854
|
return;
|
|
2158
|
-
const db =
|
|
2855
|
+
const db = getTargetDb6();
|
|
2159
2856
|
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
2160
2857
|
cleanupCaches(subgraphs);
|
|
2161
2858
|
for (const sg of subgraphs) {
|
|
@@ -2163,11 +2860,11 @@ async function startSubgraphProcessor(opts) {
|
|
|
2163
2860
|
const def = await loadSubgraphDefinition(sg);
|
|
2164
2861
|
await catchUpSubgraph(def, sg.name);
|
|
2165
2862
|
} catch (err) {
|
|
2166
|
-
const msg =
|
|
2863
|
+
const msg = getErrorMessage4(err);
|
|
2167
2864
|
if (isHandlerNotFoundError(err)) {
|
|
2168
|
-
await
|
|
2865
|
+
await updateSubgraphStatus3(db, sg.name, "error");
|
|
2169
2866
|
}
|
|
2170
|
-
|
|
2867
|
+
logger10.error("Subgraph processing failed", {
|
|
2171
2868
|
subgraph: sg.name,
|
|
2172
2869
|
error: msg
|
|
2173
2870
|
});
|
|
@@ -2184,15 +2881,15 @@ async function startSubgraphProcessor(opts) {
|
|
|
2184
2881
|
await handleSubgraphReorg(blockHeight, loadSubgraphDefinition);
|
|
2185
2882
|
}
|
|
2186
2883
|
} catch (err) {
|
|
2187
|
-
|
|
2188
|
-
error:
|
|
2884
|
+
logger10.error("Subgraph reorg handling failed", {
|
|
2885
|
+
error: getErrorMessage4(err)
|
|
2189
2886
|
});
|
|
2190
2887
|
}
|
|
2191
2888
|
}, { connectionString: sourceListenerUrl() });
|
|
2192
2889
|
const pollInterval = setInterval(async () => {
|
|
2193
2890
|
if (!running)
|
|
2194
2891
|
return;
|
|
2195
|
-
const db =
|
|
2892
|
+
const db = getTargetDb6();
|
|
2196
2893
|
const subgraphs = (await listSubgraphs2(db)).filter((v) => v.status === "active");
|
|
2197
2894
|
cleanupCaches(subgraphs);
|
|
2198
2895
|
for (const sg of subgraphs) {
|
|
@@ -2200,37 +2897,38 @@ async function startSubgraphProcessor(opts) {
|
|
|
2200
2897
|
const def = await loadSubgraphDefinition(sg);
|
|
2201
2898
|
await catchUpSubgraph(def, sg.name);
|
|
2202
2899
|
} catch (err) {
|
|
2203
|
-
|
|
2900
|
+
logger10.error("Subgraph poll processing failed", {
|
|
2204
2901
|
subgraph: sg.name,
|
|
2205
|
-
error:
|
|
2902
|
+
error: getErrorMessage4(err)
|
|
2206
2903
|
});
|
|
2207
2904
|
}
|
|
2208
2905
|
}
|
|
2209
2906
|
}, POLL_INTERVAL_MS);
|
|
2210
2907
|
const stopEmitter = await startEmitter();
|
|
2211
|
-
|
|
2908
|
+
logger10.info("Subgraph processor ready");
|
|
2212
2909
|
return async () => {
|
|
2213
2910
|
running = false;
|
|
2214
2911
|
clearInterval(pollInterval);
|
|
2215
2912
|
await stopListening();
|
|
2216
2913
|
await stopReorgListening();
|
|
2914
|
+
await stopOperations();
|
|
2217
2915
|
await stopEmitter();
|
|
2218
|
-
|
|
2916
|
+
logger10.info("Subgraph processor stopped");
|
|
2219
2917
|
};
|
|
2220
2918
|
}
|
|
2221
2919
|
|
|
2222
2920
|
// src/service.ts
|
|
2223
|
-
import { logger as
|
|
2921
|
+
import { logger as logger11 } from "@secondlayer/shared/logger";
|
|
2224
2922
|
var processor = await startSubgraphProcessor({
|
|
2225
2923
|
concurrency: Number.parseInt(process.env.SUBGRAPH_CONCURRENCY ?? "5")
|
|
2226
2924
|
});
|
|
2227
2925
|
var shutdown = async () => {
|
|
2228
|
-
|
|
2926
|
+
logger11.info("Shutting down subgraph processor...");
|
|
2229
2927
|
await processor();
|
|
2230
2928
|
process.exit(0);
|
|
2231
2929
|
};
|
|
2232
2930
|
process.on("SIGINT", shutdown);
|
|
2233
2931
|
process.on("SIGTERM", shutdown);
|
|
2234
2932
|
|
|
2235
|
-
//# debugId=
|
|
2933
|
+
//# debugId=48A79220FFF1766364756E2164756E21
|
|
2236
2934
|
//# sourceMappingURL=service.js.map
|