@secondlayer/subgraphs 3.26.1 → 3.26.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.js +371 -116
- package/dist/src/index.js.map +11 -9
- package/dist/src/runtime/block-processor.js +8 -1
- package/dist/src/runtime/block-processor.js.map +3 -3
- package/dist/src/runtime/catchup.d.ts +34 -1
- package/dist/src/runtime/catchup.js +21 -3
- package/dist/src/runtime/catchup.js.map +4 -4
- package/dist/src/runtime/context.js +8 -1
- package/dist/src/runtime/context.js.map +3 -3
- package/dist/src/runtime/processor.js +323 -305
- package/dist/src/runtime/processor.js.map +7 -7
- package/dist/src/runtime/reindex.js +321 -43
- package/dist/src/runtime/reindex.js.map +7 -5
- package/dist/src/runtime/reorg.js +21 -5
- package/dist/src/runtime/reorg.js.map +4 -4
- package/dist/src/runtime/replay.js +8 -1
- package/dist/src/runtime/replay.js.map +3 -3
- package/dist/src/schema/index.js +8 -1
- package/dist/src/schema/index.js.map +3 -3
- package/dist/src/service.js +325 -307
- package/dist/src/service.js.map +7 -7
- package/dist/src/testing/index.js +8 -1
- package/dist/src/testing/index.js.map +3 -3
- package/package.json +1 -1
package/dist/src/index.js
CHANGED
|
@@ -231,6 +231,13 @@ function quotePgIdent(name) {
|
|
|
231
231
|
return name;
|
|
232
232
|
return `"${name.replace(/"/g, '""')}"`;
|
|
233
233
|
}
|
|
234
|
+
function snakeToCamel(name) {
|
|
235
|
+
return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase());
|
|
236
|
+
}
|
|
237
|
+
function pascalCase(name) {
|
|
238
|
+
const camel = snakeToCamel(name);
|
|
239
|
+
return camel.charAt(0).toUpperCase() + camel.slice(1);
|
|
240
|
+
}
|
|
234
241
|
|
|
235
242
|
// src/schema/generator.ts
|
|
236
243
|
var TYPE_MAP = {
|
|
@@ -3259,9 +3266,278 @@ class StatsAccumulator {
|
|
|
3259
3266
|
}
|
|
3260
3267
|
}
|
|
3261
3268
|
|
|
3269
|
+
// src/runtime/catchup.ts
|
|
3270
|
+
import { getTargetDb as getTargetDb2, sql as sql4 } from "@secondlayer/shared/db";
|
|
3271
|
+
import { getSubgraph } from "@secondlayer/shared/db/queries/subgraphs";
|
|
3272
|
+
import { logger as logger8 } from "@secondlayer/shared/logger";
|
|
3273
|
+
|
|
3274
|
+
// src/runtime/catchup-leader.ts
|
|
3275
|
+
import {
|
|
3276
|
+
SUBGRAPH_CATCHUP_LOCK_KEY,
|
|
3277
|
+
createPostgresLeaderBackend,
|
|
3278
|
+
withLeaderLock
|
|
3279
|
+
} from "@secondlayer/shared/leader";
|
|
3280
|
+
import { targetListenerUrl } from "@secondlayer/shared/queue/listener";
|
|
3281
|
+
var catchUpLeader = false;
|
|
3282
|
+
function isCatchUpLeader() {
|
|
3283
|
+
return catchUpLeader;
|
|
3284
|
+
}
|
|
3285
|
+
function startCatchUpLeader(opts = {}) {
|
|
3286
|
+
return withLeaderLock(SUBGRAPH_CATCHUP_LOCK_KEY, async () => {
|
|
3287
|
+
catchUpLeader = true;
|
|
3288
|
+
await opts.onAcquire?.();
|
|
3289
|
+
return () => {
|
|
3290
|
+
catchUpLeader = false;
|
|
3291
|
+
};
|
|
3292
|
+
}, {
|
|
3293
|
+
pollMs: opts.pollMs,
|
|
3294
|
+
heartbeatMs: opts.heartbeatMs,
|
|
3295
|
+
createBackend: opts.createBackend ?? (() => createPostgresLeaderBackend(targetListenerUrl()))
|
|
3296
|
+
});
|
|
3297
|
+
}
|
|
3298
|
+
|
|
3299
|
+
// src/runtime/catchup.ts
|
|
3300
|
+
var LOG_INTERVAL = 1000;
|
|
3301
|
+
var STANDARD_CATCHUP_BATCH_CONFIG = {
|
|
3302
|
+
defaultBatchSize: 500,
|
|
3303
|
+
minBatchSize: 100,
|
|
3304
|
+
maxBatchSize: 1000,
|
|
3305
|
+
prefetch: true
|
|
3306
|
+
};
|
|
3307
|
+
var catchingUp = new Set;
|
|
3308
|
+
var subgraphBlockLockHeld = new Set;
|
|
3309
|
+
var subgraphBlockLockWaiters = new Map;
|
|
3310
|
+
var reorgEpoch = new Map;
|
|
3311
|
+
async function acquireSubgraphBlockLock(name) {
|
|
3312
|
+
if (!subgraphBlockLockHeld.has(name)) {
|
|
3313
|
+
subgraphBlockLockHeld.add(name);
|
|
3314
|
+
return () => releaseSubgraphBlockLock(name);
|
|
3315
|
+
}
|
|
3316
|
+
return new Promise((resolve) => {
|
|
3317
|
+
const waiters = subgraphBlockLockWaiters.get(name) ?? [];
|
|
3318
|
+
waiters.push(() => resolve(() => releaseSubgraphBlockLock(name)));
|
|
3319
|
+
subgraphBlockLockWaiters.set(name, waiters);
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
3322
|
+
function releaseSubgraphBlockLock(name) {
|
|
3323
|
+
const waiters = subgraphBlockLockWaiters.get(name);
|
|
3324
|
+
if (waiters && waiters.length > 0) {
|
|
3325
|
+
const next = waiters.shift();
|
|
3326
|
+
if (waiters.length === 0)
|
|
3327
|
+
subgraphBlockLockWaiters.delete(name);
|
|
3328
|
+
next();
|
|
3329
|
+
return;
|
|
3330
|
+
}
|
|
3331
|
+
subgraphBlockLockHeld.delete(name);
|
|
3332
|
+
}
|
|
3333
|
+
async function withSubgraphBlockLock(name, fn) {
|
|
3334
|
+
const release = await acquireSubgraphBlockLock(name);
|
|
3335
|
+
try {
|
|
3336
|
+
return await fn();
|
|
3337
|
+
} finally {
|
|
3338
|
+
release();
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
async function withSubgraphBlockAdvisoryLock(name, fn) {
|
|
3342
|
+
return getTargetDb2().transaction().execute(async (lockTx) => {
|
|
3343
|
+
await sql4`SELECT pg_advisory_xact_lock(hashtext(${`subgraph-block:${name}`}))`.execute(lockTx);
|
|
3344
|
+
return fn();
|
|
3345
|
+
});
|
|
3346
|
+
}
|
|
3347
|
+
async function withSubgraphBlockWriteLock(name, fn) {
|
|
3348
|
+
return withSubgraphBlockLock(name, () => withSubgraphBlockAdvisoryLock(name, fn));
|
|
3349
|
+
}
|
|
3350
|
+
function bumpReorgEpoch(name) {
|
|
3351
|
+
reorgEpoch.set(name, (reorgEpoch.get(name) ?? 0) + 1);
|
|
3352
|
+
}
|
|
3353
|
+
function getReorgEpoch(name) {
|
|
3354
|
+
return reorgEpoch.get(name) ?? 0;
|
|
3355
|
+
}
|
|
3356
|
+
|
|
3357
|
+
class ReorgEpochChangedError extends Error {
|
|
3358
|
+
constructor(name) {
|
|
3359
|
+
super(`reorg epoch changed for ${name} mid catch-up`);
|
|
3360
|
+
this.name = "ReorgEpochChangedError";
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
function parsePositiveInt(value) {
|
|
3364
|
+
if (value == null || value.trim() === "")
|
|
3365
|
+
return;
|
|
3366
|
+
const parsed = Number.parseInt(value, 10);
|
|
3367
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
3368
|
+
}
|
|
3369
|
+
function parseBoolean(value) {
|
|
3370
|
+
if (value == null || value.trim() === "")
|
|
3371
|
+
return;
|
|
3372
|
+
const normalized = value.trim().toLowerCase();
|
|
3373
|
+
if (normalized === "true")
|
|
3374
|
+
return true;
|
|
3375
|
+
if (normalized === "false")
|
|
3376
|
+
return false;
|
|
3377
|
+
return;
|
|
3378
|
+
}
|
|
3379
|
+
function resolveCatchupBatchConfig(env = process.env) {
|
|
3380
|
+
const base = STANDARD_CATCHUP_BATCH_CONFIG;
|
|
3381
|
+
const minBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MIN_BATCH_SIZE) ?? base.minBatchSize;
|
|
3382
|
+
const maxBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_MAX_BATCH_SIZE) ?? base.maxBatchSize;
|
|
3383
|
+
const defaultBatchSize = parsePositiveInt(env.SUBGRAPH_CATCHUP_BATCH_SIZE) ?? base.defaultBatchSize;
|
|
3384
|
+
return {
|
|
3385
|
+
minBatchSize,
|
|
3386
|
+
maxBatchSize,
|
|
3387
|
+
defaultBatchSize: Math.min(Math.max(defaultBatchSize, minBatchSize), maxBatchSize),
|
|
3388
|
+
prefetch: parseBoolean(env.SUBGRAPH_CATCHUP_PREFETCH) ?? base.prefetch
|
|
3389
|
+
};
|
|
3390
|
+
}
|
|
3391
|
+
function adjustBatchSize(current, avgEvents, config) {
|
|
3392
|
+
if (avgEvents > 50)
|
|
3393
|
+
return Math.max(Math.round(current * 0.5), config.minBatchSize);
|
|
3394
|
+
if (avgEvents < 10)
|
|
3395
|
+
return Math.min(Math.round(current * 1.5), config.maxBatchSize);
|
|
3396
|
+
return current;
|
|
3397
|
+
}
|
|
3398
|
+
async function catchUpSubgraph(subgraph, subgraphName) {
|
|
3399
|
+
if (catchingUp.has(subgraphName))
|
|
3400
|
+
return 0;
|
|
3401
|
+
catchingUp.add(subgraphName);
|
|
3402
|
+
try {
|
|
3403
|
+
const source = resolveBlockSource(subgraph);
|
|
3404
|
+
const targetDb = getTargetDb2();
|
|
3405
|
+
const subgraphRow = await getSubgraph(targetDb, subgraphName);
|
|
3406
|
+
if (!subgraphRow)
|
|
3407
|
+
return 0;
|
|
3408
|
+
const lastProcessedBlock = Number(subgraphRow.last_processed_block);
|
|
3409
|
+
const chainTip = await source.getTip();
|
|
3410
|
+
if (chainTip <= 0 || lastProcessedBlock >= chainTip)
|
|
3411
|
+
return 0;
|
|
3412
|
+
const subgraphStart = Number(subgraphRow.start_block) || 1;
|
|
3413
|
+
const startBlock = Math.max(lastProcessedBlock + 1, subgraphStart);
|
|
3414
|
+
const totalBlocks = chainTip - lastProcessedBlock;
|
|
3415
|
+
logger8.info("Subgraph catch-up starting", {
|
|
3416
|
+
subgraph: subgraphName,
|
|
3417
|
+
from: startBlock,
|
|
3418
|
+
to: chainTip,
|
|
3419
|
+
blocks: totalBlocks
|
|
3420
|
+
});
|
|
3421
|
+
const stats = new StatsAccumulator(subgraphName, true);
|
|
3422
|
+
let processed = 0;
|
|
3423
|
+
const batchConfig = resolveCatchupBatchConfig();
|
|
3424
|
+
let batchSize = batchConfig.defaultBatchSize;
|
|
3425
|
+
let currentHeight = startBlock;
|
|
3426
|
+
const walkEpoch = getReorgEpoch(subgraphName);
|
|
3427
|
+
let prefetchedBatchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
3428
|
+
let nextBatchPromise = batchConfig.prefetch ? source.loadBlockRange(currentHeight, prefetchedBatchEnd) : undefined;
|
|
3429
|
+
while (currentHeight <= chainTip) {
|
|
3430
|
+
if (!isCatchUpLeader()) {
|
|
3431
|
+
logger8.info("Lost catch-up leadership, stopping catch-up", {
|
|
3432
|
+
subgraph: subgraphName,
|
|
3433
|
+
processed
|
|
3434
|
+
});
|
|
3435
|
+
break;
|
|
3436
|
+
}
|
|
3437
|
+
const currentRow = await getSubgraph(targetDb, subgraphName);
|
|
3438
|
+
if (!currentRow || currentRow.status !== "active") {
|
|
3439
|
+
logger8.info("Subgraph status changed, stopping catch-up", {
|
|
3440
|
+
subgraph: subgraphName,
|
|
3441
|
+
status: currentRow?.status ?? "deleted"
|
|
3442
|
+
});
|
|
3443
|
+
break;
|
|
3444
|
+
}
|
|
3445
|
+
let batchEnd;
|
|
3446
|
+
let batch;
|
|
3447
|
+
if (nextBatchPromise) {
|
|
3448
|
+
batch = await nextBatchPromise;
|
|
3449
|
+
batchEnd = prefetchedBatchEnd;
|
|
3450
|
+
const nextStart = batchEnd + 1;
|
|
3451
|
+
if (nextStart <= chainTip) {
|
|
3452
|
+
prefetchedBatchEnd = Math.min(nextStart + batchSize - 1, chainTip);
|
|
3453
|
+
nextBatchPromise = source.loadBlockRange(nextStart, prefetchedBatchEnd);
|
|
3454
|
+
} else {
|
|
3455
|
+
nextBatchPromise = undefined;
|
|
3456
|
+
}
|
|
3457
|
+
} else {
|
|
3458
|
+
batchEnd = Math.min(currentHeight + batchSize - 1, chainTip);
|
|
3459
|
+
batch = await source.loadBlockRange(currentHeight, batchEnd);
|
|
3460
|
+
}
|
|
3461
|
+
let stopCatchup = false;
|
|
3462
|
+
for (let height = currentHeight;height <= batchEnd; height++) {
|
|
3463
|
+
let blockData = batch.get(height);
|
|
3464
|
+
if (!blockData) {
|
|
3465
|
+
blockData = (await source.loadBlockRange(height, height)).get(height);
|
|
3466
|
+
}
|
|
3467
|
+
if (!blockData) {
|
|
3468
|
+
logger8.warn("Block missing during catch-up, deferring to next tick", {
|
|
3469
|
+
subgraph: subgraphName,
|
|
3470
|
+
blockHeight: height
|
|
3471
|
+
});
|
|
3472
|
+
stopCatchup = true;
|
|
3473
|
+
break;
|
|
3474
|
+
}
|
|
3475
|
+
const preloaded = blockData;
|
|
3476
|
+
let result;
|
|
3477
|
+
try {
|
|
3478
|
+
result = await withSubgraphBlockWriteLock(subgraphName, async () => {
|
|
3479
|
+
if (getReorgEpoch(subgraphName) !== walkEpoch) {
|
|
3480
|
+
throw new ReorgEpochChangedError(subgraphName);
|
|
3481
|
+
}
|
|
3482
|
+
return processBlockWithRetry(subgraph, subgraphName, height, {
|
|
3483
|
+
preloaded
|
|
3484
|
+
});
|
|
3485
|
+
});
|
|
3486
|
+
} catch (err) {
|
|
3487
|
+
if (err instanceof ReorgEpochChangedError) {
|
|
3488
|
+
logger8.info("Reorg detected mid catch-up, aborting walk without writing further progress", { subgraph: subgraphName, blockHeight: height });
|
|
3489
|
+
stopCatchup = true;
|
|
3490
|
+
break;
|
|
3491
|
+
}
|
|
3492
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
3493
|
+
logger8.error("Block processing failed persistently during catch-up", {
|
|
3494
|
+
subgraph: subgraphName,
|
|
3495
|
+
blockHeight: height,
|
|
3496
|
+
error: errorMsg
|
|
3497
|
+
});
|
|
3498
|
+
const { updateSubgraphStatus: updateSubgraphStatus2, recordSubgraphProcessed: recordSubgraphProcessed2 } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
3499
|
+
await recordSubgraphProcessed2(targetDb, subgraphName, 0, 1, `catch-up halted at block ${height}: ${errorMsg}`).catch(() => {});
|
|
3500
|
+
await updateSubgraphStatus2(targetDb, subgraphName, "error").catch(() => {});
|
|
3501
|
+
stopCatchup = true;
|
|
3502
|
+
break;
|
|
3503
|
+
}
|
|
3504
|
+
processed++;
|
|
3505
|
+
if (result.timing) {
|
|
3506
|
+
stats.record(result.timing, result.processed);
|
|
3507
|
+
if (stats.shouldFlush()) {
|
|
3508
|
+
await stats.flush(targetDb);
|
|
3509
|
+
}
|
|
3510
|
+
}
|
|
3511
|
+
if (processed % LOG_INTERVAL === 0) {
|
|
3512
|
+
logger8.info("Subgraph catch-up progress", {
|
|
3513
|
+
subgraph: subgraphName,
|
|
3514
|
+
processed,
|
|
3515
|
+
total: totalBlocks,
|
|
3516
|
+
currentBlock: height,
|
|
3517
|
+
pct: Math.round(processed / totalBlocks * 100)
|
|
3518
|
+
});
|
|
3519
|
+
}
|
|
3520
|
+
}
|
|
3521
|
+
if (stopCatchup)
|
|
3522
|
+
break;
|
|
3523
|
+
const avg = avgEventsPerBlock(batch);
|
|
3524
|
+
batchSize = adjustBatchSize(batchSize, avg, batchConfig);
|
|
3525
|
+
currentHeight = batchEnd + 1;
|
|
3526
|
+
}
|
|
3527
|
+
await stats.flush(targetDb);
|
|
3528
|
+
logger8.info("Subgraph catch-up complete", {
|
|
3529
|
+
subgraph: subgraphName,
|
|
3530
|
+
processed
|
|
3531
|
+
});
|
|
3532
|
+
return processed;
|
|
3533
|
+
} finally {
|
|
3534
|
+
catchingUp.delete(subgraphName);
|
|
3535
|
+
}
|
|
3536
|
+
}
|
|
3537
|
+
|
|
3262
3538
|
// src/runtime/reindex.ts
|
|
3263
3539
|
import { getErrorMessage as getErrorMessage3 } from "@secondlayer/shared";
|
|
3264
|
-
import { getRawClient, getTargetDb as
|
|
3540
|
+
import { getRawClient, getTargetDb as getTargetDb3 } from "@secondlayer/shared/db";
|
|
3265
3541
|
import {
|
|
3266
3542
|
recordGapBatch,
|
|
3267
3543
|
resolveGaps
|
|
@@ -3274,11 +3550,11 @@ import {
|
|
|
3274
3550
|
recordSubgraphProcessed as recordSubgraphProcessed2,
|
|
3275
3551
|
updateSubgraphStatus as updateSubgraphStatus2
|
|
3276
3552
|
} from "@secondlayer/shared/db/queries/subgraphs";
|
|
3277
|
-
import { logger as
|
|
3553
|
+
import { logger as logger10 } from "@secondlayer/shared/logger";
|
|
3278
3554
|
|
|
3279
3555
|
// src/runtime/reindex-notify.ts
|
|
3280
3556
|
import { getErrorMessage as getErrorMessage2 } from "@secondlayer/shared";
|
|
3281
|
-
import { logger as
|
|
3557
|
+
import { logger as logger9 } from "@secondlayer/shared/logger";
|
|
3282
3558
|
async function notifyReindexComplete(db, subgraphName, stats) {
|
|
3283
3559
|
try {
|
|
3284
3560
|
const subgraph = await db.selectFrom("subgraphs").select(["account_id"]).where("name", "=", subgraphName).executeTakeFirst();
|
|
@@ -3289,7 +3565,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
3289
3565
|
return;
|
|
3290
3566
|
const resendKey = process.env.RESEND_API_KEY;
|
|
3291
3567
|
if (!resendKey) {
|
|
3292
|
-
|
|
3568
|
+
logger9.warn("RESEND_API_KEY unset — skipping reindex-complete email", {
|
|
3293
3569
|
subgraph: subgraphName
|
|
3294
3570
|
});
|
|
3295
3571
|
return;
|
|
@@ -3311,14 +3587,14 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
3311
3587
|
});
|
|
3312
3588
|
if (!res.ok) {
|
|
3313
3589
|
const text = await res.text().catch(() => "");
|
|
3314
|
-
|
|
3590
|
+
logger9.warn("reindex-complete email failed", {
|
|
3315
3591
|
subgraph: subgraphName,
|
|
3316
3592
|
status: res.status,
|
|
3317
3593
|
body: text.slice(0, 200)
|
|
3318
3594
|
});
|
|
3319
3595
|
}
|
|
3320
3596
|
} catch (err) {
|
|
3321
|
-
|
|
3597
|
+
logger9.warn("reindex-complete email threw", {
|
|
3322
3598
|
subgraph: subgraphName,
|
|
3323
3599
|
error: getErrorMessage2(err)
|
|
3324
3600
|
});
|
|
@@ -3326,7 +3602,7 @@ async function notifyReindexComplete(db, subgraphName, stats) {
|
|
|
3326
3602
|
}
|
|
3327
3603
|
|
|
3328
3604
|
// src/runtime/reindex.ts
|
|
3329
|
-
var
|
|
3605
|
+
var LOG_INTERVAL2 = 1000;
|
|
3330
3606
|
var HEALTH_FLUSH_INTERVAL = 1000;
|
|
3331
3607
|
var PROGRESS_FLUSH_INTERVAL_MS = 5000;
|
|
3332
3608
|
var EMPTY_BATCH_HALT_THRESHOLD = 3;
|
|
@@ -3346,7 +3622,7 @@ function resolveReindexResumeBlock(row) {
|
|
|
3346
3622
|
const reindexFromBlock = Number(row.reindex_from_block);
|
|
3347
3623
|
return Math.max(lastProcessedBlock + 1, reindexFromBlock);
|
|
3348
3624
|
}
|
|
3349
|
-
function
|
|
3625
|
+
function parsePositiveInt2(value) {
|
|
3350
3626
|
if (value == null || value.trim() === "")
|
|
3351
3627
|
return;
|
|
3352
3628
|
const parsed = Number.parseInt(value, 10);
|
|
@@ -3354,9 +3630,9 @@ function parsePositiveInt(value) {
|
|
|
3354
3630
|
}
|
|
3355
3631
|
function resolveReindexBatchConfig(env = process.env) {
|
|
3356
3632
|
const base = STANDARD_REINDEX_BATCH_CONFIG;
|
|
3357
|
-
const minBatchSize =
|
|
3358
|
-
const maxBatchSize =
|
|
3359
|
-
const defaultBatchSize =
|
|
3633
|
+
const minBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_MIN_BATCH_SIZE) ?? base.minBatchSize;
|
|
3634
|
+
const maxBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_MAX_BATCH_SIZE) ?? base.maxBatchSize;
|
|
3635
|
+
const defaultBatchSize = parsePositiveInt2(env.SUBGRAPH_REINDEX_BATCH_SIZE) ?? base.defaultBatchSize;
|
|
3360
3636
|
return {
|
|
3361
3637
|
minBatchSize,
|
|
3362
3638
|
maxBatchSize,
|
|
@@ -3387,7 +3663,7 @@ function coalesceFailedBlocks(blocks) {
|
|
|
3387
3663
|
}
|
|
3388
3664
|
async function processBlockRange(def, opts) {
|
|
3389
3665
|
const source = resolveBlockSource(def);
|
|
3390
|
-
const targetDb =
|
|
3666
|
+
const targetDb = getTargetDb3();
|
|
3391
3667
|
const subgraphName = def.name;
|
|
3392
3668
|
const { fromBlock, toBlock, status } = opts;
|
|
3393
3669
|
const totalBlocks = toBlock - fromBlock + 1;
|
|
@@ -3430,7 +3706,7 @@ async function processBlockRange(def, opts) {
|
|
|
3430
3706
|
while (currentHeight <= toBlock) {
|
|
3431
3707
|
if (opts.signal?.aborted) {
|
|
3432
3708
|
aborted = true;
|
|
3433
|
-
|
|
3709
|
+
logger10.info("Block processing aborted", {
|
|
3434
3710
|
subgraph: subgraphName,
|
|
3435
3711
|
currentBlock: currentHeight,
|
|
3436
3712
|
reason: String(opts.signal.reason ?? "unknown")
|
|
@@ -3472,14 +3748,16 @@ async function processBlockRange(def, opts) {
|
|
|
3472
3748
|
}
|
|
3473
3749
|
let result;
|
|
3474
3750
|
try {
|
|
3475
|
-
|
|
3751
|
+
const mayWrite = blockData.txs.length > 0 || blockData.events.length > 0;
|
|
3752
|
+
const runBlock = () => processBlockWithRetry(def, subgraphName, height, {
|
|
3476
3753
|
skipProgressUpdate: true,
|
|
3477
3754
|
atomicProgress,
|
|
3478
3755
|
preloaded: blockData
|
|
3479
3756
|
});
|
|
3757
|
+
result = mayWrite ? await withSubgraphBlockWriteLock(subgraphName, runBlock) : await runBlock();
|
|
3480
3758
|
} catch (err) {
|
|
3481
3759
|
const errorMsg = getErrorMessage3(err);
|
|
3482
|
-
|
|
3760
|
+
logger10.error("Block processing failed persistently", {
|
|
3483
3761
|
subgraph: subgraphName,
|
|
3484
3762
|
blockHeight: height,
|
|
3485
3763
|
error: errorMsg
|
|
@@ -3524,8 +3802,8 @@ async function processBlockRange(def, opts) {
|
|
|
3524
3802
|
}
|
|
3525
3803
|
lastProgressFlushAt = now;
|
|
3526
3804
|
}
|
|
3527
|
-
if (blocksProcessed %
|
|
3528
|
-
|
|
3805
|
+
if (blocksProcessed % LOG_INTERVAL2 === 0) {
|
|
3806
|
+
logger10.info(`${status === "reindexing" ? "Reindex" : "Backfill"} progress`, {
|
|
3529
3807
|
subgraph: subgraphName,
|
|
3530
3808
|
processed: blocksProcessed,
|
|
3531
3809
|
total: totalBlocks,
|
|
@@ -3545,7 +3823,7 @@ async function processBlockRange(def, opts) {
|
|
|
3545
3823
|
if (batchFailedBlocks.length > 0 && opts.subgraphId) {
|
|
3546
3824
|
const gaps = coalesceFailedBlocks(batchFailedBlocks);
|
|
3547
3825
|
await recordGapBatch(targetDb, opts.subgraphId, subgraphName, gaps).catch((err) => {
|
|
3548
|
-
|
|
3826
|
+
logger10.warn("Failed to record subgraph gaps", {
|
|
3549
3827
|
subgraph: subgraphName,
|
|
3550
3828
|
error: err instanceof Error ? err.message : String(err)
|
|
3551
3829
|
});
|
|
@@ -3562,7 +3840,7 @@ async function processBlockRange(def, opts) {
|
|
|
3562
3840
|
} else {
|
|
3563
3841
|
await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
|
|
3564
3842
|
}
|
|
3565
|
-
|
|
3843
|
+
logger10.info("Sparse skip", {
|
|
3566
3844
|
subgraph: subgraphName,
|
|
3567
3845
|
from: batchEnd + 1,
|
|
3568
3846
|
to: jumpTo - 1,
|
|
@@ -3597,16 +3875,16 @@ async function clearReindexMetadata(db, subgraphName) {
|
|
|
3597
3875
|
}
|
|
3598
3876
|
async function reindexSubgraph(def, opts) {
|
|
3599
3877
|
const source = resolveBlockSource(def);
|
|
3600
|
-
const targetDb =
|
|
3878
|
+
const targetDb = getTargetDb3();
|
|
3601
3879
|
const client = getRawClient("target");
|
|
3602
3880
|
const subgraphName = def.name;
|
|
3603
3881
|
const schemaName = opts?.schemaName ?? pgSchemaName(subgraphName);
|
|
3604
3882
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing");
|
|
3605
|
-
|
|
3883
|
+
logger10.info("Reindex starting", { subgraph: subgraphName });
|
|
3606
3884
|
try {
|
|
3607
3885
|
const { fromBlock, toBlock } = await resolveBlockRange(source, def, opts);
|
|
3608
3886
|
if (fromBlock > toBlock) {
|
|
3609
|
-
|
|
3887
|
+
logger10.info("No blocks to reindex", {
|
|
3610
3888
|
subgraph: subgraphName,
|
|
3611
3889
|
fromBlock,
|
|
3612
3890
|
toBlock
|
|
@@ -3620,7 +3898,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3620
3898
|
await client.unsafe(stmt);
|
|
3621
3899
|
}
|
|
3622
3900
|
await updateSubgraphStatus2(targetDb, subgraphName, "reindexing", Math.max(0, fromBlock - 1));
|
|
3623
|
-
|
|
3901
|
+
logger10.info("Schema recreated for reindex", {
|
|
3624
3902
|
subgraph: subgraphName,
|
|
3625
3903
|
cursorResetTo: Math.max(0, fromBlock - 1)
|
|
3626
3904
|
});
|
|
@@ -3634,7 +3912,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3634
3912
|
last_error_at: null,
|
|
3635
3913
|
updated_at: new Date
|
|
3636
3914
|
}).where("name", "=", subgraphName).execute();
|
|
3637
|
-
|
|
3915
|
+
logger10.info("Reindexing blocks", {
|
|
3638
3916
|
subgraph: subgraphName,
|
|
3639
3917
|
fromBlock,
|
|
3640
3918
|
toBlock,
|
|
@@ -3656,9 +3934,9 @@ async function reindexSubgraph(def, opts) {
|
|
|
3656
3934
|
if (reason === "user-cancelled") {
|
|
3657
3935
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3658
3936
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3659
|
-
|
|
3937
|
+
logger10.info("Reindex cancelled by user", { subgraph: subgraphName });
|
|
3660
3938
|
} else {
|
|
3661
|
-
|
|
3939
|
+
logger10.info("Reindex interrupted by shutdown, will resume", {
|
|
3662
3940
|
subgraph: subgraphName
|
|
3663
3941
|
});
|
|
3664
3942
|
}
|
|
@@ -3666,7 +3944,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3666
3944
|
}
|
|
3667
3945
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3668
3946
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3669
|
-
|
|
3947
|
+
logger10.info("Reindex complete", {
|
|
3670
3948
|
subgraph: subgraphName,
|
|
3671
3949
|
blocks: result.blocksProcessed,
|
|
3672
3950
|
events: result.totalEventsProcessed,
|
|
@@ -3679,7 +3957,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3679
3957
|
});
|
|
3680
3958
|
return { processed: result.blocksProcessed };
|
|
3681
3959
|
} catch (err) {
|
|
3682
|
-
|
|
3960
|
+
logger10.error("Reindex failed", {
|
|
3683
3961
|
subgraph: subgraphName,
|
|
3684
3962
|
error: getErrorMessage3(err)
|
|
3685
3963
|
});
|
|
@@ -3688,7 +3966,7 @@ async function reindexSubgraph(def, opts) {
|
|
|
3688
3966
|
}
|
|
3689
3967
|
}
|
|
3690
3968
|
async function resumeReindex(def, opts) {
|
|
3691
|
-
const targetDb =
|
|
3969
|
+
const targetDb = getTargetDb3();
|
|
3692
3970
|
const subgraphName = def.name;
|
|
3693
3971
|
const row = await targetDb.selectFrom("subgraphs").select([
|
|
3694
3972
|
"id",
|
|
@@ -3701,7 +3979,7 @@ async function resumeReindex(def, opts) {
|
|
|
3701
3979
|
const fromBlock = resolveReindexResumeBlock(row);
|
|
3702
3980
|
const toBlock = Number(row.reindex_to_block);
|
|
3703
3981
|
if (fromBlock == null) {
|
|
3704
|
-
|
|
3982
|
+
logger10.info("No reindex metadata, starting fresh reindex", {
|
|
3705
3983
|
subgraph: subgraphName
|
|
3706
3984
|
});
|
|
3707
3985
|
return reindexSubgraph(def, {
|
|
@@ -3710,12 +3988,12 @@ async function resumeReindex(def, opts) {
|
|
|
3710
3988
|
});
|
|
3711
3989
|
}
|
|
3712
3990
|
if (fromBlock > toBlock) {
|
|
3713
|
-
|
|
3991
|
+
logger10.info("Resume: no remaining blocks", { subgraph: subgraphName });
|
|
3714
3992
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3715
3993
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3716
3994
|
return { processed: 0 };
|
|
3717
3995
|
}
|
|
3718
|
-
|
|
3996
|
+
logger10.info("Resuming reindex", {
|
|
3719
3997
|
subgraph: subgraphName,
|
|
3720
3998
|
fromBlock,
|
|
3721
3999
|
toBlock,
|
|
@@ -3737,9 +4015,9 @@ async function resumeReindex(def, opts) {
|
|
|
3737
4015
|
if (reason === "user-cancelled") {
|
|
3738
4016
|
await updateSubgraphStatus2(targetDb, subgraphName, "active");
|
|
3739
4017
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3740
|
-
|
|
4018
|
+
logger10.info("Resume cancelled by user", { subgraph: subgraphName });
|
|
3741
4019
|
} else {
|
|
3742
|
-
|
|
4020
|
+
logger10.info("Resume interrupted by shutdown, will resume again", {
|
|
3743
4021
|
subgraph: subgraphName
|
|
3744
4022
|
});
|
|
3745
4023
|
}
|
|
@@ -3747,13 +4025,13 @@ async function resumeReindex(def, opts) {
|
|
|
3747
4025
|
}
|
|
3748
4026
|
await updateSubgraphStatus2(targetDb, subgraphName, "active", toBlock);
|
|
3749
4027
|
await clearReindexMetadata(targetDb, subgraphName);
|
|
3750
|
-
|
|
4028
|
+
logger10.info("Resumed reindex complete", {
|
|
3751
4029
|
subgraph: subgraphName,
|
|
3752
4030
|
blocks: result.blocksProcessed
|
|
3753
4031
|
});
|
|
3754
4032
|
return { processed: result.blocksProcessed };
|
|
3755
4033
|
} catch (err) {
|
|
3756
|
-
|
|
4034
|
+
logger10.error("Resumed reindex failed", {
|
|
3757
4035
|
subgraph: subgraphName,
|
|
3758
4036
|
error: getErrorMessage3(err)
|
|
3759
4037
|
});
|
|
@@ -3762,9 +4040,9 @@ async function resumeReindex(def, opts) {
|
|
|
3762
4040
|
}
|
|
3763
4041
|
}
|
|
3764
4042
|
async function backfillSubgraph(def, opts) {
|
|
3765
|
-
const targetDb =
|
|
4043
|
+
const targetDb = getTargetDb3();
|
|
3766
4044
|
const subgraphName = def.name;
|
|
3767
|
-
|
|
4045
|
+
logger10.info("Backfill starting", {
|
|
3768
4046
|
subgraph: subgraphName,
|
|
3769
4047
|
from: opts.fromBlock,
|
|
3770
4048
|
to: opts.toBlock
|
|
@@ -3782,17 +4060,17 @@ async function backfillSubgraph(def, opts) {
|
|
|
3782
4060
|
signal: opts.signal
|
|
3783
4061
|
});
|
|
3784
4062
|
if (result.aborted) {
|
|
3785
|
-
|
|
4063
|
+
logger10.info("Backfill aborted", { subgraph: subgraphName });
|
|
3786
4064
|
return { processed: result.blocksProcessed };
|
|
3787
4065
|
}
|
|
3788
4066
|
const resolved = await resolveGaps(targetDb, subgraphName, opts.fromBlock, opts.toBlock).catch(() => 0);
|
|
3789
4067
|
if (resolved > 0) {
|
|
3790
|
-
|
|
4068
|
+
logger10.info("Resolved subgraph gaps via backfill", {
|
|
3791
4069
|
subgraph: subgraphName,
|
|
3792
4070
|
resolved
|
|
3793
4071
|
});
|
|
3794
4072
|
}
|
|
3795
|
-
|
|
4073
|
+
logger10.info("Backfill complete", {
|
|
3796
4074
|
subgraph: subgraphName,
|
|
3797
4075
|
blocks: result.blocksProcessed,
|
|
3798
4076
|
events: result.totalEventsProcessed,
|
|
@@ -3800,7 +4078,7 @@ async function backfillSubgraph(def, opts) {
|
|
|
3800
4078
|
});
|
|
3801
4079
|
return { processed: result.blocksProcessed };
|
|
3802
4080
|
} catch (err) {
|
|
3803
|
-
|
|
4081
|
+
logger10.error("Backfill failed", {
|
|
3804
4082
|
subgraph: subgraphName,
|
|
3805
4083
|
error: getErrorMessage3(err)
|
|
3806
4084
|
});
|
|
@@ -4165,13 +4443,6 @@ var SYSTEM_FIELDS = [
|
|
|
4165
4443
|
line: "DateTime @default(now()) @db.Timestamptz"
|
|
4166
4444
|
}
|
|
4167
4445
|
];
|
|
4168
|
-
function snakeToCamel(name) {
|
|
4169
|
-
return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase());
|
|
4170
|
-
}
|
|
4171
|
-
function pascalCase(name) {
|
|
4172
|
-
const camel = snakeToCamel(name);
|
|
4173
|
-
return camel.charAt(0).toUpperCase() + camel.slice(1);
|
|
4174
|
-
}
|
|
4175
4446
|
function quotePrismaDefault(value) {
|
|
4176
4447
|
if (typeof value === "string")
|
|
4177
4448
|
return `@default("${value.replace(/"/g, "\\\"")}")`;
|
|
@@ -4296,13 +4567,6 @@ var BUILDERS_USED = {
|
|
|
4296
4567
|
timestamp: "timestamp",
|
|
4297
4568
|
jsonb: "jsonb"
|
|
4298
4569
|
};
|
|
4299
|
-
function snakeToCamel2(name) {
|
|
4300
|
-
return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase());
|
|
4301
|
-
}
|
|
4302
|
-
function pascalCase2(name) {
|
|
4303
|
-
const camel = snakeToCamel2(name);
|
|
4304
|
-
return camel.charAt(0).toUpperCase() + camel.slice(1);
|
|
4305
|
-
}
|
|
4306
4570
|
function literalDefault(value) {
|
|
4307
4571
|
return typeof value === "string" ? `"${value.replace(/"/g, "\\\"")}"` : `${value}`;
|
|
4308
4572
|
}
|
|
@@ -4314,7 +4578,7 @@ function renderTable2(varName, tableName, table) {
|
|
|
4314
4578
|
` createdAt: timestamp("_created_at", { withTimezone: true }).notNull().defaultNow(),`
|
|
4315
4579
|
];
|
|
4316
4580
|
for (const [colName, col] of Object.entries(table.columns)) {
|
|
4317
|
-
let line = ` ${
|
|
4581
|
+
let line = ` ${snakeToCamel(colName)}: ${DRIZZLE_BUILDER[col.type](colName)}`;
|
|
4318
4582
|
if (!col.nullable)
|
|
4319
4583
|
line += ".notNull()";
|
|
4320
4584
|
if (col.default !== undefined)
|
|
@@ -4324,16 +4588,16 @@ function renderTable2(varName, tableName, table) {
|
|
|
4324
4588
|
const extras = [];
|
|
4325
4589
|
for (const [colName, col] of Object.entries(table.columns)) {
|
|
4326
4590
|
if (col.indexed) {
|
|
4327
|
-
const f =
|
|
4591
|
+
const f = snakeToCamel(colName);
|
|
4328
4592
|
extras.push(` ${f}Idx: index("idx_${tableName}_${colName}").on(t.${f})`);
|
|
4329
4593
|
}
|
|
4330
4594
|
}
|
|
4331
4595
|
table.indexes?.forEach((idxCols, i) => {
|
|
4332
|
-
const on = idxCols.map((c) => `t.${
|
|
4596
|
+
const on = idxCols.map((c) => `t.${snakeToCamel(c)}`).join(", ");
|
|
4333
4597
|
extras.push(` idx${i}: index("idx_${tableName}_${i}").on(${on})`);
|
|
4334
4598
|
});
|
|
4335
4599
|
table.uniqueKeys?.forEach((uqCols, i) => {
|
|
4336
|
-
const on = uqCols.map((c) => `t.${
|
|
4600
|
+
const on = uqCols.map((c) => `t.${snakeToCamel(c)}`).join(", ");
|
|
4337
4601
|
extras.push(` uq${i}: uniqueIndex("uq_${tableName}_${i}").on(${on})`);
|
|
4338
4602
|
});
|
|
4339
4603
|
const extrasBlock = extras.length ? `, (t) => ({
|
|
@@ -4351,19 +4615,19 @@ function renderRelations(schema) {
|
|
|
4351
4615
|
for (const rel of table.relations ?? []) {
|
|
4352
4616
|
const list = back.get(rel.references) ?? [];
|
|
4353
4617
|
list.push({
|
|
4354
|
-
field: `${
|
|
4355
|
-
from:
|
|
4618
|
+
field: `${snakeToCamel(owning)}${pascalCase(rel.name)}`,
|
|
4619
|
+
from: snakeToCamel(owning)
|
|
4356
4620
|
});
|
|
4357
4621
|
back.set(rel.references, list);
|
|
4358
4622
|
}
|
|
4359
4623
|
}
|
|
4360
4624
|
const out = [];
|
|
4361
4625
|
for (const [tableName, table] of Object.entries(schema)) {
|
|
4362
|
-
const v =
|
|
4626
|
+
const v = snakeToCamel(tableName);
|
|
4363
4627
|
const ones = (table.relations ?? []).map((rel) => {
|
|
4364
|
-
const localFields = rel.fields.map((f) => `${v}.${
|
|
4365
|
-
const refFields = rel.referencedColumns.map((c) => `${
|
|
4366
|
-
return ` ${rel.name}: one(${
|
|
4628
|
+
const localFields = rel.fields.map((f) => `${v}.${snakeToCamel(f)}`).join(", ");
|
|
4629
|
+
const refFields = rel.referencedColumns.map((c) => `${snakeToCamel(rel.references)}.${snakeToCamel(c)}`).join(", ");
|
|
4630
|
+
return ` ${rel.name}: one(${snakeToCamel(rel.references)}, { fields: [${localFields}], references: [${refFields}] }),`;
|
|
4367
4631
|
});
|
|
4368
4632
|
const manys = (back.get(tableName) ?? []).map((b) => ` ${b.field}: many(${b.from}),`);
|
|
4369
4633
|
if (ones.length === 0 && manys.length === 0)
|
|
@@ -4400,9 +4664,9 @@ function generateDrizzleSchema(def, opts = {}) {
|
|
|
4400
4664
|
...hasRelations ? [`import { relations } from "drizzle-orm";`] : []
|
|
4401
4665
|
].join(`
|
|
4402
4666
|
`);
|
|
4403
|
-
const tables = Object.entries(def.schema).map(([name, table]) => renderTable2(
|
|
4667
|
+
const tables = Object.entries(def.schema).map(([name, table]) => renderTable2(snakeToCamel(name), name, table));
|
|
4404
4668
|
const relationDecls = renderRelations(def.schema);
|
|
4405
|
-
const typeExports = Object.keys(def.schema).map((name) => `export type ${
|
|
4669
|
+
const typeExports = Object.keys(def.schema).map((name) => `export type ${pascalCase(name)} = typeof ${snakeToCamel(name)}.$inferSelect;`);
|
|
4406
4670
|
return [
|
|
4407
4671
|
`// Generated by \`sl subgraphs codegen --target drizzle\` from subgraph "${def.name}". Do not edit by hand.`,
|
|
4408
4672
|
imports,
|
|
@@ -4438,9 +4702,6 @@ var SYSTEM_FIELDS2 = [
|
|
|
4438
4702
|
{ col: "_tx_id", type: "string" },
|
|
4439
4703
|
{ col: "_created_at", type: "Generated<Date>" }
|
|
4440
4704
|
];
|
|
4441
|
-
function pascalCase3(name) {
|
|
4442
|
-
return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase()).replace(/^[a-z]/, (c) => c.toUpperCase());
|
|
4443
|
-
}
|
|
4444
4705
|
function renderTable3(tableName, table) {
|
|
4445
4706
|
const lines = SYSTEM_FIELDS2.map((f) => ` ${f.col}: ${f.type};`);
|
|
4446
4707
|
for (const [colName, col] of Object.entries(table.columns)) {
|
|
@@ -4451,7 +4712,7 @@ function renderTable3(tableName, table) {
|
|
|
4451
4712
|
type = `Generated<${type}>`;
|
|
4452
4713
|
lines.push(` ${colName}: ${type};`);
|
|
4453
4714
|
}
|
|
4454
|
-
return `export interface ${
|
|
4715
|
+
return `export interface ${pascalCase(tableName)} {
|
|
4455
4716
|
${lines.join(`
|
|
4456
4717
|
`)}
|
|
4457
4718
|
}`;
|
|
@@ -4459,12 +4720,12 @@ ${lines.join(`
|
|
|
4459
4720
|
function generateKyselySchema(def, opts = {}) {
|
|
4460
4721
|
const schemaName = opts.schemaName ?? pgSchemaName(def.name);
|
|
4461
4722
|
const interfaces = Object.entries(def.schema).map(([name, table]) => renderTable3(name, table));
|
|
4462
|
-
const dbEntries = Object.keys(def.schema).map((name) => ` "${schemaName}.${name}": ${
|
|
4723
|
+
const dbEntries = Object.keys(def.schema).map((name) => ` "${schemaName}.${name}": ${pascalCase(name)};`);
|
|
4463
4724
|
const dbRegistry = `export interface DB {
|
|
4464
4725
|
${dbEntries.join(`
|
|
4465
4726
|
`)}
|
|
4466
4727
|
}`;
|
|
4467
|
-
const rowExports = Object.keys(def.schema).map((name) => `export type ${
|
|
4728
|
+
const rowExports = Object.keys(def.schema).map((name) => `export type ${pascalCase(name)}Row = Selectable<${pascalCase(name)}>;`);
|
|
4468
4729
|
return [
|
|
4469
4730
|
`// Generated by \`sl subgraphs codegen --target kysely\` from subgraph "${def.name}". Do not edit by hand.`,
|
|
4470
4731
|
`import type { Generated, Selectable } from "kysely";`,
|
|
@@ -4486,12 +4747,6 @@ import {
|
|
|
4486
4747
|
SOURCE_READ_PKS,
|
|
4487
4748
|
SOURCE_READ_TYPES
|
|
4488
4749
|
} from "@secondlayer/shared/db";
|
|
4489
|
-
function pascalCase4(name) {
|
|
4490
|
-
return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase()).replace(/^[a-z]/, (c) => c.toUpperCase());
|
|
4491
|
-
}
|
|
4492
|
-
function snakeToCamel3(name) {
|
|
4493
|
-
return name.replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase());
|
|
4494
|
-
}
|
|
4495
4750
|
function selectTables(opts) {
|
|
4496
4751
|
const all = SOURCE_READ_TYPES;
|
|
4497
4752
|
const names = opts.tables ?? Object.keys(all);
|
|
@@ -4516,16 +4771,16 @@ function generateKysely(entries, schemaName) {
|
|
|
4516
4771
|
const type = def.nullable ? `${KYSELY_TYPE2[def.type]} | null` : KYSELY_TYPE2[def.type];
|
|
4517
4772
|
return ` ${col}: ${type};`;
|
|
4518
4773
|
});
|
|
4519
|
-
return `export interface ${
|
|
4774
|
+
return `export interface ${pascalCase(table)} {
|
|
4520
4775
|
${lines.join(`
|
|
4521
4776
|
`)}
|
|
4522
4777
|
}`;
|
|
4523
4778
|
});
|
|
4524
4779
|
const dbEntries = entries.map(([table]) => {
|
|
4525
4780
|
const key2 = schemaName ? `${schemaName}.${table}` : table;
|
|
4526
|
-
return ` "${key2}": ${
|
|
4781
|
+
return ` "${key2}": ${pascalCase(table)};`;
|
|
4527
4782
|
});
|
|
4528
|
-
const rows = entries.map(([table]) => `export type ${
|
|
4783
|
+
const rows = entries.map(([table]) => `export type ${pascalCase(table)}Row = Selectable<${pascalCase(table)}>;`);
|
|
4529
4784
|
return [
|
|
4530
4785
|
"// Generated by `sl index codegen --target kysely`. Do not edit by hand.",
|
|
4531
4786
|
`import type { Selectable } from "kysely";`,
|
|
@@ -4571,17 +4826,17 @@ function generateDrizzle(entries, schemaName) {
|
|
|
4571
4826
|
const tableFn = schemaName ? "sg.table" : "pgTable";
|
|
4572
4827
|
const tables = entries.map(([table, cols]) => {
|
|
4573
4828
|
const lines = Object.entries(cols).map(([col, def]) => {
|
|
4574
|
-
let line = ` ${
|
|
4829
|
+
let line = ` ${snakeToCamel(col)}: ${DRIZZLE_BUILDER2[def.type](col)}`;
|
|
4575
4830
|
if (!def.nullable)
|
|
4576
4831
|
line += ".notNull()";
|
|
4577
4832
|
return `${line},`;
|
|
4578
4833
|
});
|
|
4579
|
-
return `export const ${
|
|
4834
|
+
return `export const ${snakeToCamel(table)} = ${tableFn}("${table}", {
|
|
4580
4835
|
${lines.join(`
|
|
4581
4836
|
`)}
|
|
4582
4837
|
});`;
|
|
4583
4838
|
});
|
|
4584
|
-
const rows = entries.map(([table]) => `export type ${
|
|
4839
|
+
const rows = entries.map(([table]) => `export type ${pascalCase(table)} = typeof ${snakeToCamel(table)}.$inferSelect;`);
|
|
4585
4840
|
return [
|
|
4586
4841
|
"// Generated by `sl index codegen --target drizzle`. Do not edit by hand.",
|
|
4587
4842
|
`import { ${[...used].sort().join(", ")} } from "drizzle-orm/pg-core";`,
|
|
@@ -4619,7 +4874,7 @@ function generateJsonSchema(entries) {
|
|
|
4619
4874
|
required.push(col);
|
|
4620
4875
|
}
|
|
4621
4876
|
}
|
|
4622
|
-
defs[
|
|
4877
|
+
defs[pascalCase(table)] = {
|
|
4623
4878
|
type: "object",
|
|
4624
4879
|
properties,
|
|
4625
4880
|
required,
|
|
@@ -4652,7 +4907,7 @@ function generatePrisma(entries, schemaName, datasourceEnv) {
|
|
|
4652
4907
|
}
|
|
4653
4908
|
const single = pk.length === 1 ? pk[0] : null;
|
|
4654
4909
|
const lines = Object.entries(cols).map(([col, def]) => {
|
|
4655
|
-
const field =
|
|
4910
|
+
const field = snakeToCamel(col);
|
|
4656
4911
|
const mapped = PRISMA_TYPE2[def.type];
|
|
4657
4912
|
const attrs = [];
|
|
4658
4913
|
if (single === col)
|
|
@@ -4666,12 +4921,12 @@ function generatePrisma(entries, schemaName, datasourceEnv) {
|
|
|
4666
4921
|
});
|
|
4667
4922
|
const block = [];
|
|
4668
4923
|
if (!single) {
|
|
4669
|
-
block.push(` @@id([${pk.map(
|
|
4924
|
+
block.push(` @@id([${pk.map(snakeToCamel).join(", ")}])`);
|
|
4670
4925
|
}
|
|
4671
4926
|
block.push(` @@map("${table}")`);
|
|
4672
4927
|
if (schemaName)
|
|
4673
4928
|
block.push(` @@schema("${schemaName}")`);
|
|
4674
|
-
models.push(`model ${
|
|
4929
|
+
models.push(`model ${pascalCase(table)} {
|
|
4675
4930
|
${lines.join(`
|
|
4676
4931
|
`)}
|
|
4677
4932
|
|
|
@@ -4721,7 +4976,7 @@ function generateIndexSchema(target, opts = {}) {
|
|
|
4721
4976
|
}
|
|
4722
4977
|
var INDEX_CODEGEN_TABLES = Object.keys(SOURCE_READ_TYPES);
|
|
4723
4978
|
// src/schema/deployer.ts
|
|
4724
|
-
import { sql as
|
|
4979
|
+
import { sql as sql5 } from "kysely";
|
|
4725
4980
|
function toJsonSafe(obj) {
|
|
4726
4981
|
return JSON.parse(JSON.stringify(obj, (_key, value) => typeof value === "bigint" ? value.toString() : value));
|
|
4727
4982
|
}
|
|
@@ -4836,7 +5091,7 @@ function renderDeployPlan(def, schemaName) {
|
|
|
4836
5091
|
async function deploySchema(db, def, handlerPath, opts) {
|
|
4837
5092
|
validateSubgraphDefinition(def);
|
|
4838
5093
|
const { statements, hash } = generateSubgraphSQL(def, opts?.schemaName);
|
|
4839
|
-
const { getSubgraph, registerSubgraph } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
5094
|
+
const { getSubgraph: getSubgraph2, registerSubgraph } = await import("@secondlayer/shared/db/queries/subgraphs");
|
|
4840
5095
|
const ddlDb = opts?.dataDb ?? db;
|
|
4841
5096
|
const byo = opts?.dataDb != null;
|
|
4842
5097
|
const refuseDestructiveOnByo = (reasons, diff) => {
|
|
@@ -4848,7 +5103,7 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4848
5103
|
grantScript: plan.grantScript
|
|
4849
5104
|
});
|
|
4850
5105
|
};
|
|
4851
|
-
const existing = await
|
|
5106
|
+
const existing = await getSubgraph2(db, def.name, opts?.accountId);
|
|
4852
5107
|
const schemaName = opts?.schemaName ?? pgSchemaName(def.name);
|
|
4853
5108
|
const newVersion = opts?.version ?? (existing ? bumpPatch(existing.version) : "1.0.0");
|
|
4854
5109
|
const regData = {
|
|
@@ -4873,7 +5128,7 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4873
5128
|
databaseUrlEnc: opts?.databaseUrlEnc ?? null
|
|
4874
5129
|
};
|
|
4875
5130
|
if (existing) {
|
|
4876
|
-
const schemaExists = await
|
|
5131
|
+
const schemaExists = await sql5`
|
|
4877
5132
|
SELECT EXISTS (
|
|
4878
5133
|
SELECT 1 FROM information_schema.schemata
|
|
4879
5134
|
WHERE schema_name = ${schemaName}
|
|
@@ -4881,7 +5136,7 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4881
5136
|
`.execute(ddlDb).then((r) => r.rows[0]?.exists ?? false);
|
|
4882
5137
|
if (!schemaExists) {
|
|
4883
5138
|
for (const stmt of statements) {
|
|
4884
|
-
await
|
|
5139
|
+
await sql5.raw(stmt).execute(ddlDb);
|
|
4885
5140
|
}
|
|
4886
5141
|
const sg2 = await registerSubgraph(db, regData);
|
|
4887
5142
|
return { action: "reindexed", subgraphId: sg2.id, version: newVersion };
|
|
@@ -4907,9 +5162,9 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4907
5162
|
if (existing.schema_hash === hash && opts?.forceReindex) {
|
|
4908
5163
|
if (byo)
|
|
4909
5164
|
refuseDestructiveOnByo(["force reindex"], null);
|
|
4910
|
-
await
|
|
5165
|
+
await sql5.raw(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`).execute(ddlDb);
|
|
4911
5166
|
for (const stmt of statements) {
|
|
4912
|
-
await
|
|
5167
|
+
await sql5.raw(stmt).execute(ddlDb);
|
|
4913
5168
|
}
|
|
4914
5169
|
const sg2 = await registerSubgraph(db, regData);
|
|
4915
5170
|
return { action: "reindexed", subgraphId: sg2.id, version: newVersion };
|
|
@@ -4921,9 +5176,9 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4921
5176
|
if (byo) {
|
|
4922
5177
|
refuseDestructiveOnByo(reasons.length > 0 ? reasons : ["force reindex"], diff);
|
|
4923
5178
|
}
|
|
4924
|
-
await
|
|
5179
|
+
await sql5.raw(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`).execute(ddlDb);
|
|
4925
5180
|
for (const stmt of statements) {
|
|
4926
|
-
await
|
|
5181
|
+
await sql5.raw(stmt).execute(ddlDb);
|
|
4927
5182
|
}
|
|
4928
5183
|
const sg3 = await registerSubgraph(db, regData);
|
|
4929
5184
|
const deployDiff2 = toDeployDiff(diff, reasons);
|
|
@@ -4936,16 +5191,16 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4936
5191
|
}
|
|
4937
5192
|
const addedDefs = diff.addedTables.map((tableName) => ({ tableName, tableDef: def.schema[tableName] })).filter((t) => t.tableDef !== undefined);
|
|
4938
5193
|
if (addedDefs.some(({ tableDef }) => tableNeedsTrgm(tableDef))) {
|
|
4939
|
-
await
|
|
5194
|
+
await sql5.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm").execute(ddlDb);
|
|
4940
5195
|
}
|
|
4941
5196
|
for (const { tableName, tableDef } of addedDefs) {
|
|
4942
5197
|
for (const stmt of emitTableDDL(schemaName, tableName, tableDef)) {
|
|
4943
|
-
await
|
|
5198
|
+
await sql5.raw(stmt).execute(ddlDb);
|
|
4944
5199
|
}
|
|
4945
5200
|
}
|
|
4946
5201
|
for (const { tableName, tableDef } of addedDefs) {
|
|
4947
5202
|
for (const stmt of emitForeignKeyDDL(schemaName, tableName, tableDef)) {
|
|
4948
|
-
await
|
|
5203
|
+
await sql5.raw(stmt).execute(ddlDb);
|
|
4949
5204
|
}
|
|
4950
5205
|
}
|
|
4951
5206
|
for (const [tableName, colDiff] of Object.entries(diff.tables)) {
|
|
@@ -4963,12 +5218,12 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4963
5218
|
if (!sqlType)
|
|
4964
5219
|
continue;
|
|
4965
5220
|
const nullable = col.nullable ? "" : ` NOT NULL DEFAULT ${getDefault(col.type)}`;
|
|
4966
|
-
await
|
|
5221
|
+
await sql5.raw(`ALTER TABLE ${qualifiedName} ADD COLUMN ${colName} ${sqlType}${nullable}`).execute(ddlDb);
|
|
4967
5222
|
if (col.indexed) {
|
|
4968
|
-
await
|
|
5223
|
+
await sql5.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`).execute(ddlDb);
|
|
4969
5224
|
}
|
|
4970
5225
|
if (col.search) {
|
|
4971
|
-
await
|
|
5226
|
+
await sql5.raw(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`).execute(ddlDb);
|
|
4972
5227
|
}
|
|
4973
5228
|
}
|
|
4974
5229
|
}
|
|
@@ -4976,7 +5231,7 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4976
5231
|
const flagFlips = {};
|
|
4977
5232
|
const needsTrgmForFlips = Object.entries(diff.tables).some(([tableName, colDiff]) => colDiff.indexChanged.some((colName) => def.schema[tableName]?.columns[colName]?.search && !existingSchema[tableName]?.columns[colName]?.search));
|
|
4978
5233
|
if (needsTrgmForFlips) {
|
|
4979
|
-
await
|
|
5234
|
+
await sql5.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm").execute(ddlDb);
|
|
4980
5235
|
}
|
|
4981
5236
|
for (const [tableName, colDiff] of Object.entries(diff.tables)) {
|
|
4982
5237
|
if (colDiff.indexChanged.length === 0)
|
|
@@ -4991,14 +5246,14 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
4991
5246
|
const plainIdx = `idx_${schemaName}_${tableName}_${colName}`;
|
|
4992
5247
|
const trgmIdx = `idx_${schemaName}_${tableName}_${colName}_trgm`;
|
|
4993
5248
|
if (now.indexed && !was?.indexed) {
|
|
4994
|
-
await
|
|
5249
|
+
await sql5.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${plainIdx} ON ${qualifiedName} (${colName})`).execute(ddlDb);
|
|
4995
5250
|
} else if (!now.indexed && was?.indexed) {
|
|
4996
|
-
await
|
|
5251
|
+
await sql5.raw(`DROP INDEX CONCURRENTLY IF EXISTS ${schemaName}.${plainIdx}`).execute(ddlDb);
|
|
4997
5252
|
}
|
|
4998
5253
|
if (now.search && !was?.search) {
|
|
4999
|
-
await
|
|
5254
|
+
await sql5.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${trgmIdx} ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`).execute(ddlDb);
|
|
5000
5255
|
} else if (!now.search && was?.search) {
|
|
5001
|
-
await
|
|
5256
|
+
await sql5.raw(`DROP INDEX CONCURRENTLY IF EXISTS ${schemaName}.${trgmIdx}`).execute(ddlDb);
|
|
5002
5257
|
}
|
|
5003
5258
|
}
|
|
5004
5259
|
}
|
|
@@ -5007,28 +5262,28 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
5007
5262
|
const qualifiedName = `${schemaName}.${tableName}`;
|
|
5008
5263
|
for (const cols of conDiff.addedUniqueKeys) {
|
|
5009
5264
|
const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
|
|
5010
|
-
const exists = await
|
|
5265
|
+
const exists = await sql5`
|
|
5011
5266
|
SELECT EXISTS (
|
|
5012
5267
|
SELECT 1 FROM pg_constraint
|
|
5013
5268
|
WHERE conname = ${constraintName}
|
|
5014
5269
|
) AS "exists"
|
|
5015
5270
|
`.execute(ddlDb).then((r) => r.rows[0]?.exists ?? false);
|
|
5016
5271
|
if (!exists) {
|
|
5017
|
-
await
|
|
5272
|
+
await sql5.raw(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`).execute(ddlDb);
|
|
5018
5273
|
}
|
|
5019
5274
|
addedUq[tableName] = [...addedUq[tableName] ?? [], cols];
|
|
5020
5275
|
}
|
|
5021
5276
|
for (const cols of conDiff.addedIndexes) {
|
|
5022
5277
|
const i = (def.schema[tableName]?.indexes ?? []).findIndex((entry) => JSON.stringify(entry) === JSON.stringify(cols));
|
|
5023
5278
|
const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
|
|
5024
|
-
await
|
|
5279
|
+
await sql5.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`).execute(ddlDb);
|
|
5025
5280
|
}
|
|
5026
5281
|
for (const cols of conDiff.removedIndexes) {
|
|
5027
5282
|
const i = (existingSchema[tableName]?.indexes ?? []).findIndex((entry) => JSON.stringify(entry) === JSON.stringify(cols));
|
|
5028
5283
|
if (i < 0)
|
|
5029
5284
|
continue;
|
|
5030
5285
|
const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
|
|
5031
|
-
await
|
|
5286
|
+
await sql5.raw(`DROP INDEX CONCURRENTLY IF EXISTS ${schemaName}.${idxName}`).execute(ddlDb);
|
|
5032
5287
|
}
|
|
5033
5288
|
}
|
|
5034
5289
|
const sg2 = await registerSubgraph(db, regData);
|
|
@@ -5054,7 +5309,7 @@ async function deploySchema(db, def, handlerPath, opts) {
|
|
|
5054
5309
|
}
|
|
5055
5310
|
}
|
|
5056
5311
|
for (const stmt of statements) {
|
|
5057
|
-
await
|
|
5312
|
+
await sql5.raw(stmt).execute(ddlDb);
|
|
5058
5313
|
}
|
|
5059
5314
|
const sg = await registerSubgraph(db, regData);
|
|
5060
5315
|
return { action: "created", subgraphId: sg.id, version: newVersion };
|
|
@@ -5106,5 +5361,5 @@ export {
|
|
|
5106
5361
|
ByoBreakingChangeError
|
|
5107
5362
|
};
|
|
5108
5363
|
|
|
5109
|
-
//# debugId=
|
|
5364
|
+
//# debugId=720FEFA20DA031E864756E2164756E21
|
|
5110
5365
|
//# sourceMappingURL=index.js.map
|