@secondlayer/subgraphs 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/src/index.js +11 -11
- package/dist/src/index.js.map +9 -9
- package/dist/src/runtime/block-processor.js +11 -11
- package/dist/src/runtime/block-processor.js.map +9 -9
- package/dist/src/runtime/catchup.js +11 -11
- package/dist/src/runtime/catchup.js.map +9 -9
- package/dist/src/runtime/context.d.ts +1 -1
- package/dist/src/runtime/context.js.map +1 -1
- package/dist/src/runtime/emitter.d.ts +10 -10
- package/dist/src/runtime/emitter.js +68 -44
- package/dist/src/runtime/emitter.js.map +14 -13
- package/dist/src/runtime/processor.js +64 -36
- package/dist/src/runtime/processor.js.map +13 -12
- package/dist/src/runtime/reindex.js +11 -11
- package/dist/src/runtime/reindex.js.map +9 -9
- package/dist/src/runtime/reorg.d.ts +2 -2
- package/dist/src/runtime/reorg.js +17 -17
- package/dist/src/runtime/reorg.js.map +10 -10
- package/dist/src/runtime/replay.d.ts +6 -6
- package/dist/src/runtime/replay.js +26 -26
- package/dist/src/runtime/replay.js.map +4 -4
- package/dist/src/runtime/runner.d.ts +1 -1
- package/dist/src/service.js +79 -36
- package/dist/src/service.js.map +14 -13
- package/dist/src/testing/index.js.map +1 -1
- package/dist/src/triggers/index.d.ts +2 -2
- package/dist/src/triggers/index.js.map +1 -1
- package/package.json +3 -3
package/dist/src/service.js
CHANGED
|
@@ -2701,7 +2701,7 @@ function stableStringify(obj) {
|
|
|
2701
2701
|
return acc;
|
|
2702
2702
|
}, {}));
|
|
2703
2703
|
}
|
|
2704
|
-
async function
|
|
2704
|
+
async function emitWebhookOutbox(tx, subgraphName, manifest, matcher, blockHeight) {
|
|
2705
2705
|
if (!isEmitOutboxEnabled()) {
|
|
2706
2706
|
if (!loggedKillSwitch) {
|
|
2707
2707
|
logger6.warn("SECONDLAYER_EMIT_OUTBOX=false — outbox emission bypassed");
|
|
@@ -2720,7 +2720,7 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2720
2720
|
const eventType = `${subgraphName}.${write.table}.${OP_VERB[write.op]}`;
|
|
2721
2721
|
for (const s of subs) {
|
|
2722
2722
|
rows.push({
|
|
2723
|
-
|
|
2723
|
+
webhook_id: s.id,
|
|
2724
2724
|
subgraph_name: subgraphName,
|
|
2725
2725
|
table_name: write.table,
|
|
2726
2726
|
block_height: blockHeight,
|
|
@@ -2734,7 +2734,7 @@ async function emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block
|
|
|
2734
2734
|
}
|
|
2735
2735
|
if (rows.length === 0)
|
|
2736
2736
|
return 0;
|
|
2737
|
-
await tx.insertInto("
|
|
2737
|
+
await tx.insertInto("webhook_outbox").values(rows).onConflict((oc) => oc.columns(["webhook_id", "dedup_key"]).doNothing()).execute();
|
|
2738
2738
|
return rows.length;
|
|
2739
2739
|
}
|
|
2740
2740
|
|
|
@@ -3047,8 +3047,8 @@ async function answerRead(hostCtx, msg) {
|
|
|
3047
3047
|
}
|
|
3048
3048
|
}
|
|
3049
3049
|
|
|
3050
|
-
// src/runtime/
|
|
3051
|
-
import {
|
|
3050
|
+
// src/runtime/webhook-state.ts
|
|
3051
|
+
import { listWebhooks } from "@secondlayer/shared/db/queries/webhooks";
|
|
3052
3052
|
import { sql as sql3 } from "kysely";
|
|
3053
3053
|
|
|
3054
3054
|
// src/runtime/emitter-matcher.ts
|
|
@@ -3168,7 +3168,7 @@ function key(subgraphName, tableName) {
|
|
|
3168
3168
|
return `${subgraphName}\x00${tableName}`;
|
|
3169
3169
|
}
|
|
3170
3170
|
|
|
3171
|
-
class
|
|
3171
|
+
class WebhookMatcher {
|
|
3172
3172
|
byKey = new Map;
|
|
3173
3173
|
byId = new Map;
|
|
3174
3174
|
setAll(subs) {
|
|
@@ -3210,11 +3210,11 @@ class SubscriptionMatcher {
|
|
|
3210
3210
|
}
|
|
3211
3211
|
}
|
|
3212
3212
|
|
|
3213
|
-
// src/runtime/
|
|
3214
|
-
var matcher = new
|
|
3213
|
+
// src/runtime/webhook-state.ts
|
|
3214
|
+
var matcher = new WebhookMatcher;
|
|
3215
3215
|
async function refreshMatcher(db) {
|
|
3216
3216
|
const rows = await sql3`
|
|
3217
|
-
SELECT * FROM
|
|
3217
|
+
SELECT * FROM webhooks WHERE status = 'active'
|
|
3218
3218
|
`.execute(db);
|
|
3219
3219
|
matcher.setAll(rows.rows);
|
|
3220
3220
|
return matcher.size();
|
|
@@ -3504,7 +3504,7 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
|
|
|
3504
3504
|
const manifest = await ctx.flush();
|
|
3505
3505
|
flushedWrites = manifest.count > 0;
|
|
3506
3506
|
if (manifest.count > 0) {
|
|
3507
|
-
await
|
|
3507
|
+
await emitWebhookOutbox(tx, subgraphName, manifest, matcher, block.height);
|
|
3508
3508
|
}
|
|
3509
3509
|
flushMs = performance.now() - flushStart;
|
|
3510
3510
|
}
|
|
@@ -4638,8 +4638,8 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
4638
4638
|
for (const [tableName, rows] of Object.entries(revertedByTable)) {
|
|
4639
4639
|
if (rows.length === 0)
|
|
4640
4640
|
continue;
|
|
4641
|
-
await targetDb.insertInto("
|
|
4642
|
-
"
|
|
4641
|
+
await targetDb.insertInto("webhook_outbox").columns([
|
|
4642
|
+
"webhook_id",
|
|
4643
4643
|
"subgraph_name",
|
|
4644
4644
|
"table_name",
|
|
4645
4645
|
"block_height",
|
|
@@ -4648,8 +4648,8 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
4648
4648
|
"dedup_key",
|
|
4649
4649
|
"row_pk",
|
|
4650
4650
|
"status"
|
|
4651
|
-
]).expression((eb) => eb.selectFrom("
|
|
4652
|
-
"id as
|
|
4651
|
+
]).expression((eb) => eb.selectFrom("webhooks").select((eb2) => [
|
|
4652
|
+
"id as webhook_id",
|
|
4653
4653
|
eb2.val(sg.name).as("subgraph_name"),
|
|
4654
4654
|
eb2.val(tableName).as("table_name"),
|
|
4655
4655
|
eb2.val(blockHeight).as("block_height"),
|
|
@@ -4667,8 +4667,8 @@ async function reorgOneSubgraph(sg, schema, blockHeight, loadSubgraphDef) {
|
|
|
4667
4667
|
reorgRoot: blockHeight
|
|
4668
4668
|
}).as("row_pk"),
|
|
4669
4669
|
eb2.val("pending").as("status")
|
|
4670
|
-
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["
|
|
4671
|
-
logger12.error("Failed to emit revert event for
|
|
4670
|
+
]).where("subgraph_name", "=", sg.name).where("table_name", "=", tableName).where("status", "=", "active")).onConflict((oc) => oc.columns(["webhook_id", "dedup_key"]).doNothing()).execute().catch((err) => {
|
|
4671
|
+
logger12.error("Failed to emit revert event for webhooks", {
|
|
4672
4672
|
event: "reorg_revert_emit_dropped",
|
|
4673
4673
|
subgraph: sg.name,
|
|
4674
4674
|
table: tableName,
|
|
@@ -4720,6 +4720,26 @@ import {
|
|
|
4720
4720
|
targetListenerUrl as targetListenerUrl2
|
|
4721
4721
|
} from "@secondlayer/shared/queue/listener";
|
|
4722
4722
|
|
|
4723
|
+
// src/runtime/hosted-meter.ts
|
|
4724
|
+
var hooks = {};
|
|
4725
|
+
function setHostedMeterHooks(next) {
|
|
4726
|
+
hooks = next;
|
|
4727
|
+
}
|
|
4728
|
+
async function meterBlocksProcessed(accountId, blocks, subgraphName) {
|
|
4729
|
+
if (!accountId || blocks <= 0)
|
|
4730
|
+
return true;
|
|
4731
|
+
if (!hooks.onBlocksProcessed)
|
|
4732
|
+
return true;
|
|
4733
|
+
return hooks.onBlocksProcessed(accountId, blocks, subgraphName);
|
|
4734
|
+
}
|
|
4735
|
+
async function meterDeliveryAttempt(accountId, webhookId) {
|
|
4736
|
+
if (!accountId)
|
|
4737
|
+
return true;
|
|
4738
|
+
if (!hooks.onDeliveryAttempt)
|
|
4739
|
+
return true;
|
|
4740
|
+
return hooks.onDeliveryAttempt(accountId, webhookId);
|
|
4741
|
+
}
|
|
4742
|
+
|
|
4723
4743
|
// src/runtime/streams-reorg-poll.ts
|
|
4724
4744
|
import { getErrorMessage as getErrorMessage5 } from "@secondlayer/shared";
|
|
4725
4745
|
import {
|
|
@@ -4795,7 +4815,8 @@ async function catchUpAll(subgraphs, db, concurrency) {
|
|
|
4795
4815
|
break;
|
|
4796
4816
|
try {
|
|
4797
4817
|
const def = await loadSubgraphDefinition(sg);
|
|
4798
|
-
await catchUpSubgraph(def, sg.name);
|
|
4818
|
+
const processed = await catchUpSubgraph(def, sg.name);
|
|
4819
|
+
await meterBlocksProcessed(sg.account_id, processed, sg.name);
|
|
4799
4820
|
} catch (err) {
|
|
4800
4821
|
const msg = getErrorMessage6(err);
|
|
4801
4822
|
if (isHandlerNotFoundError(err)) {
|
|
@@ -4893,38 +4914,45 @@ async function runSubgraphOperation(operation, signal) {
|
|
|
4893
4914
|
const subgraph = await db.selectFrom("subgraphs").selectAll().where("id", "=", operation.subgraph_id).executeTakeFirst();
|
|
4894
4915
|
if (!subgraph)
|
|
4895
4916
|
throw new Error(`Subgraph not found: ${operation.subgraph_id}`);
|
|
4917
|
+
if (subgraph.status === "paused")
|
|
4918
|
+
return 0;
|
|
4896
4919
|
const def = await loadSubgraphDefinition(subgraph);
|
|
4897
4920
|
const schemaName = subgraph.schema_name ?? pgSchemaName2(subgraph.name);
|
|
4921
|
+
let processed = 0;
|
|
4898
4922
|
if (operation.kind === "backfill") {
|
|
4899
4923
|
if (operation.from_block == null || operation.to_block == null) {
|
|
4900
4924
|
throw new Error("Backfill operation is missing from_block or to_block");
|
|
4901
4925
|
}
|
|
4902
4926
|
const resumeFrom = operation.cursor_block != null ? Math.max(Number(operation.from_block), Number(operation.cursor_block) + 1) : Number(operation.from_block);
|
|
4903
|
-
const
|
|
4927
|
+
const result = await backfillSubgraph(def, {
|
|
4904
4928
|
fromBlock: resumeFrom,
|
|
4905
4929
|
toBlock: Number(operation.to_block),
|
|
4906
4930
|
schemaName,
|
|
4907
4931
|
operationId: operation.id,
|
|
4908
4932
|
signal
|
|
4909
4933
|
});
|
|
4910
|
-
|
|
4911
|
-
}
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4934
|
+
processed = result.processed;
|
|
4935
|
+
} else {
|
|
4936
|
+
const hasResumeMetadata = subgraph.status === "reindexing" && subgraph.reindex_from_block != null && subgraph.reindex_to_block != null;
|
|
4937
|
+
if (hasResumeMetadata) {
|
|
4938
|
+
const result = await resumeReindex(def, {
|
|
4939
|
+
schemaName,
|
|
4940
|
+
operationId: operation.id,
|
|
4941
|
+
signal
|
|
4942
|
+
});
|
|
4943
|
+
processed = result.processed;
|
|
4944
|
+
} else {
|
|
4945
|
+
const result = await reindexSubgraph(def, {
|
|
4946
|
+
startBlockFloor: operation.from_block == null ? undefined : Number(operation.from_block),
|
|
4947
|
+
schemaName,
|
|
4948
|
+
operationId: operation.id,
|
|
4949
|
+
signal
|
|
4950
|
+
});
|
|
4951
|
+
processed = result.processed;
|
|
4952
|
+
}
|
|
4920
4953
|
}
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
schemaName,
|
|
4924
|
-
operationId: operation.id,
|
|
4925
|
-
signal
|
|
4926
|
-
});
|
|
4927
|
-
return result.processed;
|
|
4954
|
+
await meterBlocksProcessed(subgraph.account_id, processed, subgraph.name);
|
|
4955
|
+
return processed;
|
|
4928
4956
|
}
|
|
4929
4957
|
async function startSubgraphOperationRunner(opts) {
|
|
4930
4958
|
const concurrency = opts?.concurrency ?? DEFAULT_OPERATION_CONCURRENCY;
|
|
@@ -5130,6 +5158,7 @@ async function startSubgraphProcessor(opts) {
|
|
|
5130
5158
|
// src/service.ts
|
|
5131
5159
|
import { assertDbSplit, getDb } from "@secondlayer/shared/db";
|
|
5132
5160
|
import { logger as logger15 } from "@secondlayer/shared/logger";
|
|
5161
|
+
import { isPlatformMode } from "@secondlayer/shared/mode";
|
|
5133
5162
|
import { sql as sql7 } from "kysely";
|
|
5134
5163
|
var HEARTBEAT_INTERVAL_MS2 = 30000;
|
|
5135
5164
|
var SERVICE_NAME = "subgraph-processor";
|
|
@@ -5143,6 +5172,20 @@ async function writeHeartbeat() {
|
|
|
5143
5172
|
}
|
|
5144
5173
|
}
|
|
5145
5174
|
assertDbSplit();
|
|
5175
|
+
if (isPlatformMode()) {
|
|
5176
|
+
try {
|
|
5177
|
+
const spec = "@secondlayer/platform/hosted-meters";
|
|
5178
|
+
const mod = await import(spec);
|
|
5179
|
+
setHostedMeterHooks({
|
|
5180
|
+
onBlocksProcessed: mod.onBlocksProcessed,
|
|
5181
|
+
onDeliveryAttempt: mod.onDeliveryAttempt
|
|
5182
|
+
});
|
|
5183
|
+
} catch (err) {
|
|
5184
|
+
logger15.warn("hosted meters not installed; indexing unmetered", {
|
|
5185
|
+
error: err instanceof Error ? err.message : String(err)
|
|
5186
|
+
});
|
|
5187
|
+
}
|
|
5188
|
+
}
|
|
5146
5189
|
var processor = await startSubgraphProcessor({
|
|
5147
5190
|
concurrency: Number.parseInt(process.env.SUBGRAPH_CONCURRENCY ?? "5")
|
|
5148
5191
|
});
|
|
@@ -5157,5 +5200,5 @@ var shutdown = async () => {
|
|
|
5157
5200
|
process.on("SIGINT", shutdown);
|
|
5158
5201
|
process.on("SIGTERM", shutdown);
|
|
5159
5202
|
|
|
5160
|
-
//# debugId=
|
|
5203
|
+
//# debugId=075E602303939FCE64756E2164756E21
|
|
5161
5204
|
//# sourceMappingURL=service.js.map
|