@secondlayer/subgraphs 3.10.0 → 3.12.0

package/README.md

@@ -2,6 +2,8 @@
 
 Typed on-chain indexing for Stacks. Declare event filters + column schema with `defineSubgraph()`; the runtime decodes blocks, matches filters, runs your handlers inside a transactional context, and exposes the result as a Postgres schema you query over REST or SQL.
 
+Subgraphs have a visibility: managed deploys default `visibility: public` — anon-readable at `/v1/subgraphs/<name>/<table>`, in a single global public namespace claimed on publish (409 `PUBLIC_NAME_TAKEN` if the name is taken). BYO-database deploys default private. Override with `--visibility public|private` at deploy, or flip later with `sl subgraphs publish|unpublish <name>`.
+
 Subgraph rows fan out to HTTP subscribers through a post-flush outbox emitter — signed Standard Webhooks POSTs with retries, circuit breaker, and replay.
 
 ## Install
@@ -114,6 +116,9 @@ reads from there. Deploy with a connection string:
 sl subgraphs deploy subgraphs/my.ts --database-url "postgres://user:pass@your-host:5432/db"
 ```
 
+BYO deploys default `visibility: private` (managed deploys default public);
+pass `--visibility public` or `sl subgraphs publish <name>` to open reads.
+
 The connection string is stored encrypted at rest (AES-GCM, keyed by
 `SECONDLAYER_SECRETS_KEY`) and never returned in API responses. The server
 verifies the connection before deploying. Once deployed, query the

package/dist/src/index.d.ts

@@ -237,6 +237,9 @@ interface SubgraphDefinition {
 	description?: string;
 	/** Block height to start indexing from (default: 1) */
 	startBlock?: number;
+	/** 'concurrent' = tip-first: live at tip now, history backfills behind.
+	*  Requires order-tolerant handlers. Default 'blocking'. */
+	backfillMode?: "blocking" | "concurrent";
 	/** Named source filters — keys become handler keys */
 	sources: Record<string, SubgraphFilter>;
 	/** Tables in this subgraph */
@@ -252,6 +255,8 @@ interface ReindexOptions {
 	fromBlock?: number;
 	toBlock?: number;
 	schemaName?: string;
+	/** Op row to receive processed_events on each progress flush. */
+	operationId?: string;
 	signal?: AbortSignal;
 }
 /**
@@ -269,6 +274,7 @@ declare function reindexSubgraph(def: SubgraphDefinition, opts?: ReindexOptions)
 */
 declare function resumeReindex(def: SubgraphDefinition, opts: {
 	schemaName: string
+	operationId?: string
 	signal?: AbortSignal
 }): Promise<{
 	processed: number
@@ -282,6 +288,7 @@ declare function backfillSubgraph(def: SubgraphDefinition, opts: {
 	fromBlock: number
 	toBlock: number
 	schemaName?: string
+	operationId?: string
 	signal?: AbortSignal
 }): Promise<{
 	processed: number
@@ -727,6 +734,14 @@ interface ColumnDiff {
 * Compare two multi-table subgraph schemas and return differences.
 */
 declare function diffSchema(existing: SubgraphSchema, incoming: SubgraphSchema): TableDiff;
+/**
+* Returns true if the diff contains any breaking changes
+* (removed tables, removed columns, or changed column types).
+*/
+declare function hasBreakingChanges(diff: TableDiff): {
+	breaking: boolean
+	reasons: string[]
+};
 interface DeployDiff {
 	addedTables: string[];
 	removedTables: string[];
@@ -794,4 +809,17 @@ declare function deploySchema(db: AnyDb, def: SubgraphDefinition, handlerPath: s
 	version: string
 	diff?: DeployDiff
 }>;
-export { validateSubgraphDefinition, resumeReindex, renderDeployPlan, reindexSubgraph, pgSchemaName, generateSubgraphSQL, generatePrismaSchema, generateKyselySchema, generateIndexSchema, generateDrizzleSchema, diffSchema, deploySchema, defineSubgraph, backfillSubgraph, WriteRow, WhereInput, TypedSubgraphDefinition, TypedSubgraphContext, TypedHandlers, TxMeta, TableDiff, SystemRow, SubscribeOptions, SubgraphTableClient, SubgraphTable, SubgraphSchema, SubgraphHandler, SubgraphFilter, SubgraphDefinition, SubgraphContext, SubgraphColumn, StxTransferPayload, StxTransferFilter, StxMintPayload, StxMintFilter, StxLockPayload, StxLockFilter, StxBurnPayload, StxBurnFilter, RowValue, ReindexOptions, PrismaGenOptions, PrintEventPayload, PrintEventFor, PrintEventFilter, NftTransferPayload, NftTransferFilter, NftMintPayload, NftMintFilter, NftBurnPayload, NftBurnFilter, KyselyGenOptions, InferTableRow, InferSubgraphClient, InferColumnType, IndexCodegenTarget, IndexCodegenOptions, INDEX_CODEGEN_TABLES, GeneratedSQL, FtTransferPayload, FtTransferFilter, FtMintPayload, FtMintFilter, FtBurnPayload, FtBurnFilter, FindManyOptions, EventForFilter, DrizzleGenOptions, DeployPlan, DeployDiff, ContractDeployPayload, ContractDeployFilter, ContractCallPayload, ContractCallFilter, ContractCallEvent, ComputedValue, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff, ByoMigrationPlan, ByoBreakingChangeError, AnyEvent, AggregateSpec, AggregateResult };
+/** A (decoded event type, optional contract scope) pair the sparse probe
+*  checks. Contract scoping is what makes token-subgraph reindexes leap over
+*  everything that isn't their token. */
+type SparseProbeTarget = {
+	eventType: string
+	contractId?: string
+};
+/** Sparse scanning is sound only when EVERY source is an event-type filter —
+*  a contract_call/contract_deploy source matches transactions, which the
+*  event probe can't see. */
+declare function canSparseScan(subgraph: SubgraphDefinition): boolean;
+/** Probe targets for a subgraph's filters: decoded type + contract scope when\\n*  the filter pins one (assetIdentifier "SP….contract::asset" or contractId). */
+declare function sparseProbeTargets(subgraph: SubgraphDefinition): SparseProbeTarget[];
+export { validateSubgraphDefinition, sparseProbeTargets, resumeReindex, renderDeployPlan, reindexSubgraph, pgSchemaName, hasBreakingChanges, generateSubgraphSQL, generatePrismaSchema, generateKyselySchema, generateIndexSchema, generateDrizzleSchema, diffSchema, deploySchema, defineSubgraph, canSparseScan, backfillSubgraph, WriteRow, WhereInput, TypedSubgraphDefinition, TypedSubgraphContext, TypedHandlers, TxMeta, TableDiff, SystemRow, SubscribeOptions, SubgraphTableClient, SubgraphTable, SubgraphSchema, SubgraphHandler, SubgraphFilter, SubgraphDefinition, SubgraphContext, SubgraphColumn, StxTransferPayload, StxTransferFilter, StxMintPayload, StxMintFilter, StxLockPayload, StxLockFilter, StxBurnPayload, StxBurnFilter, SparseProbeTarget, RowValue, ReindexOptions, PrismaGenOptions, PrintEventPayload, PrintEventFor, PrintEventFilter, NftTransferPayload, NftTransferFilter, NftMintPayload, NftMintFilter, NftBurnPayload, NftBurnFilter, KyselyGenOptions, InferTableRow, InferSubgraphClient, InferColumnType, IndexCodegenTarget, IndexCodegenOptions, INDEX_CODEGEN_TABLES, GeneratedSQL, FtTransferPayload, FtTransferFilter, FtMintPayload, FtMintFilter, FtBurnPayload, FtBurnFilter, FindManyOptions, EventForFilter, DrizzleGenOptions, DeployPlan, DeployDiff, ContractDeployPayload, ContractDeployFilter, ContractCallPayload, ContractCallFilter, ContractCallEvent, ComputedValue, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff, ByoMigrationPlan, ByoBreakingChangeError, AnyEvent, AggregateSpec, AggregateResult };

package/dist/src/index.js

@@ -69,6 +69,7 @@ var SubgraphDefinitionSchema = z.object({
   version: z.string().optional(),
   description: z.string().optional(),
   startBlock: z.number().int().nonnegative().optional(),
+  backfillMode: z.enum(["blocking", "concurrent"]).optional(),
   sources: z.record(z.string(), SubgraphFilterSchema).refine((s) => Object.keys(s).length > 0, "Must have at least one source"),
   schema: SubgraphSchemaSchema,
   handlers: z.record(z.string(), z.any())
@@ -1195,6 +1196,28 @@ function reconstructEvent(e) {
 }
 
 // src/runtime/block-source.ts
+function canSparseScan(subgraph) {
+  if (Array.isArray(subgraph.sources))
+    return false;
+  const filters = sourceFilters(subgraph);
+  if (filters.length === 0)
+    return false;
+  return filters.every((f) => Boolean(EVENT_FILTER_TO_INDEX_TYPE[f.type]));
+}
+function sparseProbeTargets(subgraph) {
+  const targets = new Map;
+  for (const f of sourceFilters(subgraph)) {
+    const eventType = EVENT_FILTER_TO_INDEX_TYPE[f.type];
+    if (!eventType)
+      continue;
+    const scoped = f;
+    const contractId = scoped.contractId ?? scoped.assetIdentifier?.split("::")[0];
+    const key = `${eventType}|${contractId ?? ""}`;
+    targets.set(key, { eventType, ...contractId ? { contractId } : {} });
+  }
+  return [...targets.values()];
+}
+
 class PostgresBlockSource {
   async getTip() {
     const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
@@ -1257,9 +1280,18 @@ function isStreamsIndexEligible(subgraph) {
 class PublicApiBlockSource {
   http;
   eventTypes;
-  constructor(http, eventTypes) {
+  probeTargets;
+  constructor(http, eventTypes, probeTargets) {
     this.http = http;
     this.eventTypes = eventTypes;
+    this.probeTargets = probeTargets;
+  }
+  async nextDataHeight(afterHeight, untilHeight) {
+    if (!this.probeTargets?.length)
+      return afterHeight + 1;
+    const hits = await Promise.all(this.probeTargets.map((t) => this.http.firstEventHeight(t.eventType, afterHeight + 1, untilHeight, t.contractId)));
+    const found = hits.filter((h) => h !== null);
+    return found.length ? Math.min(...found) : null;
   }
   getTip() {
     return this.http.getIndexTip();
@@ -1323,6 +1355,15 @@ class FallbackBlockSource {
       return this.fallback.loadBlockRange(fromHeight, toHeight);
     }
   }
+  async nextDataHeight(afterHeight, untilHeight) {
+    if (!this.primary.nextDataHeight)
+      return afterHeight + 1;
+    try {
+      return await this.primary.nextDataHeight(afterHeight, untilHeight);
+    } catch {
+      return afterHeight + 1;
+    }
+  }
 }
 var postgresBlockSource = new PostgresBlockSource;
 function buildHttpClient() {
@@ -1335,7 +1376,7 @@ function buildHttpClient() {
 }
 function resolveBlockSource(subgraph) {
   if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
-    return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph)), postgresBlockSource);
+    return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph), canSparseScan(subgraph) ? sparseProbeTargets(subgraph) : undefined), postgresBlockSource);
   }
   if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
     logger3.debug("Subgraph not streams-index eligible, using DB tap", {
@@ -1818,6 +1859,7 @@ import {
   recordGapBatch,
   resolveGaps
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
+import { updateOperationProcessedEvents } from "@secondlayer/shared/db/queries/subgraph-operations";
 import {
   recordSubgraphProcessed as recordSubgraphProcessed2,
   updateSubgraphStatus as updateSubgraphStatus2
@@ -2005,6 +2047,7 @@ async function processBlockRange(def, opts) {
   let batchSize = batchConfig.defaultBatchSize;
   let currentHeight = fromBlock;
   let aborted = false;
+  const sparse = Boolean(source.nextDataHeight && canSparseScan(def));
   const flushHealth = async () => {
     if (pendingEventsProcessed === 0 && pendingErrors === 0)
       return;
@@ -2035,6 +2078,7 @@ async function processBlockRange(def, opts) {
       nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
     }
     const batchFailedBlocks = [];
+    let batchMatched = 0;
     for (let height = currentHeight;height <= batchEnd; height++) {
       const blockData = batch.get(height);
       if (!blockData) {
@@ -2064,6 +2108,7 @@ async function processBlockRange(def, opts) {
         continue;
       }
       blocksProcessed++;
+      batchMatched += result.matched;
       totalEventsProcessed += result.processed;
       totalErrors += result.errors;
       pendingEventsProcessed += result.processed;
@@ -2081,6 +2126,9 @@ async function processBlockRange(def, opts) {
       const shouldFlushProgress = blocksProcessed % 100 === 0 || now - lastProgressFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
       if (shouldFlushProgress) {
         await updateSubgraphStatus2(targetDb, subgraphName, status, height);
+        if (opts.operationId) {
+          await updateOperationProcessedEvents(targetDb, opts.operationId, totalEventsProcessed).catch(() => {});
+        }
         lastProgressFlushAt = now;
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
@@ -2109,6 +2157,27 @@ async function processBlockRange(def, opts) {
         });
       });
     }
+    if (sparse && batchMatched === 0 && batchEnd < toBlock && source.nextDataHeight) {
+      const next = await source.nextDataHeight(batchEnd, toBlock);
+      const jumpTo = next === null ? toBlock + 1 : Math.max(next, batchEnd + 1);
+      if (jumpTo > batchEnd + 1) {
+        const skipped = Math.min(jumpTo, toBlock + 1) - (batchEnd + 1);
+        blocksProcessed += skipped;
+        await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
+        logger6.info("Sparse skip", {
+          subgraph: subgraphName,
+          from: batchEnd + 1,
+          to: jumpTo - 1,
+          skipped
+        });
+        currentHeight = jumpTo;
+        if (currentHeight <= toBlock) {
+          nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
+          nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
+        }
+        continue;
+      }
+    }
     const avg = avgEventsPerBlock(batch);
     if (avg > 50)
       batchSize = Math.max(Math.round(batchSize * 0.5), batchConfig.minBatchSize);
@@ -2177,6 +2246,7 @@ async function reindexSubgraph(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: subgraphRow?.id,
+      operationId: opts?.operationId,
       signal: opts?.signal
     });
     if (result.aborted) {
@@ -2252,6 +2322,7 @@ async function resumeReindex(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: row.id,
+      operationId: opts.operationId,
       signal: opts.signal
     });
     if (result.aborted) {
@@ -2300,6 +2371,7 @@ async function backfillSubgraph(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: subgraphRow?.id,
+      operationId: opts.operationId,
       signal: opts.signal
     });
     if (result.aborted) {
@@ -3174,10 +3246,12 @@ function getDefault(type) {
 }
 export {
   validateSubgraphDefinition,
+  sparseProbeTargets,
   resumeReindex,
   renderDeployPlan,
   reindexSubgraph,
   pgSchemaName,
+  hasBreakingChanges,
   generateSubgraphSQL,
   generatePrismaSchema,
   generateKyselySchema,
@@ -3186,10 +3260,11 @@ export {
   diffSchema,
   deploySchema,
   defineSubgraph,
+  canSparseScan,
   backfillSubgraph,
   INDEX_CODEGEN_TABLES,
   ByoBreakingChangeError
 };
 
-//# debugId=44829C409B3F962864756E2164756E21
+//# debugId=415665ECDA2A436C64756E2164756E21
 //# sourceMappingURL=index.js.map