@secondlayer/subgraphs 3.9.0 → 3.11.0

package/README.md

@@ -2,6 +2,8 @@
 
 Typed on-chain indexing for Stacks. Declare event filters + column schema with `defineSubgraph()`; the runtime decodes blocks, matches filters, runs your handlers inside a transactional context, and exposes the result as a Postgres schema you query over REST or SQL.
 
+Subgraphs have a visibility: managed deploys default `visibility: public` — anon-readable at `/v1/subgraphs/<name>/<table>`, in a single global public namespace claimed on publish (409 `PUBLIC_NAME_TAKEN` if the name is taken). BYO-database deploys default private. Override with `--visibility public|private` at deploy, or flip later with `sl subgraphs publish|unpublish <name>`.
+
 Subgraph rows fan out to HTTP subscribers through a post-flush outbox emitter — signed Standard Webhooks POSTs with retries, circuit breaker, and replay.
 
 ## Install
@@ -114,6 +116,9 @@ reads from there. Deploy with a connection string:
 sl subgraphs deploy subgraphs/my.ts --database-url "postgres://user:pass@your-host:5432/db"
 ```
 
+BYO deploys default `visibility: private` (managed deploys default public);
+pass `--visibility public` or `sl subgraphs publish <name>` to open reads.
+
 The connection string is stored encrypted at rest (AES-GCM, keyed by
 `SECONDLAYER_SECRETS_KEY`) and never returned in API responses. The server
 verifies the connection before deploying. Once deployed, query the
@@ -130,8 +135,13 @@ script and verifies the connection — without writing anything.
   transaction, so a crash replays the block (at-least-once). `ctx.insert` and
   `ctx.upsert` (with a unique key) are safe — flush is replace-per-height. A
   deploy with non-idempotent `ctx.update` / `ctx.patchOrInsert` is rejected.
-- **No reindex.** Reindex would drop + rebuild the schema in your DB from a
-  background job; instead, re-deploy to rebuild (or drop the schema yourself).
+- **No reindex.** A breaking change (removed table/column, changed type, or a
+  forced reindex) would drop + rebuild the schema in your DB, so the deploy is
+  refused with HTTP `422` — nothing is touched. The refusal carries a migration
+  plan: the SDK throws a typed `ByoBreakingChangeError` (`details.reasons` +
+  `details.plan` with `dropStatement`/`statements`/`grantScript`) and the CLI
+  prints the exact `DROP SCHEMA … CASCADE` + rebuild DDL to run yourself, then
+  re-deploy. A destructive `--force` rebuild on your DB is not yet supported.
 - **Delete leaves your data.** Deleting the subgraph removes our registry row
   (and the stored connection) and pauses subscriptions, but never drops the
   schema in your database.

package/dist/src/index.d.ts

@@ -707,6 +707,7 @@ declare function generateIndexSchema(target: IndexCodegenTarget, opts?: IndexCod
 declare const INDEX_CODEGEN_TABLES: readonly string[];
 import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
 import { Database } from "@secondlayer/shared/db";
+import { ByoBreakingChangeDetails } from "@secondlayer/shared/errors";
 import { Kysely } from "kysely";
 type AnyDb = Kysely<Database>;
 interface TableDiff {
@@ -732,8 +733,28 @@ interface DeployDiff {
 	addedColumns: Record<string, string[]>;
 	breakingChanges: string[];
 }
+interface ByoMigrationPlan {
+	schemaName: string;
+	dropStatement: string;
+	statements: string[];
+	grantScript: string;
+}
+/**
+* Thrown when a BYO subgraph deploy is refused for a breaking schema change.
+* Plain `Error` with a literal `code` (not `SecondLayerError`) so the API
+* middleware matches it by code across bundle boundaries — bunup duplicates
+* classes per package, breaking cross-bundle `instanceof`. The refusal stands;
+* `details` carries the reviewable DROP + rebuild the user must run manually.
+*/
+declare class ByoBreakingChangeError extends Error {
+	readonly code: "BYO_BREAKING_CHANGE";
+	readonly details: ByoBreakingChangeDetails;
+	constructor(reasons: string[], diff: DeployDiff, plan: ByoMigrationPlan);
+}
 interface DeployPlan {
 	schemaName: string;
+	/** `DROP SCHEMA … CASCADE` a destructive rebuild would run first (shown, never auto-run on BYO). */
+	dropStatement: string;
 	/** DDL Secondlayer will run against your database. */
 	statements: string[];
 	/** Least-privilege grant script to run once, before deploying. */
@@ -773,4 +794,4 @@ declare function deploySchema(db: AnyDb, def: SubgraphDefinition, handlerPath: s
 	version: string
 	diff?: DeployDiff
 }>;
-export { validateSubgraphDefinition, resumeReindex, renderDeployPlan, reindexSubgraph, pgSchemaName, generateSubgraphSQL, generatePrismaSchema, generateKyselySchema, generateIndexSchema, generateDrizzleSchema, diffSchema, deploySchema, defineSubgraph, backfillSubgraph, WriteRow, WhereInput, TypedSubgraphDefinition, TypedSubgraphContext, TypedHandlers, TxMeta, TableDiff, SystemRow, SubscribeOptions, SubgraphTableClient, SubgraphTable, SubgraphSchema, SubgraphHandler, SubgraphFilter, SubgraphDefinition, SubgraphContext, SubgraphColumn, StxTransferPayload, StxTransferFilter, StxMintPayload, StxMintFilter, StxLockPayload, StxLockFilter, StxBurnPayload, StxBurnFilter, RowValue, ReindexOptions, PrismaGenOptions, PrintEventPayload, PrintEventFor, PrintEventFilter, NftTransferPayload, NftTransferFilter, NftMintPayload, NftMintFilter, NftBurnPayload, NftBurnFilter, KyselyGenOptions, InferTableRow, InferSubgraphClient, InferColumnType, IndexCodegenTarget, IndexCodegenOptions, INDEX_CODEGEN_TABLES, GeneratedSQL, FtTransferPayload, FtTransferFilter, FtMintPayload, FtMintFilter, FtBurnPayload, FtBurnFilter, FindManyOptions, EventForFilter, DrizzleGenOptions, DeployPlan, ContractDeployPayload, ContractDeployFilter, ContractCallPayload, ContractCallFilter, ContractCallEvent, ComputedValue, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff, AnyEvent, AggregateSpec, AggregateResult };
+export { validateSubgraphDefinition, resumeReindex, renderDeployPlan, reindexSubgraph, pgSchemaName, generateSubgraphSQL, generatePrismaSchema, generateKyselySchema, generateIndexSchema, generateDrizzleSchema, diffSchema, deploySchema, defineSubgraph, backfillSubgraph, WriteRow, WhereInput, TypedSubgraphDefinition, TypedSubgraphContext, TypedHandlers, TxMeta, TableDiff, SystemRow, SubscribeOptions, SubgraphTableClient, SubgraphTable, SubgraphSchema, SubgraphHandler, SubgraphFilter, SubgraphDefinition, SubgraphContext, SubgraphColumn, StxTransferPayload, StxTransferFilter, StxMintPayload, StxMintFilter, StxLockPayload, StxLockFilter, StxBurnPayload, StxBurnFilter, RowValue, ReindexOptions, PrismaGenOptions, PrintEventPayload, PrintEventFor, PrintEventFilter, NftTransferPayload, NftTransferFilter, NftMintPayload, NftMintFilter, NftBurnPayload, NftBurnFilter, KyselyGenOptions, InferTableRow, InferSubgraphClient, InferColumnType, IndexCodegenTarget, IndexCodegenOptions, INDEX_CODEGEN_TABLES, GeneratedSQL, FtTransferPayload, FtTransferFilter, FtMintPayload, FtMintFilter, FtBurnPayload, FtBurnFilter, FindManyOptions, EventForFilter, DrizzleGenOptions, DeployPlan, DeployDiff, ContractDeployPayload, ContractDeployFilter, ContractCallPayload, ContractCallFilter, ContractCallEvent, ComputedValue, ComparisonFilter, ColumnType, ColumnToTS, ColumnDiff, ByoMigrationPlan, ByoBreakingChangeError, AnyEvent, AggregateSpec, AggregateResult };

package/dist/src/index.js

@@ -1195,6 +1195,28 @@ function reconstructEvent(e) {
 }
 
 // src/runtime/block-source.ts
+function canSparseScan(subgraph) {
+  if (Array.isArray(subgraph.sources))
+    return false;
+  const filters = sourceFilters(subgraph);
+  if (filters.length === 0)
+    return false;
+  return filters.every((f) => Boolean(EVENT_FILTER_TO_INDEX_TYPE[f.type]));
+}
+function sparseProbeTargets(subgraph) {
+  const targets = new Map;
+  for (const f of sourceFilters(subgraph)) {
+    const eventType = EVENT_FILTER_TO_INDEX_TYPE[f.type];
+    if (!eventType)
+      continue;
+    const scoped = f;
+    const contractId = scoped.contractId ?? scoped.assetIdentifier?.split("::")[0];
+    const key = `${eventType}|${contractId ?? ""}`;
+    targets.set(key, { eventType, ...contractId ? { contractId } : {} });
+  }
+  return [...targets.values()];
+}
+
 class PostgresBlockSource {
   async getTip() {
     const progress = await getSourceDb().selectFrom("index_progress").selectAll().where("network", "=", process.env.NETWORK ?? "mainnet").executeTakeFirst();
@@ -1257,9 +1279,18 @@ function isStreamsIndexEligible(subgraph) {
 class PublicApiBlockSource {
   http;
   eventTypes;
-  constructor(http, eventTypes) {
+  probeTargets;
+  constructor(http, eventTypes, probeTargets) {
     this.http = http;
     this.eventTypes = eventTypes;
+    this.probeTargets = probeTargets;
+  }
+  async nextDataHeight(afterHeight, untilHeight) {
+    if (!this.probeTargets?.length)
+      return afterHeight + 1;
+    const hits = await Promise.all(this.probeTargets.map((t) => this.http.firstEventHeight(t.eventType, afterHeight + 1, untilHeight, t.contractId)));
+    const found = hits.filter((h) => h !== null);
+    return found.length ? Math.min(...found) : null;
   }
   getTip() {
     return this.http.getIndexTip();
@@ -1323,6 +1354,15 @@ class FallbackBlockSource {
       return this.fallback.loadBlockRange(fromHeight, toHeight);
     }
   }
+  async nextDataHeight(afterHeight, untilHeight) {
+    if (!this.primary.nextDataHeight)
+      return afterHeight + 1;
+    try {
+      return await this.primary.nextDataHeight(afterHeight, untilHeight);
+    } catch {
+      return afterHeight + 1;
+    }
+  }
 }
 var postgresBlockSource = new PostgresBlockSource;
 function buildHttpClient() {
@@ -1335,7 +1375,7 @@ function buildHttpClient() {
 }
 function resolveBlockSource(subgraph) {
   if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph && isStreamsIndexEligible(subgraph)) {
-    return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph)), postgresBlockSource);
+    return new FallbackBlockSource(new PublicApiBlockSource(buildHttpClient(), referencedIndexEventTypes(subgraph), canSparseScan(subgraph) ? sparseProbeTargets(subgraph) : undefined), postgresBlockSource);
   }
   if (process.env.SUBGRAPH_SOURCE === "streams-index" && subgraph) {
     logger3.debug("Subgraph not streams-index eligible, using DB tap", {
@@ -2005,6 +2045,7 @@ async function processBlockRange(def, opts) {
   let batchSize = batchConfig.defaultBatchSize;
   let currentHeight = fromBlock;
   let aborted = false;
+  const sparse = Boolean(source.nextDataHeight && canSparseScan(def));
   const flushHealth = async () => {
     if (pendingEventsProcessed === 0 && pendingErrors === 0)
       return;
@@ -2035,6 +2076,7 @@ async function processBlockRange(def, opts) {
       nextBatchPromise = source.loadBlockRange(nextStart, nextBatchEnd);
     }
     const batchFailedBlocks = [];
+    let batchMatched = 0;
     for (let height = currentHeight;height <= batchEnd; height++) {
       const blockData = batch.get(height);
       if (!blockData) {
@@ -2064,6 +2106,7 @@ async function processBlockRange(def, opts) {
         continue;
       }
       blocksProcessed++;
+      batchMatched += result.matched;
       totalEventsProcessed += result.processed;
       totalErrors += result.errors;
       pendingEventsProcessed += result.processed;
@@ -2109,6 +2152,27 @@ async function processBlockRange(def, opts) {
         });
       });
     }
+    if (sparse && batchMatched === 0 && batchEnd < toBlock && source.nextDataHeight) {
+      const next = await source.nextDataHeight(batchEnd, toBlock);
+      const jumpTo = next === null ? toBlock + 1 : Math.max(next, batchEnd + 1);
+      if (jumpTo > batchEnd + 1) {
+        const skipped = Math.min(jumpTo, toBlock + 1) - (batchEnd + 1);
+        blocksProcessed += skipped;
+        await updateSubgraphStatus2(targetDb, subgraphName, status, jumpTo - 1);
+        logger6.info("Sparse skip", {
+          subgraph: subgraphName,
+          from: batchEnd + 1,
+          to: jumpTo - 1,
+          skipped
+        });
+        currentHeight = jumpTo;
+        if (currentHeight <= toBlock) {
+          nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
+          nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
+        }
+        continue;
+      }
+    }
     const avg = avgEventsPerBlock(batch);
     if (avg > 50)
       batchSize = Math.max(Math.round(batchSize * 0.5), batchConfig.minBatchSize);
@@ -2961,10 +3025,29 @@ function bumpPatch(version) {
   const patch = Number.parseInt(parts[2] ?? "0", 10);
   return `${parts[0]}.${parts[1]}.${Number.isNaN(patch) ? 1 : patch + 1}`;
 }
+
+class ByoBreakingChangeError extends Error {
+  code = "BYO_BREAKING_CHANGE";
+  details;
+  constructor(reasons, diff, plan) {
+    super("Breaking schema change on a BYO subgraph would drop data in your " + "database. Review the plan and run the DROP + rebuild DDL manually.");
+    this.name = "ByoBreakingChangeError";
+    this.details = { reasons, diff, plan };
+  }
+}
+function toDeployDiff(diff, reasons) {
+  return {
+    addedTables: diff?.addedTables ?? [],
+    removedTables: diff?.removedTables ?? [],
+    addedColumns: diff ? Object.fromEntries(Object.entries(diff.tables).filter(([, c]) => c.added.length > 0).map(([t, c]) => [t, c.added])) : {},
+    breakingChanges: reasons
+  };
+}
 function renderDeployPlan(def, schemaName) {
   validateSubgraphDefinition(def);
   const { statements } = generateSubgraphSQL(def, schemaName);
   const schema = schemaName ?? pgSchemaName(def.name);
+  const dropStatement = `DROP SCHEMA IF EXISTS "${schema}" CASCADE;`;
   const grantScript = [
     "-- Run once on YOUR database as an owner/superuser, replacing <role>",
     "-- with the role whose credentials you give Secondlayer.",
@@ -2973,7 +3056,7 @@ function renderDeployPlan(def, schemaName) {
     `-- (after first deploy <role> owns "${schema}"; no further grants needed)`
   ].join(`
 `);
-  return { schemaName: schema, statements, grantScript };
+  return { schemaName: schema, dropStatement, statements, grantScript };
 }
 async function deploySchema(db, def, handlerPath, opts) {
   validateSubgraphDefinition(def);
@@ -2981,8 +3064,14 @@ async function deploySchema(db, def, handlerPath, opts) {
   const { getSubgraph, registerSubgraph } = await import("@secondlayer/shared/db/queries/subgraphs");
   const ddlDb = opts?.dataDb ?? db;
   const byo = opts?.dataDb != null;
-  const refuseDestructiveOnByo = (reason) => {
-    throw new Error(`Breaking schema change on a BYO subgraph (${reason}) would drop data in your database. Drop the schema "${opts?.schemaName ?? pgSchemaName(def.name)}" manually and re-deploy to rebuild.`);
+  const refuseDestructiveOnByo = (reasons, diff) => {
+    const plan = renderDeployPlan(def, opts?.schemaName);
+    throw new ByoBreakingChangeError(reasons, toDeployDiff(diff, reasons), {
+      schemaName: plan.schemaName,
+      dropStatement: plan.dropStatement,
+      statements: plan.statements,
+      grantScript: plan.grantScript
+    });
   };
   const existing = await getSubgraph(db, def.name, opts?.accountId);
   const schemaName = opts?.schemaName ?? pgSchemaName(def.name);
@@ -3037,7 +3126,7 @@ async function deploySchema(db, def, handlerPath, opts) {
     }
     if (existing.schema_hash === hash && opts?.forceReindex) {
       if (byo)
-        refuseDestructiveOnByo("force reindex");
+        refuseDestructiveOnByo(["force reindex"], null);
       await sql4.raw(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`).execute(ddlDb);
       for (const stmt of statements) {
         await sql4.raw(stmt).execute(ddlDb);
@@ -3050,19 +3139,14 @@ async function deploySchema(db, def, handlerPath, opts) {
       const { breaking, reasons } = hasBreakingChanges(diff);
       if (breaking || opts?.forceReindex) {
         if (byo) {
-          refuseDestructiveOnByo(reasons.length > 0 ? reasons.join("; ") : "force reindex");
+          refuseDestructiveOnByo(reasons.length > 0 ? reasons : ["force reindex"], diff);
         }
         await sql4.raw(`DROP SCHEMA IF EXISTS "${schemaName}" CASCADE`).execute(ddlDb);
         for (const stmt of statements) {
           await sql4.raw(stmt).execute(ddlDb);
         }
         const sg3 = await registerSubgraph(db, regData);
-        const deployDiff2 = {
-          addedTables: diff.addedTables,
-          removedTables: diff.removedTables,
-          addedColumns: Object.fromEntries(Object.entries(diff.tables).filter(([, c]) => c.added.length > 0).map(([t, c]) => [t, c.added])),
-          breakingChanges: reasons
-        };
+        const deployDiff2 = toDeployDiff(diff, reasons);
         return {
           action: "reindexed",
           subgraphId: sg3.id,
@@ -3167,8 +3251,9 @@ export {
   deploySchema,
   defineSubgraph,
   backfillSubgraph,
-  INDEX_CODEGEN_TABLES
+  INDEX_CODEGEN_TABLES,
+  ByoBreakingChangeError
 };
 
-//# debugId=822BF312943996F964756E2164756E21
+//# debugId=9A94DC6E9FCF6CD964756E2164756E21
 //# sourceMappingURL=index.js.map