@secondlayer/subgraphs 3.11.0 → 3.13.0

package/dist/src/index.js

@@ -69,6 +69,7 @@ var SubgraphDefinitionSchema = z.object({
   version: z.string().optional(),
   description: z.string().optional(),
   startBlock: z.number().int().nonnegative().optional(),
+  backfillMode: z.enum(["blocking", "concurrent"]).optional(),
   sources: z.record(z.string(), SubgraphFilterSchema).refine((s) => Object.keys(s).length > 0, "Must have at least one source"),
   schema: SubgraphSchemaSchema,
   handlers: z.record(z.string(), z.any())
@@ -81,6 +82,134 @@ function validateSubgraphDefinition(def) {
 import { logger } from "@secondlayer/shared/logger";
 import { formatUnits } from "@secondlayer/stacks/utils";
 import { sql } from "kysely";
+
+// src/schema/generator.ts
+import { createHash } from "node:crypto";
+
+// src/schema/utils.ts
+import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
+
+// src/schema/generator.ts
+var TYPE_MAP = {
+  text: "TEXT",
+  uint: "NUMERIC",
+  int: "NUMERIC",
+  principal: "TEXT",
+  boolean: "BOOLEAN",
+  timestamp: "TIMESTAMPTZ",
+  jsonb: "JSONB"
+};
+function escapeLiteralDefault(value) {
+  if (value === null || value === undefined)
+    return "NULL";
+  if (typeof value === "number" || typeof value === "bigint")
+    return String(value);
+  if (typeof value === "boolean")
+    return value ? "TRUE" : "FALSE";
+  return `'${String(value).replace(/'/g, "''")}'`;
+}
+function tableNeedsTrgm(tableDef) {
+  return Object.values(tableDef.columns).some((col) => col.search);
+}
+function emitTableDDL(schemaName, tableName, tableDef) {
+  const qualifiedName = `${schemaName}.${tableName}`;
+  const statements = [];
+  const columnDefs = [
+    "_id BIGSERIAL PRIMARY KEY",
+    "_block_height BIGINT NOT NULL",
+    "_tx_id TEXT NOT NULL",
+    "_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
+  ];
+  for (const [colName, col] of Object.entries(tableDef.columns)) {
+    const sqlType = TYPE_MAP[col.type];
+    const nullable = col.nullable ? "" : " NOT NULL";
+    let colDef = `${colName} ${sqlType}${nullable}`;
+    if (col.default !== undefined) {
+      colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
+    }
+    if (col.type === "uint") {
+      colDef += ` CHECK (${colName} >= 0)`;
+    }
+    columnDefs.push(colDef);
+  }
+  statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
+  ${columnDefs.join(`,
+  `)}
+)`);
+  statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
+  statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
+  for (const [colName, col] of Object.entries(tableDef.columns)) {
+    if (col.indexed) {
+      statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
+    }
+  }
+  for (const [colName, col] of Object.entries(tableDef.columns)) {
+    if (col.search) {
+      statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
+    }
+  }
+  if (tableDef.indexes) {
+    for (let i = 0;i < tableDef.indexes.length; i++) {
+      const cols = tableDef.indexes[i];
+      const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
+      statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
+    }
+  }
+  if (tableDef.uniqueKeys) {
+    for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
+      const cols = tableDef.uniqueKeys[i];
+      const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
+      statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
+    }
+  }
+  return statements;
+}
+function emitJournalDDL(schemaName) {
+  return [
+    `CREATE TABLE IF NOT EXISTS ${schemaName}._journal (
+  _jid BIGSERIAL PRIMARY KEY,
+  block_height BIGINT NOT NULL,
+  table_name TEXT NOT NULL,
+  row_key JSONB NOT NULL,
+  prev_row JSONB,
+  _created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+)`,
+    `CREATE INDEX IF NOT EXISTS idx_${schemaName}_journal_height ON ${schemaName}._journal (block_height)`
+  ];
+}
+function emitForeignKeyDDL(schemaName, tableName, tableDef) {
+  return (tableDef.relations ?? []).map((rel) => {
+    const constraintName = `fk_${schemaName}_${tableName}_${rel.name}`;
+    return `ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`;
+  });
+}
+function generateSubgraphSQL(def, schemaNameOverride) {
+  const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
+  const statements = [];
+  const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
+  if (needsTrgm) {
+    statements.push("CREATE EXTENSION IF NOT EXISTS pg_trgm");
+  }
+  statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
+  for (const [tableName, tableDef] of Object.entries(def.schema)) {
+    statements.push(...emitTableDDL(schemaName, tableName, tableDef));
+  }
+  statements.push(...emitJournalDDL(schemaName));
+  for (const [tableName, tableDef] of Object.entries(def.schema)) {
+    statements.push(...emitForeignKeyDDL(schemaName, tableName, tableDef));
+  }
+  const hashInput = JSON.stringify({
+    name: def.name,
+    schema: def.schema,
+    sources: def.sources
+  }, (_key, value) => typeof value === "bigint" ? value.toString() : value);
+  const hash = createHash("sha256").update(hashInput).digest("hex");
+  return { statements, hash };
+}
+
+// src/runtime/context.ts
+var JOURNAL_RETENTION_BLOCKS = 300;
+var journalEnsured = new Set;
 function validateColumnName(name) {
   if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
     throw new Error(`Invalid column name: ${name}`);
@@ -95,13 +224,15 @@ class SubgraphContext {
   subgraphSchema;
   ops = [];
   byo;
-  constructor(db, pgSchemaName, subgraphSchema, block, tx, byo = false) {
+  journal;
+  constructor(db, pgSchemaName2, subgraphSchema, block, tx, byo = false, journal = false) {
     this.db = db;
-    this.pgSchemaName = pgSchemaName;
+    this.pgSchemaName = pgSchemaName2;
     this.subgraphSchema = subgraphSchema;
     this.block = block;
     this._tx = tx;
     this.byo = byo;
+    this.journal = journal;
   }
   get tx() {
     return this._tx;
@@ -157,6 +288,43 @@ class SubgraphContext {
     this.validateTable(table);
     this.ops.push({ kind: "delete", table, data: where });
   }
+  increment(table, key, deltas) {
+    this.validateTable(table);
+    const tableDef = this.subgraphSchema[table];
+    const keyColumns = Object.keys(key);
+    const hasUniqueConstraint = tableDef?.uniqueKeys?.some((uk) => uk.length === keyColumns.length && uk.every((c) => keyColumns.includes(c)));
+    if (!hasUniqueConstraint) {
+      throw new Error(`increment("${table}") requires a uniqueKeys constraint on [${keyColumns.join(", ")}]`);
+    }
+    for (const [col, v] of Object.entries(deltas)) {
+      validateColumnName(col);
+      if (keyColumns.includes(col)) {
+        throw new Error(`increment("${table}"): "${col}" is a key column`);
+      }
+      if (typeof v !== "bigint" && typeof v !== "number") {
+        throw new Error(`increment("${table}"): delta for "${col}" must be bigint or number`);
+      }
+    }
+    this.ops.push({
+      kind: "increment",
+      table,
+      data: {
+        ...key,
+        _block_height: this.block.height,
+        _tx_id: this._tx.txId,
+        _upsert_keys: keyColumns
+      },
+      set: { ...deltas }
+    });
+  }
+  opsCheckpoint() {
+    return this.ops.length;
+  }
+  rollbackTo(checkpoint) {
+    if (checkpoint < 0 || checkpoint > this.ops.length)
+      return;
+    this.ops.length = checkpoint;
+  }
   patch(table, where, set) {
     this.update(table, where, set);
   }
@@ -178,7 +346,7 @@ class SubgraphContext {
     const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause} LIMIT 1`;
     const { rows } = await sql.raw(query).execute(this.db);
     const row = rows[0] ?? null;
-    return row ? this.coerceRow(table, row) : null;
+    return this.overlayOne(table, where, row ? this.coerceRow(table, row) : null);
   }
   async findMany(table, where) {
     this.validateTable(table);
@@ -186,7 +354,85 @@ class SubgraphContext {
     const { clause } = buildWhereClause(where);
     const query = `SELECT * FROM ${qualifiedTable} WHERE ${clause}`;
     const { rows } = await sql.raw(query).execute(this.db);
-    return rows.map((r) => this.coerceRow(table, r));
+    const dbRows = rows.map((r) => this.coerceRow(table, r));
+    return this.overlayMany(table, where, dbRows);
+  }
+  overlayOne(table, where, dbRow) {
+    let row = dbRow;
+    for (const op of this.ops) {
+      if (op.table !== table)
+        continue;
+      row = this.applyOpToRow(op, row, where);
+    }
+    return row;
+  }
+  overlayMany(table, where, dbRows) {
+    let result = [...dbRows];
+    for (const op of this.ops) {
+      if (op.table !== table)
+        continue;
+      if (op.kind === "update") {
+        result = result.map((r) => rowMatches(r, op.data) ? { ...r, ...op.set ?? {} } : r);
+      } else if (op.kind === "delete") {
+        result = result.filter((r) => !rowMatches(r, op.data));
+      } else {
+        const upsertKeys = op.data._upsert_keys;
+        const clean = stripControlKeys(op.data);
+        const idx = upsertKeys ? result.findIndex((r) => upsertKeys.every((k) => valEq(r[k], clean[k]))) : -1;
+        if (idx >= 0) {
+          result[idx] = this.applyOpToRow(op, result[idx], where) ?? result[idx];
+        } else {
+          const created = this.applyOpToRow(op, null, where);
+          if (created)
+            result.push(created);
+        }
+      }
+    }
+    return result;
+  }
+  applyOpToRow(op, row, where) {
+    const upsertKeys = op.data._upsert_keys;
+    const clean = stripControlKeys(op.data);
+    switch (op.kind) {
+      case "insert": {
+        if (row) {
+          if (upsertKeys?.every((k) => valEq(row[k], clean[k]))) {
+            const merged = { ...row };
+            for (const [k, v] of Object.entries(clean)) {
+              if (!upsertKeys.includes(k) && !k.startsWith("_"))
+                merged[k] = v;
+            }
+            return merged;
+          }
+          return row;
+        }
+        return rowMatches(clean, where) ? { ...clean } : null;
+      }
+      case "increment": {
+        const deltas = op.set ?? {};
+        if (row) {
+          if (upsertKeys.every((k) => valEq(row[k], clean[k]))) {
+            const merged = { ...row };
+            for (const [col, d] of Object.entries(deltas)) {
+              merged[col] = toBigIntOr0(merged[col]) + toBigIntOr0(d);
+            }
+            return merged;
+          }
+          return row;
+        }
+        if (!rowMatches(clean, where))
+          return null;
+        const created = { ...clean };
+        for (const [col, d] of Object.entries(deltas)) {
+          created[col] = toBigIntOr0(d);
+        }
+        return created;
+      }
+      case "update":
+        return row && rowMatches(row, op.data) ? { ...row, ...op.set ?? {} } : row;
+      case "delete":
+        return row && rowMatches(row, op.data) ? null : row;
+    }
   }
   async count(table, where) {
     this.validateTable(table);
@@ -247,6 +493,7 @@ class SubgraphContext {
   async flush() {
     if (this.ops.length === 0)
       return { count: 0, writes: [] };
+    await this.ensureJournalTable();
     const opsToFlush = [...this.ops];
     this.ops.length = 0;
     const statements = this.buildStatements(opsToFlush);
@@ -264,12 +511,12 @@ class SubgraphContext {
     const writes = opsToFlush.map((op, rowIndex) => {
       const blockHeight = op.data._block_height ?? this.block.height;
       const txId = op.data._tx_id ?? this._tx.txId;
-      const baseRow = op.kind === "update" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
+      const baseRow = op.kind === "update" || op.kind === "increment" ? { ...op.data, ...op.set ?? {} } : { ...op.data };
       baseRow._upsert_keys = undefined;
       baseRow._upsert_fallback_keys = undefined;
       baseRow._upsert_fallback_set = undefined;
       return {
-        op: op.kind,
+        op: op.kind === "increment" ? "update" : op.kind,
         table: op.table,
         row: jsonSafe(baseRow),
         pk: { blockHeight, txId, rowIndex }
@@ -294,6 +541,35 @@ class SubgraphContext {
     const batchKey = `${op.table}:${[...cols].sort().join(",")}:${upsertKeys ? [...upsertKeys].sort().join(",") : ""}`;
     return { data, cols, vals, upsertKeys, batchKey };
   }
+  async ensureJournalTable() {
+    if (!this.journal || journalEnsured.has(this.pgSchemaName))
+      return;
+    const { rows } = await sql.raw(`SELECT to_regclass('"${this.pgSchemaName}"."_journal"') AS r`).execute(this.db);
+    if (rows[0]?.r) {
+      journalEnsured.add(this.pgSchemaName);
+      return;
+    }
+    for (const stmt of emitJournalDDL(this.pgSchemaName)) {
+      await sql.raw(stmt).execute(this.db);
+    }
+  }
+  columnSqlType(table, col) {
+    const def = this.subgraphSchema[table]?.columns?.[col];
+    return def ? TYPE_MAP[def.type] : undefined;
+  }
+  journalCaptureSQL(table, keyCols, keyLiteralRows) {
+    const cast = (col, expr) => {
+      const t = this.columnSqlType(table, col);
+      return t ? `CAST(${expr} AS ${t})` : expr;
+    };
+    const keyObj = keyCols.map((k) => `'${k}', ${cast(k, `v."${k}"`)}`).join(", ");
+    const joinCond = keyCols.map((k) => `t."${k}" = ${cast(k, `v."${k}"`)}`).join(" AND ");
+    const valuesList = keyLiteralRows.map((r) => `(${r.join(", ")})`).join(", ");
+    return `INSERT INTO "${this.pgSchemaName}"."_journal" ("block_height", "table_name", "row_key", "prev_row") ` + `SELECT ${this.block.height}, '${table}', jsonb_build_object(${keyObj}), to_jsonb(t.*) ` + `FROM (VALUES ${valuesList}) AS v(${keyCols.map((k) => `"${k}"`).join(", ")}) ` + `LEFT JOIN "${this.pgSchemaName}"."${table}" t ON ${joinCond}`;
+  }
+  journalCaptureByWhereSQL(table, clause) {
+    return `INSERT INTO "${this.pgSchemaName}"."_journal" ("block_height", "table_name", "row_key", "prev_row") ` + `SELECT ${this.block.height}, '${table}', jsonb_build_object('_id', t."_id"), to_jsonb(t.*) ` + `FROM "${this.pgSchemaName}"."${table}" t WHERE ${clause}`;
+  }
   buildStatements(ops) {
     const statements = [];
     if (this.byo) {
@@ -307,6 +583,38 @@ class SubgraphContext {
     }
     let currentBatch = null;
     let currentBatchKey = "";
+    let incBatch = null;
+    let incBatchKey = "";
+    const flushIncrementBatch = () => {
+      if (!incBatch)
+        return;
+      const batch = incBatch;
+      const qualifiedTable = `"${this.pgSchemaName}"."${batch.table}"`;
+      const cols = [
+        ...batch.keyCols,
+        ...batch.deltaCols,
+        "_block_height",
+        "_tx_id",
+        "_created_at"
+      ];
+      const valuesList = Array.from(batch.rows.values()).map((r) => {
+        const vals = [
+          ...batch.keyCols.map((k) => escapeLiteral(r.keys[k])),
+          ...batch.deltaCols.map((c) => String(r.deltas[c] ?? 0n)),
+          escapeLiteral(r.meta.blockHeight),
+          escapeLiteral(r.meta.txId),
+          "NOW()"
+        ];
+        return `(${vals.join(", ")})`;
+      }).join(", ");
+      const setClauses = batch.deltaCols.map((c) => `"${c}" = COALESCE("${batch.table}"."${c}", 0) + EXCLUDED."${c}"`);
+      if (this.journal) {
+        statements.push(this.journalCaptureSQL(batch.table, batch.keyCols, Array.from(batch.rows.values()).map((r) => batch.keyCols.map((k) => escapeLiteral(r.keys[k])))));
+      }
+      statements.push(`INSERT INTO ${qualifiedTable} (${cols.map((c) => `"${c}"`).join(", ")}) VALUES ${valuesList} ` + `ON CONFLICT (${batch.keyCols.map((k) => `"${k}"`).join(", ")}) DO UPDATE SET ${setClauses.join(", ")}`);
+      incBatch = null;
+      incBatchKey = "";
+    };
     const flushInsertBatch = () => {
       if (!currentBatch)
         return;
@@ -328,6 +636,11 @@ class SubgraphContext {
       }
       const valuesList = rows.map((r) => `(${r.join(", ")})`).join(", ");
       let stmt = `INSERT INTO ${qualifiedTable} (${colList}) VALUES ${valuesList}`;
+      if (this.journal && batch.upsertKeys && batch.upsertKeys.length > 0) {
+        const uKeys = batch.upsertKeys;
+        const keyIndices = uKeys.map((k) => batch.cols.indexOf(k));
+        statements.push(this.journalCaptureSQL(batch.table, uKeys, rows.map((r) => keyIndices.map((ki) => r[ki]))));
+      }
       if (batch.upsertKeys && batch.upsertKeys.length > 0) {
         const batchKeys = batch.upsertKeys;
         const updateCols = batch.cols.filter((c) => !batchKeys.includes(c) && !c.startsWith("_"));
@@ -345,6 +658,7 @@ class SubgraphContext {
     for (const op of ops) {
       const qualifiedTable = `"${this.pgSchemaName}"."${op.table}"`;
       if (op.kind === "insert") {
+        flushIncrementBatch();
         const { cols, vals, upsertKeys, batchKey } = this.prepareInsert(op);
         if (batchKey === currentBatchKey && currentBatch) {
           currentBatch.rows.push(vals);
@@ -353,22 +667,60 @@ class SubgraphContext {
           currentBatch = { table: op.table, cols, rows: [vals], upsertKeys };
           currentBatchKey = batchKey;
         }
+      } else if (op.kind === "increment") {
+        flushInsertBatch();
+        const keyCols = [...op.data._upsert_keys].sort();
+        const deltaCols = Object.keys(op.set ?? {}).sort();
+        const batchKey = `inc:${op.table}:${keyCols.join(",")}:${deltaCols.join(",")}`;
+        if (batchKey !== incBatchKey || !incBatch) {
+          flushIncrementBatch();
+          incBatch = { table: op.table, keyCols, deltaCols, rows: new Map };
+          incBatchKey = batchKey;
+        }
+        const clean = stripControlKeys(op.data);
+        const keySig = keyCols.map((k) => escapeLiteral(clean[k])).join("\x00");
+        const existing = incBatch.rows.get(keySig);
+        if (existing) {
+          for (const c of deltaCols) {
+            existing.deltas[c] = (existing.deltas[c] ?? 0n) + toBigIntOr0(op.set?.[c]);
+          }
+        } else {
+          const deltas = {};
+          for (const c of deltaCols)
+            deltas[c] = toBigIntOr0(op.set?.[c]);
+          incBatch.rows.set(keySig, {
+            keys: clean,
+            deltas,
+            meta: {
+              blockHeight: op.data._block_height ?? this.block.height,
+              txId: op.data._tx_id ?? this._tx.txId
+            }
+          });
+        }
       } else {
         flushInsertBatch();
+        flushIncrementBatch();
         if (op.kind === "update") {
           const setEntries = Object.entries(op.set ?? {});
           for (const [k] of setEntries)
             validateColumnName(k);
           const setClauses = setEntries.map(([k, v]) => `"${k}" = ${escapeLiteral(v)}`);
           const { clause } = buildWhereClause(op.data);
+          if (this.journal) {
+            statements.push(this.journalCaptureByWhereSQL(op.table, clause));
+          }
           statements.push(`UPDATE ${qualifiedTable} SET ${setClauses.join(", ")} WHERE ${clause}`);
         } else if (op.kind === "delete") {
           const { clause } = buildWhereClause(op.data);
+          if (this.journal) {
+            statements.push(this.journalCaptureByWhereSQL(op.table, clause));
+          }
           statements.push(`DELETE FROM ${qualifiedTable} WHERE ${clause}`);
         }
       }
     }
     flushInsertBatch();
+    flushIncrementBatch();
     return statements;
   }
   validateTable(table) {
@@ -377,6 +729,36 @@ class SubgraphContext {
     }
   }
 }
+function stripControlKeys(data) {
+  const {
+    _upsert_keys: _a,
+    _upsert_fallback_keys: _b,
+    _upsert_fallback_set: _c,
+    ...clean
+  } = data;
+  return clean;
+}
+function valEq(a, b) {
+  if (a === b)
+    return true;
+  if (a == null || b == null)
+    return false;
+  return String(a) === String(b);
+}
+function rowMatches(row, where) {
+  return Object.entries(where).every(([k, v]) => valEq(row[k], v));
+}
+function toBigIntOr0(v) {
+  if (typeof v === "bigint")
+    return v;
+  if (v == null)
+    return 0n;
+  try {
+    return BigInt(String(v));
+  } catch {
+    return 0n;
+  }
+}
 function jsonSafe(row) {
   const out = {};
   for (const [k, v] of Object.entries(row)) {
@@ -695,7 +1077,25 @@ async function runHandlers(subgraph, matched, ctx, opts) {
       filterLookup.set(name, filter);
     }
   }
+  const units = [];
   for (const { tx, events, sourceName } of matched) {
+    if (events.length === 0) {
+      units.push({ tx, sourceName, event: null });
+    } else {
+      for (const event of events)
+        units.push({ tx, sourceName, event });
+    }
+  }
+  units.sort((a, b) => (a.tx.tx_index ?? 0) - (b.tx.tx_index ?? 0) || (a.event?.event_index ?? -1) - (b.event?.event_index ?? -1));
+  for (const { tx, event, sourceName } of units) {
+    if (errors >= threshold) {
+      logger2.error("Subgraph error threshold reached, skipping remaining events", {
+        subgraph: subgraph.name,
+        errors,
+        threshold
+      });
+      return { processed, errors };
+    }
     const handler = subgraph.handlers[sourceName] ?? subgraph.handlers["*"] ?? null;
     if (!handler) {
       logger2.warn("No handler found for source", {
@@ -714,9 +1114,29 @@ async function runHandlers(subgraph, matched, ctx, opts) {
       functionName: tx.function_name ?? null
     });
     const filter = filterLookup.get(sourceName);
-    if (events.length === 0) {
-      try {
-        const payload = filter ? buildEventPayload(filter, tx, null) : {
+    const checkpoint = ctx.opsCheckpoint();
+    try {
+      let payload;
+      if (event === null) {
+        payload = filter ? buildEventPayload(filter, tx, null) : {
+          tx: {
+            txId: tx.tx_id,
+            sender: tx.sender,
+            type: tx.type,
+            status: tx.status,
+            contractId: tx.contract_id,
+            functionName: tx.function_name
+          }
+        };
+      } else if (filter) {
+        payload = buildEventPayload(filter, tx, event);
+      } else {
+        const decoded = decodeEventData(event.data);
+        payload = {
+          ...decoded,
+          _eventId: event.id,
+          _eventType: event.type,
+          _eventIndex: event.event_index,
           tx: {
             txId: tx.tx_id,
             sender: tx.sender,
@@ -726,62 +1146,22 @@ async function runHandlers(subgraph, matched, ctx, opts) {
             functionName: tx.function_name
           }
         };
-        await handler(payload, ctx);
-        processed++;
-      } catch (err) {
-        errors++;
-        logger2.error("Subgraph handler error", {
-          subgraph: subgraph.name,
-          sourceName,
-          txId: tx.tx_id,
-          error: getErrorMessage(err)
-        });
-      }
-      continue;
-    }
-    for (const event of events) {
-      if (errors >= threshold) {
-        logger2.error("Subgraph error threshold reached, skipping remaining events", {
-          subgraph: subgraph.name,
-          errors,
-          threshold
-        });
-        return { processed, errors };
       }
-      try {
-        const payload = filter ? buildEventPayload(filter, tx, event) : (() => {
-          const decoded = decodeEventData(event.data);
-          return {
-            ...decoded,
-            _eventId: event.id,
-            _eventType: event.type,
-            _eventIndex: event.event_index,
-            tx: {
-              txId: tx.tx_id,
-              sender: tx.sender,
-              type: tx.type,
-              status: tx.status,
-              contractId: tx.contract_id,
-              functionName: tx.function_name
-            }
-          };
-        })();
-        if (filter?.type === "print_event" && filter.topic && payload.topic !== filter.topic) {
-          continue;
-        }
-        await handler(payload, ctx);
-        processed++;
-      } catch (err) {
-        errors++;
-        logger2.error("Subgraph handler error", {
-          subgraph: subgraph.name,
-          sourceName,
-          txId: tx.tx_id,
-          eventId: event.id,
-          eventType: event.type,
-          error: getErrorMessage(err)
-        });
+      if (event !== null && filter?.type === "print_event" && filter.topic && payload.topic !== filter.topic) {
+        continue;
       }
+      await handler(payload, ctx);
+      processed++;
+    } catch (err) {
+      ctx.rollbackTo(checkpoint);
+      errors++;
+      logger2.error("Subgraph handler error", {
+        subgraph: subgraph.name,
+        sourceName,
+        txId: tx.tx_id,
+        ...event !== null ? { eventId: event.id, eventType: event.type } : {},
+        error: getErrorMessage(err)
+      });
     }
   }
   return { processed, errors };
@@ -1038,9 +1418,6 @@ import {
 import { logger as logger5 } from "@secondlayer/shared/logger";
 import { sql as sql3 } from "kysely";
 
-// src/schema/utils.ts
-import { pgSchemaName } from "@secondlayer/shared/db/queries/subgraphs";
-
 // src/runtime/block-source.ts
 import { getSourceDb } from "@secondlayer/shared/db";
 import { IndexHttpClient } from "@secondlayer/shared/index-http";
@@ -1386,7 +1763,7 @@ function resolveBlockSource(subgraph) {
 }
 
 // src/runtime/outbox-emit.ts
-import { createHash } from "node:crypto";
+import { createHash as createHash2 } from "node:crypto";
 import { logger as logger4 } from "@secondlayer/shared/logger";
 var loggedKillSwitch = false;
 var OP_VERB = {
@@ -1399,7 +1776,7 @@ function isEmitOutboxEnabled() {
 }
 function dedupKey(subgraphName, tableName, blockHeight, txId, rowIndex, row) {
   const canonical = `${subgraphName}:${tableName}:${blockHeight}:${txId}:${rowIndex}:${stableStringify(row)}`;
-  return createHash("sha256").update(canonical).digest("hex").slice(0, 32);
+  return createHash2("sha256").update(canonical).digest("hex").slice(0, 32);
 }
 function stableStringify(obj) {
   const keys = Object.keys(obj).sort();
@@ -1651,6 +2028,32 @@ async function resolveTraitContracts(subgraph, blockHeight, db) {
   }
   return resolved;
 }
+var BLOCK_RETRY_DELAYS_MS = [500, 2000, 5000];
+function journalEnabled(opts) {
+  return !opts?.skipProgressUpdate;
+}
+async function processBlockWithRetry(subgraph, subgraphName, blockHeight, opts, retryDelaysMs = BLOCK_RETRY_DELAYS_MS) {
+  let lastError;
+  for (let attempt = 0;attempt <= retryDelaysMs.length; attempt++) {
+    try {
+      return await processBlock(subgraph, subgraphName, blockHeight, opts);
+    } catch (err) {
+      lastError = err;
+      const delay = retryDelaysMs[attempt];
+      if (delay === undefined)
+        break;
+      logger5.warn("Block processing failed, retrying", {
+        subgraph: subgraphName,
+        blockHeight,
+        attempt: attempt + 1,
+        retryInMs: delay,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      await new Promise((r) => setTimeout(r, delay));
+    }
+  }
+  throw lastError;
+}
 async function processBlock(subgraph, subgraphName, blockHeight, opts) {
   const targetDb = getTargetDb();
   const blockStart = performance.now();
@@ -1718,10 +2121,17 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
     }
   };
   if (route.byo) {
+    if (opts?.atomicProgress) {
+      const row = await targetDb.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
+      if (row && Number(row.last_processed_block) >= blockHeight) {
+        result.skipped = true;
+        return result;
+      }
+    }
     let runResult = { processed: 0, errors: 0 };
     let manifest;
     await route.dataDb.transaction().execute(async (tx) => {
-      const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, true);
+      const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, true, journalEnabled(opts));
       const handlerStart = performance.now();
       runResult = await runHandlers(subgraph, matched, ctx);
       handlerMs = performance.now() - handlerStart;
@@ -1737,24 +2147,39 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
       if (manifest && manifest.count > 0) {
         await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
       }
+      if (opts?.atomicProgress && manifest && manifest.count > 0) {
+        await updateSubgraphStatus(tx, subgraphName, opts.atomicProgress.status, blockHeight);
+      }
       await applyProgress(tx, runResult);
     });
   } else {
     await targetDb.transaction().execute(async (tx) => {
-      const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx);
+      if (opts?.atomicProgress) {
+        const row = await tx.selectFrom("subgraphs").select("last_processed_block").where("name", "=", subgraphName).executeTakeFirst();
+        if (row && Number(row.last_processed_block) >= blockHeight) {
+          result.skipped = true;
+          return;
+        }
+      }
+      const ctx = new SubgraphContext(tx, schemaName, subgraph.schema, blockMeta, initialTx, false, journalEnabled(opts));
       const handlerStart = performance.now();
       const runResult = await runHandlers(subgraph, matched, ctx);
       handlerMs = performance.now() - handlerStart;
       result.processed = runResult.processed;
       result.errors = runResult.errors;
+      let flushedWrites = false;
       if (ctx.pendingOps > 0) {
         const flushStart = performance.now();
         const manifest = await ctx.flush();
+        flushedWrites = manifest.count > 0;
         if (manifest.count > 0) {
           await emitSubscriptionOutbox(tx, subgraphName, manifest, matcher, block.height);
         }
         flushMs = performance.now() - flushStart;
       }
+      if (opts?.atomicProgress && flushedWrites) {
+        await updateSubgraphStatus(tx, subgraphName, opts.atomicProgress.status, blockHeight);
+      }
       await applyProgress(tx, runResult);
     });
   }
@@ -1784,6 +2209,9 @@ async function processBlock(subgraph, subgraphName, blockHeight, opts) {
         error: err instanceof Error ? err.message : String(err)
       });
     }
+    if (journalEnabled(opts)) {
+      await sql3.raw(`DELETE FROM "${schemaName}"."_journal" WHERE "block_height" < ${blockHeight - JOURNAL_RETENTION_BLOCKS}`).execute(route.dataDb).catch(() => {});
+    }
   }
   return result;
 }
@@ -1858,118 +2286,16 @@ import {
   recordGapBatch,
   resolveGaps
 } from "@secondlayer/shared/db/queries/subgraph-gaps";
+import { updateOperationProcessedEvents } from "@secondlayer/shared/db/queries/subgraph-operations";
 import {
   recordSubgraphProcessed as recordSubgraphProcessed2,
   updateSubgraphStatus as updateSubgraphStatus2
 } from "@secondlayer/shared/db/queries/subgraphs";
 import { logger as logger6 } from "@secondlayer/shared/logger";
-
-// src/schema/generator.ts
-import { createHash as createHash2 } from "node:crypto";
-var TYPE_MAP = {
-  text: "TEXT",
-  uint: "NUMERIC",
-  int: "NUMERIC",
-  principal: "TEXT",
-  boolean: "BOOLEAN",
-  timestamp: "TIMESTAMPTZ",
-  jsonb: "JSONB"
-};
-function escapeLiteralDefault(value) {
-  if (value === null || value === undefined)
-    return "NULL";
-  if (typeof value === "number" || typeof value === "bigint")
-    return String(value);
-  if (typeof value === "boolean")
-    return value ? "TRUE" : "FALSE";
-  return `'${String(value).replace(/'/g, "''")}'`;
-}
-function tableNeedsTrgm(tableDef) {
-  return Object.values(tableDef.columns).some((col) => col.search);
-}
-function emitTableDDL(schemaName, tableName, tableDef) {
-  const qualifiedName = `${schemaName}.${tableName}`;
-  const statements = [];
-  const columnDefs = [
-    "_id BIGSERIAL PRIMARY KEY",
-    "_block_height BIGINT NOT NULL",
-    "_tx_id TEXT NOT NULL",
-    "_created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
-  ];
-  for (const [colName, col] of Object.entries(tableDef.columns)) {
-    const sqlType = TYPE_MAP[col.type];
-    const nullable = col.nullable ? "" : " NOT NULL";
-    let colDef = `${colName} ${sqlType}${nullable}`;
-    if (col.default !== undefined) {
-      colDef += ` DEFAULT ${escapeLiteralDefault(col.default)}`;
-    }
-    columnDefs.push(colDef);
-  }
-  statements.push(`CREATE TABLE IF NOT EXISTS ${qualifiedName} (
-  ${columnDefs.join(`,
-  `)}
-)`);
-  statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_block_height ON ${qualifiedName} (_block_height)`);
-  statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_tx_id ON ${qualifiedName} (_tx_id)`);
-  for (const [colName, col] of Object.entries(tableDef.columns)) {
-    if (col.indexed) {
-      statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName} ON ${qualifiedName} (${colName})`);
-    }
-  }
-  for (const [colName, col] of Object.entries(tableDef.columns)) {
-    if (col.search) {
-      statements.push(`CREATE INDEX IF NOT EXISTS idx_${schemaName}_${tableName}_${colName}_trgm ON ${qualifiedName} USING gin (${colName} gin_trgm_ops)`);
-    }
-  }
-  if (tableDef.indexes) {
-    for (let i = 0;i < tableDef.indexes.length; i++) {
-      const cols = tableDef.indexes[i];
-      const idxName = `idx_${schemaName}_${tableName}_composite_${i}`;
-      statements.push(`CREATE INDEX IF NOT EXISTS ${idxName} ON ${qualifiedName} (${cols.join(", ")})`);
-    }
-  }
-  if (tableDef.uniqueKeys) {
-    for (let i = 0;i < tableDef.uniqueKeys.length; i++) {
-      const cols = tableDef.uniqueKeys[i];
-      const constraintName = `uq_${schemaName}_${tableName}_${cols.join("_")}`;
-      statements.push(`ALTER TABLE ${qualifiedName} ADD CONSTRAINT ${constraintName} UNIQUE (${cols.join(", ")})`);
-    }
-  }
-  return statements;
-}
-function emitForeignKeyDDL(schemaName, tableName, tableDef) {
-  return (tableDef.relations ?? []).map((rel) => {
-    const constraintName = `fk_${schemaName}_${tableName}_${rel.name}`;
-    return `ALTER TABLE ${schemaName}.${tableName} ADD CONSTRAINT ${constraintName} ` + `FOREIGN KEY (${rel.fields.join(", ")}) ` + `REFERENCES ${schemaName}.${rel.references} (${rel.referencedColumns.join(", ")})`;
-  });
-}
-function generateSubgraphSQL(def, schemaNameOverride) {
-  const schemaName = schemaNameOverride ?? pgSchemaName(def.name);
-  const statements = [];
-  const needsTrgm = Object.values(def.schema).some((table) => Object.values(table.columns).some((col) => col.search));
-  if (needsTrgm) {
-    statements.push("CREATE EXTENSION IF NOT EXISTS pg_trgm");
-  }
-  statements.push(`CREATE SCHEMA IF NOT EXISTS ${schemaName}`);
-  for (const [tableName, tableDef] of Object.entries(def.schema)) {
-    statements.push(...emitTableDDL(schemaName, tableName, tableDef));
-  }
-  for (const [tableName, tableDef] of Object.entries(def.schema)) {
-    statements.push(...emitForeignKeyDDL(schemaName, tableName, tableDef));
-  }
-  const hashInput = JSON.stringify({
-    name: def.name,
-    schema: def.schema,
-    sources: def.sources
-  }, (_key, value) => typeof value === "bigint" ? value.toString() : value);
-  const hash = createHash2("sha256").update(hashInput).digest("hex");
-  return { statements, hash };
-}
-
-// src/runtime/reindex.ts
 var LOG_INTERVAL = 1000;
 var HEALTH_FLUSH_INTERVAL = 1000;
 var PROGRESS_FLUSH_INTERVAL_MS = 5000;
+var EMPTY_BATCH_HALT_THRESHOLD = 3;
 var STANDARD_REINDEX_BATCH_CONFIG = {
   defaultBatchSize: 500,
   minBatchSize: 100,
@@ -2045,6 +2371,7 @@ async function processBlockRange(def, opts) {
   let batchSize = batchConfig.defaultBatchSize;
   let currentHeight = fromBlock;
   let aborted = false;
+  let consecutiveEmptyBatches = 0;
   const sparse = Boolean(source.nextDataHeight && canSparseScan(def));
   const flushHealth = async () => {
     if (pendingEventsProcessed === 0 && pendingErrors === 0)
@@ -2056,6 +2383,13 @@ async function processBlockRange(def, opts) {
     lastHealthFlushBlock = blocksProcessed;
     lastHealthFlushAt = Date.now();
   };
+  const haltRange = async (errorMsg, height) => {
+    pendingErrors++;
+    pendingLastError = errorMsg;
+    await flushHealth().catch(() => {});
+    await updateSubgraphStatus2(targetDb, subgraphName, "error").catch(() => {});
+    throw new Error(`${subgraphName}: halted at block ${height}: ${errorMsg}`);
+  };
   let nextBatchEnd = Math.min(currentHeight + batchSize - 1, toBlock);
   let nextBatchPromise = source.loadBlockRange(currentHeight, nextBatchEnd);
   while (currentHeight <= toBlock) {
@@ -2070,6 +2404,14 @@ async function processBlockRange(def, opts) {
     }
     const batch = await nextBatchPromise;
     const batchEnd = nextBatchEnd;
+    if (batch.size === 0 && batchEnd >= currentHeight) {
+      consecutiveEmptyBatches++;
+      if (consecutiveEmptyBatches >= EMPTY_BATCH_HALT_THRESHOLD) {
+        await haltRange(`block source returned ${consecutiveEmptyBatches} consecutive empty batches (ending ${currentHeight}..${batchEnd}) — source degraded`, currentHeight);
+      }
+    } else {
+      consecutiveEmptyBatches = 0;
+    }
     const nextStart = batchEnd + 1;
     if (nextStart <= toBlock) {
       nextBatchEnd = Math.min(nextStart + batchSize - 1, toBlock);
@@ -2077,28 +2419,39 @@ async function processBlockRange(def, opts) {
     }
     const batchFailedBlocks = [];
     let batchMatched = 0;
+    const atomicProgress = status === "reindexing" ? { status } : undefined;
     for (let height = currentHeight;height <= batchEnd; height++) {
-      const blockData = batch.get(height);
+      let blockData = batch.get(height);
       if (!blockData) {
+        blockData = (await source.loadBlockRange(height, height)).get(height);
+      }
+      if (!blockData) {
+        if (status === "reindexing") {
+          const errorMsg = `block ${height} missing from source — halting reindex (cursor stays at ${height - 1})`;
+          await haltRange(errorMsg, height);
+        }
         batchFailedBlocks.push({ height, reason: "block_missing" });
         blocksProcessed++;
         continue;
       }
       let result;
       try {
-        result = await processBlock(def, subgraphName, height, {
+        result = await processBlockWithRetry(def, subgraphName, height, {
           skipProgressUpdate: true,
+          atomicProgress,
           preloaded: blockData
         });
       } catch (err) {
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        logger6.error("Block processing error", {
+        const errorMsg = getErrorMessage2(err);
+        logger6.error("Block processing failed persistently", {
           subgraph: subgraphName,
           blockHeight: height,
           error: errorMsg
         });
+        if (status === "reindexing") {
+          await haltRange(`block ${height} failed persistently: ${errorMsg}`, height);
+        }
         batchFailedBlocks.push({ height, reason: "processing_error" });
-        await updateSubgraphStatus2(targetDb, subgraphName, status, height).catch(() => {});
         blocksProcessed++;
         totalErrors++;
         pendingErrors++;
@@ -2124,6 +2477,9 @@ async function processBlockRange(def, opts) {
       const shouldFlushProgress = blocksProcessed % 100 === 0 || now - lastProgressFlushAt >= PROGRESS_FLUSH_INTERVAL_MS;
       if (shouldFlushProgress) {
         await updateSubgraphStatus2(targetDb, subgraphName, status, height);
+        if (opts.operationId) {
+          await updateOperationProcessedEvents(targetDb, opts.operationId, totalEventsProcessed).catch(() => {});
+        }
         lastProgressFlushAt = now;
       }
       if (blocksProcessed % LOG_INTERVAL === 0) {
@@ -2241,6 +2597,7 @@ async function reindexSubgraph(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: subgraphRow?.id,
+      operationId: opts?.operationId,
       signal: opts?.signal
     });
     if (result.aborted) {
@@ -2316,6 +2673,7 @@ async function resumeReindex(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: row.id,
+      operationId: opts.operationId,
       signal: opts.signal
     });
     if (result.aborted) {
@@ -2364,6 +2722,7 @@ async function backfillSubgraph(def, opts) {
       isCatchup: false,
       apiKeyId: null,
       subgraphId: subgraphRow?.id,
+      operationId: opts.operationId,
       signal: opts.signal
     });
     if (result.aborted) {
@@ -3238,10 +3597,12 @@ function getDefault(type) {
 }
 export {
   validateSubgraphDefinition,
+  sparseProbeTargets,
   resumeReindex,
   renderDeployPlan,
   reindexSubgraph,
   pgSchemaName,
+  hasBreakingChanges,
   generateSubgraphSQL,
   generatePrismaSchema,
   generateKyselySchema,
@@ -3250,10 +3611,11 @@ export {
   diffSchema,
   deploySchema,
   defineSubgraph,
+  canSparseScan,
   backfillSubgraph,
   INDEX_CODEGEN_TABLES,
   ByoBreakingChangeError
 };
 
-//# debugId=9A94DC6E9FCF6CD964756E2164756E21
+//# debugId=A7DC4F802366B29864756E2164756E21
 //# sourceMappingURL=index.js.map