@thru/indexer 0.2.30 → 0.2.32

package/dist/index.mjs

@@ -358,7 +358,8 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
     safetyMargin = 64,
     pageSize = 512,
     logLevel = "info",
-    validateParse = false
+    validateParse = false,
+    observer
   } = options;
   const log = (level, msg) => {
     if (logLevel === "debug" || level !== "debug") {
@@ -368,6 +369,10 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
   log("info", `Starting stream processor: ${stream.description}`);
   const checkpoint = await getCheckpoint(db, stream.name);
   const startSlot = checkpoint ? checkpoint.slot : defaultStartSlot;
+  observer?.onStart?.({
+    startSlot,
+    checkpointSlot: checkpoint?.slot ?? null
+  });
   log(
     "info",
     `Starting from slot ${startSlot}${checkpoint ? " (resuming)" : " (fresh start)"}`
@@ -423,31 +428,43 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
           );
         }
       } catch (filterErr) {
+        observer?.onError?.("filterBatch", filterErr);
         log(
           "error",
           `filterBatch hook failed: ${filterErr instanceof Error ? filterErr.message : String(filterErr)}`
         );
-        return;
+        throw filterErr;
       }
     }
     const lastEventToCommit = eventsToCommit[eventsToCommit.length - 1];
     const lastEventToCommitId = lastEventToCommit?.id ?? null;
     let committedEvents = eventsToCommit;
-    await db.transaction(async (tx) => {
-      committedEvents = await tx.insert(stream.table).values(eventsToCommit).onConflictDoNothing().returning();
-      await updateCheckpoint(
-        tx,
-        stream.name,
-        batch.slot,
-        lastEventToCommitId
-      );
-    });
+    try {
+      await db.transaction(async (tx) => {
+        committedEvents = await tx.insert(stream.table).values(eventsToCommit).onConflictDoNothing().returning();
+        await updateCheckpoint(
+          tx,
+          stream.name,
+          batch.slot,
+          lastEventToCommitId
+        );
+      });
+    } catch (commitErr) {
+      observer?.onError?.("commit", commitErr);
+      throw commitErr;
+    }
     stats.batchesCommitted++;
     stats.lastSlot = batch.slot;
+    observer?.onBatchCommitted?.({
+      slot: batch.slot,
+      count: eventsToCommit.length
+    });
+    observer?.onCheckpoint?.({ slot: batch.slot });
     if (stream.onCommit && committedEvents.length > 0) {
       try {
         await stream.onCommit({ ...batch, events: committedEvents }, { db });
       } catch (hookErr) {
+        observer?.onError?.("onCommit", hookErr);
         log(
           "error",
           `onCommit hook failed: ${hookErr instanceof Error ? hookErr.message : String(hookErr)}`
@@ -455,6 +472,11 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
       }
     }
   };
+  let rejectFlushFailure = () => {
+  };
+  const flushFailure = new Promise((_, reject) => {
+    rejectFlushFailure = reject;
+  });
   const flushInterval = setInterval(async () => {
     const batch = batcher.flushIfStale();
     if (batch) {
@@ -469,21 +491,38 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
           "error",
           `Timeout flush failed: ${err instanceof Error ? err.message : String(err)}`
         );
+        clearInterval(flushInterval);
+        rejectFlushFailure(err);
       }
     }
   }, 1e3);
-  try {
+  const processReplay = async () => {
     for await (const event of replay) {
       if (abortSignal?.aborted) {
         log("info", "Abort signal received, stopping...");
         break;
       }
       eventsReceivedSinceLastLog++;
-      const parsed = stream.parse(event);
-      if (!parsed) continue;
+      observer?.onRecord?.({
+        slot: event.slot ?? null,
+        id: event.eventId ?? null
+      });
+      let parsed;
+      try {
+        parsed = stream.parse(event);
+      } catch (parseErr) {
+        observer?.onParserError?.(parseErr);
+        observer?.onError?.("parse", parseErr);
+        throw parseErr;
+      }
+      if (!parsed) {
+        observer?.onParserNull?.();
+        continue;
+      }
       if (validateParse) {
         const validation = validateParsedData(stream.schema, parsed, stream.name);
         if (!validation.success) {
+          observer?.onParseValidationError?.(validation.error);
           log("error", validation.error);
           continue;
         }
@@ -515,6 +554,9 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
         `Final flush: ${finalBatch.events.length} event(s) at slot ${finalBatch.slot}`
       );
     }
+  };
+  try {
+    await Promise.race([processReplay(), flushFailure]);
   } catch (err) {
     log(
       "error",
@@ -523,6 +565,8 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
     throw err;
   } finally {
     clearInterval(flushInterval);
+    rejectFlushFailure = () => {
+    };
   }
   log(
     "info",
@@ -571,7 +615,7 @@ function shouldLog(level, minLevel) {
   return levels.indexOf(level) >= levels.indexOf(minLevel);
 }
 async function runAccountStreamProcessor(stream, options, abortSignal) {
-  const { clientFactory, db, logLevel = "info", validateParse = false } = options;
+  const { clientFactory, db, logLevel = "info", validateParse = false, observer } = options;
   const checkpointName = `account:${stream.name}`;
   const log = (level, msg, meta) => {
     if (shouldLog(level, logLevel)) {
@@ -587,6 +631,10 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
   };
   const checkpoint = await getCheckpoint(db, checkpointName);
   const minUpdatedSlot = checkpoint?.slot;
+  observer?.onStart?.({
+    startSlot: minUpdatedSlot,
+    checkpointSlot: checkpoint?.slot ?? null
+  });
   if (minUpdatedSlot) {
     log("info", `Resuming from checkpoint: slot ${minUpdatedSlot}`);
   }
@@ -630,6 +678,10 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
       if (event.type === "account") {
         const account = event.account;
         stats.accountsProcessed++;
+        observer?.onRecord?.({
+          slot: account.slot,
+          id: account.addressHex
+        });
         if (stats.accountsProcessed <= 3) {
           log(
             "info",
@@ -644,17 +696,28 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
           try {
             await db.delete(stream.table).where(eq(table[idField], idValue));
             stats.accountsDeleted++;
+            observer?.onCheckpoint?.({ slot: account.slot });
             log("info", `Deleted row for account ${account.addressHex}`);
             if (account.slot > lastProcessedSlot) {
               lastProcessedSlot = account.slot;
             }
           } catch (err) {
+            observer?.onError?.("commit", err);
             log("error", `Failed to delete account ${account.addressHex}: ${err}`);
+            throw err;
           }
           continue;
         }
-        const parsed = stream.parse(account);
+        let parsed;
+        try {
+          parsed = stream.parse(account);
+        } catch (parseErr) {
+          observer?.onParserError?.(parseErr);
+          observer?.onError?.("parse", parseErr);
+          throw parseErr;
+        }
         if (!parsed) {
+          observer?.onParserNull?.();
           log(
             "debug",
             `Skipped account ${account.addressHex} - parser returned null (dataLen=${account.data.length})`
@@ -667,6 +730,7 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
         if (validateParse) {
           const validation = validateParsedData(stream.schema, parsed, stream.name);
           if (!validation.success) {
+            observer?.onParseValidationError?.(validation.error);
             log("error", validation.error);
             continue;
           }
@@ -689,10 +753,12 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
             );
           }
         } catch (err) {
+          observer?.onError?.("commit", err);
           log(
             "error",
             `Failed to upsert account ${account.addressHex}: ${err}`
           );
+          throw err;
         }
         if (upserted && account.slot > lastProcessedSlot) {
           lastProcessedSlot = account.slot;
@@ -707,6 +773,7 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
         const slot = event.block.slot;
         if (lastProcessedSlot > 0n) {
           await updateCheckpoint(db, checkpointName, lastProcessedSlot, null);
+          observer?.onCheckpoint?.({ slot: lastProcessedSlot });
           log(
             "debug",
             `Block finished: slot ${slot}, checkpoint saved at account slot ${lastProcessedSlot}`
@@ -721,6 +788,7 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
     }
     if (lastProcessedSlot > 0n) {
       await updateCheckpoint(db, checkpointName, lastProcessedSlot, null);
+      observer?.onCheckpoint?.({ slot: lastProcessedSlot });
       log("info", `Final checkpoint saved: slot ${lastProcessedSlot}`);
     }
   } catch (err) {
@@ -740,19 +808,83 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
   );
   return stats;
 }
+
+// src/runtime/status.ts
+function emptyStreamCounters() {
+  return {
+    eventsReceived: 0,
+    parserNulls: 0,
+    parserErrors: 0,
+    parseValidationErrors: 0,
+    commitErrors: 0,
+    filterBatchErrors: 0,
+    onCommitErrors: 0,
+    recordsProcessed: 0,
+    batchesCommitted: 0
+  };
+}
+function cloneStreamStatus(status) {
+  return {
+    ...status,
+    lastError: status.lastError ? { ...status.lastError } : null,
+    counters: { ...status.counters }
+  };
+}
+function normalizeIndexerError(input) {
+  const err = input.error;
+  const maybeRecord = err && typeof err === "object" ? err : {};
+  const name = err instanceof Error ? err.name : typeof maybeRecord.name === "string" ? maybeRecord.name : "Error";
+  const message = err instanceof Error ? err.message : String(err);
+  const code = typeof maybeRecord.code === "string" || typeof maybeRecord.code === "number" ? maybeRecord.code : void 0;
+  const retryable = isRetryablePhase(input.phase);
+  return {
+    name,
+    message,
+    code,
+    phase: input.phase,
+    retryable,
+    streamName: input.streamName,
+    streamKind: input.streamKind,
+    startSlot: input.startSlot === void 0 || input.startSlot === null ? void 0 : input.startSlot.toString(),
+    checkpointSlot: input.checkpointSlot === void 0 || input.checkpointSlot === null ? void 0 : input.checkpointSlot.toString(),
+    endpointLabel: input.endpointLabel
+  };
+}
+function isRetryablePhase(phase) {
+  switch (phase) {
+    case "starting":
+    case "backfill":
+    case "live":
+    case "commit":
+    case "supervisor":
+      return true;
+    case "parse":
+    case "filterBatch":
+    case "onCommit":
+      return false;
+  }
+}
+
+// src/runtime/indexer.ts
 var Indexer = class {
   config;
   abortController = null;
   running = false;
   shutdownRequested = false;
+  startedAtMs = null;
+  streamStatuses = /* @__PURE__ */ new Map();
   constructor(config) {
     this.config = {
       defaultStartSlot: 0n,
       safetyMargin: 64,
       pageSize: 512,
       logLevel: "info",
+      supervisorInitialBackoffMs: 1e3,
+      supervisorMaxBackoffMs: 3e4,
+      streamStaleMs: 3e5,
       ...config
     };
+    this.initializeStreamStatuses();
   }
   /**
    * Check if the checkpoint table exists in the database.
@@ -795,7 +927,9 @@ var Indexer = class {
     await this.checkCheckpointTable();
     this.running = true;
     this.shutdownRequested = false;
+    this.startedAtMs = Date.now();
     this.abortController = new AbortController();
+    this.initializeStreamStatuses();
     const {
       db,
       clientFactory,
@@ -805,7 +939,10 @@ var Indexer = class {
       safetyMargin,
       pageSize,
       logLevel,
-      validateParse
+      validateParse,
+      endpointLabel,
+      supervisorInitialBackoffMs = 1e3,
+      supervisorMaxBackoffMs = 3e4
     } = this.config;
     console.log("[indexer] Starting indexer...");
     console.log(
@@ -815,92 +952,41 @@ var Indexer = class {
       `[indexer] Running ${accountStreams.length} account stream(s): ${accountStreams.map((s) => s.name).join(", ") || "none"}`
     );
     try {
-      const eventProcessorOptions = {
-        clientFactory,
-        db,
-        defaultStartSlot,
-        safetyMargin,
-        pageSize,
-        logLevel,
-        validateParse
-      };
-      const accountProcessorOptions = {
-        clientFactory,
-        db,
-        logLevel,
-        validateParse
+      const supervisorOptions = {
+        endpointLabel,
+        initialBackoffMs: supervisorInitialBackoffMs,
+        maxBackoffMs: supervisorMaxBackoffMs
       };
-      const eventStreamPromises = eventStreams.map(
-        (stream) => runEventStreamProcessor(
-          stream,
-          eventProcessorOptions,
-          this.abortController.signal
-        )
+      const eventSupervisors = eventStreams.map(
+        (stream) => this.runEventStreamSupervisor(stream, {
+          clientFactory,
+          db,
+          defaultStartSlot,
+          safetyMargin,
+          pageSize,
+          logLevel,
+          validateParse
+        }, supervisorOptions)
       );
-      const accountStreamPromises = accountStreams.map(
-        (stream) => runAccountStreamProcessor(
-          stream,
-          accountProcessorOptions,
-          this.abortController.signal
-        )
+      const accountSupervisors = accountStreams.map(
+        (stream) => this.runAccountStreamSupervisor(stream, {
+          clientFactory,
+          db,
+          logLevel,
+          validateParse
+        }, supervisorOptions)
       );
-      const [eventResults, accountResults] = await Promise.all([
-        Promise.allSettled(eventStreamPromises),
-        Promise.allSettled(accountStreamPromises)
-      ]);
+      await Promise.all([...eventSupervisors, ...accountSupervisors]);
       const result = {
-        eventStreams: eventStreams.map((stream, i) => {
-          const r = eventResults[i];
-          if (r.status === "fulfilled") {
-            console.log(
-              `[indexer] Event stream "${stream.name}" completed: ${r.value.eventsProcessed} events in ${r.value.batchesCommitted} batches`
-            );
-            return {
-              name: stream.name,
-              status: "fulfilled",
-              result: r.value
-            };
-          } else {
-            console.error(
-              `[indexer] Event stream "${stream.name}" failed:`,
-              r.reason
-            );
-            return {
-              name: stream.name,
-              status: "rejected",
-              error: r.reason
-            };
-          }
-        }),
-        accountStreams: accountStreams.map((stream, i) => {
-          const r = accountResults[i];
-          if (r.status === "fulfilled") {
-            console.log(
-              `[indexer] Account stream "${stream.name}" completed: ${r.value.accountsUpdated} accounts updated, ${r.value.accountsDeleted} deleted`
-            );
-            return {
-              name: stream.name,
-              status: "fulfilled",
-              result: r.value
-            };
-          } else {
-            console.error(
-              `[indexer] Account stream "${stream.name}" failed:`,
-              r.reason
-            );
-            return {
-              name: stream.name,
-              status: "rejected",
-              error: r.reason
-            };
-          }
-        })
+        eventStreams: eventStreams.map((stream) => this.resultForStream(stream.name)),
+        accountStreams: accountStreams.map((stream) => this.resultForStream(stream.name))
       };
       console.log("[indexer] All streams stopped.");
       return result;
     } finally {
       this.running = false;
       this.abortController = null;
+      this.startedAtMs = null;
     }
   }
   /**
@@ -928,6 +1014,219 @@ var Indexer = class {
   isRunning() {
     return this.running;
   }
+  /**
+   * Get the current in-memory runtime status for every configured stream.
+   */
+  getStatus() {
+    const now = Date.now();
+    const streams = Array.from(this.streamStatuses.values()).map((status) => {
+      const stream = cloneStreamStatus(status);
+      stream.stale = this.isStreamStale(stream, now);
+      return stream;
+    });
+    const healthy = this.running && !this.shutdownRequested && streams.length > 0 && streams.every((stream) => stream.state === "running" && !stream.stale);
+    return {
+      running: this.running,
+      shutdownRequested: this.shutdownRequested,
+      startedAt: this.startedAtMs === null ? null : new Date(this.startedAtMs).toISOString(),
+      uptimeMs: this.startedAtMs === null ? 0 : now - this.startedAtMs,
+      healthy,
+      streams
+    };
+  }
+  initializeStreamStatuses() {
+    this.streamStatuses = /* @__PURE__ */ new Map();
+    for (const stream of this.config.eventStreams ?? []) {
+      this.streamStatuses.set(this.statusKey("event", stream.name), this.createInitialStreamStatus("event", stream.name));
+    }
+    for (const stream of this.config.accountStreams ?? []) {
+      this.streamStatuses.set(this.statusKey("account", stream.name), this.createInitialStreamStatus("account", stream.name));
+    }
+  }
+  createInitialStreamStatus(kind, name) {
+    return {
+      name,
+      kind,
+      state: "idle",
+      checkpointSlot: null,
+      lastProcessedSlot: null,
+      lastEventAt: null,
+      stale: false,
+      restartCount: 0,
+      lastStartedAt: null,
+      lastErrorAt: null,
+      lastError: null,
+      counters: emptyStreamCounters()
+    };
+  }
+  statusKey(kind, name) {
+    return `${kind}:${name}`;
+  }
+  statusFor(kind, name) {
+    const key = this.statusKey(kind, name);
+    let status = this.streamStatuses.get(key);
+    if (!status) {
+      status = this.createInitialStreamStatus(kind, name);
+      this.streamStatuses.set(key, status);
+    }
+    return status;
+  }
+  resultForStream(name) {
+    return {
+      name,
+      status: "fulfilled"
+    };
+  }
+  createObserver(kind, name, endpointLabel) {
+    const status = this.statusFor(kind, name);
+    let startSlot = null;
+    let checkpointSlot = null;
+    return {
+      onStart: (info) => {
+        startSlot = info.startSlot ?? null;
+        checkpointSlot = info.checkpointSlot ?? null;
+        status.state = "running";
+        status.checkpointSlot = checkpointSlot === null ? null : checkpointSlot.toString();
+        status.lastStartedAt = (/* @__PURE__ */ new Date()).toISOString();
+      },
+      onRecord: (info) => {
+        status.counters.eventsReceived++;
+        status.lastEventAt = (/* @__PURE__ */ new Date()).toISOString();
+        if (info.slot !== void 0 && info.slot !== null) {
+          status.lastProcessedSlot = info.slot.toString();
+        }
+      },
+      onParserNull: () => {
+        status.counters.parserNulls++;
+      },
+      onParserError: () => {
+        status.counters.parserErrors++;
+      },
+      onParseValidationError: () => {
+        status.counters.parseValidationErrors++;
+      },
+      onBatchCommitted: (info) => {
+        status.counters.batchesCommitted++;
+        status.counters.recordsProcessed += info.count;
+        status.lastProcessedSlot = info.slot.toString();
+        status.lastEventAt = (/* @__PURE__ */ new Date()).toISOString();
+      },
+      onCheckpoint: (info) => {
+        status.checkpointSlot = info.slot.toString();
+        status.lastProcessedSlot = info.slot.toString();
+      },
+      onError: (phase, error) => {
+        if (phase === "commit") status.counters.commitErrors++;
+        if (phase === "filterBatch") status.counters.filterBatchErrors++;
+        if (phase === "onCommit") status.counters.onCommitErrors++;
+        status.lastErrorAt = (/* @__PURE__ */ new Date()).toISOString();
+        status.lastError = normalizeIndexerError({
+          error,
+          phase,
+          streamName: name,
+          streamKind: kind,
+          startSlot,
+          checkpointSlot,
+          endpointLabel
+        });
+      }
+    };
+  }
+  async runEventStreamSupervisor(stream, processorOptions, supervisorOptions) {
+    await this.runStreamSupervisor("event", stream.name, supervisorOptions, async (observer) => {
+      const result = await runEventStreamProcessor(
+        stream,
+        { ...processorOptions, observer },
+        this.abortController.signal
+      );
+      return `processed ${result.eventsProcessed} event(s) in ${result.batchesCommitted} batch(es)`;
+    });
+  }
+  async runAccountStreamSupervisor(stream, processorOptions, supervisorOptions) {
+    await this.runStreamSupervisor("account", stream.name, supervisorOptions, async (observer) => {
+      const result = await runAccountStreamProcessor(
+        stream,
+        { ...processorOptions, observer },
+        this.abortController.signal
+      );
+      return `processed ${result.accountsProcessed} account event(s), updated ${result.accountsUpdated}, deleted ${result.accountsDeleted}`;
+    });
+  }
+  async runStreamSupervisor(kind, name, options, runOnce) {
+    let attempt = 0;
+    const status = this.statusFor(kind, name);
+    while (!this.abortController?.signal.aborted) {
+      const observer = this.createObserver(kind, name, options.endpointLabel);
+      status.state = attempt === 0 ? "starting" : "retrying";
+      status.lastStartedAt = (/* @__PURE__ */ new Date()).toISOString();
+      try {
+        const summary = await runOnce(observer);
+        if (this.abortController?.signal.aborted) {
+          status.state = "stopped";
+          console.log(`[indexer] ${kind} stream "${name}" stopped: ${summary}`);
+          return;
+        }
+        throw new Error(`${kind} stream "${name}" completed unexpectedly: ${summary}`);
+      } catch (error) {
+        if (this.abortController?.signal.aborted) {
+          status.state = "stopped";
+          return;
+        }
+        status.restartCount++;
+        status.state = "retrying";
+        status.lastErrorAt = (/* @__PURE__ */ new Date()).toISOString();
+        if (!status.lastError || status.lastError.phase === "supervisor") {
+          status.lastError = normalizeIndexerError({
+            error,
+            phase: "supervisor",
+            streamName: name,
+            streamKind: kind,
+            endpointLabel: options.endpointLabel
+          });
+        }
+        const backoffMs = this.supervisorBackoffMs(attempt, options.initialBackoffMs, options.maxBackoffMs);
+        console.error(
+          `[indexer] ${kind} stream "${name}" failed; restarting in ${backoffMs}ms:`,
+          error
+        );
+        attempt++;
+        await this.delay(backoffMs, this.abortController.signal);
+      }
+    }
+    status.state = "stopped";
+  }
+  supervisorBackoffMs(attempt, initialMs, maxMs) {
+    const base = Math.min(maxMs, initialMs * Math.pow(2, attempt));
+    const jitter = Math.floor(base * 0.2 * Math.random());
+    return base + jitter;
+  }
+  isStreamStale(stream, nowMs) {
+    const staleMs = this.config.streamStaleMs ?? 3e5;
+    if (staleMs <= 0 || stream.state !== "running") {
+      return false;
+    }
+    const activityAt = stream.lastEventAt ?? stream.lastStartedAt;
+    if (!activityAt) {
+      return true;
+    }
+    const activityMs = Date.parse(activityAt);
+    return !Number.isFinite(activityMs) || nowMs - activityMs > staleMs;
+  }
+  delay(ms, signal) {
+    if (ms <= 0 || signal.aborted) return Promise.resolve();
+    return new Promise((resolve) => {
+      const onAbort = () => {
+        clearTimeout(timer);
+        resolve();
+      };
+      const timer = setTimeout(() => {
+        signal.removeEventListener("abort", onAbort);
+        resolve();
+      }, ms);
+      timer.unref?.();
+      signal.addEventListener("abort", onAbort, { once: true });
+    });
+  }
 };
 
 export { Indexer, checkpointTable, columnBuilder, defineAccountStream, defineEventStream, deleteCheckpoint, generateZodSchema, getAllCheckpoints, getCheckpoint, getSchemaExports, t, updateCheckpoint, validateParsedData };