@thru/indexer 0.2.18 → 0.2.20

package/dist/index.mjs

@@ -369,7 +369,7 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
   };
   log("info", `Starting stream processor: ${stream.description}`);
   const checkpoint = await getCheckpoint(db, stream.name);
-  const startSlot = checkpoint ? checkpoint.slot + 1n : defaultStartSlot;
+  const startSlot = checkpoint ? checkpoint.slot : defaultStartSlot;
   log(
     "info",
     `Starting from slot ${startSlot}${checkpoint ? " (resuming)" : " (fresh start)"}`
@@ -384,6 +384,7 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
   const replay = createEventReplay({
     clientFactory,
     startSlot,
+    resumeAfter: checkpoint?.eventId ? { slot: checkpoint.slot, eventId: checkpoint.eventId } : void 0,
     safetyMargin: BigInt(safetyMargin),
     pageSize,
     filter: stream.getFilter(),
@@ -399,6 +400,8 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
   let lastLogTime = Date.now();
   let eventsReceivedSinceLastLog = 0;
   const commitBatch = async (batch) => {
+    const lastSeenEvent = batch.events[batch.events.length - 1];
+    const lastSeenEventId = lastSeenEvent?.id ?? null;
     let eventsToCommit = batch.events;
     if (stream.filterBatch) {
       try {
@@ -411,6 +414,8 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
             "debug",
             `All ${batch.events.length} events filtered out at slot ${batch.slot}`
           );
+          await updateCheckpoint(db, stream.name, batch.slot, lastSeenEventId);
+          stats.lastSlot = batch.slot;
           return;
         }
         if (eventsToCommit.length < batch.events.length) {
@@ -427,21 +432,23 @@ async function runEventStreamProcessor(stream, options, abortSignal) {
         return;
       }
     }
+    const lastEventToCommit = eventsToCommit[eventsToCommit.length - 1];
+    const lastEventToCommitId = lastEventToCommit?.id ?? null;
+    let committedEvents = eventsToCommit;
     await db.transaction(async (tx) => {
-      await tx.insert(stream.table).values(eventsToCommit).onConflictDoNothing();
-      const lastEvent = eventsToCommit[eventsToCommit.length - 1];
+      committedEvents = await tx.insert(stream.table).values(eventsToCommit).onConflictDoNothing().returning();
       await updateCheckpoint(
         tx,
         stream.name,
         batch.slot,
-        lastEvent.id
+        lastEventToCommitId
       );
     });
     stats.batchesCommitted++;
     stats.lastSlot = batch.slot;
-    if (stream.onCommit) {
+    if (stream.onCommit && committedEvents.length > 0) {
       try {
-        await stream.onCommit({ ...batch, events: eventsToCommit }, { db });
+        await stream.onCommit({ ...batch, events: committedEvents }, { db });
       } catch (hookErr) {
         log(
           "error",
@@ -615,7 +622,6 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
           "info",
           `Backfill complete. Highest slot: ${highestSlot}, accounts processed: ${stats.accountsProcessed}`
         );
-        lastProcessedSlot = highestSlot;
       }
     });
     for await (const event of replay) {
@@ -641,6 +647,9 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
             await db.delete(stream.table).where(eq(table[idField], idValue));
             stats.accountsDeleted++;
             log("info", `Deleted row for account ${account.addressHex}`);
+            if (account.slot > lastProcessedSlot) {
+              lastProcessedSlot = account.slot;
+            }
           } catch (err) {
             log("error", `Failed to delete account ${account.addressHex}: ${err}`);
           }
@@ -652,6 +661,9 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
             "debug",
             `Skipped account ${account.addressHex} - parser returned null (dataLen=${account.data.length})`
           );
+          if (account.slot > lastProcessedSlot) {
+            lastProcessedSlot = account.slot;
+          }
           continue;
         }
         if (validateParse) {
@@ -661,14 +673,18 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
             continue;
           }
         }
+        let upserted = false;
         try {
-          await db.insert(stream.table).values(parsed).onConflictDoUpdate({
+          const upsertedRows = await db.insert(stream.table).values(parsed).onConflictDoUpdate({
             target: table[idField],
             set: parsed,
             where: sql`${table.slot} <= ${parsed.slot}`
-          });
-          stats.accountsUpdated++;
-          if (stats.accountsUpdated <= 3) {
+          }).returning();
+          upserted = upsertedRows.length > 0;
+          if (upserted) {
+            stats.accountsUpdated++;
+          }
+          if (upserted && stats.accountsUpdated <= 3) {
             log(
               "info",
               `Successfully inserted account ${stats.accountsUpdated}`
@@ -680,7 +696,7 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
             `Failed to upsert account ${account.addressHex}: ${err}`
           );
         }
-        if (account.slot > lastProcessedSlot) {
+        if (upserted && account.slot > lastProcessedSlot) {
           lastProcessedSlot = account.slot;
         }
         if (stats.accountsProcessed % 100 === 0) {
@@ -691,11 +707,18 @@ async function runAccountStreamProcessor(stream, options, abortSignal) {
         }
       } else if (event.type === "blockFinished") {
         const slot = event.block.slot;
-        if (slot > lastProcessedSlot) {
-          lastProcessedSlot = slot;
+        if (lastProcessedSlot > 0n) {
+          await updateCheckpoint(db, checkpointName, lastProcessedSlot, null);
+          log(
+            "debug",
+            `Block finished: slot ${slot}, checkpoint saved at account slot ${lastProcessedSlot}`
+          );
+        } else {
+          log(
+            "debug",
+            `Block finished: slot ${slot}, no checkpoint yet (no accounts handled)`
+          );
         }
-        await updateCheckpoint(db, checkpointName, lastProcessedSlot, null);
-        log("debug", `Block finished: slot ${slot}, checkpoint saved`);
       }
     }
     if (lastProcessedSlot > 0n) {