@voidifydao/sdk 1.0.0 → 2.0.1

package/dist/substream/chain/registry.d.ts

@@ -1,7 +1,7 @@
 import type { Context } from "../../context.js";
 import type { ChainEventRecord, EventProjection, EventScope, EventScopeType, EventStore, ProjectionStore } from "../../substream/types.js";
 import type { VoidifyProgram } from "../../voidify/program.js";
-import type { EventStreamSpec } from "./index.js";
+import type { ChainSyncOptions, EventStreamSpec } from "./index.js";
 export interface LiveEventContext {
     ctx: Context;
     voidify: VoidifyProgram;
@@ -18,7 +18,7 @@ export interface SubstreamModuleRuntime {
     voidify: VoidifyProgram;
     events: EventStore;
     projections: ProjectionStore;
-    sync(scope: EventScope): Promise<void>;
+    sync(scope: EventScope, options?: ChainSyncOptions): Promise<void>;
     rebuildProjection(scope: EventScope, projectionId: string): Promise<void>;
 }
 export interface SubstreamModule {

package/dist/substream/chain/utils.d.ts

@@ -1,4 +1,5 @@
 import { Connection, ConfirmedSignatureInfo, PublicKey } from "@solana/web3.js";
+import type { SyncProgress } from "../../substream/types.js";
 export declare function fetchSignaturesForAddress(connection: Connection, address: PublicKey, lastSignature?: string, batchSize?: number): Promise<ConfirmedSignatureInfo[]>;
 export interface TransactionEvent {
     signature: string;
@@ -6,4 +7,4 @@ export interface TransactionEvent {
     slot: number | null;
     blockTime: number | null;
 }
-export declare function syncTransactions(connection: Connection, address: PublicKey, lastSignature?: string, batchSize?: number): Promise<TransactionEvent[]>;
+export declare function syncTransactionBatches(connection: Connection, address: PublicKey, lastSignature?: string, signatureBatchSize?: number, transactionBatchSize?: number, updateProgress?: (progress: SyncProgress) => void): AsyncGenerator<TransactionEvent[]>;

package/dist/substream/chain/utils.js

@@ -1,12 +1,17 @@
+import pRetry from "p-retry";
+const retryConnection = (operation) => pRetry(operation, {
+    minTimeout: 1000,
+    retries: 5,
+});
 export async function fetchSignaturesForAddress(connection, address, lastSignature, batchSize = 1000) {
     const signatures = [];
     let beforeSignature = undefined;
     while (true) {
-        const batch = await connection.getSignaturesForAddress(address, {
+        const batch = await retryConnection(() => connection.getSignaturesForAddress(address, {
             limit: batchSize,
             before: beforeSignature,
             until: lastSignature,
-        });
+        }));
         if (batch.length === 0)
             break;
         signatures.push(...batch);
@@ -17,25 +22,44 @@ export async function fetchSignaturesForAddress(connection, address, lastSignatu
     signatures.reverse();
     return signatures;
 }
-export async function syncTransactions(connection, address, lastSignature, batchSize = 1000) {
-    const signatures = await fetchSignaturesForAddress(connection, address, lastSignature, batchSize);
+export async function* syncTransactionBatches(connection, address, lastSignature, signatureBatchSize = 1000, transactionBatchSize = 50, updateProgress) {
+    const signatures = await fetchSignaturesForAddress(connection, address, lastSignature, signatureBatchSize);
     if (signatures.length === 0) {
-        return [];
+        return;
     }
     const events = [];
+    let current = 0;
     for (const sigInfo of signatures) {
-        const tx = await connection.getTransaction(sigInfo.signature, {
-            maxSupportedTransactionVersion: 0,
+        const tx = await retryConnection(async () => {
+            const transaction = await connection.getTransaction(sigInfo.signature, {
+                maxSupportedTransactionVersion: 0,
+            });
+            if (!transaction?.meta) {
+                throw new Error(`transaction unavailable: ${sigInfo.signature}`);
+            }
+            return transaction;
         });
-        if (!tx || !tx.meta) {
-            continue;
+        const meta = tx.meta;
+        if (!meta) {
+            throw new Error(`transaction unavailable: ${sigInfo.signature}`);
         }
         events.push({
             signature: sigInfo.signature,
-            logs: tx.meta.logMessages || [],
+            logs: meta.logMessages || [],
             slot: tx.slot ?? null,
             blockTime: tx.blockTime ?? null,
         });
+        current += 1;
+        updateProgress?.({
+            current,
+            total: signatures.length,
+            signature: sigInfo.signature,
+        });
+        if (events.length >= transactionBatchSize) {
+            yield events.splice(0, events.length);
+        }
+    }
+    if (events.length > 0) {
+        yield events.splice(0, events.length);
     }
-    return events;
 }

package/dist/substream/client.d.ts

@@ -1,5 +1,5 @@
 import type { Context, SubstreamMode } from "../context.js";
-import type { ChainEventWire, SubstreamRepos } from "../substream/types.js";
+import type { ChainEventWire, SubstreamRepos, SyncStatus } from "../substream/types.js";
 import type { BuiltinSubstreamModuleApis } from "../substream/modules/index.js";
 export interface SubstreamCliConfig {
     timeout?: number;
@@ -7,6 +7,8 @@ export interface SubstreamCliConfig {
     healthCacheMs?: number;
 }
 export interface CursorWire {
+    scopeType: string;
+    scopeKey: string;
     lastIndex: string | null;
     lastSignature: string | null;
     lastSyncAt: number;
@@ -15,6 +17,9 @@ export interface EventsApiResponse {
     events: ChainEventWire[];
     total: number;
     cursor: CursorWire | null;
+    syncStatus: (Omit<SyncStatus, "cursor"> & {
+        cursor: CursorWire | null;
+    }) | null;
 }
 export declare class SubstreamCliClient {
     private readonly runtime;

package/dist/substream/database/indexeddb.js

@@ -198,6 +198,9 @@ class IndexedDBProjectionStore {
     async put(record) {
         await this.getDb().projection_states.put(projectionStateToRow(record));
     }
+    async delete(projectionId, key) {
+        await this.getDb().projection_states.delete([projectionId, key]);
+    }
     async list(projectionId) {
         const rows = await this.getDb()
             .projection_states.where("projection_id")

package/dist/substream/database/sqlite.d.ts

@@ -20,6 +20,7 @@ export declare class SQLiteProjectionStore implements ProjectionStore {
     initialize(): Promise<void>;
     get(projectionId: string, key: string): Promise<ProjectionStateRecord | null>;
     put(record: ProjectionStateRecord): Promise<void>;
+    delete(projectionId: string, key: string): Promise<void>;
     list(projectionId: string): Promise<ProjectionStateRecord[]>;
     clear(projectionId: string): Promise<void>;
 }

package/dist/substream/database/sqlite.js

@@ -241,6 +241,12 @@ export class SQLiteProjectionStore {
             ? null
             : eventIndexToNumber(record.lastEventIndex));
     }
+    async delete(projectionId, key) {
+        this.db
+            .prepare(`DELETE FROM projection_states
+         WHERE projection_id = ? AND entity_key = ?`)
+            .run(projectionId, key);
+    }
     async list(projectionId) {
         const rows = this.db
             .prepare(`SELECT * FROM projection_states

package/dist/substream/modules/deposit.d.ts

@@ -1,7 +1,8 @@
 import type { DepositRecord, EventCursor, EventScope } from "../../substream/types.js";
 import type { SubstreamModule } from "../../substream/chain/registry.js";
+import type { ChainSyncOptions } from "../../substream/chain/index.js";
 export interface DepositModuleApi {
-    sync(denomination: bigint): Promise<void>;
+    sync(denomination: bigint, options?: ChainSyncOptions): Promise<void>;
     list(denomination: bigint, opts?: {
         offset?: number;
         limit?: number;

package/dist/substream/modules/deposit.js

@@ -1,7 +1,7 @@
 import { toBigInt } from "../../utils/anchor-events.js";
 import { bytesToBigInt, bytesToHex } from "../../utils/bytes.js";
 import { parseEventsFromLogs } from "../../utils/anchor-events.js";
-import { syncTransactions } from "../../substream/chain/utils.js";
+import { syncTransactionBatches, } from "../../substream/chain/utils.js";
 export function depositScope(denomination) {
     return { scopeType: "deposit", scopeKey: denomination.toString() };
 }
@@ -35,8 +35,8 @@ export const depositModule = {
     ],
     createClientApi(runtime) {
         return {
-            async sync(denomination) {
-                await runtime.sync(depositScope(denomination));
+            async sync(denomination, options) {
+                await runtime.sync(depositScope(denomination), options);
             },
             async list(denomination, opts) {
                 const rows = await runtime.events.list(depositScope(denomination), opts);
@@ -65,27 +65,31 @@ function createDepositStream(ctx, voidify, denomination) {
             const poolAccount = await voidify.program.account.pool.fetch(address);
             return BigInt(poolAccount.merkleTree.nextIndex) - 1n;
         },
-        collect: async (lastSignature) => {
-            const transactions = await syncTransactions(ctx.connection, address, lastSignature);
-            const records = [];
-            for (const tx of transactions) {
-                const decoded = parseEventsFromLogs(tx.logs, voidify.program.coder.events);
-                for (const item of decoded) {
-                    if (item.name !== "depositEvent")
-                        continue;
-                    records.push(depositEventToChainEvent({
-                        event: item.data,
-                        signature: tx.signature,
-                        slot: tx.slot,
-                        blockTime: tx.blockTime,
-                        address,
-                    }));
-                }
+        collectBatches: async function* (lastSignature, run) {
+            for await (const transactions of syncTransactionBatches(ctx.connection, address, lastSignature, undefined, undefined, run?.updateProgress)) {
+                yield collectDepositRecords(transactions, voidify, address);
             }
-            return records;
         },
     };
 }
+function collectDepositRecords(transactions, voidify, address) {
+    const records = [];
+    for (const tx of transactions) {
+        const decoded = parseEventsFromLogs(tx.logs, voidify.program.coder.events);
+        for (const item of decoded) {
+            if (item.name !== "depositEvent")
+                continue;
+            records.push(depositEventToChainEvent({
+                event: item.data,
+                signature: tx.signature,
+                slot: tx.slot,
+                blockTime: tx.blockTime,
+                address,
+            }));
+        }
+    }
+    return records;
+}
 function depositEventToChainEvent(args) {
     const event = args.event;
     const denomination = toBigInt(event.denomination);

package/dist/substream/modules/relayer.d.ts

@@ -1,7 +1,8 @@
 import type { EventScope, RelayerRecord } from "../../substream/types.js";
 import type { SubstreamModule } from "../../substream/chain/registry.js";
+import type { ChainSyncOptions } from "../../substream/chain/index.js";
 export interface RelayerModuleApi {
-    sync(): Promise<void>;
+    sync(options?: ChainSyncOptions): Promise<void>;
     list(): Promise<RelayerRecord[]>;
     get(pubkey: string): Promise<RelayerRecord | null>;
     rebuild(): Promise<void>;

package/dist/substream/modules/relayer.js

@@ -2,7 +2,7 @@ import { PublicKey } from "@solana/web3.js";
 import { normalizePayload } from "../../substream/chain/events.js";
 import { parseEventsFromLogs, } from "../../utils/anchor-events.js";
 import { substreamLogger } from "../../utils/logger.js";
-import { syncTransactions } from "../../substream/chain/utils.js";
+import { syncTransactionBatches, } from "../../substream/chain/utils.js";
 export const RELAYER_SCOPE = {
     scopeType: "relayer",
     scopeKey: "global",
@@ -32,7 +32,7 @@ export const relayerModule = {
             const decoded = { name: eventName, data: event };
             const pubkey = decoded.data.relayer.toBase58();
             const cfg = decoded.name === "relayerRegisteredEvent"
-                ? await fetchRelayerConfig(voidify, pubkey)
+                ? await fetchRelayerConfig(voidify, pubkey, false)
                 : null;
             return relayerEventToChainEvent({
                 event: decoded,
@@ -48,8 +48,8 @@ export const relayerModule = {
     },
     createClientApi(runtime) {
         return {
-            async sync() {
-                await runtime.sync(RELAYER_SCOPE);
+            async sync(options) {
+                await runtime.sync(RELAYER_SCOPE, options);
             },
             async list() {
                 const rows = await runtime.projections.list("relayer");
@@ -75,33 +75,37 @@ function createRelayerStream(ctx, voidify) {
             const counter = await voidify.program.account.relayerEventCounter.fetch(counterPda);
             return BigInt(counter.count.toString()) - 1n;
         },
-        collect: async (lastSignature) => {
-            const transactions = await syncTransactions(ctx.connection, ctx.programId, lastSignature);
-            const records = [];
-            for (const tx of transactions) {
-                const decoded = parseEventsFromLogs(tx.logs, voidify.program.coder.events);
-                for (const item of decoded) {
-                    if (!isIndexedRelayerEventName(item.name))
-                        continue;
-                    const event = item;
-                    const pubkey = event.data.relayer.toBase58();
-                    const cfg = event.name === "relayerRegisteredEvent"
-                        ? await fetchRelayerConfig(voidify, pubkey)
-                        : null;
-                    records.push(relayerEventToChainEvent({
-                        event,
-                        signature: tx.signature,
-                        slot: tx.slot,
-                        blockTime: tx.blockTime,
-                        address: ctx.programId,
-                        config: cfg,
-                    }));
-                }
+        collectBatches: async function* (lastSignature, run) {
+            for await (const transactions of syncTransactionBatches(ctx.connection, ctx.programId, lastSignature, undefined, undefined, run?.updateProgress)) {
+                yield await collectRelayerRecords(transactions, voidify, ctx.programId);
             }
-            return records;
         },
     };
 }
+async function collectRelayerRecords(transactions, voidify, address) {
+    const records = [];
+    for (const tx of transactions) {
+        const decoded = parseEventsFromLogs(tx.logs, voidify.program.coder.events);
+        for (const item of decoded) {
+            if (!isIndexedRelayerEventName(item.name))
+                continue;
+            const event = item;
+            const pubkey = event.data.relayer.toBase58();
+            const cfg = event.name === "relayerRegisteredEvent"
+                ? await fetchRelayerConfig(voidify, pubkey, true)
+                : null;
+            records.push(relayerEventToChainEvent({
+                event,
+                signature: tx.signature,
+                slot: tx.slot,
+                blockTime: tx.blockTime,
+                address,
+                config: cfg,
+            }));
+        }
+    }
+    return records;
+}
 function relayerEventToChainEvent(args) {
     const payload = normalizePayload(args.event.data);
     if (args.event.name === "relayerRegisteredEvent") {
@@ -122,14 +126,16 @@ function relayerEventToChainEvent(args) {
 function isIndexedRelayerEventName(name) {
     return INDEXED_RELAYER_EVENT_NAMES.has(name);
 }
-async function fetchRelayerConfig(voidify, relayerPubkey) {
+async function fetchRelayerConfig(voidify, relayerPubkey, quietNotFound = false) {
     try {
         const cfgKey = voidify.relayerConfig(new PublicKey(relayerPubkey));
         const cfg = await voidify.program.account.relayerConfig.fetch(cfgKey);
         return { name: cfg.name, url: cfg.url, feeBps: cfg.feeBps };
     }
     catch (err) {
-        substreamLogger.warn({ err, relayerPubkey }, "fetchRelayerConfig failed");
+        if (!quietNotFound) {
+            substreamLogger.warn({ err, relayerPubkey }, "fetchRelayerConfig failed");
+        }
         return null;
     }
 }
@@ -151,8 +157,10 @@ class RelayerProjection {
             const existing = await this.store.get(this.id, pubkey);
             const current = existing ? valueToRelayerState(existing.value) : null;
             const next = reduceRelayerEvent(current, event);
-            if (!next)
+            if (!next) {
+                await this.store.delete(this.id, pubkey);
                 continue;
+            }
             await this.store.put({
                 projectionId: this.id,
                 key: pubkey,
@@ -202,9 +210,7 @@ function reduceRelayerEvent(existing, event) {
         }
         case "relayerSlashedEvent":
         case "relayerUnregisteredEvent": {
-            record.stakeAmount = 0n;
-            record.isActive = false;
-            break;
+            return null;
         }
         case "relayerUpdatedEvent": {
             if (event.payload.feeBps !== null && event.payload.feeBps !== undefined) {

package/dist/substream/runtime.d.ts

@@ -1,6 +1,6 @@
 import type { Context, SubstreamMode } from "../context.js";
-import type { EventScope, EventStore, ProjectionStore, SubstreamStores } from "../substream/types.js";
-import { ChainEventSyncer, ProjectionRegistry } from "../substream/chain/index.js";
+import type { EventScope, EventStore, ProjectionStore, SyncStatus, SubstreamStores } from "../substream/types.js";
+import { type ChainSyncOptions, ChainEventSyncer, ProjectionRegistry } from "../substream/chain/index.js";
 import { SubstreamModuleRegistry, type SubstreamModule, type SubstreamModuleRuntime } from "../substream/chain/registry.js";
 import { VoidifyProgram } from "../voidify/program.js";
 export interface SubstreamRuntimeConfig {
@@ -19,20 +19,35 @@ export declare class SubstreamRuntime implements SubstreamModuleRuntime {
     private readonly mode;
     private readonly timeout;
     private readonly healthCacheMs;
+    private readonly syncInFlight;
+    private readonly syncStatuses;
     private healthState;
     private initialized;
     constructor(ctx: Context, stores: SubstreamStores, config?: SubstreamRuntimeConfig, modules?: readonly SubstreamModule[]);
     initialize(): Promise<void>;
     module(id: string): unknown;
-    sync(scope: EventScope): Promise<void>;
-    syncLocal(scope: EventScope): Promise<void>;
+    sync(scope: EventScope, options?: ChainSyncOptions): Promise<void>;
+    syncLocal(scope: EventScope, options?: ChainSyncOptions): Promise<void>;
+    syncLocalInBackground(scope: EventScope): SyncStatus;
     applyLiveEvent(eventScope: EventScope): Promise<void>;
     applyLiveRecord(scope: EventScope, record: Parameters<ChainEventSyncer["applyLiveEvent"]>[1]): Promise<void>;
     rebuildProjection(scope: EventScope, projectionId: string): Promise<void>;
+    getSyncStatus(scope: EventScope, mode?: "local" | "remote"): SyncStatus;
     private get baseUrl();
     private resolveMode;
     private healthCheck;
     private syncRemote;
+    private applyRemoteEvents;
+    private runScopedSync;
+    private syncKey;
+    private ensureSyncStatus;
+    private setSyncStatus;
+    private updateSyncStatus;
+    private waitForRemoteSync;
+    private sleep;
     private fetchRemoteEvents;
+    private remoteEventsUrl;
+    private fetchWithTimeout;
+    private fetchRemoteSyncStatus;
 }
 export declare function createSubstreamRuntime(ctx: Context, stores: SubstreamStores, config?: SubstreamRuntimeConfig, modules?: readonly SubstreamModule[]): SubstreamRuntime;