@voidifydao/sdk 1.0.0

package/dist/substream/chain/index.d.ts

@@ -0,0 +1,24 @@
+import type { PublicKey } from "@solana/web3.js";
+import type { ChainEventRecord, EventProjection, EventScope, EventStore } from "../../substream/types.js";
+export interface EventStreamSpec {
+    id: string;
+    scope: EventScope;
+    address: PublicKey;
+    getChainLastIndex(): Promise<bigint>;
+    collect(lastSignature?: string): Promise<ChainEventRecord[]>;
+}
+export declare class ProjectionRegistry {
+    private projections;
+    register(projection: EventProjection): void;
+    apply(events: ChainEventRecord[]): Promise<void>;
+    get(id: string): EventProjection | null;
+}
+export declare class ChainEventSyncer {
+    private events;
+    private projections;
+    constructor(events: EventStore, projections?: ProjectionRegistry);
+    registerProjection(projection: EventProjection): void;
+    checkAndSync(spec: EventStreamSpec): Promise<boolean>;
+    applyLiveEvent(spec: EventStreamSpec, event: ChainEventRecord): Promise<void>;
+    private sync;
+}

package/dist/substream/chain/index.js

@@ -0,0 +1,79 @@
+import { substreamLogger } from "../../utils/logger.js";
+export class ProjectionRegistry {
+    projections = [];
+    register(projection) {
+        const existing = this.projections.find((p) => p.id === projection.id);
+        if (existing) {
+            throw new Error(`Projection already registered: ${projection.id}`);
+        }
+        this.projections.push(projection);
+    }
+    async apply(events) {
+        if (events.length === 0)
+            return;
+        for (const projection of this.projections) {
+            const matched = events.filter((event) => projection.matches(event));
+            if (matched.length > 0)
+                await projection.apply(matched);
+        }
+    }
+    get(id) {
+        return this.projections.find((projection) => projection.id === id) ?? null;
+    }
+}
+export class ChainEventSyncer {
+    events;
+    projections;
+    constructor(events, projections = new ProjectionRegistry()) {
+        this.events = events;
+        this.projections = projections;
+    }
+    registerProjection(projection) {
+        this.projections.register(projection);
+    }
+    async checkAndSync(spec) {
+        const cursor = await this.events.getCursor(spec.scope);
+        const localIndex = cursor?.lastIndex ?? -1n;
+        let chainLastIndex;
+        try {
+            chainLastIndex = await spec.getChainLastIndex();
+        }
+        catch (error) {
+            substreamLogger.warn({ err: error, streamId: spec.id }, "substream alignment check failed");
+            await this.sync(spec);
+            return false;
+        }
+        if (localIndex >= chainLastIndex)
+            return true;
+        await this.sync(spec);
+        return false;
+    }
+    async applyLiveEvent(spec, event) {
+        const outcome = await this.events.apply(spec.scope, event);
+        if (outcome.kind === "gap") {
+            await this.sync(spec);
+            return;
+        }
+        if (outcome.kind === "applied") {
+            await this.projections.apply([event]);
+        }
+    }
+    async sync(spec) {
+        const cursor = await this.events.getCursor(spec.scope);
+        const records = await spec.collect(cursor?.lastSignature ?? undefined);
+        if (records.length === 0)
+            return;
+        const outcome = await this.events.applyBatch(spec.scope, records);
+        if (outcome.kind === "gap") {
+            substreamLogger.warn({
+                streamId: spec.id,
+                scopeType: spec.scope.scopeType,
+                scopeKey: spec.scope.scopeKey,
+                expected: outcome.expected.toString(),
+                got: outcome.got.toString(),
+            }, "chain event batch apply found a gap; RPC history window may be exhausted");
+            return;
+        }
+        await this.projections.apply(records);
+    }
+}

package/dist/substream/chain/registry.d.ts

@@ -0,0 +1,44 @@
+import type { Context } from "../../context.js";
+import type { ChainEventRecord, EventProjection, EventScope, EventScopeType, EventStore, ProjectionStore } from "../../substream/types.js";
+import type { VoidifyProgram } from "../../voidify/program.js";
+import type { EventStreamSpec } from "./index.js";
+export interface LiveEventContext {
+    ctx: Context;
+    voidify: VoidifyProgram;
+    event: unknown;
+    signature: string;
+    slot: number;
+}
+export interface LiveEventAdapter {
+    eventName: string;
+    toRecord(args: LiveEventContext): Promise<ChainEventRecord>;
+}
+export interface SubstreamModuleRuntime {
+    ctx: Context;
+    voidify: VoidifyProgram;
+    events: EventStore;
+    projections: ProjectionStore;
+    sync(scope: EventScope): Promise<void>;
+    rebuildProjection(scope: EventScope, projectionId: string): Promise<void>;
+}
+export interface SubstreamModule {
+    id: string;
+    scopeType: EventScopeType;
+    parseScopeKey(scopeKey: string): EventScope | null;
+    createStream(ctx: Context, voidify: VoidifyProgram, scope: EventScope): EventStreamSpec;
+    liveEvents?: LiveEventAdapter[];
+    createProjections?(store: ProjectionStore): EventProjection[];
+    createClientApi?(runtime: SubstreamModuleRuntime): unknown;
+}
+export declare class SubstreamModuleRegistry {
+    private modules;
+    constructor(modules: SubstreamModule[]);
+    getByScopeType(scopeType: string): SubstreamModule | null;
+    getById(id: string): SubstreamModule | null;
+    parseScope(scopeType: string, scopeKey: string): EventScope | null;
+    createStream(ctx: Context, voidify: VoidifyProgram, scope: EventScope): EventStreamSpec;
+    liveEvents(): Array<LiveEventAdapter & {
+        module: SubstreamModule;
+    }>;
+    createProjections(store: ProjectionStore): EventProjection[];
+}

package/dist/substream/chain/registry.js

@@ -0,0 +1,28 @@
+export class SubstreamModuleRegistry {
+    modules;
+    constructor(modules) {
+        this.modules = modules;
+    }
+    getByScopeType(scopeType) {
+        return this.modules.find((m) => m.scopeType === scopeType) ?? null;
+    }
+    getById(id) {
+        return this.modules.find((m) => m.id === id) ?? null;
+    }
+    parseScope(scopeType, scopeKey) {
+        return this.getByScopeType(scopeType)?.parseScopeKey(scopeKey) ?? null;
+    }
+    createStream(ctx, voidify, scope) {
+        const module = this.getByScopeType(scope.scopeType);
+        if (!module) {
+            throw new Error(`No substream module registered for ${scope.scopeType}`);
+        }
+        return module.createStream(ctx, voidify, scope);
+    }
+    liveEvents() {
+        return this.modules.flatMap((module) => (module.liveEvents ?? []).map((event) => ({ ...event, module })));
+    }
+    createProjections(store) {
+        return this.modules.flatMap((module) => module.createProjections ? module.createProjections(store) : []);
+    }
+}

package/dist/substream/chain/utils.d.ts

@@ -0,0 +1,9 @@
+import { Connection, ConfirmedSignatureInfo, PublicKey } from "@solana/web3.js";
+export declare function fetchSignaturesForAddress(connection: Connection, address: PublicKey, lastSignature?: string, batchSize?: number): Promise<ConfirmedSignatureInfo[]>;
+export interface TransactionEvent {
+    signature: string;
+    logs: string[];
+    slot: number | null;
+    blockTime: number | null;
+}
+export declare function syncTransactions(connection: Connection, address: PublicKey, lastSignature?: string, batchSize?: number): Promise<TransactionEvent[]>;

package/dist/substream/chain/utils.js

@@ -0,0 +1,41 @@
+export async function fetchSignaturesForAddress(connection, address, lastSignature, batchSize = 1000) {
+    const signatures = [];
+    let beforeSignature = undefined;
+    while (true) {
+        const batch = await connection.getSignaturesForAddress(address, {
+            limit: batchSize,
+            before: beforeSignature,
+            until: lastSignature,
+        });
+        if (batch.length === 0)
+            break;
+        signatures.push(...batch);
+        if (batch.length < batchSize)
+            break;
+        beforeSignature = batch[batch.length - 1].signature;
+    }
+    signatures.reverse();
+    return signatures;
+}
+export async function syncTransactions(connection, address, lastSignature, batchSize = 1000) {
+    const signatures = await fetchSignaturesForAddress(connection, address, lastSignature, batchSize);
+    if (signatures.length === 0) {
+        return [];
+    }
+    const events = [];
+    for (const sigInfo of signatures) {
+        const tx = await connection.getTransaction(sigInfo.signature, {
+            maxSupportedTransactionVersion: 0,
+        });
+        if (!tx || !tx.meta) {
+            continue;
+        }
+        events.push({
+            signature: sigInfo.signature,
+            logs: tx.meta.logMessages || [],
+            slot: tx.slot ?? null,
+            blockTime: tx.blockTime ?? null,
+        });
+    }
+    return events;
+}

package/dist/substream/client.d.ts

@@ -0,0 +1,27 @@
+import type { Context, SubstreamMode } from "../context.js";
+import type { ChainEventWire, SubstreamRepos } from "../substream/types.js";
+import type { BuiltinSubstreamModuleApis } from "../substream/modules/index.js";
+export interface SubstreamCliConfig {
+    timeout?: number;
+    mode?: SubstreamMode;
+    healthCacheMs?: number;
+}
+export interface CursorWire {
+    lastIndex: string | null;
+    lastSignature: string | null;
+    lastSyncAt: number;
+}
+export interface EventsApiResponse {
+    events: ChainEventWire[];
+    total: number;
+    cursor: CursorWire | null;
+}
+export declare class SubstreamCliClient {
+    private readonly runtime;
+    private initialized;
+    constructor(ctx: Context, repos?: SubstreamRepos, config?: SubstreamCliConfig);
+    init(): Promise<void>;
+    module<K extends keyof BuiltinSubstreamModuleApis>(id: K): BuiltinSubstreamModuleApis[K];
+    module(id: string): unknown;
+    private ensureInitialized;
+}

package/dist/substream/client.js

@@ -0,0 +1,28 @@
+import { createSubstreamRuntime, } from "../substream/runtime.js";
+export class SubstreamCliClient {
+    runtime;
+    initialized = false;
+    constructor(ctx, repos, config) {
+        const resolved = repos ?? ctx.substream.makeRepos?.();
+        if (!resolved) {
+            throw new Error("SubstreamCliClient: repos not provided and ctx.substream.makeRepos is not configured. " +
+                "Pass stores explicitly or set ctx.substream.makeRepos (Node: makeSQLiteStores(path); Browser: makeIndexedDBStores(dbName)).");
+        }
+        this.runtime = createSubstreamRuntime(ctx, resolved, config);
+    }
+    async init() {
+        if (this.initialized)
+            return;
+        await this.runtime.initialize();
+        this.initialized = true;
+    }
+    module(id) {
+        this.ensureInitialized();
+        return this.runtime.module(id);
+    }
+    ensureInitialized() {
+        if (!this.initialized) {
+            throw new Error("SubstreamCliClient not initialized. Call init() first.");
+        }
+    }
+}

package/dist/substream/database/indexeddb.d.ts

@@ -0,0 +1,2 @@
+import type { SubstreamStores } from "../../substream/types.js";
+export declare function makeIndexedDBStores(dbName: string, _programId?: string): SubstreamStores;

package/dist/substream/database/indexeddb.js

@@ -0,0 +1,242 @@
+import { Dexie } from "dexie";
+class VoidifyDexie extends Dexie {
+    events;
+    projection_states;
+    cursors;
+    constructor(name) {
+        super(name);
+        this.version(1).stores({
+            events: "++id, &[scope_type+scope_key+event_index], event_name, signature, address",
+            projection_states: "[projection_id+entity_key], projection_id",
+            cursors: "[scope_type+scope_key]",
+        });
+    }
+}
+const dbCache = new Map();
+async function getDb(dbName) {
+    let pending = dbCache.get(dbName);
+    if (!pending) {
+        pending = (async () => {
+            const db = new VoidifyDexie(dbName);
+            await db.open();
+            return db;
+        })();
+        dbCache.set(dbName, pending);
+    }
+    return pending;
+}
+function guardEvent(cursor, eventIndex) {
+    const last = cursor?.lastIndex ?? -1n;
+    if (eventIndex <= last)
+        return "duplicate";
+    if (eventIndex > last + 1n) {
+        return { kind: "gap", expected: last + 1n, got: eventIndex };
+    }
+    return "applied";
+}
+function rowToCursor(row) {
+    return {
+        scopeType: row.scope_type,
+        scopeKey: row.scope_key,
+        lastIndex: row.last_index === null ? null : BigInt(row.last_index),
+        lastSignature: row.last_signature,
+        lastSyncAt: row.last_sync_at,
+    };
+}
+function eventToRow(event) {
+    return {
+        scope_type: event.scopeType,
+        scope_key: event.scopeKey,
+        event_name: event.eventName,
+        event_index: eventIndexToNumber(event.eventIndex),
+        signature: event.signature,
+        slot: event.slot,
+        block_time: event.blockTime,
+        address: event.address,
+        payload: event.payload,
+        created_at: event.createdAt,
+    };
+}
+function rowToEvent(row) {
+    return {
+        scopeType: row.scope_type,
+        scopeKey: row.scope_key,
+        eventName: row.event_name,
+        eventIndex: BigInt(row.event_index),
+        signature: row.signature,
+        slot: row.slot,
+        blockTime: row.block_time,
+        address: row.address,
+        payload: row.payload,
+        createdAt: row.created_at,
+    };
+}
+function compareEvents(a, b) {
+    return a.eventIndex < b.eventIndex ? -1 : a.eventIndex > b.eventIndex ? 1 : 0;
+}
+function eventIndexToNumber(eventIndex) {
+    if (eventIndex < 0n || eventIndex > BigInt(Number.MAX_SAFE_INTEGER)) {
+        throw new RangeError(`Event index ${eventIndex.toString()} is outside the safe integer range`);
+    }
+    return Number(eventIndex);
+}
+class IndexedDBEventStore {
+    dbName;
+    db = null;
+    constructor(dbName) {
+        this.dbName = dbName;
+    }
+    async initialize() {
+        this.db = await getDb(this.dbName);
+    }
+    getDb() {
+        if (!this.db) {
+            throw new Error("Database not initialized. Call initialize() first.");
+        }
+        return this.db;
+    }
+    async apply(scope, event) {
+        return this.applyBatch(scope, [event]);
+    }
+    async applyBatch(scope, events) {
+        if (events.length === 0) {
+            const cursor = await this.getCursor(scope);
+            return { kind: "applied", cursor: cursor?.lastIndex ?? -1n };
+        }
+        const db = this.getDb();
+        const sorted = [...events].sort(compareEvents);
+        return db.transaction("rw", [db.events, db.cursors], async () => {
+            const cursorRow = await db.cursors.get([scope.scopeType, scope.scopeKey]);
+            let cursor = cursorRow ? rowToCursor(cursorRow) : null;
+            let last = {
+                kind: "applied",
+                cursor: cursor?.lastIndex ?? -1n,
+            };
+            for (const event of sorted) {
+                const guard = guardEvent(cursor, event.eventIndex);
+                if (typeof guard !== "string") {
+                    Dexie.currentTransaction?.abort();
+                    return guard;
+                }
+                if (guard === "duplicate") {
+                    last = { kind: "duplicate", cursor: cursor?.lastIndex ?? -1n };
+                    continue;
+                }
+                await db.events.add(eventToRow(event));
+                const now = Date.now();
+                await db.cursors.put({
+                    scope_type: scope.scopeType,
+                    scope_key: scope.scopeKey,
+                    last_index: eventIndexToNumber(event.eventIndex),
+                    last_signature: event.signature,
+                    last_sync_at: now,
+                });
+                cursor = {
+                    scopeType: scope.scopeType,
+                    scopeKey: scope.scopeKey,
+                    lastIndex: event.eventIndex,
+                    lastSignature: event.signature,
+                    lastSyncAt: now,
+                };
+                last = { kind: "applied", cursor: event.eventIndex };
+            }
+            return last;
+        });
+    }
+    async list(scope, opts) {
+        const db = this.getDb();
+        const rows = await db.events
+            .where("[scope_type+scope_key+event_index]")
+            .between([scope.scopeType, scope.scopeKey, Dexie.minKey], [scope.scopeType, scope.scopeKey, Dexie.maxKey])
+            .reverse()
+            .toArray();
+        const start = opts?.offset ?? 0;
+        const end = opts?.limit === undefined ? undefined : start + opts.limit;
+        return rows.slice(start, end).map(rowToEvent);
+    }
+    async getAfter(scope, afterIndex) {
+        const db = this.getDb();
+        const cutoff = afterIndex === undefined ? -Infinity : eventIndexToNumber(afterIndex);
+        const rows = await db.events
+            .where("[scope_type+scope_key+event_index]")
+            .between([scope.scopeType, scope.scopeKey, cutoff], [scope.scopeType, scope.scopeKey, Dexie.maxKey], false, true)
+            .toArray();
+        return rows.map(rowToEvent);
+    }
+    async count(scope) {
+        const db = this.getDb();
+        return db.events
+            .where("[scope_type+scope_key+event_index]")
+            .between([scope.scopeType, scope.scopeKey, Dexie.minKey], [scope.scopeType, scope.scopeKey, Dexie.maxKey])
+            .count();
+    }
+    async getCursor(scope) {
+        const db = this.getDb();
+        const row = await db.cursors.get([scope.scopeType, scope.scopeKey]);
+        return row ? rowToCursor(row) : null;
+    }
+}
+class IndexedDBProjectionStore {
+    dbName;
+    db = null;
+    constructor(dbName) {
+        this.dbName = dbName;
+    }
+    async initialize() {
+        this.db = await getDb(this.dbName);
+    }
+    getDb() {
+        if (!this.db) {
+            throw new Error("Database not initialized. Call initialize() first.");
+        }
+        return this.db;
+    }
+    async get(projectionId, key) {
+        const row = await this.getDb().projection_states.get([projectionId, key]);
+        return row ? rowToProjectionState(row) : null;
+    }
+    async put(record) {
+        await this.getDb().projection_states.put(projectionStateToRow(record));
+    }
+    async list(projectionId) {
+        const rows = await this.getDb()
+            .projection_states.where("projection_id")
+            .equals(projectionId)
+            .toArray();
+        return rows.map(rowToProjectionState);
+    }
+    async clear(projectionId) {
+        await this.getDb()
+            .projection_states.where("projection_id")
+            .equals(projectionId)
+            .delete();
+    }
+}
+function projectionStateToRow(record) {
+    return {
+        projection_id: record.projectionId,
+        entity_key: record.key,
+        value: record.value,
+        updated_at: record.updatedAt,
+        last_event_index: record.lastEventIndex === null
+            ? null
+            : eventIndexToNumber(record.lastEventIndex),
+    };
+}
+function rowToProjectionState(row) {
+    return {
+        projectionId: row.projection_id,
+        key: row.entity_key,
+        value: row.value,
+        updatedAt: row.updated_at,
+        lastEventIndex: row.last_event_index === null ? null : BigInt(row.last_event_index),
+    };
+}
+export function makeIndexedDBStores(dbName, _programId) {
+    const events = new IndexedDBEventStore(dbName);
+    const projections = new IndexedDBProjectionStore(dbName);
+    return {
+        events,
+        projections,
+    };
+}

package/dist/substream/database/sqlite.d.ts

@@ -0,0 +1,26 @@
+import Database from "better-sqlite3";
+import type { ApplyOutcome, ChainEventRecord, EventCursor, EventScope, EventStore, ProjectionStateRecord, ProjectionStore, SubstreamStores } from "../../substream/types.js";
+export declare class SQLiteEventStore implements EventStore {
+    private db;
+    constructor(db: Database.Database);
+    initialize(): Promise<void>;
+    apply(scope: EventScope, event: ChainEventRecord): Promise<ApplyOutcome>;
+    applyBatch(scope: EventScope, events: ChainEventRecord[]): Promise<ApplyOutcome>;
+    list(scope: EventScope, opts?: {
+        offset?: number;
+        limit?: number;
+    }): Promise<ChainEventRecord[]>;
+    getAfter(scope: EventScope, afterIndex?: bigint): Promise<ChainEventRecord[]>;
+    count(scope: EventScope): Promise<number>;
+    getCursor(scope: EventScope): Promise<EventCursor | null>;
+}
+export declare class SQLiteProjectionStore implements ProjectionStore {
+    private db;
+    constructor(db: Database.Database);
+    initialize(): Promise<void>;
+    get(projectionId: string, key: string): Promise<ProjectionStateRecord | null>;
+    put(record: ProjectionStateRecord): Promise<void>;
+    list(projectionId: string): Promise<ProjectionStateRecord[]>;
+    clear(projectionId: string): Promise<void>;
+}
+export declare function makeSQLiteStores(path: string, _programId?: string): SubstreamStores;