@lunarislab/state-sync 1.0.1

package/README.md

@@ -0,0 +1,3 @@
+# StateSync
+
+Work in progress

package/classes/Client.d.ts

@@ -0,0 +1,30 @@
+import { AbiEvent, GetLogsReturnType, PublicClient, WalletClient } from "viem";
+import { IClientConfig, IStateSyncEvents } from "../types/Client";
+import { Watcher } from "./Watcher";
+import { ISimplifiedWatcher, IWatcherConfig, LevelDBConstructor } from "../types/Watcher";
+import { EventSignature } from "abitype/dist/types/human-readable/types/signatures";
+import { EventEmitter } from "events";
+/**
+ * The StateSync client class.
+ */
+export declare class StateSync implements IClientConfig {
+    public: PublicClient;
+    wallet: WalletClient;
+    level: LevelDBConstructor;
+    levelPath: string;
+    timeout: number;
+    protected watchers: Map<string, Watcher<any>>;
+    protected emitter: EventEmitter;
+    protected startBlockNum: bigint;
+    protected syncedBlockNum: bigint;
+    protected gap: bigint;
+    protected lastBlockNum: bigint;
+    protected eventCount: bigint;
+    constructor(config: IClientConfig);
+    protected _setInitialBlockNumbers(): Promise<import("viem").WatchBlocksReturnType>;
+    protected _processLogs(logs: GetLogsReturnType<undefined, AbiEvent[]>): Promise<void>;
+    protected _sync(): Promise<void>;
+    on<T extends keyof IStateSyncEvents>(eventName: T, callback: IStateSyncEvents[T]): EventEmitter;
+    start(): Promise<import("viem").WatchBlocksReturnType>;
+    createWatcher<Signature extends EventSignature>(config: IWatcherConfig<Signature>): Promise<ISimplifiedWatcher>;
+}

package/classes/Client.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StateSync = void 0;
+const viem_1 = require("viem");
+const Watcher_1 = require("./Watcher");
+const events_1 = require("events");
+/**
+ * The StateSync client class.
+ */
+class StateSync {
+    constructor(config) {
+        this.watchers = new Map();
+        this.startBlockNum = 0n;
+        this.syncedBlockNum = 0n;
+        this.gap = 500000n;
+        this.lastBlockNum = 0n;
+        this.eventCount = 0n;
+        this.public = config.public;
+        this.wallet = config.wallet;
+        this.level = config.level;
+        this.levelPath = config.levelPath || "";
+        this.emitter = new events_1.EventEmitter();
+        this.timeout = config.timeout || 50;
+    }
+    ;
+    async _setInitialBlockNumbers() {
+        const lastSyncedBlocksNumbers = Array.from(this.watchers.values()).map(s => s.lastSyncedBlockNumber);
+        const smalledSyncedBlockNumber = lastSyncedBlocksNumbers.reduce((min, value) => (value < min ? value : min));
+        this.startBlockNum = smalledSyncedBlockNumber;
+        this.syncedBlockNum = smalledSyncedBlockNumber;
+        this.lastBlockNum = await this.public.getBlockNumber();
+        return this.public.watchBlocks({ onBlock: (block) => this.lastBlockNum = block.number });
+    }
+    ;
+    async _processLogs(logs) {
+        // Send logs to watcher classes to handle the storing logic.
+        for (const watcher of this.watchers.values()) {
+            const eventName = watcher.event.name;
+            const watcherLogs = logs.filter(log => watcher.addresses.includes((0, viem_1.checksumAddress)(log.address)) && log.eventName === eventName);
+            // make sure logs are well ordered by logIndex
+            watcherLogs.sort((a, b) => a.logIndex - b.logIndex);
+            await watcher.onLogs(watcherLogs);
+            await watcher.setSyncedBlockNumber(this.syncedBlockNum);
+        }
+    }
+    ;
+    async _sync() {
+        const stopInitialWatcher = await this._setInitialBlockNumbers();
+        const contractsAddresses = Array.from(this.watchers.values()).map(s => s.addresses).flat(1);
+        const eventsABI = Array.from(this.watchers.values()).map(s => s.event);
+        while (this.syncedBlockNum < this.lastBlockNum) {
+            try {
+                await new Promise(x => setTimeout(x, this.timeout));
+                // Fetch logs from the network.
+                const logs = await this.public.getLogs({
+                    fromBlock: this.syncedBlockNum,
+                    toBlock: this.syncedBlockNum + this.gap,
+                    address: contractsAddresses,
+                    events: eventsABI,
+                });
+                await this._processLogs(logs);
+                this.syncedBlockNum + this.gap > this.lastBlockNum ?
+                    this.syncedBlockNum = this.lastBlockNum :
+                    this.syncedBlockNum += this.gap;
+                this.eventCount += BigInt(logs.length);
+                this.emitter.emit("sync", {
+                    startBlockNum: this.startBlockNum,
+                    endBlockNum: this.lastBlockNum,
+                    syncedBlockNum: this.syncedBlockNum,
+                    eventCount: this.eventCount
+                });
+            }
+            catch (e) {
+                const error = e;
+                if (error.name === "LimitExceededRpcError") {
+                    this.gap = this.gap / 2n;
+                    this.emitter.emit("syncRateLimitExeeded", this.gap);
+                    continue;
+                }
+                throw error;
+            }
+        }
+        stopInitialWatcher();
+    }
+    ;
+    on(eventName, callback) {
+        return this.emitter.on(eventName, callback);
+    }
+    ;
+    async start() {
+        await this._sync();
+        this.emitter.emit('ready');
+        return this.public.watchBlocks({
+            onBlock: async (block) => {
+                this.syncedBlockNum = block.number;
+                this.emitter.emit('newBlock', block.number);
+                const contractsAddresses = Array.from(this.watchers.values()).map(s => s.addresses).flat(1);
+                const eventsABI = Array.from(this.watchers.values()).map(s => s.event);
+                // Fetch logs from the network.
+                const logs = await this.public.getLogs({
+                    blockHash: block.hash,
+                    address: contractsAddresses,
+                    events: eventsABI,
+                });
+                await this._processLogs(logs);
+            }
+        });
+    }
+    async createWatcher(config) {
+        const watcher = new Watcher_1.Watcher(this, config);
+        await watcher.connect();
+        this.watchers.set(config.dbID, watcher);
+        return watcher;
+    }
+    ;
+}
+exports.StateSync = StateSync;

package/classes/Watcher.d.ts

@@ -0,0 +1,27 @@
+import { AbiEvent, Address } from "viem";
+import { IWatcherEvents, Log } from "../types/Watcher";
+import { IAbstractLevel, IWatcherConfig, WatcherSchema } from "../types/Watcher";
+import { StateSync } from "./Client";
+import { EventSignature } from "abitype/dist/types/human-readable/types/signatures";
+import { EventEmitter } from "events";
+export declare class Watcher<Signature extends EventSignature> {
+    protected client: StateSync;
+    protected config: IWatcherConfig<Signature>;
+    protected id: string;
+    event: AbiEvent;
+    addresses: Address[];
+    lastSyncedBlockNumber: bigint;
+    protected emitter: EventEmitter;
+    db: IAbstractLevel<string, WatcherSchema>;
+    constructor(client: StateSync, config: IWatcherConfig<Signature>);
+    protected _updateEnv(): Promise<void>;
+    on<T extends keyof IWatcherEvents>(eventName: T, callback: IWatcherEvents[T]): EventEmitter;
+    setSyncedBlockNumber(blockNumber: bigint): Promise<void>;
+    connect(): Promise<void>;
+    onLog(log: Log<any>): Promise<void>;
+    /**
+     * onLogs is optimized to not call mutiple times the db for the same contract.
+     * Instead it will calculate the state in memory and save the final result in the db.
+     */
+    onLogs(logs: Log<any>[]): Promise<void>;
+}

package/classes/Watcher.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Watcher = void 0;
+const viem_1 = require("viem");
+const events_1 = require("events");
+class Watcher {
+    constructor(client, config) {
+        this.client = client;
+        this.config = config;
+        this.db = new this.client.level(client.levelPath + `/${config.dbID}`.replace('//', "/"), { valueEncoding: "json" });
+        this.id = config.dbID;
+        this.event = (0, viem_1.parseAbiItem)(config.event);
+        this.addresses = config.addresses.map(add => (0, viem_1.checksumAddress)(add));
+        this.lastSyncedBlockNumber = 0n;
+        this.emitter = new events_1.EventEmitter();
+    }
+    ;
+    // save some env variables in the db
+    async _updateEnv() {
+        const env = {
+            lastSyncedBlockNumber: this.lastSyncedBlockNumber.toString()
+        };
+        await this.db.put("ENV", env);
+    }
+    ;
+    on(eventName, callback) {
+        return this.emitter.on(eventName, callback);
+    }
+    ;
+    async setSyncedBlockNumber(blockNumber) {
+        this.lastSyncedBlockNumber = blockNumber;
+        await this._updateEnv();
+    }
+    ;
+    async connect() {
+        await this.db.open();
+        const env = await this.db.get("ENV").catch(e => undefined);
+        this.lastSyncedBlockNumber = env?.lastSyncedBlockNumber ? BigInt(env.lastSyncedBlockNumber) : 0n;
+    }
+    ;
+    async onLog(log) {
+        const oldData = await this.db.get((0, viem_1.checksumAddress)(log.address));
+        const newData = this.config.onLog(log, oldData);
+        this.lastSyncedBlockNumber = log.blockNumber;
+        await this.db.put(log.address, newData);
+        await this._updateEnv();
+        this.emitter.emit("change", {
+            id: log.address,
+            state: newData
+        });
+    }
+    ;
+    /**
+     * onLogs is optimized to not call mutiple times the db for the same contract.
+     * Instead it will calculate the state in memory and save the final result in the db.
+     */
+    async onLogs(logs) {
+        if (logs.length === 0)
+            return;
+        const addresses = logs.map((l) => (0, viem_1.checksumAddress)(l.address));
+        const oldData = await this.db.getMany(addresses);
+        // generate Watchers
+        const newDatas = new Map();
+        for (const index in addresses) {
+            const watcher = oldData[index];
+            const log = logs[index];
+            const identifier = this.config.extractIdentifier ? this.config.extractIdentifier(log) : addresses[index];
+            const newData = this.config.onLog(log, watcher);
+            newDatas.set(identifier, newData);
+            this.emitter.emit("change", {
+                id: identifier,
+                state: newData
+            });
+        }
+        ;
+        // save results
+        const arr = Array.from(newDatas.entries());
+        const payload = arr.map(([address, newData]) => ({
+            type: "put",
+            key: address,
+            value: newData
+        }));
+        await this.db.batch(payload);
+        await this._updateEnv();
+    }
+    ;
+}
+exports.Watcher = Watcher;
+;

package/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./classes/Client";
+export * from "./classes/Watcher";
+export * from "./types/Client";
+export * from "./types/Watcher";

package/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./classes/Client"), exports);
+__exportStar(require("./classes/Watcher"), exports);
+__exportStar(require("./types/Client"), exports);
+__exportStar(require("./types/Watcher"), exports);

package/package.json

@@ -0,0 +1,25 @@
+{
+  "dependencies": {
+    "@types/events": "^3.0.3",
+    "abitype": "^1.0.8",
+    "abstract-level": "^2.0.2",
+    "events": "^3.3.0",
+    "level": "^9.0.0",
+    "path": "^0.12.7",
+    "viem": "^2.21.58"
+  },
+  "name": "@lunarislab/state-sync",
+  "version": "1.0.1",
+  "main": "index.js",
+  "scripts": {
+    "test": "ts-node ./test.ts",
+    "build": "tsc && cp package.json dist/ && cp README.md dist/",
+    "sendNpm": "cd ./dist && npm version patch && npm publish --access=public"
+  },
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "devDependencies": {
+    "typescript": "^5.7.3"
+  }
+}

package/types/Client.d.ts

@@ -0,0 +1,21 @@
+import { PublicClient, WalletClient } from "viem";
+import { LevelDBConstructor } from "./Watcher";
+export interface IClientConfig {
+    public: PublicClient;
+    wallet: WalletClient;
+    level: LevelDBConstructor;
+    levelPath?: string;
+    timeout?: number;
+}
+export interface SyncParams {
+    startBlockNum: bigint;
+    endBlockNum: bigint;
+    syncedBlockNum: bigint;
+    eventCount: bigint;
+}
+export interface IStateSyncEvents {
+    sync: (params: SyncParams) => void;
+    ready: () => void;
+    newBlock: (blockNumber: bigint) => void;
+    syncRateLimitExceeded: (newBlockRange: number) => void;
+}

package/types/Client.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+;
+;

package/types/Watcher.d.ts

@@ -0,0 +1,131 @@
+import { AbstractBatchOperation, AbstractBatchOptions, AbstractChainedBatch, AbstractIterator, AbstractIteratorOptions, AbstractKeyIterator, AbstractKeyIteratorOptions, AbstractOpenOptions, AbstractValueIterator, AbstractValueIteratorOptions } from "abstract-level";
+import { AbiEvent, Address, Log as viemLog } from "viem";
+import { ParseAbiItem, AbiParametersToPrimitiveTypes, AbiParameter } from 'abitype';
+import { EventSignature } from "abitype/dist/types/human-readable/types/signatures";
+/**
+ * Convert the signature to ABI type with `ParseAbiItem` and extract the inputs as `AbiParameter[]`.
+ * So the returned type can be used in `AbiParametersToPrimitiveTypes`.
+ */
+export type EventAbiInputs<signature extends EventSignature> = ParseAbiItem<signature> extends {
+    inputs: infer Inputs;
+} ? Inputs extends readonly AbiParameter[] ? Inputs : never : never;
+/**
+ * Not necessary but more readable, it just wrap the `AbiParametersToPrimitiveTypes` with the previous type `EventAbiInputs`.
+ */
+export type EventArgsTypes<signature extends EventSignature> = AbiParametersToPrimitiveTypes<EventAbiInputs<signature>>;
+/**
+ * Check if all the keys have a name. To decide if final type is object or array.
+ */
+type HasAllNames<Inputs extends {
+    name: string;
+}[]> = undefined extends Inputs[number]['name'] ? false : true;
+/**
+ * Create the object using both EventAbiInputs for keys and the converted types for values.
+ */
+export type EventArgs<signature extends EventSignature> = HasAllNames<EventAbiInputs<signature>> extends true ? {
+    [K in EventAbiInputs<signature>[number]['name']]: EventArgsTypes<signature>[number];
+} : EventArgsTypes<signature>;
+export type Log<signature extends EventSignature> = viemLog & {
+    args: EventArgs<signature>;
+};
+export interface ISimplifiedWatcher {
+    db: IAbstractLevel<string, WatcherSchema>;
+    event: AbiEvent;
+    addresses: Address[];
+    lastSyncedBlockNumber: bigint;
+    on: <T extends keyof IWatcherEvents>(eventName: T, callback: IWatcherEvents[T]) => void;
+}
+export interface IWatcherEvents {
+    change: (data: {
+        id: string;
+        state: any;
+    }) => void;
+}
+export type WatcherSchema = {
+    [key: string]: number | boolean | string;
+};
+export interface IWatcherConfig<signature extends EventSignature> {
+    dbID: string;
+    event: signature;
+    extractIdentifier?: (event: Log<signature>) => string;
+    onLog: (event: Log<signature>, oldWatcher?: WatcherSchema) => WatcherSchema;
+    addresses: Address[];
+}
+export interface IDBEnvironment {
+    lastSyncedBlockNumber?: string;
+}
+/**
+ * For more modularity, LevelDB instance is injected by user, so we need to provide a generic type for LevelDB.
+ * Every LevelDB implementation must implement this (minimized) interface to avoid errors.
+ */
+export interface LevelDBConstructor<K = any, V = any> {
+    new (path: string, options?: any): IAbstractLevel<K, V>;
+}
+export declare class IAbstractLevel<KDefault = string, VDefault = string> {
+    /**
+    * Open the database.
+    */
+    open(): Promise<void>;
+    open(options: AbstractOpenOptions): Promise<void>;
+    /**
+     * Get a value from the database by {@link key}.
+     */
+    get(key: KDefault): Promise<VDefault>;
+    get(key: KDefault, callback: (err: any, value: VDefault) => void): Promise<VDefault>;
+    /**
+     * Get multiple values from the database by an array of {@link keys}.
+     */
+    getMany(keys: KDefault[]): Promise<VDefault[]>;
+    /**
+     * Add a new entry or overwrite an existing entry.
+     */
+    put(key: KDefault, value: VDefault): Promise<void>;
+    /**
+     * Delete an entry by {@link key}.
+     */
+    del(key: KDefault): Promise<void>;
+    /**
+     * Perform multiple _put_ and/or _del_ operations in bulk.
+     */
+    batch(operations: Array<AbstractBatchOperation<typeof this, KDefault, VDefault>>): Promise<void>;
+    batch<K = KDefault, V = VDefault>(operations: Array<AbstractBatchOperation<typeof this, K, V>>, options: AbstractBatchOptions<K, V>): Promise<void>;
+    batch(): AbstractChainedBatch<typeof this, KDefault, VDefault>;
+    /**
+        * Create an iterator. For example:
+        *
+        * ```js
+        * for await (const [key, value] of db.iterator({ gte: 'a' })) {
+        *   console.log([key, value])
+        * }
+        * ```
+        */
+    iterator(): AbstractIterator<typeof this, KDefault, VDefault>;
+    iterator<K = KDefault, V = VDefault>(options: AbstractIteratorOptions<K, V>): AbstractIterator<typeof this, K, V>;
+    /**
+     * Create a key iterator. For example:
+     *
+     * ```js
+     * for await (const key of db.keys({ gte: 'a' })) {
+     *   console.log(key)
+     * }
+     * ```
+     */
+    keys(): AbstractKeyIterator<typeof this, KDefault>;
+    keys<K = KDefault>(options: AbstractKeyIteratorOptions<K>): AbstractKeyIterator<typeof this, K>;
+    /**
+     * Create a value iterator. For example:
+     *
+     * ```js
+     * for await (const value of db.values({ gte: 'a' })) {
+     *   console.log(value)
+     * }
+     * ```
+     */
+    values(): AbstractValueIterator<typeof this, KDefault, VDefault>;
+    values<K = KDefault, V = VDefault>(options: AbstractValueIteratorOptions<K, V>): AbstractValueIterator<typeof this, K, V>;
+    /**
+     * Delete all entries or a range.
+     */
+    clear(): Promise<void>;
+}
+export {};

package/types/Watcher.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+;
+;
+;