@subsquid/batch-processor 0.2.0-portal-api.493495 → 0.2.0-portal-api.19316d

package/lib/template-registry.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TemplateRegistry = void 0;
+class HandlerTemplates {
+    constructor(range, registry) {
+        this.range = range;
+        this.registry = registry;
+        this.data = [];
+    }
+    add(key, value, blockNumber) {
+        this.mutate('add', key, value, blockNumber);
+    }
+    remove(key, value, blockNumber) {
+        this.mutate('delete', key, value, blockNumber);
+    }
+    has(key, value, blockNumber) {
+        return this.registry.has(key, value, blockNumber);
+    }
+    mutate(type, key, value, blockNumber) {
+        if (blockNumber < this.range.from) {
+            throw new RangeError(`blockNumber ${blockNumber} is before batch start ${this.range.from}`);
+        }
+        let mutation = { type, key, value, blockNumber };
+        this.registry.apply(mutation);
+        this.data.push(mutation);
+    }
+}
+class TemplateRegistry {
+    constructor() {
+        this.byKey = new Map();
+        this.baseMutations = [];
+        this.undoLog = [];
+        this.pendingTemplates = [];
+    }
+    /**
+     * Rebuild state from finalized mutations and optionally replay hot block
+     * mutations to reconstruct the undo log.
+     *
+     * Finalized mutations form the base state. Hot block mutations are stored
+     * in the undo log so rollbackTo() works for hot block heights after restart.
+     */
+    init(mutations, hotBlocks) {
+        this.baseMutations = mutations;
+        this.undoLog = hotBlocks ?? [];
+        this.pendingTemplates = [];
+        this.rebuild();
+    }
+    get(key) {
+        let m = this.byKey.get(key);
+        if (m == null)
+            return [];
+        let out = [];
+        for (let [value, ranges] of m) {
+            for (let range of ranges) {
+                out.push({ value, range });
+            }
+        }
+        return out;
+    }
+    has(key, value, block) {
+        let byVal = this.byKey.get(key);
+        if (byVal == null)
+            return false;
+        let ranges = byVal.get(value);
+        if (ranges == null)
+            return false;
+        for (let range of ranges) {
+            if (block >= range.from && (range.to == null || block <= range.to)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    apply(mutation) {
+        let changed = this.applyToState(mutation);
+        if (changed) {
+            this.pendingTemplates.push(mutation);
+        }
+        return changed;
+    }
+    async transact(range, fn) {
+        let templates = new HandlerTemplates(range, this);
+        try {
+            await fn(templates);
+        }
+        catch (e) {
+            this.pendingTemplates = [];
+            this.rebuild();
+            throw e;
+        }
+        let changed = this.pendingTemplates.length > 0;
+        if (changed) {
+            this.undoLog.push({ blockNumber: range.to, templates: this.pendingTemplates });
+        }
+        this.pendingTemplates = [];
+        return { data: templates.data, changed };
+    }
+    rollbackTo(blockNumber) {
+        let splitIdx = upperBound(this.undoLog, blockNumber);
+        if (splitIdx >= this.undoLog.length)
+            return;
+        this.undoLog.length = splitIdx;
+        this.pendingTemplates = [];
+        this.rebuild();
+    }
+    prune(blockNumber) {
+        let splitIdx = upperBound(this.undoLog, blockNumber);
+        for (let i = 0; i < splitIdx; i++) {
+            this.baseMutations.push(...this.undoLog[i].templates);
+        }
+        this.undoLog = this.undoLog.slice(splitIdx);
+    }
+    applyToState({ type, key, value, blockNumber }) {
+        if (type === 'add') {
+            let byVal = this.byKey.get(key);
+            if (byVal == null) {
+                byVal = new Map();
+                this.byKey.set(key, byVal);
+            }
+            let ranges = byVal.get(value);
+            if (ranges == null) {
+                byVal.set(value, [{ from: blockNumber }]);
+                return true;
+            }
+            let last = ranges[ranges.length - 1];
+            if (last.to != null) {
+                if (blockNumber < last.to)
+                    return false;
+                ranges.push({ from: blockNumber });
+                return true;
+            }
+            if (blockNumber >= last.from)
+                return false;
+            last.from = blockNumber;
+            return true;
+        }
+        else {
+            let byVal = this.byKey.get(key);
+            if (!byVal)
+                return false;
+            let ranges = byVal.get(value);
+            if (!ranges || ranges.length === 0)
+                return false;
+            let last = ranges[ranges.length - 1];
+            if (last.to != null)
+                return false;
+            if (blockNumber < last.from)
+                return false;
+            last.to = blockNumber;
+            return true;
+        }
+    }
+    rebuild() {
+        this.byKey.clear();
+        let allMutations = [...this.baseMutations];
+        for (let entry of this.undoLog) {
+            allMutations.push(...entry.templates);
+        }
+        allMutations.sort((a, b) => {
+            if (a.blockNumber !== b.blockNumber)
+                return a.blockNumber - b.blockNumber;
+            if (a.type !== b.type)
+                return a.type === 'delete' ? -1 : 1;
+            return 0;
+        });
+        for (let m of allMutations) {
+            this.applyToState(m);
+        }
+    }
+}
+exports.TemplateRegistry = TemplateRegistry;
+function upperBound(log, target) {
+    let lo = 0;
+    let hi = log.length;
+    while (lo < hi) {
+        let mid = (lo + hi) >> 1;
+        if (log[mid].blockNumber > target) {
+            hi = mid;
+        }
+        else {
+            lo = mid + 1;
+        }
+    }
+    return lo;
+}
+//# sourceMappingURL=template-registry.js.map

package/lib/template-registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"template-registry.js","sourceRoot":"","sources":["../src/template-registry.ts"],"names":[],"mappings":";;;AAUA,MAAM,gBAAgB;IAGlB,YACqB,KAAkB,EAClB,QAA0B;QAD1B,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAkB;QAJtC,SAAI,GAAuB,EAAE,CAAA;IAKnC,CAAC;IAEJ,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,WAAmB;QAC/C,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,CAAC,GAAW,EAAE,KAAa,EAAE,WAAmB;QAClD,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,CAAC,CAAA;IAClD,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,WAAmB;QAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,WAAW,CAAC,CAAA;IACrD,CAAC;IAEO,MAAM,CAAC,IAA8B,EAAE,GAAW,EAAE,KAAa,EAAE,WAAmB;QAC1F,IAAI,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAChC,MAAM,IAAI,UAAU,CAAC,eAAe,WAAW,0BAA0B,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/F,CAAC;QACD,IAAI,QAAQ,GAAqB,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAC,CAAA;QAChE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAC7B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5B,CAAC;CACJ;AAED,MAAa,gBAAgB;IAA7B;QACY,UAAK,GAAG,IAAI,GAAG,EAAgC,CAAA;QAC/C,kBAAa,GAAuB,EAAE,CAAA;QACtC,YAAO,GAAgE,EAAE,CAAA;QACzE,qBAAgB,GAAuB,EAAE,CAAA;IA4IrD,CAAC;IA1IG;;;;;;OAMG;IACH,IAAI,CACA,SAA6B,EAC7B,SAAkE;QAElE,IAAI,CAAC,aAAa,GAAG,SAAS,CAAA;QAC9B,IAAI,CAAC,OAAO,GAAG,SAAS,IAAI,EAAE,CAAA;QAC9B,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC1B,IAAI,CAAC,OAAO,EAAE,CAAA;IAClB,CAAC;IAED,GAAG,CAAC,GAAW;QACX,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,EAAE,CAAA;QACxB,IAAI,GAAG,GAAoB,EAAE,CAAA;QAC7B,KAAK,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACvB,GAAG,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,KAAK,EAAC,CAAC,CAAA;YAC5B,CAAC;QACL,CAAC;QACD,OAAO,GAAG,CAAA;IACd,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,KAAa,EAAE,KAAa;QACzC,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,KAAK,IAAI,IAAI;YAAE,OAAO,KAAK,CAAA;QAC/B,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAI,MAAM,IAAI,IAAI;YAAE,OAAO,KAAK,CAAA;QAChC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;gBACjE,OAAO,IAAI,CAAA;YACf,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,QAA0B;QAC5B,IAAI,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;QACzC,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxC,CAAC;QACD,OAAO,OAAO,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,QAAQ,CACV,KAAkB,EAClB,EAAiD;QAEjD,IAAI,SAAS,GAAG,IAAI,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACjD,IAAI,CAAC;YACD,MAAM,EAAE,CAAC,SAAS,CAAC,CAAA;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;YAC1B,IAAI,CAAC,OAAO,EAAE,CAAA;YACd,MAAM,CAAC,CAAA;QACX,CAAC;QACD,IAAI,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAA;QAC9C,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,gBAAgB,EAAC,CAAC,CAAA;QAChF,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC1B,OAAO,EAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,OAAO,EAAC,CAAA;IAC1C,CAAC;IAED,UAAU,CAAC,WAAmB;QAC1B,IAAI,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QACpD,IAAI,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAM;QAC3C,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAA;QAC9B,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC1B,IAAI,CAAC,OAAO,EAAE,CAAA;IAClB,CAAC;IAED,KAAK,CAAC,WAAmB;QACrB,IAAI,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;QACzD,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC;IAEO,YAAY,CAAC,EAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAmB;QAClE,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACjB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC/B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAChB,KAAK,GAAG,IAAI,GAAG,EAAE,CAAA;gBACjB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YAC9B,CAAC;YAED,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACjB,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAC,IAAI,EAAE,WAAW,EAAC,CAAC,CAAC,CAAA;gBACvC,OAAO,IAAI,CAAA;YACf,CAAC;YAED,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;gBAClB,IAAI,WAAW,GAAG,IAAI,CAAC,EAAE;oBAAE,OAAO,KAAK,CAAA;gBACvC,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,WAAW,EAAC,CAAC,CAAA;gBAChC,OAAO,IAAI,CAAA;YACf,CAAC;YAED,IAAI,WAAW,IAAI,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAA;YAC1C,IAAI,CAAC,IAAI,GAAG,WAAW,CAAA;YACvB,OAAO,IAAI,CAAA;QACf,CAAC;aAAM,CAAC;YACJ,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC/B,IAAI,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAA;YACxB,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YAChD,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAA;YACjC,IAAI,WAAW,GAAG,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAA;YACzC,IAAI,CAAC,EAAE,GAAG,WAAW,CAAA;YACrB,OAAO,IAAI,CAAA;QACf,CAAC;IACL,CAAC;IAEO,OAAO;QACX,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QAClB,IAAI,YAAY,GAAuB,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAA;QAC9D,KAAK,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAA;QACzC,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACvB,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;gBAAE,OAAO,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAA;YACzE,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;gBAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAC1D,OAAO,CAAC,CAAA;QACZ,CAAC,CAAC,CAAA;QACF,KAAK,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACL,CAAC;CACJ;AAhJD,4CAgJC;AAED,SAAS,UAAU,CAAC,GAAiC,EAAE,MAAc;IACjE,IAAI,EAAE,GAAG,CAAC,CAAA;IACV,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAA;IACnB,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACb,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;QACxB,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,WAAW,GAAG,MAAM,EAAE,CAAC;YAChC,EAAE,GAAG,GAAG,CAAA;QACZ,CAAC;aAAM,CAAC;YACJ,EAAE,GAAG,GAAG,GAAG,CAAC,CAAA;QAChB,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAA;AACb,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subsquid/batch-processor",
-  "version": "0.2.0-portal-api.493495",
+  "version": "0.2.0-portal-api.19316d",
   "description": "ETL processor",
   "license": "GPL-3.0-or-later",
   "repository": "git@github.com:subsquid/squid-sdk.git",
@@ -20,9 +20,10 @@
     "@subsquid/util-internal-counters": "^1.3.2",
     "@subsquid/util-internal-prometheus-server": "^1.3.0",
     "@subsquid/util-internal-range": "^0.3.0",
-    "@subsquid/util-internal-data-source": "0.0.1-portal-api.493495"
+    "@subsquid/util-internal-data-source": "0.0.1-portal-api.19316d"
   },
   "devDependencies": {
+    "@types/mocha": "^10.0.7",
     "@types/node": "^18.18.14",
     "typescript": "~5.5.4"
   },

package/src/database.ts

@@ -14,12 +14,22 @@ export interface FinalTxInfo {
 
 export interface FinalDatabase<S> {
     supportsHotBlocks?: false
+    supportsTemplates?: boolean
     connect(): Promise<FinalDatabaseState>
-    transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
+    transact(info: FinalTxInfo, cb: (store: S) => Promise<DatabaseTransactResult | void>): Promise<void>
 }
 
 
-export interface FinalDatabaseState extends HashAndHeight {}
+export interface DatabaseTransactResult {
+    templates?: TemplateMutation[]
+}
+
+
+export interface FinalDatabaseState {
+    height: number
+    hash: string
+    templates?: TemplateMutation[]
+}
 
 
 export interface HotTxInfo {
@@ -31,22 +41,42 @@ export interface HotTxInfo {
 
 export interface HotDatabase<S> {
     supportsHotBlocks: true
+    supportsTemplates?: boolean
     connect(): Promise<HotDatabaseState>
-    transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
+    transact(info: FinalTxInfo, cb: (store: S) => Promise<DatabaseTransactResult | void>): Promise<void>
 
     transactHot2(
         info: HotTxInfo,
-        cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>
+        cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<DatabaseTransactResult | void>
     ): Promise<void>
 }
 
 
+export interface HotBlock extends HashAndHeight {
+    templates?: TemplateMutation[]
+}
+
+
 export interface HotDatabaseState extends HashAndHeight {
-    top: HashAndHeight[]
+    top: HotBlock[]
+    templates?: TemplateMutation[]
+}
+
+
+export interface FinalDatabaseState extends HashAndHeight {
+    templates?: TemplateMutation[]
+}
+
+
+export interface TemplateMutation {
+    type: 'add' | 'delete'
+    key: string
+    value: string
+    blockNumber: number
 }
 
 
 export interface HashAndHeight {
-    height: number
     hash: string
+    height: number
 }