@typeberry/lib 0.11.0-829b7ea → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typeberry/lib",
-  "version": "0.11.0-829b7ea",
+  "version": "0.11.0",
   "description": "Typeberry Library",
   "main": "./bin/lib/index.js",
   "types": "./bin/lib/index.d.ts",
@@ -214,6 +214,8 @@
     "#@typeberry/database/*": "./packages/jam/database/*",
     "#@typeberry/database-fjall": "./packages/jam/database-fjall/index.js",
     "#@typeberry/database-fjall/*": "./packages/jam/database-fjall/*",
+    "#@typeberry/database-lmdb": "./packages/jam/database-lmdb/index.js",
+    "#@typeberry/database-lmdb/*": "./packages/jam/database-lmdb/*",
     "#@typeberry/executor": "./packages/jam/executor/index.js",
     "#@typeberry/executor/*": "./packages/jam/executor/*",
     "#@typeberry/fuzz-proto": "./packages/jam/fuzz-proto/index.js",

package/packages/jam/config-node/node-config.d.ts

@@ -20,13 +20,21 @@ export declare enum KnownChainSpec {
     /** Full chain spec. */
     Full = "full"
 }
+/** Persistent regular-node database backend. */
+export declare enum RegularStateBackend {
+    /** @deprecated lmdb remains available as an explicit fallback, but fjall is the default backend. */
+    Lmdb = "lmdb",
+    Fjall = "fjall"
+}
 export declare const knownChainSpecFromJson: FromJson<KnownChainSpec>;
+export declare const regularStateBackendFromJson: FromJson<RegularStateBackend>;
 type NodeConfigurationJson = {
     $schema: string;
     version: number;
     flavor: KnownChainSpec;
     chain_spec: JipChainSpec;
     database_base_path?: string;
+    state_backend?: RegularStateBackend;
     authorship: AuthorshipOptions;
     rpc?: RpcOptions;
 };
@@ -37,11 +45,13 @@ export declare class NodeConfiguration {
     readonly chainSpec: JipChainSpec;
     /** If database path is not provided, we load an in-memory db. */
     readonly databaseBasePath: string | undefined;
+    /** Persistent database backend used when `databaseBasePath` is set. */
+    readonly stateBackend: RegularStateBackend;
     readonly authorship: AuthorshipOptions;
     /** Optional RPC server configuration. When present, an in-process RPC server is started. */
     readonly rpc: RpcOptions | undefined;
     static fromJson: import("@typeberry/json-parser").FromJsonWithParser<unknown, NodeConfiguration>;
-    static new({ $schema, version, flavor, chain_spec, database_base_path, authorship, rpc }: NodeConfigurationJson): NodeConfiguration;
+    static new({ $schema, version, flavor, chain_spec, database_base_path, state_backend, authorship, rpc, }: NodeConfigurationJson): NodeConfiguration;
     private constructor();
 }
 export declare function loadConfig(config: string[], withRelPath: (p: string) => string): NodeConfiguration;

package/packages/jam/config-node/node-config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"node-config.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/config-node/node-config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,KAAK,QAAQ,EAAuB,MAAM,wBAAwB,CAAC;AAG5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAItC,+DAA+D;AAC/D,eAAO,MAAM,eAAe,QAAQ,CAAC;AACrC,eAAO,MAAM,eAAe,aAAa,CAAC;AAE1C,2BAA2B;AAC3B,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC,eAAO,MAAM,aAAa;;;;CAIzB,CAAC;AAEF,0BAA0B;AAC1B,oBAAY,cAAc;IACxB,uBAAuB;IACvB,IAAI,SAAS;IACb,uBAAuB;IACvB,IAAI,SAAS;CACd;AAED,eAAO,MAAM,sBAAsB,EAS7B,QAAQ,CAAC,cAAc,CAAC,CAAC;AAE/B,KAAK,qBAAqB,GAAG;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,YAAY,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,GAAG,CAAC,EAAE,UAAU,CAAC;CAClB,CAAC;AAEF,qBAAa,iBAAiB;aA8BV,OAAO,EAAE,MAAM;aACf,OAAO,EAAE,MAAM;aACf,MAAM,EAAE,cAAc;aACtB,SAAS,EAAE,YAAY;IACvC,iEAAiE;aACjD,gBAAgB,EAAE,MAAM,GAAG,SAAS;aACpC,UAAU,EAAE,iBAAiB;IAC7C,4FAA4F;aAC5E,GAAG,EAAE,UAAU,GAAG,SAAS;IArC7C,MAAM,CAAC,QAAQ,kFAWb;IAEF,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE,qBAAqB;IAe/G,OAAO;CAWR;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,iBAAiB,CAuDlG"}
+{"version":3,"file":"node-config.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/config-node/node-config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,KAAK,QAAQ,EAAuB,MAAM,wBAAwB,CAAC;AAG5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAItC,+DAA+D;AAC/D,eAAO,MAAM,eAAe,QAAQ,CAAC;AACrC,eAAO,MAAM,eAAe,aAAa,CAAC;AAE1C,2BAA2B;AAC3B,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC,eAAO,MAAM,aAAa;;;;CAIzB,CAAC;AAEF,0BAA0B;AAC1B,oBAAY,cAAc;IACxB,uBAAuB;IACvB,IAAI,SAAS;IACb,uBAAuB;IACvB,IAAI,SAAS;CACd;AAED,gDAAgD;AAChD,oBAAY,mBAAmB;IAC7B,oGAAoG;IACpG,IAAI,SAAS;IACb,KAAK,UAAU;CAChB;AAED,eAAO,MAAM,sBAAsB,EAS7B,QAAQ,CAAC,cAAc,CAAC,CAAC;AAE/B,eAAO,MAAM,2BAA2B,EASlC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;AAEpC,KAAK,qBAAqB,GAAG;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,YAAY,CAAC;IACzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,EAAE,iBAAiB,CAAC;IAC9B,GAAG,CAAC,EAAE,UAAU,CAAC;CAClB,CAAC;AAEF,qBAAa,iBAAiB;aAyCV,OAAO,EAAE,MAAM;aACf,OAAO,EAAE,MAAM;aACf,MAAM,EAAE,cAAc;aACtB,SAAS,EAAE,YAAY;IACvC,iEAAiE;aACjD,gBAAgB,EAAE,MAAM,GAAG,SAAS;IACpD,uEAAuE;aACvD,YAAY,EAAE,mBAAmB;aACjC,UAAU,EAAE,iBAAiB;IAC7C,4FAA4F;aAC5E,GAAG,EAAE,UAAU,GAAG,SAAS;IAlD7C,MAAM,CAAC,QAAQ,kFAYb;IAEF,MAAM,CAAC,GAAG,CAAC,EACT,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,EACV,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,GAAG,GACJ,EAAE,qBAAqB;IAgBxB,OAAO;CAaR;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,iBAAiB,CAuDlG"}

package/packages/jam/config-node/node-config.js

@@ -27,6 +27,13 @@ export var KnownChainSpec;
     /** Full chain spec. */
     KnownChainSpec["Full"] = "full";
 })(KnownChainSpec || (KnownChainSpec = {}));
+/** Persistent regular-node database backend. */
+export var RegularStateBackend;
+(function (RegularStateBackend) {
+    /** @deprecated lmdb remains available as an explicit fallback, but fjall is the default backend. */
+    RegularStateBackend["Lmdb"] = "lmdb";
+    RegularStateBackend["Fjall"] = "fjall";
+})(RegularStateBackend || (RegularStateBackend = {}));
 export const knownChainSpecFromJson = json.fromString((input, ctx) => {
     switch (input) {
         case KnownChainSpec.Tiny:
@@ -37,12 +44,23 @@ export const knownChainSpecFromJson = json.fromString((input, ctx) => {
             throw Error(`unknown network flavor: ${input} at ${ctx}`);
     }
 });
+export const regularStateBackendFromJson = json.fromString((input, ctx) => {
+    switch (input) {
+        case RegularStateBackend.Lmdb:
+            return RegularStateBackend.Lmdb;
+        case RegularStateBackend.Fjall:
+            return RegularStateBackend.Fjall;
+        default:
+            throw Error(`unknown state backend: ${input} at ${ctx}`);
+    }
+});
 export class NodeConfiguration {
     $schema;
     version;
     flavor;
     chainSpec;
     databaseBasePath;
+    stateBackend;
     authorship;
     rpc;
     static fromJson = json.object({
@@ -51,18 +69,21 @@ export class NodeConfiguration {
         flavor: knownChainSpecFromJson,
         chain_spec: JipChainSpec.fromJson,
         database_base_path: json.optional("string"),
+        state_backend: json.optional(regularStateBackendFromJson),
         authorship: AuthorshipOptions.fromJson,
         rpc: json.optional(RpcOptions.fromJson),
     }, NodeConfiguration.new);
-    static new({ $schema, version, flavor, chain_spec, database_base_path, authorship, rpc }) {
+    static new({ $schema, version, flavor, chain_spec, database_base_path, state_backend, authorship, rpc, }) {
         if (version !== 1) {
             throw new Error("Only version=1 config is supported.");
         }
-        return new NodeConfiguration($schema, version, flavor, chain_spec, database_base_path ?? undefined, authorship, rpc ?? undefined);
+        return new NodeConfiguration($schema, version, flavor, chain_spec, database_base_path ?? undefined, state_backend ?? RegularStateBackend.Fjall, authorship, rpc ?? undefined);
     }
     constructor($schema, version, flavor, chainSpec, 
     /** If database path is not provided, we load an in-memory db. */
-    databaseBasePath, authorship, 
+    databaseBasePath, 
+    /** Persistent database backend used when `databaseBasePath` is set. */
+    stateBackend, authorship, 
     /** Optional RPC server configuration. When present, an in-process RPC server is started. */
     rpc) {
         this.$schema = $schema;
@@ -70,6 +91,7 @@ export class NodeConfiguration {
         this.flavor = flavor;
         this.chainSpec = chainSpec;
         this.databaseBasePath = databaseBasePath;
+        this.stateBackend = stateBackend;
         this.authorship = authorship;
         this.rpc = rpc;
     }

package/packages/jam/config-node/node-config.test.js

@@ -5,7 +5,7 @@ import { configs } from "#@typeberry/configs";
 import polkajamConfig from "#@typeberry/configs/typeberry-polkajam-dev.json" with { type: "json" };
 import { parseFromJson } from "#@typeberry/json-parser";
 import { workspacePathFix } from "#@typeberry/utils";
-import { KnownChainSpec, loadConfig, NodeConfiguration } from "./node-config.js";
+import { KnownChainSpec, loadConfig, NodeConfiguration, RegularStateBackend } from "./node-config.js";
 import { RpcOptions } from "./rpc.js";
 const withRelPath = workspacePathFix(`${import.meta.dirname}/../..`);
 const NODE_CONFIG_TEST = {
@@ -41,6 +41,9 @@ describe("Importing Node Configuration", () => {
     it("should read the database base path", () => {
         assert.deepStrictEqual(config.databaseBasePath, "/tmp/jam-node-db");
     });
+    it("defaults to the fjall state backend", () => {
+        assert.deepStrictEqual(config.stateBackend, RegularStateBackend.Fjall);
+    });
     it("defaults to no rpc config", () => {
         assert.deepStrictEqual(config.rpc, undefined);
     });
@@ -48,8 +51,9 @@ describe("Importing Node Configuration", () => {
         const withRpc = parseFromJson({ ...NODE_CONFIG_TEST, rpc: { port: 9999 } }, NodeConfiguration.fromJson);
         assert.deepStrictEqual(withRpc.rpc, RpcOptions.new({ port: 9999 }));
     });
-    it("should reject unknown config keys", () => {
-        assert.throws(() => parseFromJson({ ...NODE_CONFIG_TEST, state_backend: "fjall" }, NodeConfiguration.fromJson));
+    it("should read the state backend", () => {
+        const lmdbConfig = parseFromJson({ ...NODE_CONFIG_TEST, state_backend: "lmdb" }, NodeConfiguration.fromJson);
+        assert.deepStrictEqual(lmdbConfig.stateBackend, RegularStateBackend.Lmdb);
     });
     it("should read the chain spec", () => {
         assert.deepStrictEqual(config.chainSpec.id, "testnet");

package/packages/jam/database/states.d.ts

@@ -14,7 +14,14 @@ export interface InitStatesDb<T = State> {
     /** Insert a pre-defined initial state directly into the database. */
     insertInitialState(headerHash: HeaderHash, initialState: T): Promise<Result<OK, StateUpdateError>>;
 }
-/** Interface for accessing states stored in the database. */
+/**
+ * Interface for accessing states stored in the database.
+ *
+ * NOTE that the design of this interface is heavily influenced
+ * by the LMDB implementation, so that we can implement it efficiently.
+ *
+ * See the documentation there for more detailed reasoning.
+ */
 export interface StatesDb<T extends State = State> {
     /** Compute a state root for given state. */
     getStateRoot(state: T): Promise<StateRootHash>;

package/packages/jam/database/states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED,6DAA6D;AAC7D,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC;;;;;;;;OAQG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE7C,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAI7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}
+{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAe,MAAM,kBAAkB,CAAC;AAE/F,OAAO,EAAe,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE3D,0DAA0D;AAC1D,oBAAY,gBAAgB;IAC1B,oDAAoD;IACpD,QAAQ,IAAI;IACZ,iDAAiD;IACjD,MAAM,IAAI;CACX;AAED,wFAAwF;AACxF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,KAAK;IACrC,qEAAqE;IACrE,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;CACpG;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ,CAAC,CAAC,SAAS,KAAK,GAAG,KAAK;IAC/C,4CAA4C;IAC5C,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/C;;;;OAIG;IACH,iBAAiB,CACf,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,CAAC,EACR,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEzC,gDAAgD;IAChD,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC;IAEvC;;;;;;;;OAQG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAE7C,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC;IAErC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;IAElC,6CAA6C;IAC7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,qBAAa,cAAe,YAAW,QAAQ,CAAC,aAAa,CAAC;IAQxC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAPzC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmE;IACtF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAE3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS;IAI1B,OAAO;IAID,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAgBlC,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAKhE,6CAA6C;IACvC,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAM7G,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAStD,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAI7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAI9B,KAAK;CACZ"}

package/packages/jam/database/value-refs.d.ts

@@ -10,9 +10,10 @@ export interface ValueDelta {
 /**
  * Synchronous, read-only view of the persisted refcounting state.
  *
- * fjall offers synchronous reads, so reads can go straight to the backing
- * store. All writes go through `ValueRefsUpdate` batches instead, since
- * persistent backends can only write asynchronously (fjall inserts + persist).
+ * Both LMDB and fjall offer synchronous reads, so reads can go straight
+ * to the backing store. All writes go through `ValueRefsUpdate` batches
+ * instead, since persistent backends can only write asynchronously
+ * (LMDB transactions, fjall inserts + persist).
  */
 export interface ValueRefsReader {
     /** How many leaves of the finalized-tip state reference the value. Missing keys read as `0`. */
@@ -27,7 +28,7 @@ export interface ValueRefsReader {
  *
  * The backend is responsible for applying the batch using its own write
  * primitive - ideally atomically with the state write that triggered it
- * (one fjall persist).
+ * (one LMDB transaction, one fjall persist).
  *
  * Counts are absolute values rather than increments, so applying the same
  * update more than once (e.g. on crash-replay) is harmless.

package/packages/jam/database/value-refs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"value-refs.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/value-refs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,sEAAsE;IACtE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,sEAAsE;IACtE,OAAO,EAAE,SAAS,EAAE,CAAC;CACtB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,gGAAgG;IAChG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3C,iGAAiG;IACjG,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IACzC,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;CACtD;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,eAAe;IAC9B,kFAAkF;IAClF,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACvC,gFAAgF;IAChF,aAAa,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACrC,oDAAoD;IACpD,SAAS,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;IACtC,uDAAuD;IACvD,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,kFAAkF;IAClF,YAAY,EAAE,SAAS,EAAE,CAAC;CAC3B;AAED,+DAA+D;AAC/D,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAQ9D;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,eAAe;IAEpD,yEAAyE;IACzE,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,eAAe;IAQjD;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,eAAe;IAYhE;;;;;OAKG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,eAAe;IAoBvD;;;;;;;OAOG;IACH,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe;CAYxD;AA0ED;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,eAAe;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2D;IACrF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2D;IACnF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgE;IAEvF,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAI1C,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS;IAIpD,KAAK,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;CAUrC"}
+{"version":3,"file":"value-refs.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database/value-refs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,sEAAsE;IACtE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,sEAAsE;IACtE,OAAO,EAAE,SAAS,EAAE,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAC9B,gGAAgG;IAChG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3C,iGAAiG;IACjG,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IACzC,8DAA8D;IAC9D,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;CACtD;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,eAAe;IAC9B,kFAAkF;IAClF,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACvC,gFAAgF;IAChF,aAAa,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IACrC,oDAAoD;IACpD,SAAS,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;IACtC,uDAAuD;IACvD,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,kFAAkF;IAClF,YAAY,EAAE,SAAS,EAAE,CAAC;CAC3B;AAED,+DAA+D;AAC/D,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAQ9D;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,eAAe;IAEpD,yEAAyE;IACzE,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,eAAe;IAQjD;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,eAAe;IAYhE;;;;;OAKG;IACH,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,eAAe;IAoBvD;;;;;;;OAOG;IACH,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe;CAYxD;AA0ED;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,eAAe;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA2D;IACrF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2D;IACnF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgE;IAEvF,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAI1C,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIxC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS;IAIpD,KAAK,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI;CAUrC"}

package/packages/jam/database-lmdb/blocks.d.ts

@@ -0,0 +1,24 @@
+import { type BlockView, type ExtrinsicView, type HeaderHash, type HeaderView, type StateRootHash } from "#@typeberry/block";
+import type { ChainSpec } from "#@typeberry/config";
+import type { BlocksDb } from "#@typeberry/database/blocks.js";
+import { type WithHash } from "#@typeberry/hash";
+import type { LmdbRoot } from "./root.js";
+export declare class LmdbBlocks implements BlocksDb {
+    private readonly chainSpec;
+    private readonly root;
+    private readonly extrinsics;
+    private readonly headers;
+    private readonly postStateRoots;
+    static new(chainSpec: ChainSpec, root: LmdbRoot): LmdbBlocks;
+    private constructor();
+    setPostStateRoot(hash: HeaderHash, postStateRoot: StateRootHash): Promise<void>;
+    getPostStateRoot(hash: HeaderHash): StateRootHash | null;
+    insertBlock(block: WithHash<HeaderHash, BlockView>): Promise<void>;
+    setBestHeaderHash(hash: HeaderHash): Promise<void>;
+    getBestHeaderHash(): HeaderHash;
+    getHeader(hash: HeaderHash): HeaderView | null;
+    getExtrinsic(hash: HeaderHash): ExtrinsicView | null;
+    markUnused(hash: HeaderHash): void;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=blocks.d.ts.map

package/packages/jam/database-lmdb/blocks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"blocks.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/blocks.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,aAAa,EAElB,KAAK,UAAU,EACf,KAAK,UAAU,EACf,KAAK,aAAa,EACnB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAQjD,qBAAa,UAAW,YAAW,QAAQ;IAUvC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IAEvC,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ;IAI/C,OAAO;IAeD,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrF,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQlD,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IASlE,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxD,iBAAiB,IAAI,UAAU;IAS/B,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,GAAG,IAAI;IAS9C,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,aAAa,GAAG,IAAI;IAQpD,UAAU,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI;IAM5B,KAAK;CAGZ"}

package/packages/jam/database-lmdb/blocks.js

@@ -0,0 +1,82 @@
+import { Extrinsic, Header, } from "#@typeberry/block";
+import { Bytes } from "#@typeberry/bytes";
+import { Decoder } from "#@typeberry/codec";
+import { HASH_SIZE } from "#@typeberry/hash";
+const BEST_BLOCK = "best hash and posterior state root";
+// TODO [ToDr] consider having a changeset for transactions,
+// where we store all `insert ++ key ++ value` and `remove ++ key`
+// in a single `Uint8Array` JAM-encoded. That could then
+// be efficiently transferred between threads.
+export class LmdbBlocks {
+    chainSpec;
+    root;
+    extrinsics;
+    headers;
+    postStateRoots;
+    static new(chainSpec, root) {
+        return new LmdbBlocks(chainSpec, root);
+    }
+    constructor(chainSpec, root) {
+        this.chainSpec = chainSpec;
+        this.root = root;
+        // NOTE [ToDr] Extrinsics are stored under header hash, not their hash. Revise if it's an issue.
+        this.extrinsics = this.root.subDb("extrinsics");
+        this.headers = this.root.subDb("headers");
+        // NOTE [ToDr] We currently store all posterior state roots, however it's
+        // most likely very redundant. We probably only need to store the posterior
+        // state roots of recent blocks to be able to quickly resolve forks
+        // OR we need a way to be able to traverse the blocks history forward
+        // (i.e. know what next block(s) is).
+        this.postStateRoots = this.root.subDb("postStateRoots");
+    }
+    async setPostStateRoot(hash, postStateRoot) {
+        await this.postStateRoots.put(hash.raw, postStateRoot.raw);
+    }
+    getPostStateRoot(hash) {
+        const postStateRoot = this.postStateRoots.get(hash.raw);
+        if (postStateRoot === undefined) {
+            return null;
+        }
+        return Bytes.fromBlob(postStateRoot, HASH_SIZE).asOpaque();
+    }
+    async insertBlock(block) {
+        const header = block.data.header.view().encoded();
+        const extrinsic = block.data.extrinsic.view().encoded();
+        await this.root.db.transaction(() => {
+            this.headers.put(block.hash.raw, header.raw);
+            this.extrinsics.put(block.hash.raw, extrinsic.raw);
+        });
+    }
+    async setBestHeaderHash(hash) {
+        await this.root.db.put(BEST_BLOCK, hash.raw);
+    }
+    getBestHeaderHash() {
+        const bestHeaderHash = this.root.db.get(BEST_BLOCK);
+        if (bestHeaderHash === undefined) {
+            return Bytes.zero(HASH_SIZE).asOpaque();
+        }
+        return Bytes.fromBlob(bestHeaderHash, HASH_SIZE).asOpaque();
+    }
+    getHeader(hash) {
+        const data = this.headers.get(hash.raw);
+        if (data === undefined) {
+            return null;
+        }
+        return Decoder.decodeObject(Header.Codec.View, data, this.chainSpec);
+    }
+    getExtrinsic(hash) {
+        const data = this.extrinsics.get(hash.raw);
+        if (data === undefined) {
+            return null;
+        }
+        return Decoder.decodeObject(Extrinsic.Codec.View, data, this.chainSpec);
+    }
+    markUnused(hash) {
+        this.headers.removeSync(hash.raw);
+        this.extrinsics.removeSync(hash.raw);
+        this.postStateRoots.removeSync(hash.raw);
+    }
+    async close() {
+        await Promise.all([this.headers.close(), this.extrinsics.close(), this.postStateRoots.close()]);
+    }
+}

package/packages/jam/database-lmdb/hybrid-states.d.ts

@@ -0,0 +1,56 @@
+import type { HeaderHash, StateRootHash } from "#@typeberry/block";
+import type { ChainSpec } from "#@typeberry/config";
+import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
+import type { Blake2b } from "#@typeberry/hash";
+import type { ServicesUpdate, State } from "#@typeberry/state";
+import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
+import { OK, Result } from "#@typeberry/utils";
+/**
+ * Hybrid serialized-states db.
+ *
+ * States (leafs) are kept in-memory, but large values are persisted to lmdb.
+ * Reads go straight to lmdb, which keeps its own page cache.
+ * NOTE: this DB is designed for long fuzzing and to be used with pruning to
+ * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
+ */
+export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
+    private readonly spec;
+    private readonly blake2b;
+    private readonly root;
+    private readonly inMemStates;
+    private readonly lmdbValues;
+    private readonly valuesDb;
+    private readonly refsStore;
+    private readonly refs;
+    private pendingCleanup;
+    static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }: {
+        spec: ChainSpec;
+        blake2b: Blake2b;
+        dbPath: string;
+        readOnly?: boolean;
+        ephemeral?: boolean;
+        compression?: boolean;
+    }): HybridSerializedStates;
+    private constructor();
+    insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
+    updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
+    getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
+    getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(headers: HeaderHash[]): void;
+    markUnused(header: HeaderHash): void;
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    private applyRefs;
+    diskSizeInBytes(): number | null;
+    close(): Promise<void>;
+    /** Write new large values to LMDB in one transaction. */
+    private writeValues;
+    /** Read a value from LMDB. */
+    private readValue;
+}
+//# sourceMappingURL=hybrid-states.d.ts.map

package/packages/jam/database-lmdb/hybrid-states.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EAEjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAKjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAK9C;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IA8BxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IA/BvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IAEtD,OAAO,CAAC,cAAc,CAAuC;IAE7D,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,EACT,WAAW,GACZ,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAexG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAmBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAI5C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAOpC,mFAAmF;IACnF,OAAO,CAAC,SAAS;IAmBjB,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;IAMX,yDAAyD;YAC3C,WAAW;IAqBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-lmdb/hybrid-states.js

@@ -0,0 +1,149 @@
+import { HashDictionary, SortedSet } from "#@typeberry/collections";
+import { InMemoryValueRefsStore, LeafDb, StateUpdateError, updateLeafs, ValueRefs, } from "#@typeberry/database";
+import { Logger } from "#@typeberry/logger";
+import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
+import { leafComparator } from "#@typeberry/trie";
+import { OK, Result } from "#@typeberry/utils";
+import { LmdbRoot } from "./root.js";
+const logger = Logger.new(import.meta.filename, "db");
+/**
+ * Hybrid serialized-states db.
+ *
+ * States (leafs) are kept in-memory, but large values are persisted to lmdb.
+ * Reads go straight to lmdb, which keeps its own page cache.
+ * NOTE: this DB is designed for long fuzzing and to be used with pruning to
+ * keep the heap usage bounded.
+ *
+ * Values that no longer belong to any surviving state are removed from lmdb,
+ * decided by in-memory refcounting (`ValueRefs`) driven by the importer's
+ * finality signal. Counts are not persisted: this db cannot resume from disk
+ * anyway (the leaf sets live in memory), so values left over by a previous
+ * run are never collected.
+ */
+export class HybridSerializedStates {
+    spec;
+    blake2b;
+    root;
+    inMemStates = HashDictionary.new();
+    lmdbValues;
+    // A single shared values accessor reused by every `LeafDb` we hand out.
+    valuesDb;
+    refsStore = new InMemoryValueRefsStore();
+    refs = new ValueRefs(this.refsStore);
+    // Queue of not-yet-committed value removals, awaited on close.
+    pendingCleanup = Promise.resolve();
+    static new({ spec, blake2b, dbPath, readOnly, ephemeral, compression, }) {
+        const root = LmdbRoot.new(dbPath, { readOnly, ephemeral, compression });
+        return new HybridSerializedStates(spec, blake2b, root);
+    }
+    constructor(spec, blake2b, root) {
+        this.spec = spec;
+        this.blake2b = blake2b;
+        this.root = root;
+        this.lmdbValues = this.root.subDb("values");
+        this.valuesDb = { get: (key) => this.readValue(key) };
+    }
+    async insertInitialState(headerHash, entries) {
+        const { values, leafs } = updateLeafs(SortedSet.fromArray(leafComparator, []), this.blake2b, Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
+        const res = await this.writeValues(values);
+        if (res.isError) {
+            return res;
+        }
+        this.inMemStates.set(headerHash, leafs);
+        this.applyRefs(this.refs.onInitial(values.map((v) => v[0])));
+        return Result.ok(OK);
+    }
+    async updateAndSetState(header, state, update) {
+        const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
+        // Clone the leaf set before mutating: the previous state keeps using its own.
+        const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
+        const { values, removed, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const res = await this.writeValues(values);
+        if (res.isError) {
+            // Leave the caller's state untouched: its new leaves would reference
+            // values that never reached disk.
+            return res;
+        }
+        // Re-create the lookup with the shared values accessor only once the new
+        // values are durably written.
+        state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
+        this.inMemStates.set(header, leafs);
+        this.applyRefs(this.refs.onImport(header, { inserted: values.map((v) => v[0]), removed }));
+        return Result.ok(OK);
+    }
+    async getStateRoot(state) {
+        return state.backend.getStateRoot(this.blake2b);
+    }
+    getState(header) {
+        const leafs = this.inMemStates.get(header);
+        if (leafs === undefined) {
+            return null;
+        }
+        const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
+        return SerializedState.new(this.spec, this.blake2b, leafDb);
+    }
+    commitFinalized(headers) {
+        this.applyRefs(this.refs.commitFinalized(headers));
+    }
+    markUnused(header) {
+        // Release the speculative references first (a no-op for finalized states,
+        // whose deltas were already consumed by `commitFinalized`).
+        this.applyRefs(this.refs.releaseUnfinalized(header));
+        this.inMemStates.delete(header);
+    }
+    /** Apply a refcounting update and remove values that lost their last reference. */
+    applyRefs(update) {
+        this.refsStore.apply(update);
+        if (update.removeValues.length === 0) {
+            return;
+        }
+        // Queued, not awaited: a failed removal only leaks a value.
+        this.pendingCleanup = this.pendingCleanup
+            .then(() => this.lmdbValues.transaction(() => {
+            for (const v of update.removeValues) {
+                this.lmdbValues.remove(v.raw);
+            }
+        }))
+            .catch((e) => {
+            logger.error `Failed to remove unreferenced values: ${e}`;
+        });
+    }
+    diskSizeInBytes() {
+        return this.root.sizeInBytes();
+    }
+    async close() {
+        await this.pendingCleanup;
+        await this.lmdbValues.close();
+        await this.root.close();
+    }
+    /** Write new large values to LMDB in one transaction. */
+    async writeValues(values) {
+        if (values.length === 0) {
+            return Result.ok(OK);
+        }
+        try {
+            // Flush queued removals first: a pending cleanup might be about to delete
+            // a content hash we are re-inserting now. Writing behind it keeps the
+            // re-inserted value on disk (removals are only queued at refcount 0, so
+            // none can be queued for a value still referenced).
+            await this.pendingCleanup;
+            await this.lmdbValues.transaction(() => {
+                for (const [hash, val] of values) {
+                    this.lmdbValues.put(hash.raw, val.raw);
+                }
+            });
+        }
+        catch (e) {
+            return Result.error(StateUpdateError.Commit, () => `Failed to commit values: ${e}`);
+        }
+        return Result.ok(OK);
+    }
+    /** Read a value from LMDB. */
+    readValue(key) {
+        const val = this.lmdbValues.get(key.raw);
+        if (val === undefined) {
+            throw new Error(`Missing value at key: ${key}`);
+        }
+        return val;
+    }
+}

package/packages/jam/database-lmdb/hybrid-states.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=hybrid-states.test.d.ts.map

package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.test.ts"],"names":[],"mappings":""}