@lodestar/db 1.35.0-dev.c85be4e26c → 1.35.0-dev.c9deb9b59f

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lodestar/db",
-  "version": "1.35.0-dev.c85be4e26c",
+  "version": "1.35.0-dev.c9deb9b59f",
   "description": "DB modules of Lodestar",
   "author": "ChainSafe Systems",
   "homepage": "https://github.com/ChainSafe/lodestar#readme",
@@ -12,14 +12,27 @@
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
   "type": "module",
-  "exports": "./lib/index.js",
+  "exports": {
+    ".": {
+      "bun": "./src/index.ts",
+      "import": "./lib/index.js"
+    },
+    "./controller/level": {
+      "bun": "./src/controller/level_bun.ts",
+      "import": "./lib/controller/level.js"
+    }
+  },
+  "imports": {
+    "#controller/level": {
+      "bun": "./src/controller/level_bun.ts",
+      "import": "./src/controller/level.js"
+    }
+  },
   "types": "./lib/index.d.ts",
   "files": [
-    "lib/**/*.d.ts",
-    "lib/**/*.js",
-    "lib/**/*.js.map",
-    "*.d.ts",
-    "*.js"
+    "src",
+    "lib",
+    "!**/*.tsbuildinfo"
   ],
   "scripts": {
     "clean": "rm -rf lib && rm -f *.tsbuildinfo",
@@ -37,13 +50,14 @@
   },
   "dependencies": {
     "@chainsafe/ssz": "^1.2.2",
-    "@lodestar/config": "1.35.0-dev.c85be4e26c",
-    "@lodestar/utils": "1.35.0-dev.c85be4e26c",
+    "@lodestar/bun": "git+https://github.com/ChainSafe/lodestar-bun.git",
+    "@lodestar/config": "1.35.0-dev.c9deb9b59f",
+    "@lodestar/utils": "1.35.0-dev.c9deb9b59f",
     "classic-level": "^1.4.1",
     "it-all": "^3.0.4"
   },
   "devDependencies": {
-    "@lodestar/logger": "1.35.0-dev.c85be4e26c"
+    "@lodestar/logger": "1.35.0-dev.c9deb9b59f"
   },
-  "gitHead": "8e54be62e05b31670c0b51b0337198cfc8650fd0"
+  "gitHead": "9f18a0bd68957d2324cc3b05f36910ab08f28672"
 }

package/src/abstractPrefixedRepository.ts

@@ -0,0 +1,244 @@
+import {Type} from "@chainsafe/ssz";
+import {ChainForkConfig} from "@lodestar/config";
+import {BUCKET_LENGTH} from "./const.js";
+import {KeyValue} from "./controller/index.js";
+import {Db, DbReqOpts, FilterOptions} from "./controller/interface.js";
+import {encodeKey} from "./util.js";
+
+type Id = Uint8Array | string | number | bigint;
+
+/**
+ * Repository is a high level kv storage
+ * This abstract repository is designed in a way to store items with different prefixed
+ * Specially when those prefixed data is not available in the object to be stored
+ *
+ * By default, SSZ-encoded values,
+ */
+export abstract class PrefixedRepository<P, I extends Id, T> {
+  private readonly dbReqOpts: DbReqOpts;
+  /** Inclusive range for the minimum key for the bucket */
+  private readonly minKey: Uint8Array;
+  /** Exclusive range for the maximum key for the bucket */
+  private readonly maxKey: Uint8Array;
+
+  protected constructor(
+    protected config: ChainForkConfig,
+    protected db: Db,
+    protected bucket: number,
+    protected type: Type<T>,
+    private readonly bucketId: string
+  ) {
+    this.dbReqOpts = {bucketId: this.bucketId};
+    this.minKey = encodeKey(bucket, Buffer.alloc(0));
+    this.maxKey = encodeKey(bucket + 1, Buffer.alloc(0));
+  }
+
+  abstract encodeKeyRaw(prefix: P, id: I): Uint8Array;
+  abstract decodeKeyRaw(raw: Uint8Array): {prefix: P; id: I};
+  /**
+   * Max key is inclusive
+   * */
+  abstract getMaxKeyRaw(prefix: P): Uint8Array;
+  /**
+   * Min key is inclusive
+   * */
+  abstract getMinKeyRaw(prefix: P): Uint8Array;
+
+  protected encodeValue(value: T): Uint8Array {
+    return this.type.serialize(value);
+  }
+
+  protected decodeValue(data: Uint8Array): T {
+    return this.type.deserialize(data);
+  }
+
+  protected wrapKey(raw: Uint8Array): Uint8Array {
+    return encodeKey(this.bucket, raw);
+  }
+
+  protected unwrapKey(key: Uint8Array): Uint8Array {
+    return key.slice(BUCKET_LENGTH);
+  }
+
+  // The Id can be inferred from the value
+  getId(value: T): I {
+    return this.type.hashTreeRoot(value) as I;
+  }
+
+  async get(prefix: P, id: I): Promise<T | null> {
+    const key = this.wrapKey(this.encodeKeyRaw(prefix, id));
+    const v = await this.db.get(key, this.dbReqOpts);
+    return v ? this.decodeValue(v) : null;
+  }
+
+  async getMany(prefix: P, ids: I[]): Promise<(T | undefined)[]> {
+    const keys = [];
+    for (const id of ids) {
+      keys.push(this.wrapKey(this.encodeKeyRaw(prefix, id)));
+    }
+    const values = await this.db.getMany(keys, this.dbReqOpts);
+
+    const result = [];
+    for (const value of values) {
+      result.push(value ? this.decodeValue(value) : undefined);
+    }
+
+    return result;
+  }
+
+  async getManyBinary(prefix: P, ids: I[]): Promise<(Uint8Array | undefined)[]> {
+    const keys = [];
+    for (const id of ids) {
+      keys.push(this.wrapKey(this.encodeKeyRaw(prefix, id)));
+    }
+    return await this.db.getMany(keys, this.dbReqOpts);
+  }
+
+  async getBinary(prefix: P, id: I): Promise<Uint8Array | null> {
+    const key = this.wrapKey(this.encodeKeyRaw(prefix, id));
+    return await this.db.get(key, this.dbReqOpts);
+  }
+
+  async put(prefix: P, item: T): Promise<void> {
+    const id = this.getId(item);
+    const key = this.wrapKey(this.encodeKeyRaw(prefix, id));
+    await this.db.put(key, this.encodeValue(item), this.dbReqOpts);
+  }
+
+  async putMany(prefix: P, items: T[]): Promise<void> {
+    const batch: KeyValue<Uint8Array, Uint8Array>[] = [];
+    for (const item of items) {
+      const id = this.getId(item);
+      const key = this.wrapKey(this.encodeKeyRaw(prefix, id));
+      batch.push({key, value: this.encodeValue(item)});
+    }
+    await this.db.batchPut(batch, this.dbReqOpts);
+  }
+
+  async putBinary(prefix: P, id: I, bytes: Uint8Array): Promise<void> {
+    const key = this.wrapKey(this.encodeKeyRaw(prefix, id));
+    await this.db.put(key, bytes, this.dbReqOpts);
+  }
+
+  async putManyBinary(prefix: P, items: KeyValue<I, Uint8Array>[]): Promise<void> {
+    const batch: KeyValue<Uint8Array, Uint8Array>[] = [];
+    for (const {key, value} of items) {
+      batch.push({key: this.wrapKey(this.encodeKeyRaw(prefix, key)), value: value});
+    }
+    await this.db.batchPut(batch, this.dbReqOpts);
+  }
+
+  async delete(prefix: P, id: I): Promise<void> {
+    const key = this.wrapKey(this.encodeKeyRaw(prefix, id));
+    await this.db.delete(key, this.dbReqOpts);
+  }
+
+  async deleteMany(prefix: P | P[]): Promise<void> {
+    const keys: Uint8Array[][] = [];
+
+    for (const p of Array.isArray(prefix) ? prefix : [prefix]) {
+      const prefixedKeys = await this.db.keys({
+        gte: this.wrapKey(this.getMinKeyRaw(p)),
+        lte: this.wrapKey(this.getMaxKeyRaw(p)),
+        bucketId: this.bucketId,
+      });
+      keys.push(prefixedKeys);
+    }
+
+    await this.db.batchDelete(keys.flat(), this.dbReqOpts);
+  }
+
+  async *valuesStream(prefix: P | P[]): AsyncIterable<T> {
+    for (const p of Array.isArray(prefix) ? prefix : [prefix]) {
+      for await (const vb of this.db.valuesStream({
+        gte: this.wrapKey(this.getMinKeyRaw(p)),
+        lte: this.wrapKey(this.getMaxKeyRaw(p)),
+        bucketId: this.bucketId,
+      })) {
+        yield this.decodeValue(vb);
+      }
+    }
+  }
+
+  async *valuesStreamBinary(prefix: P | P[]): AsyncIterable<{prefix: P; id: I; value: Uint8Array}> {
+    for (const p of Array.isArray(prefix) ? prefix : [prefix]) {
+      for await (const {key, value} of this.db.entriesStream({
+        gte: this.wrapKey(this.getMinKeyRaw(p)),
+        lte: this.wrapKey(this.getMaxKeyRaw(p)),
+        bucketId: this.bucketId,
+      })) {
+        const {prefix, id} = this.decodeKeyRaw(this.unwrapKey(key));
+
+        yield {
+          prefix,
+          id,
+          value,
+        };
+      }
+    }
+  }
+
+  async *entriesStream(prefix: P | P[]): AsyncIterable<{prefix: P; id: I; value: T}> {
+    for (const v of Array.isArray(prefix) ? prefix : [prefix]) {
+      for await (const {key, value} of this.db.entriesStream({
+        gte: this.wrapKey(this.getMinKeyRaw(v)),
+        lte: this.wrapKey(this.getMaxKeyRaw(v)),
+        bucketId: this.bucketId,
+      })) {
+        const {prefix, id} = this.decodeKeyRaw(this.unwrapKey(key));
+
+        yield {
+          prefix,
+          id,
+          value: this.decodeValue(value),
+        };
+      }
+    }
+  }
+
+  async *entriesStreamBinary(prefix: P | P[]): AsyncIterable<{prefix: P; id: I; value: Uint8Array}> {
+    for (const v of Array.isArray(prefix) ? prefix : [prefix]) {
+      for await (const {key, value} of this.db.entriesStream({
+        gte: this.wrapKey(this.getMinKeyRaw(v)),
+        lte: this.wrapKey(this.getMaxKeyRaw(v)),
+        bucketId: this.bucketId,
+      })) {
+        const {prefix, id} = this.decodeKeyRaw(this.unwrapKey(key));
+
+        yield {
+          prefix,
+          id: id,
+          value,
+        };
+      }
+    }
+  }
+
+  async keys(opts?: FilterOptions<{prefix: P; id: I}>): Promise<{prefix: P; id: I}[]> {
+    const optsBuff: FilterOptions<Uint8Array> = {
+      bucketId: this.bucketId,
+    };
+
+    if (opts?.gte !== undefined) {
+      optsBuff.gte = this.wrapKey(this.encodeKeyRaw(opts.gte.prefix, opts.gte.id));
+    } else if (opts?.gt !== undefined) {
+      optsBuff.gt = this.wrapKey(this.encodeKeyRaw(opts.gt.prefix, opts.gt.id));
+    } else {
+      optsBuff.gte = this.minKey;
+    }
+
+    if (opts?.lte !== undefined) {
+      optsBuff.lte = this.wrapKey(this.encodeKeyRaw(opts.lte.prefix, opts.lte.id));
+    } else if (opts?.lt !== undefined) {
+      optsBuff.lt = this.wrapKey(this.encodeKeyRaw(opts.lt.prefix, opts.lt.id));
+    } else {
+      optsBuff.lt = this.maxKey;
+    }
+
+    if (opts?.reverse !== undefined) optsBuff.reverse = opts.reverse;
+    if (opts?.limit !== undefined) optsBuff.limit = opts.limit;
+
+    const data = await this.db.keys(optsBuff);
+    return (data ?? []).map((data) => this.decodeKeyRaw(this.unwrapKey(data)));
+  }
+}

package/src/abstractRepository.ts

@@ -0,0 +1,283 @@
+import {Type} from "@chainsafe/ssz";
+import {ChainForkConfig} from "@lodestar/config";
+import {BUCKET_LENGTH} from "./const.js";
+import {FilterOptions, KeyValue} from "./controller/index.js";
+import {Db, DbReqOpts} from "./controller/interface.js";
+import {encodeKey as _encodeKey} from "./util.js";
+
+export type Id = Uint8Array | string | number | bigint;
+
+/**
+ * Repository is a high level kv storage
+ * managing a Uint8Array to Uint8Array kv database
+ * It translates typed keys and values to Uint8Arrays required by the underlying database
+ *
+ * By default, SSZ-encoded values,
+ * indexed by root
+ */
+export abstract class Repository<I extends Id, T> {
+  private readonly dbReqOpts: DbReqOpts;
+
+  private readonly minKey: Uint8Array;
+  private readonly maxKey: Uint8Array;
+
+  protected constructor(
+    protected config: ChainForkConfig,
+    protected db: Db,
+    protected bucket: number,
+    protected type: Type<T>,
+    protected readonly bucketId: string
+  ) {
+    this.dbReqOpts = {bucketId: this.bucketId};
+    this.minKey = _encodeKey(bucket, Buffer.alloc(0));
+    this.maxKey = _encodeKey(bucket + 1, Buffer.alloc(0));
+  }
+
+  encodeValue(value: T): Uint8Array {
+    return this.type.serialize(value);
+  }
+
+  decodeValue(data: Uint8Array): T {
+    return this.type.deserialize(data);
+  }
+
+  encodeKey(id: I): Uint8Array {
+    return _encodeKey(this.bucket, id);
+  }
+
+  decodeKey(key: Uint8Array): I {
+    return key.slice(BUCKET_LENGTH) as I;
+  }
+
+  async get(id: I): Promise<T | null> {
+    const value = await this.db.get(this.encodeKey(id), this.dbReqOpts);
+    if (!value) return null;
+    return this.decodeValue(value);
+  }
+
+  async getBinary(id: I): Promise<Uint8Array | null> {
+    const value = await this.db.get(this.encodeKey(id), this.dbReqOpts);
+    if (!value) return null;
+    return value;
+  }
+
+  async has(id: I): Promise<boolean> {
+    return (await this.get(id)) !== null;
+  }
+
+  async put(id: I, value: T): Promise<void> {
+    await this.db.put(this.encodeKey(id), this.encodeValue(value), this.dbReqOpts);
+  }
+
+  async putBinary(id: I, value: Uint8Array): Promise<void> {
+    await this.db.put(this.encodeKey(id), value, this.dbReqOpts);
+  }
+
+  async delete(id: I): Promise<void> {
+    await this.db.delete(this.encodeKey(id), this.dbReqOpts);
+  }
+
+  // The Id can be inferred from the value
+  getId(value: T): I {
+    return this.type.hashTreeRoot(value) as I;
+  }
+
+  async add(value: T): Promise<void> {
+    await this.put(this.getId(value), value);
+  }
+
+  async remove(value: T): Promise<void> {
+    await this.delete(this.getId(value));
+  }
+
+  async batchPut(items: KeyValue<I, T>[]): Promise<void> {
+    if (items.length === 1) {
+      return this.put(items[0].key, items[0].value);
+    }
+
+    await this.db.batchPut(
+      Array.from({length: items.length}, (_, i) => ({
+        key: this.encodeKey(items[i].key),
+        value: this.encodeValue(items[i].value),
+      })),
+      this.dbReqOpts
+    );
+  }
+
+  // Similar to batchPut but we support value as Uint8Array
+  async batchPutBinary(items: KeyValue<I, Uint8Array>[]): Promise<void> {
+    if (items.length === 1) {
+      return this.db.put(this.encodeKey(items[0].key), items[0].value, this.dbReqOpts);
+    }
+
+    await this.db.batchPut(
+      Array.from({length: items.length}, (_, i) => ({
+        key: this.encodeKey(items[i].key),
+        value: items[i].value,
+      })),
+      this.dbReqOpts
+    );
+  }
+
+  async batchDelete(ids: I[]): Promise<void> {
+    if (ids.length === 1) {
+      return this.delete(ids[0]);
+    }
+
+    await this.db.batchDelete(
+      Array.from({length: ids.length}, (_, i) => this.encodeKey(ids[i])),
+      this.dbReqOpts
+    );
+  }
+
+  async batchAdd(values: T[]): Promise<void> {
+    // handle single value in batchPut
+    await this.batchPut(
+      Array.from({length: values.length}, (_, i) => ({
+        key: this.getId(values[i]),
+        value: values[i],
+      }))
+    );
+  }
+
+  async batchRemove(values: T[]): Promise<void> {
+    // handle single value in batchDelete
+    await this.batchDelete(Array.from({length: values.length}, (_ignored, i) => this.getId(values[i])));
+  }
+
+  async keys(opts?: FilterOptions<I>): Promise<I[]> {
+    const data = await this.db.keys(this.dbFilterOptions(opts));
+    return (data ?? []).map((data) => this.decodeKey(data));
+  }
+
+  async *keysStream(opts?: FilterOptions<I>): AsyncIterable<I> {
+    const keysStream = this.db.keysStream(this.dbFilterOptions(opts));
+    const decodeKey = this.decodeKey.bind(this);
+    for await (const key of keysStream) {
+      yield decodeKey(key);
+    }
+  }
+
+  async values(opts?: FilterOptions<I>): Promise<T[]> {
+    const data = await this.db.values(this.dbFilterOptions(opts));
+    return (data ?? []).map((data) => this.decodeValue(data));
+  }
+
+  async *valuesStream(opts?: FilterOptions<I>): AsyncIterable<T> {
+    const valuesStream = this.db.valuesStream(this.dbFilterOptions(opts));
+    const decodeValue = this.decodeValue.bind(this);
+    for await (const value of valuesStream) {
+      yield decodeValue(value);
+    }
+  }
+
+  async *binaryEntriesStream(opts?: FilterOptions<I>): AsyncIterable<KeyValue<Uint8Array, Uint8Array>> {
+    yield* this.db.entriesStream(this.dbFilterOptions(opts));
+  }
+
+  async entries(opts?: FilterOptions<I>): Promise<KeyValue<I, T>[]> {
+    const data = await this.db.entries(this.dbFilterOptions(opts));
+    return (data ?? []).map((data) => ({
+      key: this.decodeKey(data.key),
+      value: this.decodeValue(data.value),
+    }));
+  }
+
+  async *entriesStream(opts?: FilterOptions<I>): AsyncIterable<KeyValue<I, T>> {
+    const entriesStream = this.db.entriesStream(this.dbFilterOptions(opts));
+    const decodeKey = this.decodeKey.bind(this);
+    const decodeValue = this.decodeValue.bind(this);
+    for await (const entry of entriesStream) {
+      yield {
+        key: decodeKey(entry.key),
+        value: decodeValue(entry.value),
+      };
+    }
+  }
+
+  async firstKey(): Promise<I | null> {
+    // Metrics accounted in this.keys()
+    const keys = await this.keys({limit: 1, bucketId: this.bucketId});
+    if (!keys.length) {
+      return null;
+    }
+    return keys[0];
+  }
+
+  async lastKey(): Promise<I | null> {
+    // Metrics accounted in this.keys()
+    const keys = await this.keys({limit: 1, reverse: true, bucketId: this.bucketId});
+    if (!keys.length) {
+      return null;
+    }
+    return keys[0];
+  }
+
+  async firstValue(): Promise<T | null> {
+    // Metrics accounted in this.values()
+    const values = await this.values({limit: 1, bucketId: this.bucketId});
+    if (!values.length) {
+      return null;
+    }
+    return values[0];
+  }
+
+  async lastValue(): Promise<T | null> {
+    // Metrics accounted in this.values()
+    const values = await this.values({limit: 1, reverse: true, bucketId: this.bucketId});
+    if (!values.length) {
+      return null;
+    }
+    return values[0];
+  }
+
+  async firstEntry(): Promise<KeyValue<I, T> | null> {
+    // Metrics accounted in this.entries()
+    const entries = await this.entries({limit: 1, bucketId: this.bucketId});
+    if (!entries.length) {
+      return null;
+    }
+    return entries[0];
+  }
+
+  async lastEntry(): Promise<KeyValue<I, T> | null> {
+    // Metrics accounted in this.entries()
+    const entries = await this.entries({limit: 1, reverse: true, bucketId: this.bucketId});
+    if (!entries.length) {
+      return null;
+    }
+    return entries[0];
+  }
+
+  /**
+   * Transforms opts from I to Uint8Array
+   */
+  protected dbFilterOptions(opts?: FilterOptions<I>): FilterOptions<Uint8Array> {
+    const optsBuff: FilterOptions<Uint8Array> = {
+      bucketId: this.bucketId,
+    };
+
+    // Set at least one min key
+    if (opts?.lt !== undefined) {
+      optsBuff.lt = this.encodeKey(opts.lt);
+    } else if (opts?.lte !== undefined) {
+      optsBuff.lte = this.encodeKey(opts.lte);
+    } else {
+      optsBuff.lt = this.maxKey;
+    }
+
+    // Set at least on max key
+    if (opts?.gt !== undefined) {
+      optsBuff.gt = this.encodeKey(opts.gt);
+    } else if (opts?.gte !== undefined) {
+      optsBuff.gte = this.encodeKey(opts.gte);
+    } else {
+      optsBuff.gte = this.minKey;
+    }
+
+    if (opts?.reverse !== undefined) optsBuff.reverse = opts.reverse;
+    if (opts?.limit !== undefined) optsBuff.limit = opts.limit;
+
+    return optsBuff;
+  }
+}

package/src/const.ts

@@ -0,0 +1,2 @@
+export const BUCKET_LENGTH = 1;
+export const DB_PREFIX_LENGTH = BUCKET_LENGTH;

package/src/controller/index.ts

@@ -0,0 +1,2 @@
+export type {DatabaseController, Db, DbReqOpts, FilterOptions, KeyValue} from "./interface.js";
+export type {LevelDbControllerMetrics} from "./metrics.js";

package/src/controller/interface.ts

@@ -0,0 +1,62 @@
+import {LevelDbControllerMetrics} from "./metrics.js";
+
+/** Shortcut for Uint8Array based DatabaseController */
+export type Db = DatabaseController<Uint8Array, Uint8Array>;
+
+export type DatabaseOptions = {
+  name: string;
+};
+
+export interface FilterOptions<K> {
+  gt?: K;
+  gte?: K;
+  lt?: K;
+  lte?: K;
+  reverse?: boolean;
+  limit?: number;
+  /** For metrics */
+  bucketId?: string;
+}
+
+export type DbReqOpts = {
+  /** For metrics */
+  bucketId?: string;
+};
+
+export interface KeyValue<K, V> {
+  key: K;
+  value: V;
+}
+
+export interface DatabaseController<K, V> {
+  // service start / stop
+
+  close(): Promise<void>;
+
+  /** To inject metrics after CLI initialization */
+  setMetrics(metrics: LevelDbControllerMetrics): void;
+
+  // Core API
+
+  get(key: K, opts?: DbReqOpts): Promise<V | null>;
+  getMany(key: K[], opts?: DbReqOpts): Promise<(V | undefined)[]>;
+
+  put(key: K, value: V, opts?: DbReqOpts): Promise<void>;
+  delete(key: K, opts?: DbReqOpts): Promise<void>;
+
+  // Batch operations
+
+  batchPut(items: KeyValue<K, V>[], opts?: DbReqOpts): Promise<void>;
+  batchDelete(keys: K[], opts?: DbReqOpts): Promise<void>;
+
+  // Iterate over entries
+
+  keysStream(opts?: FilterOptions<K>): AsyncIterable<K>;
+  keys(opts?: FilterOptions<K>): Promise<K[]>;
+
+  valuesStream(opts?: FilterOptions<K>): AsyncIterable<V>;
+  values(opts?: FilterOptions<K>): Promise<V[]>;
+
+  entriesStream(opts?: FilterOptions<K>): AsyncIterable<KeyValue<K, V>>;
+  entries(opts?: FilterOptions<K>): Promise<KeyValue<K, V>[]>;
+}