extra-disk-store 0.5.0 → 0.5.1

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "extra-disk-store",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "",
   "keywords": [],
   "files": [
-    "lib"
+    "lib",
+    "src"
   ],
   "type": "module",
   "main": "lib/index.js",

package/src/converters/index-key-converter.ts

@@ -0,0 +1,13 @@
+import { IKeyConverter } from '@src/types.js'
+
+export class IndexKeyConverter implements IKeyConverter<number> {
+  constructor(private radix: number = 10) {}
+
+  toString(value: number): string {
+    return value.toString(this.radix)
+  }
+
+  fromString(value: string): number {
+    return Number.parseInt(value, this.radix)
+  }
+}

package/src/converters/index.ts

@@ -0,0 +1,11 @@
+export * from './json-key-converter.js'
+export * from './json-value-converter.js'
+
+export * from './passthrough-key-converter.js'
+export * from './passthrough-value-converter.js'
+
+export * from './index-key-converter.js'
+export * from './prefix-key-converter.js'
+
+export * from './lz4-value-converter.js'
+export * from './zstandard-value-converter.js'

package/src/converters/json-key-converter.ts

@@ -0,0 +1,11 @@
+import { IKeyConverter } from '@src/types.js'
+
+export class JSONKeyConverter<T> implements IKeyConverter<T> {
+  fromString(value: string): T {
+    return JSON.parse(value)
+  }
+
+  toString(value: T): string {
+    return JSON.stringify(value)
+  }
+}

package/src/converters/json-value-converter.ts

@@ -0,0 +1,13 @@
+import { IValueConverter } from '@src/types.js'
+
+export class JSONValueConverter<T> implements IValueConverter<T> {
+  constructor(private encoding: BufferEncoding = 'utf-8') {}
+
+  fromBuffer(buffer: Buffer): T {
+    return JSON.parse(buffer.toString(this.encoding))
+  }
+
+  toBuffer(value: T): Buffer {
+    return Buffer.from(JSON.stringify(value), this.encoding)
+  }
+}

package/src/converters/lz4-value-converter.ts

@@ -0,0 +1,18 @@
+import * as lz4 from 'lz4-wasm-nodejs'
+import { IValueConverter } from '@src/types.js'
+
+export class LZ4ValueConverter<T> implements IValueConverter<T> {
+  constructor(
+    private valueConverter: IValueConverter<T>
+  ) {}
+
+  async toBuffer(value: T): Promise<Buffer> {
+    const buffer = await this.valueConverter.toBuffer(value)
+    return Buffer.from(lz4.compress(buffer))
+  }
+
+  async fromBuffer(value: Buffer): Promise<T> {
+    const buffer = Buffer.from(lz4.decompress(value))
+    return await this.valueConverter.fromBuffer(buffer)
+  }
+}

package/src/converters/passthrough-key-converter.ts

@@ -0,0 +1,11 @@
+import { IKeyConverter } from '@src/types.js'
+
+export class PassthroughKeyConverter implements IKeyConverter<string> {
+  toString(value: string): string {
+    return value
+  }
+
+  fromString(value: string): string {
+    return value
+  }
+}

package/src/converters/passthrough-value-converter.ts

@@ -0,0 +1,11 @@
+import { IValueConverter } from '@src/types.js'
+
+export class PassthroughValueConverter implements IValueConverter<Buffer> {
+  toBuffer(value: Buffer): Buffer {
+    return value
+  }
+
+  fromBuffer(value: Buffer): Buffer {
+    return value
+  }
+}

package/src/converters/prefix-key-converter.ts

@@ -0,0 +1,22 @@
+import { IKeyConverter } from '@src/types.js'
+
+export class PrefixKeyConverter<T> implements IKeyConverter<T> {
+  constructor(
+    private keyConverter: IKeyConverter<T>
+  , private prefix: string
+  ) {}
+
+  async toString(value: T): Promise<string> {
+    const key = await this.keyConverter.toString(value)
+    return this.prefix + key
+  }
+
+  async fromString(value: string): Promise<T | undefined> {
+    if (value.startsWith(this.prefix)) {
+      const key = await this.keyConverter.fromString(value.slice(this.prefix.length))
+      return key
+    } else {
+      return undefined
+    }
+  }
+}

package/src/converters/zstandard-value-converter.ts

@@ -0,0 +1,19 @@
+import * as zstd from '@mongodb-js/zstd'
+import { IValueConverter } from '@src/types.js'
+
+export class ZstandardValueConverter<T> implements IValueConverter<T> {
+  constructor(
+    private valueConverter: IValueConverter<T>
+  , private level: number
+  ) {}
+
+  async toBuffer(value: T): Promise<Buffer> {
+    const buffer = await this.valueConverter.toBuffer(value)
+    return Buffer.from(await zstd.compress(buffer, this.level))
+  }
+
+  async fromBuffer(value: Buffer): Promise<T> {
+    const buffer = Buffer.from(await zstd.decompress(value))
+    return await this.valueConverter.fromBuffer(buffer)
+  }
+}

package/src/disk-store-view.ts

@@ -0,0 +1,62 @@
+import { DiskStore } from '@src/disk-store.js'
+import { DiskStoreWithCache } from '@src/disk-store-with-cache.js'
+import { pipe } from 'extra-utils'
+import { mapAsync, filterAsync } from 'iterable-operator'
+import { Awaitable, isntUndefined } from '@blackglory/prelude'
+
+export interface IKeyConverter<T> {
+  toString: (value: T) => Awaitable<string>
+  fromString: (value: string) => Awaitable<T | undefined>
+}
+
+export interface IValueConverter<T> {
+  toBuffer: (value: T) => Awaitable<Buffer>
+  fromBuffer: (value: Buffer) => Awaitable<T>
+}
+
+export class DiskStoreView<K, V> {
+  constructor(
+    private store: DiskStore | DiskStoreWithCache
+  , private keyConverter: IKeyConverter<K>
+  , private valueConverter: IValueConverter<V>
+  ) {}
+
+  async has(key: K): Promise<boolean> {
+    return this.store.has(await this.keyConverter.toString(key))
+  }
+
+  async get(key: K): Promise<V | undefined> {
+    const buffer = this.store.get(await this.keyConverter.toString(key))
+
+    if (buffer) {
+      return this.valueConverter.fromBuffer(buffer)
+    } else {
+      return undefined
+    }
+  }
+
+  async set(key: K, value: V): Promise<void> {
+    await this.store.set(
+      ...await Promise.all([
+        await this.keyConverter.toString(key)
+      , await this.valueConverter.toBuffer(value)
+      ])
+    )
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.store.delete(await this.keyConverter.toString(key))
+  }
+
+  async clear(): Promise<void> {
+    await this.store.clear()
+  }
+
+  keys(): AsyncIterableIterator<K> {
+    return pipe(
+      this.store.keys()
+    , iter => mapAsync(iter, async key => await this.keyConverter.fromString(key))
+    , iter => filterAsync(iter, isntUndefined)
+    )
+  }
+}

package/src/disk-store-with-cache.ts

@@ -0,0 +1,78 @@
+import { isntUndefined } from '@blackglory/prelude'
+import { DiskStore } from './disk-store.js'
+
+export interface ICache {
+  set(key: string, value: Buffer | false): void
+  get(key: string): Buffer | false | undefined
+  delete(key: string): void
+  clear(): void
+}
+
+export class DiskStoreWithCache {
+  constructor(
+    private store: DiskStore
+  , private cache: ICache
+  ) {}
+
+  async close(): Promise<void> {
+    await this.store.close()
+  }
+
+  has(key: string): boolean {
+    const result = this.cache.get(key)
+    if (result === false) {
+      return result
+    } else if (isntUndefined(result)) {
+      return true
+    } else {
+      const result = this.store.get(key)
+      if (result) {
+        this.cache.set(key, result)
+        return true
+      } else {
+        this.cache.set(key, false)
+        return false
+      }
+    }
+  }
+
+  get(key: string): Buffer | undefined {
+    const result = this.cache.get(key)
+    if (result === false) {
+      return undefined
+    } else if (isntUndefined(result)) {
+      return result
+    } else {
+      const result = this.store.get(key)
+      if (result) {
+        this.cache.set(key, result)
+        return result
+      } else {
+        this.cache.set(key, false)
+        return result
+      }
+    }
+  }
+
+  async set(key: string, value: Buffer): Promise<void> {
+    await this.store.set(key, value)
+
+    this.cache.delete(key)
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.store.delete(key)
+
+    this.cache.delete(key)
+  }
+
+  async clear(): Promise<void> {
+    await this.store.clear()
+
+    this.cache.clear()
+  }
+
+  keys(): IterableIterator<string> {
+    return this.store.keys()
+  }
+}

package/src/disk-store.ts

@@ -0,0 +1,59 @@
+import * as Iter from 'iterable-operator'
+import * as LMDB from 'lmdb'
+import { createTempNameSync, remove } from 'extra-filesystem'
+import { isUndefined } from '@blackglory/prelude'
+
+export class DiskStore {
+  public _db: LMDB.RootDatabase
+  public _dirname: string
+  private isTempPathname: boolean
+
+  constructor(dirname?: string) {
+    if (isUndefined(dirname)) {
+      this._dirname = createTempNameSync()
+      this.isTempPathname = true
+    } else {
+      this._dirname = dirname
+      this.isTempPathname = false
+    }
+
+    this._db = LMDB.open<Buffer, string>(this._dirname, {
+      // 采用其他编码方式可能遇到错误.
+      // 尤其是采用与lmdb-js同作者的msgpackr一定会出现错误, 因为它是一个有问题的实现.
+      encoding: 'binary'
+    , compression: false
+    })
+  }
+
+  async close(): Promise<void> {
+    await this._db.close()
+
+    if (this.isTempPathname) {
+      await remove(this._dirname)
+    }
+  }
+
+  has(key: string): boolean {
+    return this._db.doesExist(key)
+  }
+
+  get(key: string): Buffer | undefined {
+    return this._db.getBinary(key)
+  }
+
+  async set(key: string, value: Buffer): Promise<void> {
+    await this._db.put(key, value)
+  }
+
+  async delete(key: string): Promise<void> {
+    await this._db.remove(key)
+  }
+
+  async clear(): Promise<void> {
+    await this._db.clearAsync()
+  }
+
+  keys(): IterableIterator<string> {
+    return Iter.map(this._db.getKeys(), key => key as string)
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './disk-store.js'
+export { DiskStoreWithCache, ICache } from './disk-store-with-cache.js'
+export { DiskStoreView, IKeyConverter, IValueConverter } from './disk-store-view.js'
+export * from './converters/index.js'

package/src/types.ts

@@ -0,0 +1,18 @@
+import { Awaitable } from '@blackglory/prelude'
+
+export interface ICache {
+  set(key: string, value: Buffer | boolean | undefined): void
+  get(key: string): Buffer | boolean | undefined
+  delete(key: string): void
+  clear(): void
+}
+
+export interface IKeyConverter<T> {
+  toString: (value: T) => Awaitable<string>
+  fromString: (value: string) => Awaitable<T | undefined>
+}
+
+export interface IValueConverter<T> {
+  toBuffer: (value: T) => Awaitable<Buffer>
+  fromBuffer: (value: Buffer) => Awaitable<T>
+}