@rangedb/js 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,168 @@
+export type Compression = number;
+/**
+ * @typedef {Object} RangeDBOptions
+ * @property {number} [firstReadSize] Specify how much should be read from the file on first call.
+ * If known, it can be set to size of header + index. It will save an additional request.
+ */
+/**
+ * @typedef {Object} Range
+ * @property {bigint} [start]
+ * @property {bigint} [end]
+ */
+/**
+ * @enum {number}
+ */
+export const Compression: Readonly<{
+    none: 0;
+    gzip: 1;
+    brotli: 2;
+}>;
+export type ContentType = number;
+/**
+ * @enum {number}
+ */
+export const ContentType: Readonly<{
+    unknown: 0;
+    json: 1;
+    xml: 2;
+}>;
+/**
+ * @typedef {Object} Header
+ * @property {number} specVersion
+ * @property {bigint} metadataOffset
+ * @property {number} metadataLength
+ * @property {bigint} indexOffset
+ * @property {number} indexLength
+ * @property {bigint} dataOffset
+ * @property {bigint} dataLength
+ * @property {Compression} compression
+ * @property {ContentType} contentType
+ */
+/**
+ * @typedef {string | number | boolean | null | JSONObject | JSONArray} JSONValue
+ */
+/**
+ * @typedef {Array<JSONValue>} JSONArray
+ */
+/**
+ * @typedef {{ [key: string]: JSONValue }} JSONObject
+ */
+export class RangeDB {
+    /**
+     * Traverse chunk consisting of mulitple key/value pairs and returns value
+     * for given key or null if not founded
+     * @private
+     * @param {ArrayBuffer} chunk
+     * @param {bigint} key
+     *
+     * @returns {ArrayBuffer | null}
+     */
+    private static findInChunk;
+    /**
+     * Binary search in index for a given key and return value
+     * @param {bigint} key
+     * @param {BigUint64Array} index
+     * @param {bigint} dataEndOffset one behind data ends
+     *
+     * @return {Range | null} Offset of data
+     */
+    static binarySearch(key: bigint, index: BigUint64Array, dataEndOffset: bigint): Range | null;
+    /**
+     * Initialize database by providing url of rangedb file.
+     *
+     * @param {string} url
+     * @param {RangeDBOptions} options
+     */
+    constructor(url: string, options?: RangeDBOptions);
+    /** @protected @type {string}  */
+    protected url: string;
+    /** @protected @type {string| null}  */
+    protected etag: string | null;
+    /** @private @type {Header | null} */
+    private header;
+    /** @private @type {BigUint64Array | null}  */
+    private index;
+    /** @private @type {JSONObject| JSONArray | null}  */
+    private metadata;
+    /** @private @type {number}  */
+    private firstReadSize;
+    /**
+     * Invalidate header and index.
+     *
+     * @returns {void}
+     */
+    invalidate(): void;
+    /**
+     * Perform HTTP range request.
+     *
+     * @protected
+     * @param {bigint} start
+     * @param {bigint} end
+     * @returns {Promise<ArrayBuffer>}
+     */
+    protected readRange(start: bigint, end: bigint): Promise<ArrayBuffer>;
+    /**
+     * Get header from database or return cached.
+     *
+     * @returns {Promise<Header>}
+     */
+    getHeader(): Promise<Header>;
+    /**
+     * Load index from database or return cached
+     *
+     * @private
+     * @returns {Promise<number>}
+     */
+    private getIndex;
+    /**
+     * Get metadata from database or return cached
+     *
+     * @return {Promise<JSONObject | JSONArray>}
+     */
+    getMetadata(): Promise<JSONObject | JSONArray>;
+    /**
+     * Get a raw ArrayBuffer from database for given key or null if not exists
+     *
+     * @param {bigint} key
+     *
+     * @returns {Promise<ArrayBuffer | null>}
+     * */
+    getRaw(key: bigint): Promise<ArrayBuffer | null>;
+    /**
+     * Get a JSON from database for a given key or null if not exists
+     * It may throw JSON parsing error
+     *
+     * @param {bigint} key
+     *
+     * @returns {Promise<JSONValue | null>}
+     * @throws {SyntaxError}
+     */
+    getJson(key: bigint): Promise<JSONValue | null>;
+}
+export type RangeDBOptions = {
+    /**
+     * Specify how much should be read from the file on first call.
+     * If known, it can be set to size of header + index. It will save an additional request.
+     */
+    firstReadSize?: number;
+};
+export type Range = {
+    start?: bigint;
+    end?: bigint;
+};
+export type Header = {
+    specVersion: number;
+    metadataOffset: bigint;
+    metadataLength: number;
+    indexOffset: bigint;
+    indexLength: number;
+    dataOffset: bigint;
+    dataLength: bigint;
+    compression: Compression;
+    contentType: ContentType;
+};
+export type JSONValue = string | number | boolean | null | JSONObject | JSONArray;
+export type JSONArray = Array<JSONValue>;
+export type JSONObject = {
+    [key: string]: JSONValue;
+};

package/index.js

@@ -0,0 +1,332 @@
+// @ts-check
+
+/**
+ * @typedef {Object} RangeDBOptions
+ * @property {number} [firstReadSize] Specify how much should be read from the file on first call.
+ * If known, it can be set to size of header + index. It will save an additional request.
+ */
+
+/**
+ * @typedef {Object} Range
+ * @property {bigint} [start]
+ * @property {bigint} [end]
+ */
+
+/**
+ * @enum {number}
+ */
+export const Compression = Object.freeze({
+  none: 0,
+  gzip: 1,
+  brotli: 2,
+})
+
+/**
+ * @enum {number}
+ */
+export const ContentType = Object.freeze({
+  unknown: 0,
+  json: 1,
+  xml: 2,
+})
+
+/**
+ * @typedef {Object} Header
+ * @property {number} specVersion
+ * @property {bigint} metadataOffset
+ * @property {number} metadataLength
+ * @property {bigint} indexOffset
+ * @property {number} indexLength
+ * @property {bigint} dataOffset
+ * @property {bigint} dataLength
+ * @property {Compression} compression
+ * @property {ContentType} contentType
+ */
+
+/**
+ * @typedef {string | number | boolean | null | JSONObject | JSONArray} JSONValue
+ */
+
+/**
+ * @typedef {Array<JSONValue>} JSONArray
+ */
+
+/**
+ * @typedef {{ [key: string]: JSONValue }} JSONObject
+ */
+
+export class RangeDB {
+  /**
+   * Initialize database by providing url of rangedb file.
+   *
+   * @param {string} url
+   * @param {RangeDBOptions} options
+   */
+  constructor(url, options = {}) {
+    /** @protected @type {string}  */
+    this.url = url
+
+    /** @protected @type {string| null}  */
+    this.etag = null
+
+    /** @private @type {Header | null} */
+    this.header = null
+
+    /** @private @type {BigUint64Array | null}  */
+    this.index = null
+
+    /** @private @type {JSONObject| JSONArray | null}  */
+    this.metadata = null
+
+    /** @private @type {number}  */
+    this.firstReadSize = options.firstReadSize ?? 64 * 1024
+  }
+
+  /**
+   * Invalidate header and index.
+   *
+   * @returns {void}
+   */
+  invalidate() {
+    this.header = null
+    this.index = null
+  }
+
+  /**
+   * Perform HTTP range request.
+   *
+   * @protected
+   * @param {bigint} start
+   * @param {bigint} end
+   * @returns {Promise<ArrayBuffer>}
+   */
+  async readRange(start, end) {
+    const {headers, arrayBuffer} = await fetch(this.url, {
+      headers: {
+        range: `bytes=${start}-${end}`,
+      },
+    })
+
+    const etag = headers.get('etag')
+    if (this.etag && this.etag !== etag) {
+      this.invalidate()
+      throw new Error('Database file has changed based on ETag.')
+    }
+    this.etag = etag
+
+    return arrayBuffer()
+  }
+
+  /**
+   * Get header from database or return cached.
+   *
+   * @returns {Promise<Header>}
+   */
+  async getHeader() {
+    if (this.header) {
+      return this.header
+    }
+    const buffer = await this.readRange(0n, BigInt(this.firstReadSize) - 1n)
+    const view = new DataView(buffer)
+
+    const magicNumber =
+      view.getUint32(0, false) === 0x52616e67 && // Rang
+      view.getUint16(4, false) === 0x6544 && // eD
+      view.getUint8(6) === 0x42 // B
+
+    if (!magicNumber) {
+      throw new Error(
+        'Invalid Magic Number: Expected file starting with "RangeDB" or [0x52	0x61	0x6E	0x67	0x65	0x44	0x42]',
+      )
+    }
+
+    const specVersion = view.getUint8(7)
+    if (specVersion !== 1) {
+      throw new Error(`Unsupported spec version. Expected 1 got ${specVersion}`)
+    }
+    const metadataOffset = view.getBigUint64(8, true)
+    const metadataLength = view.getUint32(16, true)
+    const indexOffset = view.getBigUint64(20, true)
+    const indexLength = view.getUint32(28, true)
+    const dataOffset = view.getBigUint64(32, true)
+    const dataLength = view.getBigUint64(40, true)
+    const compression = view.getInt8(48)
+    const contentType = view.getInt8(49)
+
+    this.header = {
+      specVersion,
+      metadataOffset,
+      metadataLength,
+      indexOffset,
+      indexLength,
+      dataOffset,
+      dataLength,
+      compression,
+      contentType,
+    }
+    return this.header
+  }
+
+  /**
+   * Load index from database or return cached
+   *
+   * @private
+   * @returns {Promise<number>}
+   */
+  async getIndex() {
+    if (this.index) {
+      return this.index.length / 2
+    }
+
+    const { indexLength, indexOffset } = await this.getHeader()
+
+    const buffer = await this.readRange(
+      indexOffset,
+      indexOffset + BigInt(indexLength) - 1n,
+    )
+    const view = new DataView(buffer)
+
+    const indexType = view.getUint8(0)
+    if (indexType !== 1) {
+      throw new Error(`Unsuported index type: Expected 1 got ${indexType}`)
+    }
+    const count = view.getUint32(1, true)
+
+    this.index = new BigUint64Array(buffer, 1 + 4 + 3, count * 2)
+    return count
+  }
+
+  /**
+   * Get metadata from database or return cached
+   *
+   * @return {Promise<JSONObject | JSONArray>}
+   */
+  async getMetadata() {
+    if (this.metadata) {
+      return this.metadata
+    }
+
+    const { metadataOffset, metadataLength } = await this.getHeader()
+    const buffer = await this.readRange(
+      metadataOffset,
+      metadataOffset + BigInt(metadataLength) - 1n,
+    )
+    const text = new TextDecoder().decode(buffer)
+    this.metadata = JSON.parse(text)
+    return this.metadata
+  }
+
+  /**
+   * Traverse chunk consisting of mulitple key/value pairs and returns value
+   * for given key or null if not founded
+   * @private
+   * @param {ArrayBuffer} chunk
+   * @param {bigint} key
+   *
+   * @returns {ArrayBuffer | null}
+   */
+  static findInChunk(key, chunk) {
+    const view = new DataView(chunk)
+    const length = chunk.byteLength
+    let offset = 0
+
+    // Chunk format
+    // [Key: BigUint64][Data Length: UInt32][Data bytes]
+    while (offset < length) {
+      const recordKey = view.getBigUint64(offset, true)
+      if (key < recordKey) {
+        return null
+      }
+      offset += 8
+
+      const dataLength = view.getUint32(offset, true)
+      offset += 4
+      if (recordKey === key) {
+        return chunk.slice(offset, offset + dataLength)
+      }
+      offset += dataLength
+    }
+    return null
+  }
+
+  /**
+   * Binary search in index for a given key and return value
+   * @param {bigint} key
+   * @param {BigUint64Array} index
+   * @param {bigint} dataEndOffset one behind data ends
+   *
+   * @return {Range | null} Offset of data
+   */
+  static binarySearch(key, index, dataEndOffset) {
+    let low = 0,
+      high = (index.length >> 1) - 1,
+      blockIndex = -1
+
+    while (low <= high) {
+      const midPair = (low + high) >>> 1
+      const midIdx = midPair * 2
+      const midKey = index[midIdx]
+
+      if (midKey === key) {
+        blockIndex = midIdx
+        break
+      } else if (midKey < key) {
+        blockIndex = midIdx
+        low = midPair + 1
+      } else {
+        high = midPair - 1
+      }
+    }
+
+    if (blockIndex === -1) {
+      return null
+    }
+    const start = index[blockIndex + 1]
+    const end =
+      blockIndex + 2 < index.length ? index[blockIndex + 3] : dataEndOffset
+    return {
+      start,
+      end,
+    }
+  }
+
+  /**
+   * Get a raw ArrayBuffer from database for given key or null if not exists
+   *
+   * @param {bigint} key
+   *
+   * @returns {Promise<ArrayBuffer | null>}
+   * */
+  async getRaw(key) {
+    if (!this.index) {
+      await this.getIndex()
+    }
+    const { dataOffset, dataLength } = this.header
+    const range = RangeDB.binarySearch(key, this.index, dataOffset + dataLength)
+    if (!range) {
+      return null
+    }
+    const { start, end } = range
+    const chunkBuffer = await this.readRange(start, end - 1n)
+
+    return RangeDB.findInChunk(key, chunkBuffer)
+  }
+
+  /**
+   * Get a JSON from database for a given key or null if not exists
+   * It may throw JSON parsing error
+   *
+   * @param {bigint} key
+   *
+   * @returns {Promise<JSONValue | null>}
+   * @throws {SyntaxError}
+   */
+  async getJson(key) {
+    const buffer = await this.getRaw(key)
+    if (buffer === null) {
+      return null
+    }
+    const string = new TextDecoder().decode(buffer)
+    return JSON.parse(string)
+  }
+}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@rangedb/js",
+  "version": "1.0.0",
+  "type": "module",
+  "main": "./index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./index.js"
+    }
+  },
+  "scripts": {
+    "build:types": "npx -p typescript tsc",
+    "prepare": "npm run build:types",
+    "test": "node --test --experimental-test-coverage",
+    "test:watch": "node --test --watch"
+  },
+  "engines": {
+    "node": "^22.0.0 || >=23.6.0"
+  },
+  "files": [
+    "index.js",
+    "dist/index.d.ts"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "provenance": false
+  }
+}