@thi.ng/column-store 0.1.0

package/api.d.ts

@@ -0,0 +1,101 @@
+import type { FloatType, Fn3, IntType, Maybe, TypedArray, UintType } from "@thi.ng/api";
+import type { BitmapIndex } from "./bitmap.js";
+import type { QueryCtx } from "./query.js";
+import type { Table } from "./table.js";
+export type ColumnSchema = Record<string, ColumnSpec>;
+export type NumericType = IntType | UintType | FloatType;
+export type Cardinality = [number, number];
+export interface ColumnSpec {
+    type: NumericType | "num" | "str" | string;
+    cardinality: Cardinality;
+    flags: number;
+    default?: any;
+}
+export interface ColumnTypeSpec {
+    /**
+     * Factory function to instantiate a colum type for given table, column name
+     * & spec.
+     */
+    impl: Fn3<Table, string, ColumnSpec, IColumn>;
+    /**
+     * Bit mask of supported flags (default: 0, i.e. none allowed). During
+     * column validation, the (inverted) mask will be applied to the user
+     * defined flags assigned to a column and will throw an error if the result
+     * is non-zero.
+     */
+    flags?: number;
+    /**
+     * Supported `[min, max]` number of values per row.
+     */
+    cardinality: Cardinality;
+    /**
+     * Only used if min cardinality of this column type is zero, and the type
+     * requires at least a (non-null) default value.
+     */
+    required?: boolean;
+}
+export declare const REQUIRED: Cardinality;
+export declare const OPTIONAL: Cardinality;
+export declare const ONE_PLUS: Cardinality;
+export declare const ZERO_PLUS: Cardinality;
+export declare const FLAG_DICT: number;
+export declare const FLAG_BITMAP: number;
+export declare const FLAG_UNIQUE: number;
+export declare const FLAG_RLE: number;
+export interface IColumn {
+    values: any[] | TypedArray;
+    bitmap?: BitmapIndex;
+    readonly isArray: boolean;
+    load(spec: SerializedColumn): void;
+    reindex(): void;
+    validate(value: any): boolean;
+    getRow(i: number): any;
+    /**
+     * Sets the column's value at given row index. Assumes the value has been
+     * pre-validated (via {@link IColumn.validate}). If value is null-ish and
+     * the column has a configured default, it will be replaced with said value,
+     * otherwise an error will be thrown (if the column requires a value, but
+     * has no default).
+     *
+     * @param i
+     * @param value
+     */
+    setRow(i: number, value: any): void;
+    removeRow(i: number): void;
+    encode(value: any): any;
+    decode(value: any): any;
+    replaceValue(currValue: any, newValue: any): boolean;
+}
+export interface SerializedTable {
+    schema: ColumnSchema;
+    columns: Record<string, SerializedColumn>;
+    length: number;
+}
+export interface SerializedColumn {
+    dict?: SerializedIndex;
+    values: any[];
+}
+export interface SerializedIndex {
+    index: any[];
+    next: number;
+}
+export type Row = Record<string, any>;
+export interface QueryTerm {
+    type: string;
+    column?: string;
+    value: any;
+    params?: any;
+}
+export type QueryTermOp = Fn3<QueryCtx, QueryTerm, Maybe<IColumn>, void>;
+export interface QueryTermOpSpec {
+    /**
+     * Default mode is: "col";
+     */
+    mode?: "col" | "row";
+    /**
+     * Query op implementation. Unless {@link QueryTermOpSpec.mode} is `row`,
+     * the provided `column` arg is guaranteed to be defined.
+     */
+    fn: QueryTermOp;
+}
+//# sourceMappingURL=api.d.ts.map

package/api.js

@@ -0,0 +1,18 @@
+const REQUIRED = [1, 1];
+const OPTIONAL = [0, 1];
+const ONE_PLUS = [1, -1 >>> 0];
+const ZERO_PLUS = [0, -1 >>> 0];
+const FLAG_DICT = 1 << 0;
+const FLAG_BITMAP = 1 << 1;
+const FLAG_UNIQUE = 1 << 2;
+const FLAG_RLE = 1 << 3;
+export {
+  FLAG_BITMAP,
+  FLAG_DICT,
+  FLAG_RLE,
+  FLAG_UNIQUE,
+  ONE_PLUS,
+  OPTIONAL,
+  REQUIRED,
+  ZERO_PLUS
+};

package/bitmap.d.ts

@@ -0,0 +1,35 @@
+export declare class BitmapIndex {
+    index: Map<any, Bitfield>;
+    /**
+     * Returns iterator of row IDs (i.e. bit positions) which have been marked
+     * as used by given `key`. If `max` is given, only positions < `max` are
+     * considered.
+     *
+     * @param key
+     * @param max
+     */
+    rowIDs(key: any, max?: number): Generator<number, void, unknown>;
+    clear(): void;
+    ensure(key: any): Bitfield;
+    setBit(key: any, rowID: number): void;
+    clearBit(key: any, rowID: number): void;
+    /**
+     * Removes bit at given position in _all_ indices by clearing it and
+     * shifting all successive bits 1 bit to the right (towards the start).
+     * Essentially similar to `Array.splice()`.
+     *
+     * @param id
+     */
+    removeBit(rowID: number): void;
+    toJSON(): Record<string, number[]>;
+}
+export declare class Bitfield {
+    buffer?: Uint32Array | undefined;
+    constructor(buffer?: Uint32Array | undefined);
+    ones(max?: number): Generator<number, void, unknown>;
+    setBit(id: number): void;
+    clearBit(id: number): void;
+    ensure(size: number): Uint32Array<ArrayBufferLike>;
+    removeBit(id: number): void;
+}
+//# sourceMappingURL=bitmap.d.ts.map

package/bitmap.js

@@ -0,0 +1,104 @@
+class BitmapIndex {
+  index = /* @__PURE__ */ new Map();
+  /**
+   * Returns iterator of row IDs (i.e. bit positions) which have been marked
+   * as used by given `key`. If `max` is given, only positions < `max` are
+   * considered.
+   *
+   * @param key
+   * @param max
+   */
+  rowIDs(key, max = Infinity) {
+    return this.index.get(key)?.ones(max) ?? (function* () {
+    })();
+  }
+  clear() {
+    this.index.clear();
+  }
+  ensure(key) {
+    let bitmap = this.index.get(key);
+    if (!bitmap) this.index.set(key, bitmap = new Bitfield());
+    return bitmap;
+  }
+  setBit(key, rowID) {
+    this.ensure(key).setBit(rowID);
+  }
+  clearBit(key, rowID) {
+    this.index.get(key)?.clearBit(rowID);
+  }
+  /**
+   * Removes bit at given position in _all_ indices by clearing it and
+   * shifting all successive bits 1 bit to the right (towards the start).
+   * Essentially similar to `Array.splice()`.
+   *
+   * @param id
+   */
+  removeBit(rowID) {
+    for (let bitmap of this.index.values()) bitmap.removeBit(rowID);
+  }
+  toJSON() {
+    const res = {};
+    for (let [k, bits] of this.index) {
+      if (bits.buffer) res[k] = Array.from(bits.buffer);
+    }
+    return res;
+  }
+}
+class Bitfield {
+  constructor(buffer) {
+    this.buffer = buffer;
+  }
+  *ones(max = Infinity) {
+    const buf = this.buffer;
+    if (!buf) return;
+    for (let i = 0, n = buf.length; i < n; i++) {
+      let bits = buf[i];
+      while (bits) {
+        const lsb = bits & -bits;
+        const x = (i << 5) + (Math.clz32(lsb) ^ 31);
+        if (x >= max) return;
+        yield x;
+        bits ^= lsb;
+      }
+    }
+  }
+  setBit(id) {
+    const w = id >>> 5;
+    this.ensure(w)[w] |= 1 << (id & 31);
+  }
+  clearBit(id) {
+    const w = id >>> 5;
+    this.ensure(w)[w] &= ~(1 << (id & 31));
+  }
+  ensure(size) {
+    const b = this.buffer;
+    if (!b || size >= b.length) {
+      const tmp = new Uint32Array(size + 1);
+      if (b) tmp.set(b);
+      return this.buffer = tmp;
+    }
+    return b;
+  }
+  removeBit(id) {
+    const b = this.buffer;
+    if (!b) return;
+    const w = id >>> 5;
+    const bit = id & 31;
+    const end = b.length - 1;
+    if (bit === 0) {
+      b[w] >>>= 1;
+    } else {
+      const mask = (1 << bit) - 1;
+      b[w] = bit < 31 ? b[w] & mask | b[w] >>> 1 & ~mask : b[w] & mask;
+    }
+    if (w < end && b[w + 1] & 1) b[w] |= 1 << 31;
+    for (let i = w + 1; i <= end; i++) {
+      b[i] >>>= 1;
+      if (i < end && b[i + 1] & 1) b[i] |= 1 << 31;
+    }
+  }
+}
+export {
+  Bitfield,
+  BitmapIndex
+};

package/columns/acolumn.d.ts

@@ -0,0 +1,19 @@
+import type { BidirIndex } from "@thi.ng/bidir-index";
+import { type ColumnSpec, type SerializedIndex } from "../api.js";
+import { BitmapIndex } from "../bitmap.js";
+import type { Table } from "../table.js";
+export declare abstract class AColumn {
+    readonly id: string;
+    table: Table;
+    spec: ColumnSpec;
+    bitmap?: BitmapIndex;
+    dict?: BidirIndex<any>;
+    constructor(id: string, table: Table);
+    encode(value: any): any;
+    decode(value: any): any;
+    protected loadDict(serialized: SerializedIndex): void;
+    protected updateBitmap(rows: ArrayLike<any>): void;
+    protected ensureValue(val: any): any;
+    protected ensureBitmap(): void;
+}
+//# sourceMappingURL=acolumn.d.ts.map

package/columns/acolumn.js

@@ -0,0 +1,55 @@
+import { isArray } from "@thi.ng/checks";
+import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
+import { FLAG_BITMAP } from "../api.js";
+import { BitmapIndex } from "../bitmap.js";
+class AColumn {
+  constructor(id, table) {
+    this.id = id;
+    this.table = table;
+    this.spec = table.schema[id];
+    this.ensureBitmap();
+  }
+  spec;
+  bitmap;
+  dict;
+  encode(value) {
+    return value;
+  }
+  decode(value) {
+    return value;
+  }
+  loadDict(serialized) {
+    const { dict } = this;
+    if (!dict) return;
+    dict.clear();
+    const index = serialized.index;
+    for (let i = 0; i < index.length; i++) {
+      if (index[i] != null) {
+        dict.fwd.set(index[i], i);
+        dict.rev.set(i, index[i]);
+      }
+    }
+    dict.nextID = serialized.next;
+  }
+  updateBitmap(rows) {
+    this.ensureBitmap();
+    const { bitmap } = this;
+    if (!bitmap) return;
+    bitmap.clear();
+    for (let i = 0; i < rows.length; i++) {
+      const value = rows[i];
+      if (isArray(value)) {
+        for (let x of value) bitmap.setBit(x, i);
+      } else bitmap.setBit(value, i);
+    }
+  }
+  ensureValue(val) {
+    return val != null ? val : this.spec.cardinality[0] > 0 ? this.spec.default ?? illegalArgs(`missing value for column: ${this.id}`) : this.spec.default ?? null;
+  }
+  ensureBitmap() {
+    if (this.spec.flags & FLAG_BITMAP) this.bitmap = new BitmapIndex();
+  }
+}
+export {
+  AColumn
+};

package/columns/array.d.ts

@@ -0,0 +1,19 @@
+import type { Nullable } from "@thi.ng/api";
+import { type IColumn, type SerializedColumn } from "../api.js";
+import { AColumn } from "./acolumn.js";
+export declare class ArrayColumn extends AColumn implements IColumn {
+    values: Nullable<number[]>[];
+    readonly isArray = true;
+    load(spec: SerializedColumn): void;
+    reindex(): void;
+    encode(value: any): any[];
+    validate(value: any): boolean;
+    setRow(i: number, value: any[]): void;
+    getRow(i: number): Nullable<number[]>;
+    removeRow(i: number): void;
+    replaceValue(currValue: any, newValue: any): boolean;
+    toJSON(): {
+        values: Nullable<number[]>[];
+    };
+}
+//# sourceMappingURL=array.d.ts.map

package/columns/array.js

@@ -0,0 +1,62 @@
+import { isArray } from "@thi.ng/checks/is-array";
+import { FLAG_UNIQUE } from "../api.js";
+import { __validateArrayValue } from "../internal/checks.js";
+import { AColumn } from "./acolumn.js";
+class ArrayColumn extends AColumn {
+  values = [];
+  isArray = true;
+  load(spec) {
+    this.values = spec.values;
+    this.reindex();
+  }
+  reindex() {
+    super.updateBitmap(this.values);
+  }
+  encode(value) {
+    return isArray(value) ? value : [value];
+  }
+  validate(value) {
+    return __validateArrayValue(this.spec, value);
+  }
+  setRow(i, value) {
+    value = this.ensureValue(value);
+    const { values, bitmap } = this;
+    const old = values[i];
+    const row = values[i] = value != null ? this.table.schema[this.id].flags & FLAG_UNIQUE ? [...new Set(value)] : value : null;
+    if (bitmap) {
+      if (old) for (let x of old) bitmap.clearBit(x, i);
+      if (row) for (let x of row) bitmap.setBit(x, i);
+    }
+  }
+  getRow(i) {
+    return this.values[i];
+  }
+  removeRow(i) {
+    this.values.splice(i, 1);
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue(currValue, newValue) {
+    const { values, bitmap } = this;
+    const isUnique = this.spec.flags & FLAG_UNIQUE;
+    const bits = bitmap?.ensure(newValue);
+    bitmap?.index.delete(currValue);
+    let result = false;
+    for (let i = 0; i < values.length; i++) {
+      const row = values[i];
+      if (row?.includes(currValue)) {
+        let $values = row.map((x) => x === currValue ? newValue : x);
+        if (isUnique) $values = [...new Set($values)];
+        values[i] = $values;
+        bits?.setBit(i);
+        result = true;
+      }
+    }
+    return result;
+  }
+  toJSON() {
+    return { values: this.values };
+  }
+}
+export {
+  ArrayColumn
+};

package/columns/dict-array.d.ts

@@ -0,0 +1,25 @@
+import { BidirIndex } from "@thi.ng/bidir-index";
+import { type IColumn, type SerializedColumn } from "../api.js";
+import { AColumn } from "./acolumn.js";
+export declare class DictArrayColumn extends AColumn implements IColumn {
+    values: (number[] | null)[];
+    dict: BidirIndex<any>;
+    readonly isArray = true;
+    load({ dict, values }: SerializedColumn): void;
+    reindex(): void;
+    encode(value: any): (number | null)[];
+    decode(value: any[]): any[];
+    validate(value: any): boolean;
+    setRow(i: number, value: any[]): void;
+    getRow(i: number): any[] | null;
+    removeRow(i: number): void;
+    replaceValue(currValue: any, newValue: any): boolean;
+    toJSON(): {
+        dict: {
+            index: any;
+            next: number;
+        };
+        values: (number[] | null)[];
+    };
+}
+//# sourceMappingURL=dict-array.d.ts.map

package/columns/dict-array.js

@@ -0,0 +1,79 @@
+import { BidirIndex } from "@thi.ng/bidir-index";
+import { isArray } from "@thi.ng/checks/is-array";
+import { FLAG_UNIQUE } from "../api.js";
+import { __validateArrayValue } from "../internal/checks.js";
+import { __serializeDict } from "../internal/serialize.js";
+import { AColumn } from "./acolumn.js";
+class DictArrayColumn extends AColumn {
+  values = [];
+  dict = new BidirIndex();
+  isArray = true;
+  load({ dict, values }) {
+    this.values = values;
+    super.loadDict(dict);
+    super.updateBitmap(this.values);
+  }
+  reindex() {
+    const dict = this.dict;
+    const newDict = new BidirIndex();
+    this.values = this.values.map(
+      (ids) => ids ? newDict.addAll(dict.getAllIDs(ids)) : null
+    );
+    this.dict = newDict;
+    super.updateBitmap(this.values);
+  }
+  encode(value) {
+    return this.dict.getAll(isArray(value) ? value : [value], false, true);
+  }
+  decode(value) {
+    return this.dict.getAllIDs(value, false, true);
+  }
+  validate(value) {
+    return __validateArrayValue(this.spec, value);
+  }
+  setRow(i, value) {
+    value = this.ensureValue(value);
+    const { values, dict, bitmap } = this;
+    const old = values[i];
+    const encoded = values[i] = value != null ? this.table.schema[this.id].flags & FLAG_UNIQUE ? [...dict.addAllUnique(value)] : dict.addAll(value) : null;
+    if (bitmap) {
+      if (old) for (let x of old) bitmap.clearBit(x, i);
+      if (encoded) for (let x of encoded) bitmap.setBit(x, i);
+    }
+  }
+  getRow(i) {
+    const values = this.values[i];
+    return values != null ? this.dict.getAllIDs(values) : null;
+  }
+  removeRow(i) {
+    this.values.splice(i, 1);
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue(currValue, newValue) {
+    const { dict, values, bitmap } = this;
+    const res = dict.renameKey(currValue, newValue);
+    if (res === "ok") return true;
+    if (res === "missing") return false;
+    const currID = dict.get(currValue);
+    const newID = dict.get(newValue);
+    const isUnique = this.spec.flags & FLAG_UNIQUE;
+    const bits = bitmap?.ensure(newID);
+    bitmap?.index.delete(currID);
+    for (let i = 0; i < values.length; i++) {
+      const row = values[i];
+      if (row?.includes(currID)) {
+        let $values = row.map((x) => x === currID ? newID : x);
+        if (isUnique) $values = [...new Set($values)];
+        values[i] = $values;
+        bits?.setBit(i);
+      }
+    }
+    return true;
+  }
+  toJSON() {
+    return { dict: __serializeDict(this.dict), values: this.values };
+  }
+}
+export {
+  DictArrayColumn
+};

package/columns/dict.d.ts

@@ -0,0 +1,25 @@
+import { BidirIndex } from "@thi.ng/bidir-index";
+import { type IColumn, type SerializedColumn } from "../api.js";
+import { AColumn } from "./acolumn.js";
+export declare class DictColumn extends AColumn implements IColumn {
+    values: (number | null)[];
+    dict: BidirIndex<any>;
+    readonly isArray = false;
+    load({ dict, values }: SerializedColumn): void;
+    reindex(): void;
+    encode(value: any): number | undefined;
+    decode(value: any): any;
+    validate(value: any): boolean;
+    setRow(i: number, value: any): void;
+    getRow(i: number): any;
+    removeRow(i: number): void;
+    replaceValue(currValue: any, newValue: any): boolean;
+    toJSON(): {
+        dict: {
+            index: any;
+            next: number;
+        };
+        values: (number | null)[];
+    };
+}
+//# sourceMappingURL=dict.d.ts.map

package/columns/dict.js

@@ -0,0 +1,82 @@
+import { BidirIndex } from "@thi.ng/bidir-index";
+import { decode as decodeRLE, encode as encodeRLE } from "@thi.ng/rle-pack";
+import { FLAG_RLE } from "../api.js";
+import { __validateValue } from "../internal/checks.js";
+import { __serializeDict } from "../internal/serialize.js";
+import { AColumn } from "./acolumn.js";
+class DictColumn extends AColumn {
+  values = [];
+  dict = new BidirIndex();
+  isArray = false;
+  load({ dict, values }) {
+    this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeRLE(values)) : values;
+    super.loadDict(dict);
+    super.updateBitmap(this.values);
+  }
+  reindex() {
+    const dict = this.dict;
+    const newDict = new BidirIndex();
+    this.values = this.values.map(
+      (x) => x != null ? newDict.add(dict.getID(x)) : null
+    );
+    this.dict = newDict;
+    super.updateBitmap(this.values);
+  }
+  encode(value) {
+    return this.dict.get(value);
+  }
+  decode(value) {
+    return this.dict.getID(value);
+  }
+  validate(value) {
+    return __validateValue(this.spec, value);
+  }
+  setRow(i, value) {
+    value = this.ensureValue(value);
+    const { values, bitmap } = this;
+    const old = values[i];
+    const encoded = values[i] = value != null ? this.dict.add(value) : null;
+    if (bitmap) {
+      if (old != null) bitmap.clearBit(old, i);
+      if (encoded != null) bitmap?.setBit(encoded, i);
+    }
+  }
+  getRow(i) {
+    const value = this.values[i];
+    return value != null ? this.dict.getID(value) : null;
+  }
+  removeRow(i) {
+    this.values.splice(i, 1);
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue(currValue, newValue) {
+    const { dict, values, bitmap } = this;
+    const res = dict.renameKey(currValue, newValue);
+    if (res === "ok") return true;
+    if (res === "missing") return false;
+    const currID = dict.get(currValue);
+    const newID = dict.get(newValue);
+    const bits = bitmap?.ensure(newID);
+    bitmap?.index.delete(currID);
+    for (let i = 0; i < values.length; i++) {
+      if (values[i] === currID) {
+        values[i] = newID;
+        bits?.setBit(i);
+      }
+    }
+    return true;
+  }
+  toJSON() {
+    let values = this.values;
+    if (this.spec.flags & FLAG_RLE) {
+      const numBits = Math.max(1, Math.ceil(Math.log2(this.dict.size)));
+      values = Array.from(
+        encodeRLE(values, values.length, numBits)
+      );
+    }
+    return { dict: __serializeDict(this.dict), values };
+  }
+}
+export {
+  DictColumn
+};

package/columns/plain.d.ts

@@ -0,0 +1,17 @@
+import type { IColumn, SerializedColumn } from "../api.js";
+import { AColumn } from "./acolumn.js";
+export declare class PlainColumn extends AColumn implements IColumn {
+    values: any[];
+    readonly isArray = false;
+    load(spec: SerializedColumn): void;
+    reindex(): void;
+    validate(value: any): boolean;
+    setRow(i: number, value: any): void;
+    getRow(i: number): any;
+    removeRow(i: number): void;
+    replaceValue(currValue: any, newValue: any): boolean;
+    toJSON(): {
+        values: any[];
+    };
+}
+//# sourceMappingURL=plain.d.ts.map

package/columns/plain.js

@@ -0,0 +1,42 @@
+import { __validateValue } from "../internal/checks.js";
+import { __replaceValue } from "../internal/replace.js";
+import { AColumn } from "./acolumn.js";
+class PlainColumn extends AColumn {
+  values = [];
+  isArray = false;
+  load(spec) {
+    this.values = spec.values;
+    this.reindex();
+  }
+  reindex() {
+    super.updateBitmap(this.values);
+  }
+  validate(value) {
+    return __validateValue(this.spec, value);
+  }
+  setRow(i, value) {
+    const { values, bitmap } = this;
+    const old = values[i];
+    values[i] = this.ensureValue(value);
+    if (bitmap) {
+      if (old != null) bitmap.clearBit(old, i);
+      if (value != null) bitmap.setBit(value, i);
+    }
+  }
+  getRow(i) {
+    return this.values[i];
+  }
+  removeRow(i) {
+    this.values.splice(i, 1);
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue(currValue, newValue) {
+    return __replaceValue(this.bitmap, this.values, currValue, newValue);
+  }
+  toJSON() {
+    return { values: this.values };
+  }
+}
+export {
+  PlainColumn
+};