@thi.ng/column-store 0.1.1 → 0.2.0

package/README.md

@@ -17,6 +17,8 @@
 - [About](#about)
 - [Column storage](#column-storage)
   - [Column types](#column-types)
+  - [Vector column types](#vector-column-types)
+  - [Serialization options](#serialization-options)
   - [Custom column types](#custom-column-types)
   - [Cardinality](#cardinality)
   - [Default values](#default-values)
@@ -49,6 +51,60 @@ In-memory column store database with customizable column types, extensible query
 
 ## Column storage
 
+As the name indicates, data is stored in different columns, where each column
+manages its own schema, value validation, backing storage, indexing,
+serialization etc.
+
+From a user's POV, columns are not (usually) used directly, but via table, which
+acts as facade to the various configured columns. Data items are added as JS
+objects to a table, which then pulls out related values, validates them and the
+delegates them to the columns.
+
+An example table definition looks like this (explanation of column types in next
+section below):
+
+```ts
+import { Table, FLAG_DICT, FLAG_UNIQUE } from "@thi.ng/column-store";
+
+// define a table with the given columns
+const table = new Table({
+
+    // column of single numeric values
+    // (default cardinality makes values required)
+    id: { type: "num" },
+
+    // column of required string values
+    name: { type: "str" },
+
+    // optional tuples of max. 3 string values
+    // (if no value is given, the column stores null)
+    aliases: { type: "str", cardinality: [0, 3] },
+
+    // required fixed size tuples (aka vectors) of numbers
+    latlon: { type: "num", cardinality: [2, 2] },
+
+    // optional tuples of max. 10 strings, with default
+    // the given flags (explained further below) are triggering:
+    // - dictionary-based encoding
+    // - unique values (per tuple)
+    tags: {
+        type: "str",
+        cardinality: [0, 10],
+        default: ["todo"],
+        flags: FLAG_DICT | FLAG_UNIQUE
+    }
+});
+
+// add data
+table.addRow({
+    id: 1,
+    name: "karsten",
+    aliases: ["toxi"],
+    latlon: [47.421, 10.984],
+    tags: ["person", "opensource"],
+});
+```
+
 ### Column types
 
 The current built-in column types only support numeric or string values, though
@@ -62,8 +118,8 @@ Note: Booleans and `BigInt`s are still unsupported, but being worked on...
 
 | **Column type** | **Description**     | **Tuples supported** | **RLE serialization** |
 |-----------------|---------------------|----------------------|-----------------------|
-| `num`           | JS numbers          | ✅                    | ✅ <sup>(1)</sup>       |
-| `str`           | JS strings (UTF-16) | ✅                    | ✅ <sup>(1)</sup>       |
+| `num`           | JS numbers          | ✅                    | ✅ <sup>(1)</sup>      |
+| `str`           | JS strings (UTF-16) | ✅                    | ✅ <sup>(1)</sup>      |
 | `u8`            | 8bit unsigned int   | ❌                    | ✅                     |
 | `i8`            | 8bit signed int     | ❌                    | ✅                     |
 | `u16`           | 16bit unsigned int  | ❌                    | ✅                     |
@@ -73,7 +129,56 @@ Note: Booleans and `BigInt`s are still unsupported, but being worked on...
 | `f32`           | 32bit float         | ❌                    | ❌                     |
 | `f64`           | 64bit float         | ❌                    | ❌                     |
 
-- <sup>(1)</sup> only if `FLAG_DICT` is enabled, [further information](#flag_rle)
+- <sup>(1)</sup> only if max. cardinality is 1, [further information](#flag_rle)
+
+### Vector column types
+
+Columns storing fixed size n-dimensional vectors can be created vis the `vec`
+suffix for any of the typedarray based column types, i.e. `u8vec`, `i16vec`,
+`f32vec` etc.
+
+The `cardinality` column config (in the form `[min,max]`) is interpreted as follows:
+
+- if `min` is zero, the value is optional, but a `default` value MUST be defined
+  for the column. Otherwise the `min` value MUST be the same as `max`.
+- `max` defines the actual vector size
+
+Therefore, using a 3D vector as example, the only two possible `cardinality`
+configs are: `[0,3]` (with default given) or `[3,3]`.
+
+When [querying](#query-engine) vector columns using the standard
+`(n)or`/`(n)and` operators, always the entire vector is matched (by value).
+
+> [!IMPORTANT] For performance reasons, rows retrieved from vector columns
+> contain mutable data views of the underlying column storage. That means when
+> manipulating data in these views, the underlying data in the column would be
+> changed too. To avoid index corruption, always edit only copies of this vector
+> data and then use `table.updateRow()` to properly update the column storage
+> (incl. any internal indexes).
+
+### Serialization options
+
+For `f32`, `f64`, `f32vec` and `f64vec` column types, the optional `prec` column
+option can be provided to specify the number of fractional digits used in
+the JSON serialization:
+
+```ts tangle:export/readme-serialize-prec.ts
+import { Table } from "@thi.ng/column-store";
+
+const table = new Table({
+    vec: { type: "f32vec", cardinality: [3, 3], opts: { prec: 2 } }
+});
+
+table.addRow({ vec: [1.11111, 22.22222, 333.33333]});
+
+console.log(JSON.stringify(table));
+// {
+//   "schema":{"vec":{"cardinality":[3,3],"flags":0,"type":"f32vec","opts":{"prec":2}}},
+//   "columns":{"vec":{"values":[1.11,22.22,333.33]}},
+//   "length":1
+// }
+
+```
 
 ### Custom column types
 
@@ -159,15 +264,18 @@ Note: Not supported by typedarray-backed column types.
 
 (Value: 0x08)
 
-This flag enables bitwise [Run-length encoding](https://thi.ng/rle-pack) in the
+This flag enables [Run-length encoding](https://thi.ng/rle-pack) in the
 JSON serialization of a column, potentially leading to dramatic file size
 savings, esp. for dictionary-based data.
 
-Only applicable to these column types & configurations:
+Two modes of RLE compression are supported, depending on column type:
+
+- Simple RLE merely stores arrays in `[value1, count1, value2, count2...]` form
+- Binary RLE uses more advanced bitwise encoding, but is only available for
+  typedarray and vector columns
 
-- typedarray-based integer columns (see [table](#column-types))
-- dictionary-based single value columns (if the min. cardinality is zero, a
-  default value **must** be supplied)
+Tuple-based columns do not support RLE and will throw an error at creation time
+when trying to use this flag.
 
 #### Custom flags
 
@@ -298,7 +406,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.11 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.65 KB
 
 ## Dependencies
 

package/api.d.ts

@@ -1,15 +1,54 @@
-import type { FloatType, Fn3, IntType, Maybe, TypedArray, UintType } from "@thi.ng/api";
+import type { FloatType, Fn3, IntType, Maybe, UintType } from "@thi.ng/api";
 import type { BitmapIndex } from "./bitmap.js";
 import type { QueryCtx } from "./query.js";
 import type { Table } from "./table.js";
 export type ColumnSchema = Record<string, ColumnSpec>;
 export type NumericType = IntType | UintType | FloatType;
+export type VectorType = `${NumericType}vec`;
 export type Cardinality = [number, number];
 export interface ColumnSpec {
-    type: NumericType | "num" | "str" | string;
+    /**
+     * Column type ID (see readme for overview)
+     */
+    type: NumericType | VectorType | "num" | "str" | string;
+    /**
+     * `[min,max]` number of allowed values per row. The following cardinality
+     * presets are available in general (but not for all column types, see
+     * readme for overview):
+     *
+     * - {@link REQUIRED}: [1,1] (default) — value is required
+     * - {@link OPTIONAL}: [0,1] — value is optional
+     * - {@link ZERO_PLUS}: [0, (2**32)-1] — zero or more values
+     * - {@link ONE_PLUS}: [1, (2**32)-1] — one or more values
+     *
+     * Note: Some column types always require a value. So when using
+     * {@link OPTIONAL}, you might also need to provide a
+     * {@link ColumnSpec.default} value.
+     */
     cardinality: Cardinality;
+    /**
+     * Bit mask to control column behavior/encoding. The lowest 16 bits are
+     * reserved for built-in column types and internal use. The upper 16 bits
+     * are freely usable for custom purposes.
+     *
+     * @remarks
+     * The following built-in flags are available in general (but not for all
+     * column types, see readme for overview):
+     *
+     * - {@link FLAG_BITMAP}: Enable bitmap indexing
+     * - {@link FLAG_DICT}: Enable dictionary encoding of values
+     * - {@link FLAG_UNIQUE}: Enable Set-semantics, i.e. unique values per tuple
+     * - {@link FLAG_RLE}: Enable run-length encoding in serialization
+     */
     flags: number;
+    /**
+     * Default value
+     */
     default?: any;
+    /**
+     * Columntype specific options (e.g. for serialization)
+     */
+    opts?: Record<string, any>;
 }
 export interface ColumnTypeSpec {
     /**
@@ -42,8 +81,9 @@ export declare const FLAG_DICT: number;
 export declare const FLAG_BITMAP: number;
 export declare const FLAG_UNIQUE: number;
 export declare const FLAG_RLE: number;
+/** @internal */
+export declare const LIMITS: Record<NumericType, [number, number]>;
 export interface IColumn {
-    values: any[] | TypedArray;
     bitmap?: BitmapIndex;
     readonly isArray: boolean;
     load(spec: SerializedColumn): void;
@@ -65,6 +105,8 @@ export interface IColumn {
     encode(value: any): any;
     decode(value: any): any;
     replaceValue(currValue: any, newValue: any): boolean;
+    getRowKey(i: number): any;
+    valueKey(value: any): any;
 }
 export interface SerializedTable {
     schema: ColumnSchema;

package/api.js

@@ -6,11 +6,23 @@ const FLAG_DICT = 1 << 0;
 const FLAG_BITMAP = 1 << 1;
 const FLAG_UNIQUE = 1 << 2;
 const FLAG_RLE = 1 << 3;
+const LIMITS = {
+  u8: [0, 255],
+  u8c: [0, 255],
+  u16: [0, 65535],
+  u32: [0, 4294967295],
+  i8: [-128, 127],
+  i16: [-32768, 32767],
+  i32: [-2147483648, 2147483647],
+  f32: [-Infinity, Infinity],
+  f64: [-Infinity, Infinity]
+};
 export {
   FLAG_BITMAP,
   FLAG_DICT,
   FLAG_RLE,
   FLAG_UNIQUE,
+  LIMITS,
   ONE_PLUS,
   OPTIONAL,
   REQUIRED,

package/columns/acolumn.d.ts

@@ -1,18 +1,28 @@
 import type { BidirIndex } from "@thi.ng/bidir-index";
-import { type ColumnSpec, type SerializedIndex } from "../api.js";
+import { type ColumnSpec, type IColumn, type SerializedColumn, type SerializedIndex } from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
 import type { Table } from "../table.js";
-export declare abstract class AColumn {
+export declare abstract class AColumn implements IColumn {
     readonly id: string;
     table: Table;
     spec: ColumnSpec;
     bitmap?: BitmapIndex;
     dict?: BidirIndex<any>;
+    abstract isArray: boolean;
     constructor(id: string, table: Table);
+    abstract load(spec: SerializedColumn): void;
+    reindex(): void;
+    abstract validate(value: any): boolean;
+    abstract setRow(i: number, value: any): void;
+    abstract removeRow(i: number): void;
+    abstract replaceValue(currValue: any, newValue: any): boolean;
+    abstract valueKey(x: any): any;
+    abstract getRow(i: number): any;
+    abstract getRowKey(i: number): any;
     encode(value: any): any;
     decode(value: any): any;
     protected loadDict(serialized: SerializedIndex): void;
-    protected updateBitmap(rows: ArrayLike<any>): void;
+    protected updateBitmap(): void;
     protected ensureValue(val: any): any;
     protected ensureBitmap(): void;
 }

package/columns/acolumn.js

@@ -1,6 +1,7 @@
-import { isArray } from "@thi.ng/checks";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
-import { FLAG_BITMAP } from "../api.js";
+import {
+  FLAG_BITMAP
+} from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
 class AColumn {
   constructor(id, table) {
@@ -12,6 +13,9 @@ class AColumn {
   spec;
   bitmap;
   dict;
+  reindex() {
+    this.updateBitmap();
+  }
   encode(value) {
     return value;
   }
@@ -31,16 +35,17 @@ class AColumn {
     }
     dict.nextID = serialized.next;
   }
-  updateBitmap(rows) {
+  updateBitmap() {
     this.ensureBitmap();
-    const { bitmap } = this;
+    const { bitmap, isArray } = this;
     if (!bitmap) return;
     bitmap.clear();
-    for (let i = 0; i < rows.length; i++) {
-      const value = rows[i];
-      if (isArray(value)) {
+    for (let i = 0, n = this.table.length; i < n; i++) {
+      const value = this.getRow(i);
+      if (value == null) continue;
+      if (isArray) {
         for (let x of value) bitmap.setBit(x, i);
-      } else bitmap.setBit(value, i);
+      } else bitmap.setBit(this.getRowKey(i), i);
     }
   }
   ensureValue(val) {

package/columns/{dict-array.d.ts → dict-tuple.d.ts}

@@ -1,7 +1,7 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
 import { type IColumn, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class DictArrayColumn extends AColumn implements IColumn {
+export declare class DictTupleColumn extends AColumn implements IColumn {
     values: (number[] | null)[];
     dict: BidirIndex<any>;
     readonly isArray = true;
@@ -12,6 +12,8 @@ export declare class DictArrayColumn extends AColumn implements IColumn {
     validate(value: any): boolean;
     setRow(i: number, value: any[]): void;
     getRow(i: number): any[] | null;
+    getRowKey(i: number): number[] | null;
+    valueKey(value: any): (number | null)[];
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
@@ -22,4 +24,4 @@ export declare class DictArrayColumn extends AColumn implements IColumn {
         values: (number[] | null)[];
     };
 }
-//# sourceMappingURL=dict-array.d.ts.map
+//# sourceMappingURL=dict-tuple.d.ts.map

package/columns/{dict-array.js → dict-tuple.js}

@@ -4,14 +4,14 @@ import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-class DictArrayColumn extends AColumn {
+class DictTupleColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = true;
   load({ dict, values }) {
     this.values = values;
     super.loadDict(dict);
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   reindex() {
     const dict = this.dict;
@@ -20,7 +20,7 @@ class DictArrayColumn extends AColumn {
       (ids) => ids ? newDict.addAll(dict.getAllIDs(ids)) : null
     );
     this.dict = newDict;
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   encode(value) {
     return this.dict.getAll(isArray(value) ? value : [value], false, true);
@@ -45,6 +45,12 @@ class DictArrayColumn extends AColumn {
     const values = this.values[i];
     return values != null ? this.dict.getAllIDs(values) : null;
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    return this.encode(value);
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
@@ -75,5 +81,5 @@ class DictArrayColumn extends AColumn {
   }
 }
 export {
-  DictArrayColumn
+  DictTupleColumn
 };

package/columns/dict.d.ts

@@ -12,6 +12,8 @@ export declare class DictColumn extends AColumn implements IColumn {
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): any;
+    getRowKey(i: number): number | null;
+    valueKey(value: any): number | number[] | undefined;
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/dict.js

@@ -1,17 +1,19 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
-import { decode as decodeRLE, encode as encodeRLE } from "@thi.ng/rle-pack";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
+import { isArray } from "@thi.ng/checks/is-array";
 class DictColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = false;
   load({ dict, values }) {
-    this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeRLE(values)) : values;
+    this.values = this.spec.flags & FLAG_RLE ? this.spec.cardinality[0] === 0 && this.spec.default == null ? decodeSimple(values) : Array.from(decodeBinary(values)) : values;
     super.loadDict(dict);
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   reindex() {
     const dict = this.dict;
@@ -20,7 +22,7 @@ class DictColumn extends AColumn {
       (x) => x != null ? newDict.add(dict.getID(x)) : null
     );
     this.dict = newDict;
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   encode(value) {
     return this.dict.get(value);
@@ -45,6 +47,12 @@ class DictColumn extends AColumn {
     const value = this.values[i];
     return value != null ? this.dict.getID(value) : null;
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    return isArray(value) ? this.dict.getAll(value) : this.dict.get(value);
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
@@ -67,12 +75,19 @@ class DictColumn extends AColumn {
     return true;
   }
   toJSON() {
-    let values = this.values;
-    if (this.spec.flags & FLAG_RLE) {
-      const numBits = Math.max(1, Math.ceil(Math.log2(this.dict.size)));
-      values = Array.from(
-        encodeRLE(values, values.length, numBits)
-      );
+    let { values, spec } = this;
+    if (spec.flags & FLAG_RLE) {
+      if (spec.cardinality[0] == 0 && spec.default == null) {
+        values = encodeSimple(values);
+      } else {
+        const numBits = Math.max(
+          1,
+          Math.ceil(Math.log2(this.dict.size))
+        );
+        values = Array.from(
+          encodeBinary(values, values.length, numBits)
+        );
+      }
     }
     return { dict: __serializeDict(this.dict), values };
   }

package/columns/plain.d.ts

@@ -1,13 +1,14 @@
-import type { IColumn, SerializedColumn } from "../api.js";
+import { type IColumn, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
 export declare class PlainColumn extends AColumn implements IColumn {
     values: any[];
     readonly isArray = false;
-    load(spec: SerializedColumn): void;
-    reindex(): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): any;
+    getRowKey(i: number): any;
+    valueKey(x: any): any;
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/plain.js

@@ -1,16 +1,15 @@
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
+import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
 import { __replaceValue } from "../internal/replace.js";
 import { AColumn } from "./acolumn.js";
 class PlainColumn extends AColumn {
   values = [];
   isArray = false;
-  load(spec) {
-    this.values = spec.values;
+  load({ values }) {
+    this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeSimple(values)) : values;
     this.reindex();
   }
-  reindex() {
-    super.updateBitmap(this.values);
-  }
   validate(value) {
     return __validateValue(this.spec, value);
   }
@@ -26,6 +25,12 @@ class PlainColumn extends AColumn {
   getRow(i) {
     return this.values[i];
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(x) {
+    return x;
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
@@ -34,7 +39,9 @@ class PlainColumn extends AColumn {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }
   toJSON() {
-    return { values: this.values };
+    return {
+      values: this.spec.flags & FLAG_RLE ? encodeSimple(this.values) : this.values
+    };
   }
 }
 export {

package/columns/{array.d.ts → tuple.d.ts}

@@ -1,19 +1,20 @@
 import type { Nullable } from "@thi.ng/api";
 import { type IColumn, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class ArrayColumn extends AColumn implements IColumn {
+export declare class TupleColumn extends AColumn implements IColumn {
     values: Nullable<number[]>[];
     readonly isArray = true;
     load(spec: SerializedColumn): void;
-    reindex(): void;
     encode(value: any): any[];
     validate(value: any): boolean;
     setRow(i: number, value: any[]): void;
     getRow(i: number): Nullable<number[]>;
+    getRowKey(i: number): Nullable<number[]>;
+    valueKey(value: any): any[];
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         values: Nullable<number[]>[];
     };
 }
-//# sourceMappingURL=array.d.ts.map
+//# sourceMappingURL=tuple.d.ts.map

package/columns/{array.js → tuple.js}

@@ -2,16 +2,13 @@ import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
 import { AColumn } from "./acolumn.js";
-class ArrayColumn extends AColumn {
+class TupleColumn extends AColumn {
   values = [];
   isArray = true;
   load(spec) {
     this.values = spec.values;
     this.reindex();
   }
-  reindex() {
-    super.updateBitmap(this.values);
-  }
   encode(value) {
     return isArray(value) ? value : [value];
   }
@@ -31,6 +28,12 @@ class ArrayColumn extends AColumn {
   getRow(i) {
     return this.values[i];
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    return this.encode(value);
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
@@ -58,5 +61,5 @@ class ArrayColumn extends AColumn {
   }
 }
 export {
-  ArrayColumn
+  TupleColumn
 };

package/columns/typedarray.d.ts

@@ -6,13 +6,15 @@ export declare class TypedArrayColumn extends AColumn implements IColumn {
     values: TypedArray;
     type: NumericType;
     limit: [number, number];
+    protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: string, table: Table);
     load(spec: SerializedColumn): void;
-    reindex(): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): number;
+    getRowKey(i: number): number;
+    valueKey(value: any): number | number[];
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/typedarray.js

@@ -1,45 +1,35 @@
 import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
 import { isNumber } from "@thi.ng/checks/is-number";
-import { decode as decodeRLE, encode as encodeRLE } from "@thi.ng/rle-pack";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import {
-  FLAG_RLE
+  FLAG_RLE,
+  LIMITS
 } from "../api.js";
 import { __replaceValue } from "../internal/replace.js";
 import { AColumn } from "./acolumn.js";
-const LIMITS = {
-  u8: [0, 255],
-  u8c: [0, 255],
-  u16: [0, 65535],
-  u32: [0, 4294967295],
-  i8: [-128, 127],
-  i16: [-32768, 32767],
-  i32: [-2147483648, 2147483647],
-  f32: [-Infinity, Infinity],
-  f64: [-Infinity, Infinity]
-};
+import { isArray } from "@thi.ng/checks/is-array";
 class TypedArrayColumn extends AColumn {
   values;
   type;
   limit;
+  tmp;
   isArray = false;
   constructor(id, table) {
     super(id, table);
     this.type = table.schema[id].type;
     this.limit = LIMITS[this.type];
     this.values = typedArray(this.type, 8);
+    this.tmp = typedArray(this.type, 1);
   }
   load(spec) {
     if (this.spec.flags & FLAG_RLE) {
-      const values = decodeRLE(spec.values);
+      const values = decodeBinary(spec.values);
       this.values = typedArray(this.type, values.buffer);
     } else {
       this.values = typedArray(this.type, spec.values);
     }
     this.reindex();
   }
-  reindex() {
-    super.updateBitmap(this.values.subarray(0, this.table.length));
-  }
   validate(value) {
     return isNumber(value) && value >= this.limit[0] && value <= this.limit[1] || value == null && this.spec.default != null;
   }
@@ -63,6 +53,21 @@ class TypedArrayColumn extends AColumn {
   getRow(i) {
     return this.values[i];
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    const { tmp } = this;
+    if (isArray(value)) {
+      return value.map((x) => {
+        tmp[0] = x;
+        return tmp[0];
+      });
+    } else {
+      tmp[0] = value;
+      return tmp[0];
+    }
+  }
   removeRow(i) {
     this.values.copyWithin(i, i + 1, this.table.length);
     this.values[this.table.length - 1] = 0;
@@ -74,7 +79,7 @@ class TypedArrayColumn extends AColumn {
   toJSON() {
     let values = this.values.subarray(0, this.table.length);
     if (this.spec.flags & FLAG_RLE) {
-      values = encodeRLE(values, values.length, SIZEOF[this.type] * 8);
+      values = encodeBinary(values, values.length, SIZEOF[this.type] * 8);
     }
     return { values: Array.from(values) };
   }

package/columns/vector.d.ts

@@ -0,0 +1,25 @@
+import { type TypedArray } from "@thi.ng/api/typedarray";
+import { type IColumn, type NumericType, type SerializedColumn } from "../api.js";
+import type { Table } from "../table.js";
+import { AColumn } from "./acolumn.js";
+export declare class VectorColumn extends AColumn implements IColumn {
+    values: TypedArray;
+    type: NumericType;
+    size: number;
+    limit: [number, number];
+    protected tmp: TypedArray;
+    readonly isArray = false;
+    constructor(id: string, table: Table);
+    load(spec: SerializedColumn): void;
+    validate(value: any): boolean;
+    setRow(i: number, value: any): void;
+    getRow(i: number): Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRowKey(i: number): string;
+    valueKey(value: any): string | string[];
+    removeRow(i: number): void;
+    replaceValue(): boolean;
+    toJSON(): {
+        values: any[];
+    };
+}
+//# sourceMappingURL=vector.d.ts.map

package/columns/vector.js

@@ -0,0 +1,105 @@
+import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { isArrayLike } from "@thi.ng/checks/is-arraylike";
+import { isNumber } from "@thi.ng/checks/is-number";
+import { unsupportedOp } from "@thi.ng/errors/unsupported";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
+import {
+  FLAG_RLE,
+  LIMITS
+} from "../api.js";
+import { AColumn } from "./acolumn.js";
+import { isArray } from "@thi.ng/checks/is-array";
+class VectorColumn extends AColumn {
+  values;
+  type;
+  size;
+  limit;
+  tmp;
+  isArray = false;
+  constructor(id, table) {
+    super(id, table);
+    this.type = this.spec.type.split("v")[0];
+    this.size = this.spec.cardinality[1];
+    this.limit = LIMITS[this.type];
+    this.values = typedArray(this.type, 8 * this.size);
+    this.tmp = typedArray(this.type, this.size);
+  }
+  load(spec) {
+    if (this.spec.flags & FLAG_RLE) {
+      const values = decodeBinary(spec.values);
+      this.values = typedArray(this.type, values.buffer);
+    } else {
+      this.values = typedArray(this.type, spec.values);
+    }
+    this.reindex();
+  }
+  validate(value) {
+    return isArrayLike(value) && value.length == this.size || value == null && this.spec.default != null;
+  }
+  setRow(i, value) {
+    value = this.ensureValue(value);
+    const j = i * this.size;
+    let len = this.values.length;
+    if (j >= len) {
+      while (j >= len) len <<= 1;
+      const tmp = typedArray(this.type, len);
+      tmp.set(this.values);
+      this.values = tmp;
+    }
+    const { values, bitmap } = this;
+    if (bitmap) {
+      bitmap.clearBit(this.getRowKey(i), i);
+      bitmap.setBit(this.valueKey(value), i);
+    }
+    values.set(value, j);
+  }
+  getRow(i) {
+    const { size } = this;
+    i *= size;
+    return this.values.subarray(i, i + size);
+  }
+  getRowKey(i) {
+    return this.getRow(i).join("|");
+  }
+  valueKey(value) {
+    const { tmp } = this;
+    if (isArray(value) && !isNumber(value[0])) {
+      return value.map((x) => {
+        tmp.set(x);
+        return tmp.join("|");
+      });
+    } else {
+      tmp.set(value);
+      return tmp.join("|");
+    }
+  }
+  removeRow(i) {
+    const {
+      size,
+      table: { length }
+    } = this;
+    this.values.copyWithin(i, i + size, length * size);
+    this.values.fill(0, (length - 1) * size);
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue() {
+    unsupportedOp("TODO");
+  }
+  toJSON() {
+    let $values = this.values.subarray(0, this.table.length * this.size);
+    if (this.spec.flags & FLAG_RLE) {
+      $values = encodeBinary(
+        $values,
+        $values.length,
+        SIZEOF[this.type] * 8
+      );
+    }
+    let values = Array.from($values);
+    const prec = this.spec.opts?.prec;
+    if (prec != null) values = values.map((x) => +x.toFixed(prec));
+    return { values };
+  }
+}
+export {
+  VectorColumn
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.1.1",
+	"version": "0.2.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -44,7 +44,7 @@
 		"@thi.ng/bidir-index": "^1.5.0",
 		"@thi.ng/checks": "^3.8.4",
 		"@thi.ng/errors": "^2.6.3",
-		"@thi.ng/rle-pack": "^3.1.118"
+		"@thi.ng/rle-pack": "^3.2.0"
 	},
 	"devDependencies": {
 		"esbuild": "^0.27.2",
@@ -97,11 +97,8 @@
 		"./columns/acolumn": {
 			"default": "./columns/acolumn.js"
 		},
-		"./columns/array": {
-			"default": "./columns/array.js"
-		},
-		"./columns/dict-array": {
-			"default": "./columns/dict-array.js"
+		"./columns/dict-tuple": {
+			"default": "./columns/dict-tuple.js"
 		},
 		"./columns/dict": {
 			"default": "./columns/dict.js"
@@ -109,9 +106,15 @@
 		"./columns/plain": {
 			"default": "./columns/plain.js"
 		},
+		"./columns/tuple": {
+			"default": "./columns/tuple.js"
+		},
 		"./columns/typedarray": {
 			"default": "./columns/typedarray.js"
 		},
+		"./columns/vector": {
+			"default": "./columns/vector.js"
+		},
 		"./query": {
 			"default": "./query.js"
 		},
@@ -123,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "28440e6d826cce964928f4c2ffe1f0e3e2e25ab0\n"
+	"gitHead": "b90a2f41eb0b3c89391bbb7cfff940192f23a83c\n"
 }

package/query.d.ts

@@ -26,7 +26,27 @@ export declare class QueryCtx {
     bitmap?: Uint32Array;
     constructor(query: Query);
     makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
+    /**
+     * Combines the `mask` with the context's mask (combined using bitwise AND).
+     * If the context mask is still undefined, assigns `mask` as the initial
+     * context mask.
+     *
+     * @param mask
+     */
     mergeMask(mask: Uint32Array): void;
+    /**
+     * Combines the bitwise inverted `mask` with the context's mask (combined
+     * using bitwise AND). If the context mask is still undefined, first inverts
+     * `mask` in place and uses result as the initial context mask.
+     *
+     * @param mask
+     */
+    mergeInvMask(mask: Uint32Array): void;
+    /**
+     * Bitwise inverts `mask` in place and then returns it.
+     *
+     * @param mask
+     */
     invertMask(mask: Uint32Array): Uint32Array<ArrayBufferLike>;
 }
 /**

package/query.js

@@ -5,10 +5,11 @@ import { Bitfield } from "./bitmap.js";
 class Query {
   constructor(table, terms = []) {
     this.table = table;
-    this.terms = terms;
+    for (let term of terms) this.addTerm(term);
   }
   terms = [];
   addTerm(term) {
+    if (!QUERY_OPS[term.type]) unsupportedOp(`query type: ${term.type}`);
     this.terms.push(term);
     return this;
   }
@@ -53,7 +54,6 @@ class Query {
     const ctx = new QueryCtx(this);
     for (let term of this.terms) {
       const op = QUERY_OPS[term.type];
-      if (!op) unsupportedOp(`query type: ${term.type}`);
       let column;
       if (term.column) {
         column = ctx.table.columns[term.column];
@@ -85,12 +85,37 @@ class QueryCtx {
     if (seed) mask.set(seed);
     return mask;
   }
+  /**
+   * Combines the `mask` with the context's mask (combined using bitwise AND).
+   * If the context mask is still undefined, assigns `mask` as the initial
+   * context mask.
+   *
+   * @param mask
+   */
   mergeMask(mask) {
     if (this.bitmap) {
       for (let i = 0, n = this.size; i < n; i++)
         this.bitmap[i] &= mask[i];
     } else this.bitmap = mask;
   }
+  /**
+   * Combines the bitwise inverted `mask` with the context's mask (combined
+   * using bitwise AND). If the context mask is still undefined, first inverts
+   * `mask` in place and uses result as the initial context mask.
+   *
+   * @param mask
+   */
+  mergeInvMask(mask) {
+    if (this.bitmap) {
+      for (let i = 0, n = this.size; i < n; i++)
+        this.bitmap[i] &= mask[i] ^ -1;
+    } else this.bitmap = this.invertMask(mask);
+  }
+  /**
+   * Bitwise inverts `mask` in place and then returns it.
+   *
+   * @param mask
+   */
   invertMask(mask) {
     for (let i = 0, n = this.size; i < n; i++) mask[i] ^= -1;
     return mask;
@@ -98,42 +123,39 @@ class QueryCtx {
 }
 const execBitOr = (ctx, term, column) => {
   const bitmap = column.bitmap;
-  const value = column.encode(term.value);
+  const key = column.valueKey(term.value);
   let mask;
-  if (isArray(value)) {
-    for (let v of value) {
-      const b = bitmap.index.get(v)?.buffer;
+  if (isArray(key)) {
+    for (let k of key) {
+      const b = bitmap.index.get(k)?.buffer;
       if (!b) continue;
       if (mask) {
         for (let i = 0; i < b.length; i++) mask[i] |= b[i];
       } else mask = ctx.makeMask(b);
     }
   } else {
-    const b = bitmap.index.get(value)?.buffer;
+    const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
   if (mask) {
-    if (term.type === "nor") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   }
 };
 const execOr = (ctx, term, column) => {
   const n = ctx.table.length;
-  const encoded = column.encode(term.value);
-  const values = column.values;
-  const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
+  const key = column.valueKey(term.value);
+  const pred = column.isArray ? (row, k) => row.includes(k) : (row, k) => row === k;
   let mask;
-  for (let v of isArray(encoded) ? encoded : [encoded]) {
+  for (let k of isArray(key) ? key : [key]) {
     for (let i = 0; i < n; i++) {
-      if (pred(values[i], v)) {
+      if (pred(column.getRowKey(i), k)) {
         if (!mask) mask = ctx.makeMask();
         mask[i >>> 5] |= 1 << (i & 31);
       }
     }
   }
   if (mask) {
-    if (term.type === "nor") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   }
 };
 const delegateOr = (ctx, term, column) => {
@@ -145,12 +167,12 @@ const delegateOr = (ctx, term, column) => {
 };
 const execBitAnd = (ctx, term, column) => {
   const bitmap = column.bitmap;
-  const value = column.encode(term.value);
+  const key = column.valueKey(term.value);
   let mask;
-  if (isArray(value)) {
+  if (isArray(key)) {
     const colBitmaps = [];
-    for (let v of value) {
-      const b = bitmap.index.get(v)?.buffer;
+    for (let k of key) {
+      const b = bitmap.index.get(k)?.buffer;
       if (!b) {
         if (term.type === "and") ctx.bitmap = void 0;
         return;
@@ -164,26 +186,24 @@ const execBitAnd = (ctx, term, column) => {
       } else mask = ctx.makeMask(b);
     }
   } else {
-    const b = bitmap.index.get(value)?.buffer;
+    const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
   if (mask) {
-    if (term.type === "nand") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   } else {
     ctx.bitmap = void 0;
   }
 };
 const execAnd = (ctx, term, column) => {
   const n = ctx.table.length;
-  const encoded = column.encode(term.value) ?? null;
-  const values = column.values;
+  const key = column.valueKey(term.value) ?? null;
   const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
   let mask;
-  for (let v of isArray(encoded) ? encoded : [encoded]) {
+  for (let k of isArray(key) ? key : [key]) {
     let m;
     for (let i = 0; i < n; i++) {
-      if (pred(values[i], v)) {
+      if (pred(column.getRowKey(i), k)) {
         if (!m) m = ctx.makeMask();
         m[i >>> 5] |= 1 << (i & 31);
       }
@@ -198,8 +218,7 @@ const execAnd = (ctx, term, column) => {
     }
   }
   if (mask) {
-    if (term.type === "nand") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   }
 };
 const delegateAnd = (ctx, term, column) => {
@@ -216,11 +235,10 @@ const QUERY_OPS = {
   nand: { fn: delegateAnd },
   matchCol: {
     fn: (ctx, term, column) => {
-      const values = column.values;
       const pred = term.value;
       let mask;
       for (let i = 0, n = ctx.table.length; i < n; i++) {
-        if (pred(column.decode(values[i]))) {
+        if (pred(column.getRow(i))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
         }

package/table.js

@@ -6,11 +6,12 @@ import {
   FLAG_RLE,
   FLAG_UNIQUE
 } from "./api.js";
-import { ArrayColumn } from "./columns/array.js";
-import { DictArrayColumn } from "./columns/dict-array.js";
+import { DictTupleColumn } from "./columns/dict-tuple.js";
 import { DictColumn } from "./columns/dict.js";
 import { PlainColumn } from "./columns/plain.js";
+import { TupleColumn } from "./columns/tuple.js";
 import { TypedArrayColumn } from "./columns/typedarray.js";
+import { VectorColumn } from "./columns/vector.js";
 import { __columnError } from "./internal/checks.js";
 import { Query } from "./query.js";
 class Table {
@@ -140,18 +141,25 @@ const $typed = {
 };
 const $float = { ...$typed, flags: FLAG_BITMAP };
 const $untyped = {
-  impl: (table, id, { flags, cardinality: [min, max], default: d }) => {
+  impl: (table, id, { flags, cardinality: [_, max] }) => {
     const isDict = flags & FLAG_DICT;
-    if (flags & FLAG_RLE) {
-      if (!isDict || max > 1 || min === 0 && d == null) {
-        __columnError(id, `RLE encoding not supported`);
-      }
-    }
-    return max > 1 ? new (isDict ? DictArrayColumn : ArrayColumn)(id, table) : new (isDict ? DictColumn : PlainColumn)(id, table);
+    if (flags & FLAG_RLE && max > 1) __columnError(id, `RLE not supported`);
+    return max > 1 ? new (isDict ? DictTupleColumn : TupleColumn)(id, table) : new (isDict ? DictColumn : PlainColumn)(id, table);
   },
   flags: FLAG_BITMAP | FLAG_DICT | FLAG_UNIQUE | FLAG_RLE,
   cardinality: [0, -1 >>> 0]
 };
+const $vec = {
+  impl: (table, id, { cardinality: [min, max] }) => {
+    if (min > 0 && min !== max)
+      __columnError(id, `only fixed size vectors supported`);
+    return new VectorColumn(id, table);
+  },
+  flags: FLAG_BITMAP | FLAG_RLE,
+  cardinality: [0, -1 >>> 0],
+  required: true
+};
+const $fvec = { ...$vec, flags: FLAG_BITMAP };
 const COLUMN_TYPES = {
   u8: $typed,
   i8: $typed,
@@ -162,7 +170,11 @@ const COLUMN_TYPES = {
   f32: $float,
   f64: $float,
   num: $untyped,
-  str: $untyped
+  str: $untyped,
+  u8vec: $vec,
+  u16vec: $vec,
+  u32vec: $vec,
+  f32vec: $fvec
 };
 const registerColumnType = (type, spec) => {
   if (COLUMN_TYPES[type])