@thi.ng/column-store 0.1.3 → 0.3.0

package/README.md

@@ -17,6 +17,8 @@
 - [About](#about)
 - [Column storage](#column-storage)
   - [Column types](#column-types)
+  - [Vector column types](#vector-column-types)
+  - [Serialization options](#serialization-options)
   - [Custom column types](#custom-column-types)
   - [Cardinality](#cardinality)
   - [Default values](#default-values)
@@ -61,7 +63,7 @@ delegates them to the columns.
 An example table definition looks like this (explanation of column types in next
 section below):
 
-```ts
+```ts tangle:export/readme-types.ts
 import { Table, FLAG_DICT, FLAG_UNIQUE } from "@thi.ng/column-store";
 
 // define a table with the given columns
@@ -79,7 +81,7 @@ const table = new Table({
     aliases: { type: "str", cardinality: [0, 3] },
 
     // required fixed size tuples (aka vectors) of numbers
-    latlon: { type: "num", cardinality: [2, 2] },
+    latlon: { type: "f32vec", cardinality: [2, 2] },
 
     // optional tuples of max. 10 strings, with default
     // the given flags (explained further below) are triggering:
@@ -103,6 +105,11 @@ table.addRow({
 });
 ```
 
+> [!IMPORTANT]
+> Columns can be named freely, with the exception that the `__` name prefix is
+> reserved for internal use. For example, `foo` is allowed, but `__foo` is a
+> reserved name.
+
 ### Column types
 
 The current built-in column types only support numeric or string values, though
@@ -127,7 +134,57 @@ Note: Booleans and `BigInt`s are still unsupported, but being worked on...
 | `f32`           | 32bit float         | ❌                    | ❌                     |
 | `f64`           | 64bit float         | ❌                    | ❌                     |
 
-- <sup>(1)</sup> only if `FLAG_DICT` is enabled, [further information](#flag_rle)
+- <sup>(1)</sup> only if max. cardinality is 1, [further information](#flag_rle)
+
+### Vector column types
+
+Columns storing fixed size n-dimensional vectors can be created vis the `vec`
+suffix for any of the typedarray based column types, i.e. `u8vec`, `i16vec`,
+`f32vec` etc.
+
+The `cardinality` column config (in the form `[min,max]`) is interpreted as follows:
+
+- if `min` is zero, the value is optional, but a `default` value MUST be defined
+  for the column. Otherwise the `min` value MUST be the same as `max`.
+- `max` defines the actual vector size
+
+Therefore, using a 3D vector as example, the only two possible `cardinality`
+configs are: `[0,3]` (with default given) or `[3,3]`.
+
+When [querying](#query-engine) vector columns using the standard
+`(n)or`/`(n)and` operators, always the entire vector is matched (by value).
+
+> [!IMPORTANT]
+> For performance reasons, rows retrieved from vector columns contain mutable
+> data views of the underlying column storage. That means when manipulating data
+> in these views, the underlying data in the column would be changed too. To
+> avoid index corruption, always edit only copies of this vector data and then
+> use `table.updateRow()` to properly update the column storage (incl. any
+> internal indexes).
+
+### Serialization options
+
+For `f32`, `f64`, `f32vec` and `f64vec` column types, the optional `prec` column
+option can be provided to specify the number of fractional digits used in
+the JSON serialization:
+
+```ts tangle:export/readme-serialize-prec.ts
+import { Table } from "@thi.ng/column-store";
+
+const table = new Table({
+    vec: { type: "f32vec", cardinality: [3, 3], opts: { prec: 2 } }
+});
+
+table.addRow({ vec: [1.11111, 22.22222, 333.33333]});
+
+console.log(JSON.stringify(table));
+// {
+//   "schema":{"vec":{"cardinality":[3,3],"flags":0,"type":"f32vec","opts":{"prec":2}}},
+//   "columns":{"vec":{"values":[1.11,22.22,333.33]}},
+//   "length":1
+// }
+
+```
 
 ### Custom column types
 
@@ -213,15 +270,18 @@ Note: Not supported by typedarray-backed column types.
 
 (Value: 0x08)
 
-This flag enables bitwise [Run-length encoding](https://thi.ng/rle-pack) in the
+This flag enables [Run-length encoding](https://thi.ng/rle-pack) in the
 JSON serialization of a column, potentially leading to dramatic file size
 savings, esp. for dictionary-based data.
 
-Only applicable to these column types & configurations:
+Two modes of RLE compression are supported, depending on column type:
+
+- Simple RLE merely stores arrays in `[value1, count1, value2, count2...]` form
+- Binary RLE uses more advanced bitwise encoding, but is only available for
+  typedarray and vector columns
 
-- typedarray-based integer columns (see [table](#column-types))
-- dictionary-based single value columns (if the min. cardinality is zero, a
-  default value **must** be supplied)
+Tuple-based columns do not support RLE and will throw an error at creation time
+when trying to use this flag.
 
 #### Custom flags
 
@@ -256,8 +316,8 @@ TODO see code examples below
 
 The query engine works by applying a number of [query
 terms](https://docs.thi.ng/umbrella/column-store/interfaces/QueryTerm.html) in
-series, with each step intersecting its results with the results of the previous
-step(s), thereby narrowing down the result set.
+series, with each step intersecting (aka logical AND) its results with the
+results of the previous step(s), thereby narrowing down the result set.
 
 By default, individual query terms operate on a single column, but can also can
 also apply to multiple. Terms are supplied either as array given to the
@@ -352,7 +412,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.15 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.67 KB
 
 ## Dependencies
 
@@ -411,19 +471,20 @@ table.addRows([
 const unsortedImages = table.query().where("type", "img").and("tags", "unsorted");
 
 // queries are iterables and only execute when the iterator is consumed
+// each query result includes a `__row` ID
 console.log([...unsortedImages]);
-// [ { id: 102, type: "img", tags: [ "unsorted" ] } ]
+// [ { id: 102, type: "img", tags: [ "unsorted" ], __row: 2 } ]
 
 // select items with `a` OR `b` tags, intersect with those which have `c` AND `d` tags
 const complexTagQuery = table.query().or("tags", ["a", "b"]).and("tags", ["c", "d"]);
 console.log([...complexTagQuery]);
-// [ { id: 104, type: "img", tags: [ "b", "c", "d" ] } ]
+// [ { id: 104, type: "img", tags: [ "b", "c", "d" ], __row: 4 } ]
 
 // query using custom predicates
 console.log([...table.query().matchColumn("id", (id) => id > 102)]);
 // [
-//    { id: 103, type: "img", tags: [ "unsorted" ] },
-//    { id: 104, type: "img", tags: [ "b", "c", "d" ] }
+//    { id: 103, type: "img", tags: [ "unsorted" ], __row: 3 },
+//    { id: 104, type: "img", tags: [ "b", "c", "d" ], __row: 4 }
 // ]
 
 // serialize table to JSON

package/api.d.ts

@@ -1,15 +1,54 @@
-import type { FloatType, Fn3, IntType, Maybe, TypedArray, UintType } from "@thi.ng/api";
+import type { FloatType, Fn3, IntType, Maybe, UintType } from "@thi.ng/api";
 import type { BitmapIndex } from "./bitmap.js";
 import type { QueryCtx } from "./query.js";
 import type { Table } from "./table.js";
 export type ColumnSchema = Record<string, ColumnSpec>;
 export type NumericType = IntType | UintType | FloatType;
+export type VectorType = `${NumericType}vec`;
 export type Cardinality = [number, number];
 export interface ColumnSpec {
-    type: NumericType | "num" | "str" | string;
+    /**
+     * Column type ID (see readme for overview)
+     */
+    type: NumericType | VectorType | "num" | "str" | string;
+    /**
+     * `[min,max]` number of allowed values per row. The following cardinality
+     * presets are available in general (but not for all column types, see
+     * readme for overview):
+     *
+     * - {@link REQUIRED}: [1,1] (default) — value is required
+     * - {@link OPTIONAL}: [0,1] — value is optional
+     * - {@link ZERO_PLUS}: [0, (2**32)-1] — zero or more values
+     * - {@link ONE_PLUS}: [1, (2**32)-1] — one or more values
+     *
+     * Note: Some column types always require a value. So when using
+     * {@link OPTIONAL}, you might also need to provide a
+     * {@link ColumnSpec.default} value.
+     */
     cardinality: Cardinality;
+    /**
+     * Bit mask to control column behavior/encoding. The lowest 16 bits are
+     * reserved for built-in column types and internal use. The upper 16 bits
+     * are freely usable for custom purposes.
+     *
+     * @remarks
+     * The following built-in flags are available in general (but not for all
+     * column types, see readme for overview):
+     *
+     * - {@link FLAG_BITMAP}: Enable bitmap indexing
+     * - {@link FLAG_DICT}: Enable dictionary encoding of values
+     * - {@link FLAG_UNIQUE}: Enable Set-semantics, i.e. unique values per tuple
+     * - {@link FLAG_RLE}: Enable run-length encoding in serialization
+     */
     flags: number;
+    /**
+     * Default value
+     */
     default?: any;
+    /**
+     * Columntype specific options (e.g. for serialization)
+     */
+    opts?: Record<string, any>;
 }
 export interface ColumnTypeSpec {
     /**
@@ -42,8 +81,9 @@ export declare const FLAG_DICT: number;
 export declare const FLAG_BITMAP: number;
 export declare const FLAG_UNIQUE: number;
 export declare const FLAG_RLE: number;
+/** @internal */
+export declare const LIMITS: Record<NumericType, [number, number]>;
 export interface IColumn {
-    values: any[] | TypedArray;
     bitmap?: BitmapIndex;
     readonly isArray: boolean;
     load(spec: SerializedColumn): void;
@@ -65,6 +105,8 @@ export interface IColumn {
     encode(value: any): any;
     decode(value: any): any;
     replaceValue(currValue: any, newValue: any): boolean;
+    getRowKey(i: number): any;
+    valueKey(value: any): any;
 }
 export interface SerializedTable {
     schema: ColumnSchema;

package/api.js

@@ -6,11 +6,23 @@ const FLAG_DICT = 1 << 0;
 const FLAG_BITMAP = 1 << 1;
 const FLAG_UNIQUE = 1 << 2;
 const FLAG_RLE = 1 << 3;
+const LIMITS = {
+  u8: [0, 255],
+  u8c: [0, 255],
+  u16: [0, 65535],
+  u32: [0, 4294967295],
+  i8: [-128, 127],
+  i16: [-32768, 32767],
+  i32: [-2147483648, 2147483647],
+  f32: [-Infinity, Infinity],
+  f64: [-Infinity, Infinity]
+};
 export {
   FLAG_BITMAP,
   FLAG_DICT,
   FLAG_RLE,
   FLAG_UNIQUE,
+  LIMITS,
   ONE_PLUS,
   OPTIONAL,
   REQUIRED,

package/columns/acolumn.d.ts

@@ -1,8 +1,8 @@
 import type { BidirIndex } from "@thi.ng/bidir-index";
-import { type ColumnSpec, type SerializedIndex } from "../api.js";
+import { type ColumnSpec, type IColumn, type SerializedColumn, type SerializedIndex } from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
 import type { Table } from "../table.js";
-export declare abstract class AColumn {
+export declare abstract class AColumn implements IColumn {
     readonly id: string;
     table: Table;
     spec: ColumnSpec;
@@ -10,10 +10,19 @@ export declare abstract class AColumn {
     dict?: BidirIndex<any>;
     abstract isArray: boolean;
     constructor(id: string, table: Table);
+    abstract load(spec: SerializedColumn): void;
+    reindex(): void;
+    abstract validate(value: any): boolean;
+    abstract setRow(i: number, value: any): void;
+    abstract removeRow(i: number): void;
+    abstract replaceValue(currValue: any, newValue: any): boolean;
+    abstract valueKey(x: any): any;
+    abstract getRow(i: number): any;
+    abstract getRowKey(i: number): any;
     encode(value: any): any;
     decode(value: any): any;
     protected loadDict(serialized: SerializedIndex): void;
-    protected updateBitmap(rows: ArrayLike<any>): void;
+    protected updateBitmap(): void;
     protected ensureValue(val: any): any;
     protected ensureBitmap(): void;
 }

package/columns/acolumn.js

@@ -1,5 +1,7 @@
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
-import { FLAG_BITMAP } from "../api.js";
+import {
+  FLAG_BITMAP
+} from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
 class AColumn {
   constructor(id, table) {
@@ -11,6 +13,9 @@ class AColumn {
   spec;
   bitmap;
   dict;
+  reindex() {
+    this.updateBitmap();
+  }
   encode(value) {
     return value;
   }
@@ -30,17 +35,17 @@ class AColumn {
     }
     dict.nextID = serialized.next;
   }
-  updateBitmap(rows) {
+  updateBitmap() {
     this.ensureBitmap();
     const { bitmap, isArray } = this;
     if (!bitmap) return;
     bitmap.clear();
-    for (let i = 0; i < rows.length; i++) {
-      const value = rows[i];
+    for (let i = 0, n = this.table.length; i < n; i++) {
+      const value = this.getRow(i);
       if (value == null) continue;
       if (isArray) {
         for (let x of value) bitmap.setBit(x, i);
-      } else bitmap.setBit(value, i);
+      } else bitmap.setBit(this.getRowKey(i), i);
     }
   }
   ensureValue(val) {

package/columns/dict-tuple.d.ts

@@ -12,6 +12,8 @@ export declare class DictTupleColumn extends AColumn implements IColumn {
     validate(value: any): boolean;
     setRow(i: number, value: any[]): void;
     getRow(i: number): any[] | null;
+    getRowKey(i: number): number[] | null;
+    valueKey(value: any): (number | null)[];
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/dict-tuple.js

@@ -11,7 +11,7 @@ class DictTupleColumn extends AColumn {
   load({ dict, values }) {
     this.values = values;
     super.loadDict(dict);
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   reindex() {
     const dict = this.dict;
@@ -20,7 +20,7 @@ class DictTupleColumn extends AColumn {
       (ids) => ids ? newDict.addAll(dict.getAllIDs(ids)) : null
     );
     this.dict = newDict;
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   encode(value) {
     return this.dict.getAll(isArray(value) ? value : [value], false, true);
@@ -45,6 +45,12 @@ class DictTupleColumn extends AColumn {
     const values = this.values[i];
     return values != null ? this.dict.getAllIDs(values) : null;
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    return this.encode(value);
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);

package/columns/dict.d.ts

@@ -12,6 +12,8 @@ export declare class DictColumn extends AColumn implements IColumn {
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): any;
+    getRowKey(i: number): number | null;
+    valueKey(value: any): number | number[] | undefined;
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/dict.js

@@ -1,17 +1,19 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
-import { decode as decodeRLE, encode as encodeRLE } from "@thi.ng/rle-pack";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
+import { isArray } from "@thi.ng/checks/is-array";
 class DictColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = false;
   load({ dict, values }) {
-    this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeRLE(values)) : values;
+    this.values = this.spec.flags & FLAG_RLE ? this.spec.cardinality[0] === 0 && this.spec.default == null ? decodeSimple(values) : Array.from(decodeBinary(values)) : values;
     super.loadDict(dict);
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   reindex() {
     const dict = this.dict;
@@ -20,7 +22,7 @@ class DictColumn extends AColumn {
       (x) => x != null ? newDict.add(dict.getID(x)) : null
     );
     this.dict = newDict;
-    super.updateBitmap(this.values);
+    super.updateBitmap();
   }
   encode(value) {
     return this.dict.get(value);
@@ -45,6 +47,12 @@ class DictColumn extends AColumn {
     const value = this.values[i];
     return value != null ? this.dict.getID(value) : null;
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    return isArray(value) ? this.dict.getAll(value) : this.dict.get(value);
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
@@ -67,12 +75,19 @@ class DictColumn extends AColumn {
     return true;
   }
   toJSON() {
-    let values = this.values;
-    if (this.spec.flags & FLAG_RLE) {
-      const numBits = Math.max(1, Math.ceil(Math.log2(this.dict.size)));
-      values = Array.from(
-        encodeRLE(values, values.length, numBits)
-      );
+    let { values, spec } = this;
+    if (spec.flags & FLAG_RLE) {
+      if (spec.cardinality[0] == 0 && spec.default == null) {
+        values = encodeSimple(values);
+      } else {
+        const numBits = Math.max(
+          1,
+          Math.ceil(Math.log2(this.dict.size))
+        );
+        values = Array.from(
+          encodeBinary(values, values.length, numBits)
+        );
+      }
     }
     return { dict: __serializeDict(this.dict), values };
   }

package/columns/plain.d.ts

@@ -1,13 +1,14 @@
-import type { IColumn, SerializedColumn } from "../api.js";
+import { type IColumn, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
 export declare class PlainColumn extends AColumn implements IColumn {
     values: any[];
     readonly isArray = false;
-    load(spec: SerializedColumn): void;
-    reindex(): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): any;
+    getRowKey(i: number): any;
+    valueKey(x: any): any;
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/plain.js

@@ -1,16 +1,15 @@
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
+import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
 import { __replaceValue } from "../internal/replace.js";
 import { AColumn } from "./acolumn.js";
 class PlainColumn extends AColumn {
   values = [];
   isArray = false;
-  load(spec) {
-    this.values = spec.values;
+  load({ values }) {
+    this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeSimple(values)) : values;
     this.reindex();
   }
-  reindex() {
-    super.updateBitmap(this.values);
-  }
   validate(value) {
     return __validateValue(this.spec, value);
   }
@@ -26,6 +25,12 @@ class PlainColumn extends AColumn {
   getRow(i) {
     return this.values[i];
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(x) {
+    return x;
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
@@ -34,7 +39,9 @@ class PlainColumn extends AColumn {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }
   toJSON() {
-    return { values: this.values };
+    return {
+      values: this.spec.flags & FLAG_RLE ? encodeSimple(this.values) : this.values
+    };
   }
 }
 export {

package/columns/tuple.d.ts

@@ -5,11 +5,12 @@ export declare class TupleColumn extends AColumn implements IColumn {
     values: Nullable<number[]>[];
     readonly isArray = true;
     load(spec: SerializedColumn): void;
-    reindex(): void;
     encode(value: any): any[];
     validate(value: any): boolean;
     setRow(i: number, value: any[]): void;
     getRow(i: number): Nullable<number[]>;
+    getRowKey(i: number): Nullable<number[]>;
+    valueKey(value: any): any[];
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {

package/columns/tuple.js

@@ -9,9 +9,6 @@ class TupleColumn extends AColumn {
     this.values = spec.values;
     this.reindex();
   }
-  reindex() {
-    super.updateBitmap(this.values);
-  }
   encode(value) {
     return isArray(value) ? value : [value];
   }
@@ -31,6 +28,12 @@ class TupleColumn extends AColumn {
   getRow(i) {
     return this.values[i];
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    return this.encode(value);
+  }
   removeRow(i) {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);

package/columns/typedarray.d.ts

@@ -6,17 +6,19 @@ export declare class TypedArrayColumn extends AColumn implements IColumn {
     values: TypedArray;
     type: NumericType;
     limit: [number, number];
+    protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: string, table: Table);
-    load(spec: SerializedColumn): void;
-    reindex(): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): number;
+    getRowKey(i: number): number;
+    valueKey(value: any): number | number[];
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
-        values: number[];
+        values: any[];
     };
 }
 //# sourceMappingURL=typedarray.d.ts.map

package/columns/typedarray.js

@@ -1,45 +1,29 @@
-import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { typedArray } from "@thi.ng/api/typedarray";
+import { isArray } from "@thi.ng/checks/is-array";
 import { isNumber } from "@thi.ng/checks/is-number";
-import { decode as decodeRLE, encode as encodeRLE } from "@thi.ng/rle-pack";
 import {
-  FLAG_RLE
+  LIMITS
 } from "../api.js";
 import { __replaceValue } from "../internal/replace.js";
+import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-const LIMITS = {
-  u8: [0, 255],
-  u8c: [0, 255],
-  u16: [0, 65535],
-  u32: [0, 4294967295],
-  i8: [-128, 127],
-  i16: [-32768, 32767],
-  i32: [-2147483648, 2147483647],
-  f32: [-Infinity, Infinity],
-  f64: [-Infinity, Infinity]
-};
 class TypedArrayColumn extends AColumn {
   values;
   type;
   limit;
+  tmp;
   isArray = false;
   constructor(id, table) {
     super(id, table);
     this.type = table.schema[id].type;
     this.limit = LIMITS[this.type];
     this.values = typedArray(this.type, 8);
+    this.tmp = typedArray(this.type, 1);
   }
-  load(spec) {
-    if (this.spec.flags & FLAG_RLE) {
-      const values = decodeRLE(spec.values);
-      this.values = typedArray(this.type, values.buffer);
-    } else {
-      this.values = typedArray(this.type, spec.values);
-    }
+  load({ values }) {
+    this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();
   }
-  reindex() {
-    super.updateBitmap(this.values.subarray(0, this.table.length));
-  }
   validate(value) {
     return isNumber(value) && value >= this.limit[0] && value <= this.limit[1] || value == null && this.spec.default != null;
   }
@@ -63,6 +47,21 @@ class TypedArrayColumn extends AColumn {
   getRow(i) {
     return this.values[i];
   }
+  getRowKey(i) {
+    return this.values[i];
+  }
+  valueKey(value) {
+    const { tmp } = this;
+    if (isArray(value)) {
+      return value.map((x) => {
+        tmp[0] = x;
+        return tmp[0];
+      });
+    } else {
+      tmp[0] = value;
+      return tmp[0];
+    }
+  }
   removeRow(i) {
     this.values.copyWithin(i, i + 1, this.table.length);
     this.values[this.table.length - 1] = 0;
@@ -72,11 +71,11 @@ class TypedArrayColumn extends AColumn {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }
   toJSON() {
-    let values = this.values.subarray(0, this.table.length);
-    if (this.spec.flags & FLAG_RLE) {
-      values = encodeRLE(values, values.length, SIZEOF[this.type] * 8);
-    }
-    return { values: Array.from(values) };
+    return __serializeTyped(
+      this.values.subarray(0, this.table.length),
+      this.spec,
+      this.type
+    );
   }
 }
 export {

package/columns/vector.d.ts

@@ -0,0 +1,25 @@
+import { type TypedArray } from "@thi.ng/api/typedarray";
+import { type IColumn, type NumericType, type SerializedColumn } from "../api.js";
+import type { Table } from "../table.js";
+import { AColumn } from "./acolumn.js";
+export declare class VectorColumn extends AColumn implements IColumn {
+    values: TypedArray;
+    type: NumericType;
+    size: number;
+    limit: [number, number];
+    protected tmp: TypedArray;
+    readonly isArray = false;
+    constructor(id: string, table: Table);
+    load({ values }: SerializedColumn): void;
+    validate(value: any): boolean;
+    setRow(i: number, value: any): void;
+    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRowKey(i: number): string;
+    valueKey(value: any): string | string[];
+    removeRow(i: number): void;
+    replaceValue(): boolean;
+    toJSON(): {
+        values: any[];
+    };
+}
+//# sourceMappingURL=vector.d.ts.map

package/columns/vector.js

@@ -0,0 +1,92 @@
+import { typedArray } from "@thi.ng/api/typedarray";
+import { isArray } from "@thi.ng/checks/is-array";
+import { isArrayLike } from "@thi.ng/checks/is-arraylike";
+import { isNumber } from "@thi.ng/checks/is-number";
+import { unsupportedOp } from "@thi.ng/errors/unsupported";
+import {
+  LIMITS
+} from "../api.js";
+import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
+import { AColumn } from "./acolumn.js";
+class VectorColumn extends AColumn {
+  values;
+  type;
+  size;
+  limit;
+  tmp;
+  isArray = false;
+  constructor(id, table) {
+    super(id, table);
+    this.type = this.spec.type.split("v")[0];
+    this.size = this.spec.cardinality[1];
+    this.limit = LIMITS[this.type];
+    this.values = typedArray(this.type, 8 * this.size);
+    this.tmp = typedArray(this.type, this.size);
+  }
+  load({ values }) {
+    this.values = __deserializeTyped(this.type, this.spec.flags, values);
+    this.reindex();
+  }
+  validate(value) {
+    return isArrayLike(value) && value.length == this.size || value == null && this.spec.default != null;
+  }
+  setRow(i, value) {
+    value = this.ensureValue(value);
+    const j = i * this.size;
+    let len = this.values.length;
+    if (j >= len) {
+      while (j >= len) len <<= 1;
+      const tmp = typedArray(this.type, len);
+      tmp.set(this.values);
+      this.values = tmp;
+    }
+    const { values, bitmap } = this;
+    if (bitmap) {
+      bitmap.clearBit(this.getRowKey(i), i);
+      bitmap.setBit(this.valueKey(value), i);
+    }
+    values.set(value, j);
+  }
+  getRow(i) {
+    const { size } = this;
+    i *= size;
+    return this.values.subarray(i, i + size);
+  }
+  getRowKey(i) {
+    return this.getRow(i).join("|");
+  }
+  valueKey(value) {
+    const { tmp } = this;
+    if (isArray(value) && !isNumber(value[0])) {
+      return value.map((x) => {
+        tmp.set(x);
+        return tmp.join("|");
+      });
+    } else {
+      tmp.set(value);
+      return tmp.join("|");
+    }
+  }
+  removeRow(i) {
+    const {
+      size,
+      table: { length }
+    } = this;
+    this.values.copyWithin(i, i + size, length * size);
+    this.values.fill(0, (length - 1) * size);
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue() {
+    unsupportedOp("TODO");
+  }
+  toJSON() {
+    return __serializeTyped(
+      this.values.subarray(0, this.table.length * this.size),
+      this.spec,
+      this.type
+    );
+  }
+}
+export {
+  VectorColumn
+};

package/internal/serialize.d.ts

@@ -1,7 +1,15 @@
+import { type NumericArray, type Type } from "@thi.ng/api/typedarray";
 import type { BidirIndex } from "@thi.ng/bidir-index";
+import { type ColumnSpec } from "../api.js";
 /** @internal */
 export declare const __serializeDict: (dict: BidirIndex<any>) => {
     index: any;
     next: number;
 };
+/** @internal */
+export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec, type: Type) => {
+    values: any[];
+};
+/** @internal */
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/internal/serialize.js

@@ -1,8 +1,27 @@
+import {
+  SIZEOF,
+  typedArray
+} from "@thi.ng/api/typedarray";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
+import { FLAG_RLE } from "../api.js";
 const __serializeDict = (dict) => {
   const res = [];
   for (let [val, id] of dict.entries()) res[id] = val;
   return { index: res, next: dict.nextID };
 };
+const __serializeTyped = ($values, spec, type) => {
+  if (spec.flags & FLAG_RLE) {
+    $values = type[0] === "f" ? encodeSimple($values) : encodeBinary($values, $values.length, SIZEOF[type] * 8);
+  }
+  let values = Array.from($values);
+  const prec = spec.opts?.prec;
+  if (prec != null) values = values.map((x) => +x.toFixed(prec));
+  return { values };
+};
+const __deserializeTyped = (type, flags, values) => flags & FLAG_RLE ? type[0] === "f" ? typedArray(type, decodeSimple(values)) : typedArray(type, decodeBinary(values).buffer) : typedArray(type, values);
 export {
-  __serializeDict
+  __deserializeTyped,
+  __serializeDict,
+  __serializeTyped
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.1.3",
+	"version": "0.3.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -44,7 +44,7 @@
 		"@thi.ng/bidir-index": "^1.5.0",
 		"@thi.ng/checks": "^3.8.4",
 		"@thi.ng/errors": "^2.6.3",
-		"@thi.ng/rle-pack": "^3.1.118"
+		"@thi.ng/rle-pack": "^3.2.0"
 	},
 	"devDependencies": {
 		"esbuild": "^0.27.2",
@@ -112,6 +112,9 @@
 		"./columns/typedarray": {
 			"default": "./columns/typedarray.js"
 		},
+		"./columns/vector": {
+			"default": "./columns/vector.js"
+		},
 		"./query": {
 			"default": "./query.js"
 		},
@@ -123,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "634df92bac82036b7f8c861fbf63210abdef3117\n"
+	"gitHead": "e3f0a8a9868bb51ae0c0c4882763f68ec4083bc3\n"
 }

package/query.js

@@ -67,7 +67,7 @@ class Query {
     }
     if (ctx.bitmap) {
       for (let i of new Bitfield(ctx.bitmap).ones(table.length))
-        yield table.getRow(i);
+        yield table.getRow(i, false, true);
     }
   }
 }
@@ -123,18 +123,18 @@ class QueryCtx {
 }
 const execBitOr = (ctx, term, column) => {
   const bitmap = column.bitmap;
-  const value = column.encode(term.value);
+  const key = column.valueKey(term.value);
   let mask;
-  if (isArray(value)) {
-    for (let v of value) {
-      const b = bitmap.index.get(v)?.buffer;
+  if (isArray(key)) {
+    for (let k of key) {
+      const b = bitmap.index.get(k)?.buffer;
       if (!b) continue;
       if (mask) {
         for (let i = 0; i < b.length; i++) mask[i] |= b[i];
       } else mask = ctx.makeMask(b);
     }
   } else {
-    const b = bitmap.index.get(value)?.buffer;
+    const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
   if (mask) {
@@ -143,13 +143,12 @@ const execBitOr = (ctx, term, column) => {
 };
 const execOr = (ctx, term, column) => {
   const n = ctx.table.length;
-  const encoded = column.encode(term.value);
-  const values = column.values;
-  const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
+  const key = column.valueKey(term.value);
+  const pred = column.isArray ? (row, k) => row.includes(k) : (row, k) => row === k;
   let mask;
-  for (let v of isArray(encoded) ? encoded : [encoded]) {
+  for (let k of isArray(key) ? key : [key]) {
     for (let i = 0; i < n; i++) {
-      if (pred(values[i], v)) {
+      if (pred(column.getRowKey(i), k)) {
         if (!mask) mask = ctx.makeMask();
         mask[i >>> 5] |= 1 << (i & 31);
       }
@@ -168,12 +167,12 @@ const delegateOr = (ctx, term, column) => {
 };
 const execBitAnd = (ctx, term, column) => {
   const bitmap = column.bitmap;
-  const value = column.encode(term.value);
+  const key = column.valueKey(term.value);
   let mask;
-  if (isArray(value)) {
+  if (isArray(key)) {
     const colBitmaps = [];
-    for (let v of value) {
-      const b = bitmap.index.get(v)?.buffer;
+    for (let k of key) {
+      const b = bitmap.index.get(k)?.buffer;
       if (!b) {
         if (term.type === "and") ctx.bitmap = void 0;
         return;
@@ -187,7 +186,7 @@ const execBitAnd = (ctx, term, column) => {
       } else mask = ctx.makeMask(b);
     }
   } else {
-    const b = bitmap.index.get(value)?.buffer;
+    const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
   if (mask) {
@@ -198,14 +197,13 @@ const execBitAnd = (ctx, term, column) => {
 };
 const execAnd = (ctx, term, column) => {
   const n = ctx.table.length;
-  const encoded = column.encode(term.value) ?? null;
-  const values = column.values;
+  const key = column.valueKey(term.value) ?? null;
   const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
   let mask;
-  for (let v of isArray(encoded) ? encoded : [encoded]) {
+  for (let k of isArray(key) ? key : [key]) {
     let m;
     for (let i = 0; i < n; i++) {
-      if (pred(values[i], v)) {
+      if (pred(column.getRowKey(i), k)) {
         if (!m) m = ctx.makeMask();
         m[i >>> 5] |= 1 << (i & 31);
       }
@@ -237,11 +235,10 @@ const QUERY_OPS = {
   nand: { fn: delegateAnd },
   matchCol: {
     fn: (ctx, term, column) => {
-      const values = column.values;
       const pred = term.value;
       let mask;
       for (let i = 0, n = ctx.table.length; i < n; i++) {
-        if (pred(column.decode(values[i]))) {
+        if (pred(column.getRow(i))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
         }

package/table.d.ts

@@ -25,8 +25,8 @@ export declare class Table {
     addRows(rows: Iterable<Row>): void;
     updateRow(i: number, row: Row): void;
     removeRow(i: number): void;
-    getRow(i: number, safe?: boolean): Row | undefined;
-    getPartialRow(i: number, columns: string[], safe?: boolean): Row | undefined;
+    getRow(i: number, safe?: boolean, includeID?: boolean): Row | undefined;
+    getPartialRow(i: number, columns: string[], safe?: boolean, includeID?: boolean): Row | undefined;
     validateRow(row: Row): void;
     validateColumnSpec(id: string, spec: ColumnSpec): void;
     toJSON(): {

package/table.js

@@ -11,6 +11,7 @@ import { DictColumn } from "./columns/dict.js";
 import { PlainColumn } from "./columns/plain.js";
 import { TupleColumn } from "./columns/tuple.js";
 import { TypedArrayColumn } from "./columns/typedarray.js";
+import { VectorColumn } from "./columns/vector.js";
 import { __columnError } from "./internal/checks.js";
 import { Query } from "./query.js";
 class Table {
@@ -81,17 +82,17 @@ class Table {
     }
     this.length--;
   }
-  getRow(i, safe = true) {
+  getRow(i, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
-    const row = {};
+    const row = includeID ? { __row: i } : {};
     for (let id in this.columns) {
       row[id] = this.columns[id].getRow(i);
     }
     return row;
   }
-  getPartialRow(i, columns, safe = true) {
+  getPartialRow(i, columns, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
-    const row = {};
+    const row = includeID ? { __row: i } : {};
     for (let id of columns) {
       row[id] = this.columns[id]?.getRow(i);
     }
@@ -138,20 +139,25 @@ const $typed = {
   cardinality: [0, 1],
   required: true
 };
-const $float = { ...$typed, flags: FLAG_BITMAP };
 const $untyped = {
-  impl: (table, id, { flags, cardinality: [min, max], default: d }) => {
+  impl: (table, id, { flags, cardinality: [_, max] }) => {
     const isDict = flags & FLAG_DICT;
-    if (flags & FLAG_RLE) {
-      if (!isDict || max > 1 || min === 0 && d == null) {
-        __columnError(id, `RLE encoding not supported`);
-      }
-    }
+    if (flags & FLAG_RLE && max > 1) __columnError(id, `RLE not supported`);
     return max > 1 ? new (isDict ? DictTupleColumn : TupleColumn)(id, table) : new (isDict ? DictColumn : PlainColumn)(id, table);
   },
   flags: FLAG_BITMAP | FLAG_DICT | FLAG_UNIQUE | FLAG_RLE,
   cardinality: [0, -1 >>> 0]
 };
+const $vec = {
+  impl: (table, id, { cardinality: [min, max] }) => {
+    if (min > 0 && min !== max)
+      __columnError(id, `only fixed size vectors supported`);
+    return new VectorColumn(id, table);
+  },
+  flags: FLAG_BITMAP | FLAG_RLE,
+  cardinality: [0, -1 >>> 0],
+  required: true
+};
 const COLUMN_TYPES = {
   u8: $typed,
   i8: $typed,
@@ -159,10 +165,18 @@ const COLUMN_TYPES = {
   i16: $typed,
   u32: $typed,
   i32: $typed,
-  f32: $float,
-  f64: $float,
+  f32: $typed,
+  f64: $typed,
   num: $untyped,
-  str: $untyped
+  str: $untyped,
+  u8vec: $vec,
+  u16vec: $vec,
+  u32vec: $vec,
+  i8vec: $vec,
+  i16vec: $vec,
+  i32vec: $vec,
+  f32vec: $vec,
+  f64vec: $vec
 };
 const registerColumnType = (type, spec) => {
   if (COLUMN_TYPES[type])