@thi.ng/column-store 0.2.0 → 0.3.0

package/README.md

@@ -63,7 +63,7 @@ delegates them to the columns.
 An example table definition looks like this (explanation of column types in next
 section below):
 
-```ts
+```ts tangle:export/readme-types.ts
 import { Table, FLAG_DICT, FLAG_UNIQUE } from "@thi.ng/column-store";
 
 // define a table with the given columns
@@ -81,7 +81,7 @@ const table = new Table({
     aliases: { type: "str", cardinality: [0, 3] },
 
     // required fixed size tuples (aka vectors) of numbers
-    latlon: { type: "num", cardinality: [2, 2] },
+    latlon: { type: "f32vec", cardinality: [2, 2] },
 
     // optional tuples of max. 10 strings, with default
     // the given flags (explained further below) are triggering:
@@ -105,6 +105,11 @@ table.addRow({
 });
 ```
 
+> [!IMPORTANT]
+> Columns can be named freely, with the exception that the `__` name prefix is
+> reserved for internal use. For example, `foo` is allowed, but `__foo` is a
+> reserved name.
+
 ### Column types
 
 The current built-in column types only support numeric or string values, though
@@ -149,12 +154,13 @@ configs are: `[0,3]` (with default given) or `[3,3]`.
 When [querying](#query-engine) vector columns using the standard
 `(n)or`/`(n)and` operators, always the entire vector is matched (by value).
 
-> [!IMPORTANT] For performance reasons, rows retrieved from vector columns
-> contain mutable data views of the underlying column storage. That means when
-> manipulating data in these views, the underlying data in the column would be
-> changed too. To avoid index corruption, always edit only copies of this vector
-> data and then use `table.updateRow()` to properly update the column storage
-> (incl. any internal indexes).
+> [!IMPORTANT]
+> For performance reasons, rows retrieved from vector columns contain mutable
+> data views of the underlying column storage. That means when manipulating data
+> in these views, the underlying data in the column would be changed too. To
+> avoid index corruption, always edit only copies of this vector data and then
+> use `table.updateRow()` to properly update the column storage (incl. any
+> internal indexes).
 
 ### Serialization options
 
@@ -310,8 +316,8 @@ TODO see code examples below
 
 The query engine works by applying a number of [query
 terms](https://docs.thi.ng/umbrella/column-store/interfaces/QueryTerm.html) in
-series, with each step intersecting its results with the results of the previous
-step(s), thereby narrowing down the result set.
+series, with each step intersecting (aka logical AND) its results with the
+results of the previous step(s), thereby narrowing down the result set.
 
 By default, individual query terms operate on a single column, but can also can
 also apply to multiple. Terms are supplied either as array given to the
@@ -406,7 +412,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.65 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.67 KB
 
 ## Dependencies
 
@@ -465,19 +471,20 @@ table.addRows([
 const unsortedImages = table.query().where("type", "img").and("tags", "unsorted");
 
 // queries are iterables and only execute when the iterator is consumed
+// each query result includes a `__row` ID
 console.log([...unsortedImages]);
-// [ { id: 102, type: "img", tags: [ "unsorted" ] } ]
+// [ { id: 102, type: "img", tags: [ "unsorted" ], __row: 2 } ]
 
 // select items with `a` OR `b` tags, intersect with those which have `c` AND `d` tags
 const complexTagQuery = table.query().or("tags", ["a", "b"]).and("tags", ["c", "d"]);
 console.log([...complexTagQuery]);
-// [ { id: 104, type: "img", tags: [ "b", "c", "d" ] } ]
+// [ { id: 104, type: "img", tags: [ "b", "c", "d" ], __row: 4 } ]
 
 // query using custom predicates
 console.log([...table.query().matchColumn("id", (id) => id > 102)]);
 // [
-//    { id: 103, type: "img", tags: [ "unsorted" ] },
-//    { id: 104, type: "img", tags: [ "b", "c", "d" ] }
+//    { id: 103, type: "img", tags: [ "unsorted" ], __row: 3 },
+//    { id: 104, type: "img", tags: [ "b", "c", "d" ], __row: 4 }
 // ]
 
 // serialize table to JSON

package/columns/typedarray.d.ts

@@ -9,7 +9,7 @@ export declare class TypedArrayColumn extends AColumn implements IColumn {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: string, table: Table);
-    load(spec: SerializedColumn): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): number;
@@ -18,7 +18,7 @@ export declare class TypedArrayColumn extends AColumn implements IColumn {
     removeRow(i: number): void;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
-        values: number[];
+        values: any[];
     };
 }
 //# sourceMappingURL=typedarray.d.ts.map

package/columns/typedarray.js

@@ -1,13 +1,12 @@
-import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { typedArray } from "@thi.ng/api/typedarray";
+import { isArray } from "@thi.ng/checks/is-array";
 import { isNumber } from "@thi.ng/checks/is-number";
-import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import {
-  FLAG_RLE,
   LIMITS
 } from "../api.js";
 import { __replaceValue } from "../internal/replace.js";
+import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-import { isArray } from "@thi.ng/checks/is-array";
 class TypedArrayColumn extends AColumn {
   values;
   type;
@@ -21,13 +20,8 @@ class TypedArrayColumn extends AColumn {
     this.values = typedArray(this.type, 8);
     this.tmp = typedArray(this.type, 1);
   }
-  load(spec) {
-    if (this.spec.flags & FLAG_RLE) {
-      const values = decodeBinary(spec.values);
-      this.values = typedArray(this.type, values.buffer);
-    } else {
-      this.values = typedArray(this.type, spec.values);
-    }
+  load({ values }) {
+    this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();
   }
   validate(value) {
@@ -77,11 +71,11 @@ class TypedArrayColumn extends AColumn {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }
   toJSON() {
-    let values = this.values.subarray(0, this.table.length);
-    if (this.spec.flags & FLAG_RLE) {
-      values = encodeBinary(values, values.length, SIZEOF[this.type] * 8);
-    }
-    return { values: Array.from(values) };
+    return __serializeTyped(
+      this.values.subarray(0, this.table.length),
+      this.spec,
+      this.type
+    );
   }
 }
 export {

package/columns/vector.d.ts

@@ -10,10 +10,10 @@ export declare class VectorColumn extends AColumn implements IColumn {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: string, table: Table);
-    load(spec: SerializedColumn): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
-    getRow(i: number): Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;

package/columns/vector.js

@@ -1,14 +1,13 @@
-import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { typedArray } from "@thi.ng/api/typedarray";
+import { isArray } from "@thi.ng/checks/is-array";
 import { isArrayLike } from "@thi.ng/checks/is-arraylike";
 import { isNumber } from "@thi.ng/checks/is-number";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
-import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import {
-  FLAG_RLE,
   LIMITS
 } from "../api.js";
+import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-import { isArray } from "@thi.ng/checks/is-array";
 class VectorColumn extends AColumn {
   values;
   type;
@@ -24,13 +23,8 @@ class VectorColumn extends AColumn {
     this.values = typedArray(this.type, 8 * this.size);
     this.tmp = typedArray(this.type, this.size);
   }
-  load(spec) {
-    if (this.spec.flags & FLAG_RLE) {
-      const values = decodeBinary(spec.values);
-      this.values = typedArray(this.type, values.buffer);
-    } else {
-      this.values = typedArray(this.type, spec.values);
-    }
+  load({ values }) {
+    this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();
   }
   validate(value) {
@@ -86,18 +80,11 @@ class VectorColumn extends AColumn {
     unsupportedOp("TODO");
   }
   toJSON() {
-    let $values = this.values.subarray(0, this.table.length * this.size);
-    if (this.spec.flags & FLAG_RLE) {
-      $values = encodeBinary(
-        $values,
-        $values.length,
-        SIZEOF[this.type] * 8
-      );
-    }
-    let values = Array.from($values);
-    const prec = this.spec.opts?.prec;
-    if (prec != null) values = values.map((x) => +x.toFixed(prec));
-    return { values };
+    return __serializeTyped(
+      this.values.subarray(0, this.table.length * this.size),
+      this.spec,
+      this.type
+    );
   }
 }
 export {

package/internal/serialize.d.ts

@@ -1,7 +1,15 @@
+import { type NumericArray, type Type } from "@thi.ng/api/typedarray";
 import type { BidirIndex } from "@thi.ng/bidir-index";
+import { type ColumnSpec } from "../api.js";
 /** @internal */
 export declare const __serializeDict: (dict: BidirIndex<any>) => {
     index: any;
     next: number;
 };
+/** @internal */
+export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec, type: Type) => {
+    values: any[];
+};
+/** @internal */
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/internal/serialize.js

@@ -1,8 +1,27 @@
+import {
+  SIZEOF,
+  typedArray
+} from "@thi.ng/api/typedarray";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
+import { FLAG_RLE } from "../api.js";
 const __serializeDict = (dict) => {
   const res = [];
   for (let [val, id] of dict.entries()) res[id] = val;
   return { index: res, next: dict.nextID };
 };
+const __serializeTyped = ($values, spec, type) => {
+  if (spec.flags & FLAG_RLE) {
+    $values = type[0] === "f" ? encodeSimple($values) : encodeBinary($values, $values.length, SIZEOF[type] * 8);
+  }
+  let values = Array.from($values);
+  const prec = spec.opts?.prec;
+  if (prec != null) values = values.map((x) => +x.toFixed(prec));
+  return { values };
+};
+const __deserializeTyped = (type, flags, values) => flags & FLAG_RLE ? type[0] === "f" ? typedArray(type, decodeSimple(values)) : typedArray(type, decodeBinary(values).buffer) : typedArray(type, values);
 export {
-  __serializeDict
+  __deserializeTyped,
+  __serializeDict,
+  __serializeTyped
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.2.0",
+	"version": "0.3.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -126,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "b90a2f41eb0b3c89391bbb7cfff940192f23a83c\n"
+	"gitHead": "e3f0a8a9868bb51ae0c0c4882763f68ec4083bc3\n"
 }

package/query.js

@@ -67,7 +67,7 @@ class Query {
     }
     if (ctx.bitmap) {
       for (let i of new Bitfield(ctx.bitmap).ones(table.length))
-        yield table.getRow(i);
+        yield table.getRow(i, false, true);
     }
   }
 }

package/table.d.ts

@@ -25,8 +25,8 @@ export declare class Table {
     addRows(rows: Iterable<Row>): void;
     updateRow(i: number, row: Row): void;
     removeRow(i: number): void;
-    getRow(i: number, safe?: boolean): Row | undefined;
-    getPartialRow(i: number, columns: string[], safe?: boolean): Row | undefined;
+    getRow(i: number, safe?: boolean, includeID?: boolean): Row | undefined;
+    getPartialRow(i: number, columns: string[], safe?: boolean, includeID?: boolean): Row | undefined;
     validateRow(row: Row): void;
     validateColumnSpec(id: string, spec: ColumnSpec): void;
     toJSON(): {

package/table.js

@@ -82,17 +82,17 @@ class Table {
     }
     this.length--;
   }
-  getRow(i, safe = true) {
+  getRow(i, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
-    const row = {};
+    const row = includeID ? { __row: i } : {};
     for (let id in this.columns) {
       row[id] = this.columns[id].getRow(i);
     }
     return row;
   }
-  getPartialRow(i, columns, safe = true) {
+  getPartialRow(i, columns, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
-    const row = {};
+    const row = includeID ? { __row: i } : {};
     for (let id of columns) {
       row[id] = this.columns[id]?.getRow(i);
     }
@@ -139,7 +139,6 @@ const $typed = {
   cardinality: [0, 1],
   required: true
 };
-const $float = { ...$typed, flags: FLAG_BITMAP };
 const $untyped = {
   impl: (table, id, { flags, cardinality: [_, max] }) => {
     const isDict = flags & FLAG_DICT;
@@ -159,7 +158,6 @@ const $vec = {
   cardinality: [0, -1 >>> 0],
   required: true
 };
-const $fvec = { ...$vec, flags: FLAG_BITMAP };
 const COLUMN_TYPES = {
   u8: $typed,
   i8: $typed,
@@ -167,14 +165,18 @@ const COLUMN_TYPES = {
   i16: $typed,
   u32: $typed,
   i32: $typed,
-  f32: $float,
-  f64: $float,
+  f32: $typed,
+  f64: $typed,
   num: $untyped,
   str: $untyped,
   u8vec: $vec,
   u16vec: $vec,
   u32vec: $vec,
-  f32vec: $fvec
+  i8vec: $vec,
+  i16vec: $vec,
+  i32vec: $vec,
+  f32vec: $vec,
+  f64vec: $vec
 };
 const registerColumnType = (type, spec) => {
   if (COLUMN_TYPES[type])