@thi.ng/column-store 0.1.1 → 0.1.3

package/README.md

@@ -49,6 +49,60 @@ In-memory column store database with customizable column types, extensible query
 
 ## Column storage
 
+As the name indicates, data is stored in different columns, where each column
+manages its own schema, value validation, backing storage, indexing,
+serialization etc.
+
+From a user's POV, columns are not (usually) used directly, but via table, which
+acts as facade to the various configured columns. Data items are added as JS
+objects to a table, which then pulls out related values, validates them and the
+delegates them to the columns.
+
+An example table definition looks like this (explanation of column types in next
+section below):
+
+```ts
+import { Table, FLAG_DICT, FLAG_UNIQUE } from "@thi.ng/column-store";
+
+// define a table with the given columns
+const table = new Table({
+
+    // column of single numeric values
+    // (default cardinality makes values required)
+    id: { type: "num" },
+
+    // column of required string values
+    name: { type: "str" },
+
+    // optional tuples of max. 3 string values
+    // (if no value is given, the column stores null)
+    aliases: { type: "str", cardinality: [0, 3] },
+
+    // required fixed size tuples (aka vectors) of numbers
+    latlon: { type: "num", cardinality: [2, 2] },
+
+    // optional tuples of max. 10 strings, with default
+    // the given flags (explained further below) are triggering:
+    // - dictionary-based encoding
+    // - unique values (per tuple)
+    tags: {
+        type: "str",
+        cardinality: [0, 10],
+        default: ["todo"],
+        flags: FLAG_DICT | FLAG_UNIQUE
+    }
+});
+
+// add data
+table.addRow({
+    id: 1,
+    name: "karsten",
+    aliases: ["toxi"],
+    latlon: [47.421, 10.984],
+    tags: ["person", "opensource"],
+});
+```
+
 ### Column types
 
 The current built-in column types only support numeric or string values, though
@@ -62,8 +116,8 @@ Note: Booleans and `BigInt`s are still unsupported, but being worked on...
 
 | **Column type** | **Description**     | **Tuples supported** | **RLE serialization** |
 |-----------------|---------------------|----------------------|-----------------------|
-| `num`           | JS numbers          | ✅                    | ✅ <sup>(1)</sup>       |
-| `str`           | JS strings (UTF-16) | ✅                    | ✅ <sup>(1)</sup>       |
+| `num`           | JS numbers          | ✅                    | ✅ <sup>(1)</sup>      |
+| `str`           | JS strings (UTF-16) | ✅                    | ✅ <sup>(1)</sup>      |
 | `u8`            | 8bit unsigned int   | ❌                    | ✅                     |
 | `i8`            | 8bit signed int     | ❌                    | ✅                     |
 | `u16`           | 16bit unsigned int  | ❌                    | ✅                     |
@@ -298,7 +352,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.11 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.15 KB
 
 ## Dependencies
 

package/columns/acolumn.d.ts

@@ -8,6 +8,7 @@ export declare abstract class AColumn {
     spec: ColumnSpec;
     bitmap?: BitmapIndex;
     dict?: BidirIndex<any>;
+    abstract isArray: boolean;
     constructor(id: string, table: Table);
     encode(value: any): any;
     decode(value: any): any;

package/columns/acolumn.js

@@ -1,4 +1,3 @@
-import { isArray } from "@thi.ng/checks";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
 import { FLAG_BITMAP } from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
@@ -33,12 +32,13 @@ class AColumn {
   }
   updateBitmap(rows) {
     this.ensureBitmap();
-    const { bitmap } = this;
+    const { bitmap, isArray } = this;
     if (!bitmap) return;
     bitmap.clear();
     for (let i = 0; i < rows.length; i++) {
       const value = rows[i];
-      if (isArray(value)) {
+      if (value == null) continue;
+      if (isArray) {
         for (let x of value) bitmap.setBit(x, i);
       } else bitmap.setBit(value, i);
     }

package/columns/{dict-array.d.ts → dict-tuple.d.ts}

@@ -1,7 +1,7 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
 import { type IColumn, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class DictArrayColumn extends AColumn implements IColumn {
+export declare class DictTupleColumn extends AColumn implements IColumn {
     values: (number[] | null)[];
     dict: BidirIndex<any>;
     readonly isArray = true;
@@ -22,4 +22,4 @@ export declare class DictArrayColumn extends AColumn implements IColumn {
         values: (number[] | null)[];
     };
 }
-//# sourceMappingURL=dict-array.d.ts.map
+//# sourceMappingURL=dict-tuple.d.ts.map

package/columns/{dict-array.js → dict-tuple.js}

@@ -4,7 +4,7 @@ import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-class DictArrayColumn extends AColumn {
+class DictTupleColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = true;
@@ -75,5 +75,5 @@ class DictArrayColumn extends AColumn {
   }
 }
 export {
-  DictArrayColumn
+  DictTupleColumn
 };

package/columns/{array.d.ts → tuple.d.ts}

@@ -1,7 +1,7 @@
 import type { Nullable } from "@thi.ng/api";
 import { type IColumn, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class ArrayColumn extends AColumn implements IColumn {
+export declare class TupleColumn extends AColumn implements IColumn {
     values: Nullable<number[]>[];
     readonly isArray = true;
     load(spec: SerializedColumn): void;
@@ -16,4 +16,4 @@ export declare class ArrayColumn extends AColumn implements IColumn {
         values: Nullable<number[]>[];
     };
 }
-//# sourceMappingURL=array.d.ts.map
+//# sourceMappingURL=tuple.d.ts.map

package/columns/{array.js → tuple.js}

@@ -2,7 +2,7 @@ import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
 import { AColumn } from "./acolumn.js";
-class ArrayColumn extends AColumn {
+class TupleColumn extends AColumn {
   values = [];
   isArray = true;
   load(spec) {
@@ -58,5 +58,5 @@ class ArrayColumn extends AColumn {
   }
 }
 export {
-  ArrayColumn
+  TupleColumn
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.1.1",
+	"version": "0.1.3",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -97,11 +97,8 @@
 		"./columns/acolumn": {
 			"default": "./columns/acolumn.js"
 		},
-		"./columns/array": {
-			"default": "./columns/array.js"
-		},
-		"./columns/dict-array": {
-			"default": "./columns/dict-array.js"
+		"./columns/dict-tuple": {
+			"default": "./columns/dict-tuple.js"
 		},
 		"./columns/dict": {
 			"default": "./columns/dict.js"
@@ -109,6 +106,9 @@
 		"./columns/plain": {
 			"default": "./columns/plain.js"
 		},
+		"./columns/tuple": {
+			"default": "./columns/tuple.js"
+		},
 		"./columns/typedarray": {
 			"default": "./columns/typedarray.js"
 		},
@@ -123,5 +123,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "28440e6d826cce964928f4c2ffe1f0e3e2e25ab0\n"
+	"gitHead": "634df92bac82036b7f8c861fbf63210abdef3117\n"
 }

package/query.d.ts

@@ -26,7 +26,27 @@ export declare class QueryCtx {
     bitmap?: Uint32Array;
     constructor(query: Query);
     makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
+    /**
+     * Combines the `mask` with the context's mask (combined using bitwise AND).
+     * If the context mask is still undefined, assigns `mask` as the initial
+     * context mask.
+     *
+     * @param mask
+     */
     mergeMask(mask: Uint32Array): void;
+    /**
+     * Combines the bitwise inverted `mask` with the context's mask (combined
+     * using bitwise AND). If the context mask is still undefined, first inverts
+     * `mask` in place and uses result as the initial context mask.
+     *
+     * @param mask
+     */
+    mergeInvMask(mask: Uint32Array): void;
+    /**
+     * Bitwise inverts `mask` in place and then returns it.
+     *
+     * @param mask
+     */
     invertMask(mask: Uint32Array): Uint32Array<ArrayBufferLike>;
 }
 /**

package/query.js

@@ -5,10 +5,11 @@ import { Bitfield } from "./bitmap.js";
 class Query {
   constructor(table, terms = []) {
     this.table = table;
-    this.terms = terms;
+    for (let term of terms) this.addTerm(term);
   }
   terms = [];
   addTerm(term) {
+    if (!QUERY_OPS[term.type]) unsupportedOp(`query type: ${term.type}`);
     this.terms.push(term);
     return this;
   }
@@ -53,7 +54,6 @@ class Query {
     const ctx = new QueryCtx(this);
     for (let term of this.terms) {
       const op = QUERY_OPS[term.type];
-      if (!op) unsupportedOp(`query type: ${term.type}`);
       let column;
       if (term.column) {
         column = ctx.table.columns[term.column];
@@ -85,12 +85,37 @@ class QueryCtx {
     if (seed) mask.set(seed);
     return mask;
   }
+  /**
+   * Combines the `mask` with the context's mask (combined using bitwise AND).
+   * If the context mask is still undefined, assigns `mask` as the initial
+   * context mask.
+   *
+   * @param mask
+   */
   mergeMask(mask) {
     if (this.bitmap) {
       for (let i = 0, n = this.size; i < n; i++)
         this.bitmap[i] &= mask[i];
     } else this.bitmap = mask;
   }
+  /**
+   * Combines the bitwise inverted `mask` with the context's mask (combined
+   * using bitwise AND). If the context mask is still undefined, first inverts
+   * `mask` in place and uses result as the initial context mask.
+   *
+   * @param mask
+   */
+  mergeInvMask(mask) {
+    if (this.bitmap) {
+      for (let i = 0, n = this.size; i < n; i++)
+        this.bitmap[i] &= mask[i] ^ -1;
+    } else this.bitmap = this.invertMask(mask);
+  }
+  /**
+   * Bitwise inverts `mask` in place and then returns it.
+   *
+   * @param mask
+   */
   invertMask(mask) {
     for (let i = 0, n = this.size; i < n; i++) mask[i] ^= -1;
     return mask;
@@ -113,8 +138,7 @@ const execBitOr = (ctx, term, column) => {
     if (b) mask = ctx.makeMask(b);
   }
   if (mask) {
-    if (term.type === "nor") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   }
 };
 const execOr = (ctx, term, column) => {
@@ -132,8 +156,7 @@ const execOr = (ctx, term, column) => {
     }
   }
   if (mask) {
-    if (term.type === "nor") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   }
 };
 const delegateOr = (ctx, term, column) => {
@@ -168,8 +191,7 @@ const execBitAnd = (ctx, term, column) => {
     if (b) mask = ctx.makeMask(b);
   }
   if (mask) {
-    if (term.type === "nand") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   } else {
     ctx.bitmap = void 0;
   }
@@ -198,8 +220,7 @@ const execAnd = (ctx, term, column) => {
     }
   }
   if (mask) {
-    if (term.type === "nand") ctx.invertMask(mask);
-    ctx.mergeMask(mask);
+    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
   }
 };
 const delegateAnd = (ctx, term, column) => {

package/table.js

@@ -6,10 +6,10 @@ import {
   FLAG_RLE,
   FLAG_UNIQUE
 } from "./api.js";
-import { ArrayColumn } from "./columns/array.js";
-import { DictArrayColumn } from "./columns/dict-array.js";
+import { DictTupleColumn } from "./columns/dict-tuple.js";
 import { DictColumn } from "./columns/dict.js";
 import { PlainColumn } from "./columns/plain.js";
+import { TupleColumn } from "./columns/tuple.js";
 import { TypedArrayColumn } from "./columns/typedarray.js";
 import { __columnError } from "./internal/checks.js";
 import { Query } from "./query.js";
@@ -147,7 +147,7 @@ const $untyped = {
         __columnError(id, `RLE encoding not supported`);
       }
     }
-    return max > 1 ? new (isDict ? DictArrayColumn : ArrayColumn)(id, table) : new (isDict ? DictColumn : PlainColumn)(id, table);
+    return max > 1 ? new (isDict ? DictTupleColumn : TupleColumn)(id, table) : new (isDict ? DictColumn : PlainColumn)(id, table);
   },
   flags: FLAG_BITMAP | FLAG_DICT | FLAG_UNIQUE | FLAG_RLE,
   cardinality: [0, -1 >>> 0]