@thi.ng/column-store 0.2.0 → 0.4.0

package/README.md

@@ -63,7 +63,7 @@ delegates them to the columns.
 An example table definition looks like this (explanation of column types in next
 section below):
 
-```ts
+```ts tangle:export/readme-types.ts
 import { Table, FLAG_DICT, FLAG_UNIQUE } from "@thi.ng/column-store";
 
 // define a table with the given columns
@@ -81,7 +81,7 @@ const table = new Table({
     aliases: { type: "str", cardinality: [0, 3] },
 
     // required fixed size tuples (aka vectors) of numbers
-    latlon: { type: "num", cardinality: [2, 2] },
+    latlon: { type: "f32vec", cardinality: [2, 2] },
 
     // optional tuples of max. 10 strings, with default
     // the given flags (explained further below) are triggering:
@@ -105,6 +105,11 @@ table.addRow({
 });
 ```
 
+> [!IMPORTANT]
+> Columns can be named freely, with the exception that the `__` name prefix is
+> reserved for internal use. For example, `foo` is allowed, but `__foo` is a
+> reserved name.
+
 ### Column types
 
 The current built-in column types only support numeric or string values, though
@@ -149,12 +154,13 @@ configs are: `[0,3]` (with default given) or `[3,3]`.
 When [querying](#query-engine) vector columns using the standard
 `(n)or`/`(n)and` operators, always the entire vector is matched (by value).
 
-> [!IMPORTANT] For performance reasons, rows retrieved from vector columns
-> contain mutable data views of the underlying column storage. That means when
-> manipulating data in these views, the underlying data in the column would be
-> changed too. To avoid index corruption, always edit only copies of this vector
-> data and then use `table.updateRow()` to properly update the column storage
-> (incl. any internal indexes).
+> [!IMPORTANT]
+> For performance reasons, rows retrieved from vector columns contain mutable
+> data views of the underlying column storage. That means when manipulating data
+> in these views, the underlying data in the column would be changed too. To
+> avoid index corruption, always edit only copies of this vector data and then
+> use `table.updateRow()` to properly update the column storage (incl. any
+> internal indexes).
 
 ### Serialization options
 
@@ -310,8 +316,8 @@ TODO see code examples below
 
 The query engine works by applying a number of [query
 terms](https://docs.thi.ng/umbrella/column-store/interfaces/QueryTerm.html) in
-series, with each step intersecting its results with the results of the previous
-step(s), thereby narrowing down the result set.
+series, with each step intersecting (aka logical AND) its results with the
+results of the previous step(s), thereby narrowing down the result set.
 
 By default, individual query terms operate on a single column, but can also can
 also apply to multiple. Terms are supplied either as array given to the
@@ -406,7 +412,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.65 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.67 KB
 
 ## Dependencies
 
@@ -465,19 +471,20 @@ table.addRows([
 const unsortedImages = table.query().where("type", "img").and("tags", "unsorted");
 
 // queries are iterables and only execute when the iterator is consumed
+// each query result includes a `__row` ID
 console.log([...unsortedImages]);
-// [ { id: 102, type: "img", tags: [ "unsorted" ] } ]
+// [ { id: 102, type: "img", tags: [ "unsorted" ], __row: 2 } ]
 
 // select items with `a` OR `b` tags, intersect with those which have `c` AND `d` tags
 const complexTagQuery = table.query().or("tags", ["a", "b"]).and("tags", ["c", "d"]);
 console.log([...complexTagQuery]);
-// [ { id: 104, type: "img", tags: [ "b", "c", "d" ] } ]
+// [ { id: 104, type: "img", tags: [ "b", "c", "d" ], __row: 4 } ]
 
 // query using custom predicates
 console.log([...table.query().matchColumn("id", (id) => id > 102)]);
 // [
-//    { id: 103, type: "img", tags: [ "unsorted" ] },
-//    { id: 104, type: "img", tags: [ "b", "c", "d" ] }
+//    { id: 103, type: "img", tags: [ "unsorted" ], __row: 3 },
+//    { id: 104, type: "img", tags: [ "b", "c", "d" ], __row: 4 }
 // ]
 
 // serialize table to JSON

package/api.d.ts

@@ -89,6 +89,21 @@ export interface IColumn {
     load(spec: SerializedColumn): void;
     reindex(): void;
     validate(value: any): boolean;
+    /**
+     * Searches for `value` in the column data, optionally constrained to given
+     * `start`/`end` range. If found, returns row ID of first occurrence,
+     * otherwise -1.
+     *
+     * @param value
+     * @param start
+     * @param end
+     */
+    indexOf(value: any, start?: number, end?: number): number;
+    /**
+     * Returns value at given row.
+     *
+     * @param i
+     */
     getRow(i: number): any;
     /**
      * Sets the column's value at given row index. Assumes the value has been

package/bitmap.d.ts

@@ -27,6 +27,7 @@ export declare class Bitfield {
     buffer?: Uint32Array | undefined;
     constructor(buffer?: Uint32Array | undefined);
     ones(max?: number): Generator<number, void, unknown>;
+    first(start?: number, end?: number): number;
     setBit(id: number): void;
     clearBit(id: number): void;
     ensure(size: number): Uint32Array<ArrayBufferLike>;

package/bitmap.js

@@ -49,10 +49,10 @@ class Bitfield {
     this.buffer = buffer;
   }
   *ones(max = Infinity) {
-    const buf = this.buffer;
-    if (!buf) return;
-    for (let i = 0, n = buf.length; i < n; i++) {
-      let bits = buf[i];
+    const { buffer } = this;
+    if (!buffer) return;
+    for (let i = 0, n = buffer.length; i < n; i++) {
+      let bits = buffer[i];
       while (bits) {
         const lsb = bits & -bits;
         const x = (i << 5) + (Math.clz32(lsb) ^ 31);
@@ -62,6 +62,20 @@ class Bitfield {
       }
     }
   }
+  first(start = 0, end = (this.buffer?.length ?? -1) * 32) {
+    const { buffer } = this;
+    if (!buffer || start >= end) return -1;
+    for (let i = start >>> 5, n = Math.min(Math.ceil(end / 32), buffer.length); i < n; i++) {
+      let bits = buffer[i];
+      while (bits) {
+        const lsb = bits & -bits;
+        const x = (i << 5) + (Math.clz32(lsb) ^ 31);
+        if (x >= start && x < end) return x;
+        bits ^= lsb;
+      }
+    }
+    return -1;
+  }
   setBit(id) {
     const w = id >>> 5;
     this.ensure(w)[w] |= 1 << (id & 31);

package/columns/acolumn.d.ts

@@ -19,6 +19,7 @@ export declare abstract class AColumn implements IColumn {
     abstract valueKey(x: any): any;
     abstract getRow(i: number): any;
     abstract getRowKey(i: number): any;
+    abstract indexOf(value: any, start?: number, end?: number): number;
     encode(value: any): any;
     decode(value: any): any;
     protected loadDict(serialized: SerializedIndex): void;

package/columns/dict-tuple.d.ts

@@ -15,6 +15,7 @@ export declare class DictTupleColumn extends AColumn implements IColumn {
     getRowKey(i: number): number[] | null;
     valueKey(value: any): (number | null)[];
     removeRow(i: number): void;
+    indexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         dict: {

package/columns/dict-tuple.js

@@ -2,6 +2,7 @@ import { BidirIndex } from "@thi.ng/bidir-index";
 import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
+import { __indexOfTuple } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
 class DictTupleColumn extends AColumn {
@@ -55,6 +56,15 @@ class DictTupleColumn extends AColumn {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
   }
+  indexOf(value, start = 0, end = this.table.length) {
+    return __indexOfTuple(
+      value != null ? this.encode(value) : null,
+      this.values,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     const { dict, values, bitmap } = this;
     const res = dict.renameKey(currValue, newValue);

package/columns/dict.d.ts

@@ -7,7 +7,7 @@ export declare class DictColumn extends AColumn implements IColumn {
     readonly isArray = false;
     load({ dict, values }: SerializedColumn): void;
     reindex(): void;
-    encode(value: any): number | undefined;
+    encode(value: any): number | null;
     decode(value: any): any;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
@@ -15,6 +15,7 @@ export declare class DictColumn extends AColumn implements IColumn {
     getRowKey(i: number): number | null;
     valueKey(value: any): number | number[] | undefined;
     removeRow(i: number): void;
+    indexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         dict: {

package/columns/dict.js

@@ -1,11 +1,12 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
+import { isArray } from "@thi.ng/checks/is-array";
 import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
+import { __indexOfSingle } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-import { isArray } from "@thi.ng/checks/is-array";
 class DictColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
@@ -25,7 +26,7 @@ class DictColumn extends AColumn {
     super.updateBitmap();
   }
   encode(value) {
-    return this.dict.get(value);
+    return this.dict.get(value) ?? null;
   }
   decode(value) {
     return this.dict.getID(value);
@@ -57,6 +58,16 @@ class DictColumn extends AColumn {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
   }
+  indexOf(value, start = 0, end = this.table.length) {
+    return __indexOfSingle(
+      this.encode(value),
+      this.values,
+      this.bitmap,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     const { dict, values, bitmap } = this;
     const res = dict.renameKey(currValue, newValue);

package/columns/plain.d.ts

@@ -10,6 +10,7 @@ export declare class PlainColumn extends AColumn implements IColumn {
     getRowKey(i: number): any;
     valueKey(x: any): any;
     removeRow(i: number): void;
+    indexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         values: any[];

package/columns/plain.js

@@ -1,6 +1,7 @@
 import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
+import { __indexOfSingle } from "../internal/indexof.js";
 import { __replaceValue } from "../internal/replace.js";
 import { AColumn } from "./acolumn.js";
 class PlainColumn extends AColumn {
@@ -35,6 +36,16 @@ class PlainColumn extends AColumn {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
   }
+  indexOf(value, start = 0, end) {
+    return __indexOfSingle(
+      value,
+      this.values,
+      this.bitmap,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }

package/columns/tuple.d.ts

@@ -12,6 +12,7 @@ export declare class TupleColumn extends AColumn implements IColumn {
     getRowKey(i: number): Nullable<number[]>;
     valueKey(value: any): any[];
     removeRow(i: number): void;
+    indexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         values: Nullable<number[]>[];

package/columns/tuple.js

@@ -1,6 +1,7 @@
 import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
+import { __indexOfTuple } from "../internal/indexof.js";
 import { AColumn } from "./acolumn.js";
 class TupleColumn extends AColumn {
   values = [];
@@ -38,6 +39,15 @@ class TupleColumn extends AColumn {
     this.values.splice(i, 1);
     this.bitmap?.removeBit(i);
   }
+  indexOf(value, start = 0, end = this.table.length) {
+    return __indexOfTuple(
+      value,
+      this.values,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     const { values, bitmap } = this;
     const isUnique = this.spec.flags & FLAG_UNIQUE;

package/columns/typedarray.d.ts

@@ -9,16 +9,17 @@ export declare class TypedArrayColumn extends AColumn implements IColumn {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: string, table: Table);
-    load(spec: SerializedColumn): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): number;
     getRowKey(i: number): number;
     valueKey(value: any): number | number[];
     removeRow(i: number): void;
+    indexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
-        values: number[];
+        values: any[];
     };
 }
 //# sourceMappingURL=typedarray.d.ts.map

package/columns/typedarray.js

@@ -1,13 +1,13 @@
-import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { typedArray } from "@thi.ng/api/typedarray";
+import { isArray } from "@thi.ng/checks/is-array";
 import { isNumber } from "@thi.ng/checks/is-number";
-import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import {
-  FLAG_RLE,
   LIMITS
 } from "../api.js";
+import { __indexOfSingle } from "../internal/indexof.js";
 import { __replaceValue } from "../internal/replace.js";
+import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-import { isArray } from "@thi.ng/checks/is-array";
 class TypedArrayColumn extends AColumn {
   values;
   type;
@@ -21,13 +21,8 @@ class TypedArrayColumn extends AColumn {
     this.values = typedArray(this.type, 8);
     this.tmp = typedArray(this.type, 1);
   }
-  load(spec) {
-    if (this.spec.flags & FLAG_RLE) {
-      const values = decodeBinary(spec.values);
-      this.values = typedArray(this.type, values.buffer);
-    } else {
-      this.values = typedArray(this.type, spec.values);
-    }
+  load({ values }) {
+    this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();
   }
   validate(value) {
@@ -73,15 +68,25 @@ class TypedArrayColumn extends AColumn {
     this.values[this.table.length - 1] = 0;
     this.bitmap?.removeBit(i);
   }
+  indexOf(value, start = 0, end = this.table.length) {
+    return __indexOfSingle(
+      value,
+      this.values,
+      this.bitmap,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }
   toJSON() {
-    let values = this.values.subarray(0, this.table.length);
-    if (this.spec.flags & FLAG_RLE) {
-      values = encodeBinary(values, values.length, SIZEOF[this.type] * 8);
-    }
-    return { values: Array.from(values) };
+    return __serializeTyped(
+      this.values.subarray(0, this.table.length),
+      this.spec,
+      this.type
+    );
   }
 }
 export {

package/columns/vector.d.ts

@@ -10,13 +10,14 @@ export declare class VectorColumn extends AColumn implements IColumn {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: string, table: Table);
-    load(spec: SerializedColumn): void;
+    load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
-    getRow(i: number): Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;
+    indexOf(value: any, start?: number, end?: number): number;
     replaceValue(): boolean;
     toJSON(): {
         values: any[];

package/columns/vector.js

@@ -1,14 +1,13 @@
-import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { typedArray } from "@thi.ng/api/typedarray";
+import { isArray } from "@thi.ng/checks/is-array";
 import { isArrayLike } from "@thi.ng/checks/is-arraylike";
 import { isNumber } from "@thi.ng/checks/is-number";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
-import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import {
-  FLAG_RLE,
   LIMITS
 } from "../api.js";
+import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
-import { isArray } from "@thi.ng/checks/is-array";
 class VectorColumn extends AColumn {
   values;
   type;
@@ -24,13 +23,8 @@ class VectorColumn extends AColumn {
     this.values = typedArray(this.type, 8 * this.size);
     this.tmp = typedArray(this.type, this.size);
   }
-  load(spec) {
-    if (this.spec.flags & FLAG_RLE) {
-      const values = decodeBinary(spec.values);
-      this.values = typedArray(this.type, values.buffer);
-    } else {
-      this.values = typedArray(this.type, spec.values);
-    }
+  load({ values }) {
+    this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();
   }
   validate(value) {
@@ -82,22 +76,32 @@ class VectorColumn extends AColumn {
     this.values.fill(0, (length - 1) * size);
     this.bitmap?.removeBit(i);
   }
+  indexOf(value, start = 0, end = this.table.length) {
+    const { values, bitmap, size } = this;
+    start = Math.max(start, 0);
+    end = Math.min(end, this.table.length);
+    if (bitmap) {
+      return bitmap.index.get(this.valueKey(value))?.first(start, end) ?? -1;
+    }
+    end *= size;
+    let i, j;
+    outer: for (i = start * size; i < end; i += size) {
+      for (j = 0; j < size; j++) {
+        if (values[i + j] !== value[j]) continue outer;
+      }
+      return i / size;
+    }
+    return -1;
+  }
   replaceValue() {
     unsupportedOp("TODO");
   }
   toJSON() {
-    let $values = this.values.subarray(0, this.table.length * this.size);
-    if (this.spec.flags & FLAG_RLE) {
-      $values = encodeBinary(
-        $values,
-        $values.length,
-        SIZEOF[this.type] * 8
-      );
-    }
-    let values = Array.from($values);
-    const prec = this.spec.opts?.prec;
-    if (prec != null) values = values.map((x) => +x.toFixed(prec));
-    return { values };
+    return __serializeTyped(
+      this.values.subarray(0, this.table.length * this.size),
+      this.spec,
+      this.type
+    );
   }
 }
 export {

package/internal/indexof.d.ts

@@ -0,0 +1,6 @@
+import type { Maybe } from "@thi.ng/api";
+import type { BitmapIndex } from "../bitmap.js";
+/** @internal */
+export declare const __indexOfSingle: (needle: any, values: ArrayLike<any>, bitmap: Maybe<BitmapIndex>, max: number, start: number, end?: number) => number;
+export declare const __indexOfTuple: (needle: ArrayLike<any> | null, values: ArrayLike<any>, max: number, start: number, end?: number) => number;
+//# sourceMappingURL=indexof.d.ts.map

package/internal/indexof.js

@@ -0,0 +1,34 @@
+const __indexOfSingle = (needle, values, bitmap, max, start, end = max) => {
+  start = Math.max(start, 0);
+  end = Math.min(end, max);
+  if (bitmap) return bitmap.index.get(needle)?.first(start, end) ?? -1;
+  for (let i = start; i < end; i++) {
+    if (values[i] === needle) return i;
+  }
+  return -1;
+};
+const __indexOfTuple = (needle, values, max, start, end = max) => {
+  start = Math.max(start, 0);
+  end = Math.min(end, max);
+  if (needle == null) {
+    for (let i2 = start; i2 < end; i2++) {
+      if (!values[i2]) return i2;
+    }
+  }
+  const n = needle.length;
+  let i, j, row;
+  outer: for (i = start; i < end; i++) {
+    row = values[i];
+    if (row?.length === n) {
+      for (j = 0; j < n; j++) {
+        if (row[j] !== needle[j]) continue outer;
+      }
+      return i;
+    }
+  }
+  return -1;
+};
+export {
+  __indexOfSingle,
+  __indexOfTuple
+};

package/internal/serialize.d.ts

@@ -1,7 +1,15 @@
+import { type NumericArray, type Type } from "@thi.ng/api/typedarray";
 import type { BidirIndex } from "@thi.ng/bidir-index";
+import { type ColumnSpec } from "../api.js";
 /** @internal */
 export declare const __serializeDict: (dict: BidirIndex<any>) => {
     index: any;
     next: number;
 };
+/** @internal */
+export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec, type: Type) => {
+    values: any[];
+};
+/** @internal */
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/internal/serialize.js

@@ -1,8 +1,27 @@
+import {
+  SIZEOF,
+  typedArray
+} from "@thi.ng/api/typedarray";
+import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
+import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
+import { FLAG_RLE } from "../api.js";
 const __serializeDict = (dict) => {
   const res = [];
   for (let [val, id] of dict.entries()) res[id] = val;
   return { index: res, next: dict.nextID };
 };
+const __serializeTyped = ($values, spec, type) => {
+  if (spec.flags & FLAG_RLE) {
+    $values = type[0] === "f" ? encodeSimple($values) : encodeBinary($values, $values.length, SIZEOF[type] * 8);
+  }
+  let values = Array.from($values);
+  const prec = spec.opts?.prec;
+  if (prec != null) values = values.map((x) => +x.toFixed(prec));
+  return { values };
+};
+const __deserializeTyped = (type, flags, values) => flags & FLAG_RLE ? type[0] === "f" ? typedArray(type, decodeSimple(values)) : typedArray(type, decodeBinary(values).buffer) : typedArray(type, values);
 export {
-  __serializeDict
+  __deserializeTyped,
+  __serializeDict,
+  __serializeTyped
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.2.0",
+	"version": "0.4.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -126,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "b90a2f41eb0b3c89391bbb7cfff940192f23a83c\n"
+	"gitHead": "5c620a1aa00535cc7c674794790edd87b2e5e526\n"
 }

package/query.js

@@ -67,7 +67,7 @@ class Query {
     }
     if (ctx.bitmap) {
       for (let i of new Bitfield(ctx.bitmap).ones(table.length))
-        yield table.getRow(i);
+        yield table.getRow(i, false, true);
     }
   }
 }

package/table.d.ts

@@ -25,8 +25,9 @@ export declare class Table {
     addRows(rows: Iterable<Row>): void;
     updateRow(i: number, row: Row): void;
     removeRow(i: number): void;
-    getRow(i: number, safe?: boolean): Row | undefined;
-    getPartialRow(i: number, columns: string[], safe?: boolean): Row | undefined;
+    getRow(i: number, safe?: boolean, includeID?: boolean): Row | undefined;
+    getPartialRow(i: number, columns: string[], safe?: boolean, includeID?: boolean): Row | undefined;
+    indexOf(id: string, value: any, start?: number, end?: number): number;
     validateRow(row: Row): void;
     validateColumnSpec(id: string, spec: ColumnSpec): void;
     toJSON(): {

package/table.js

@@ -82,22 +82,25 @@ class Table {
     }
     this.length--;
   }
-  getRow(i, safe = true) {
+  getRow(i, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
-    const row = {};
+    const row = includeID ? { __row: i } : {};
     for (let id in this.columns) {
       row[id] = this.columns[id].getRow(i);
     }
     return row;
   }
-  getPartialRow(i, columns, safe = true) {
+  getPartialRow(i, columns, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
-    const row = {};
+    const row = includeID ? { __row: i } : {};
     for (let id of columns) {
       row[id] = this.columns[id]?.getRow(i);
     }
     return row;
   }
+  indexOf(id, value, start, end) {
+    return this.columns[id]?.indexOf(value, start, end) ?? -1;
+  }
   validateRow(row) {
     const { columns } = this;
     for (let id in columns) {
@@ -114,15 +117,13 @@ class Table {
         `unsupported flags for column type: ${spec.type}`
       );
     const [min, max] = spec.cardinality;
-    if (max < min)
+    if (max < min || min < def.cardinality[0] || max > def.cardinality[1] || max < 1)
       __columnError(id, `wrong cardinality: ${spec.cardinality}`);
-    if (min < def.cardinality[0] || max > def.cardinality[1])
-      __columnError(id, `wrong cardimality`);
     if (def.required && min === 0 && spec.default == null)
       __columnError(id, `missing default value`);
     if (spec.default != null) {
       if (max > 1 !== isArray(spec.default))
-        __columnError(id, `default value`);
+        __columnError(id, `wrong default value`);
     }
   }
   toJSON() {
@@ -139,7 +140,6 @@ const $typed = {
   cardinality: [0, 1],
   required: true
 };
-const $float = { ...$typed, flags: FLAG_BITMAP };
 const $untyped = {
   impl: (table, id, { flags, cardinality: [_, max] }) => {
     const isDict = flags & FLAG_DICT;
@@ -159,7 +159,6 @@ const $vec = {
   cardinality: [0, -1 >>> 0],
   required: true
 };
-const $fvec = { ...$vec, flags: FLAG_BITMAP };
 const COLUMN_TYPES = {
   u8: $typed,
   i8: $typed,
@@ -167,14 +166,18 @@ const COLUMN_TYPES = {
   i16: $typed,
   u32: $typed,
   i32: $typed,
-  f32: $float,
-  f64: $float,
+  f32: $typed,
+  f64: $typed,
   num: $untyped,
   str: $untyped,
   u8vec: $vec,
   u16vec: $vec,
   u32vec: $vec,
-  f32vec: $fvec
+  i8vec: $vec,
+  i16vec: $vec,
+  i32vec: $vec,
+  f32vec: $vec,
+  f64vec: $vec
 };
 const registerColumnType = (type, spec) => {
   if (COLUMN_TYPES[type])