@thi.ng/column-store 0.4.1 → 0.6.0

package/README.md

@@ -34,6 +34,8 @@
     - [AND](#and)
     - [Negation](#negation)
     - [Predicate-based matchers](#predicate-based-matchers)
+    - [Row ranges](#row-ranges)
+    - [Value ranges](#value-ranges)
   - [Custom operators](#custom-operators)
   - [Result aggregation](#result-aggregation)
   - [Query ranges](#query-ranges)
@@ -55,10 +57,11 @@ As the name indicates, data is stored in different columns, where each column
 manages its own schema, value validation, backing storage, indexing,
 serialization etc.
 
-From a user's POV, columns are not (usually) used directly, but via table, which
-acts as facade to the various configured columns. Data items are added as JS
-objects to a table, which then pulls out related values, validates them and the
-delegates them to the columns.
+From a user's POV, columns are not (usually) used directly, but accessed via a
+table, which provides row-based access to compound data records and acts as
+facade to the various configured columns. Data records are added as JS objects
+to a table, which then pulls out related values, validates them and the
+delegates them to the individual columns for encoding & storage.
 
 An example table definition looks like this (explanation of column types in next
 section below):
@@ -66,8 +69,17 @@ section below):
 ```ts tangle:export/readme-types.ts
 import { Table, FLAG_DICT, FLAG_UNIQUE } from "@thi.ng/column-store";
 
+// schema for a single row item
+interface Item {
+    id: number;
+    name: string;
+    aliases?: string[];
+    latlon: number[] | Float32Array;
+    tags: string[];
+}
+
 // define a table with the given columns
-const table = new Table({
+const table = new Table<Item>({
 
     // column of single numeric values
     // (default cardinality makes values required)
@@ -367,6 +379,30 @@ can be used, otherwise the behavior is:
 - [`matchPartialRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchpartialrow):
   apply predicate to partial row (only selected columns)
 
+#### Row ranges
+
+The
+[`rowRange`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#rowrange)
+operator selects the given `start` (inclusive) .. `end` (exclusive) range of row
+indices.
+
+```ts
+// select first ten rows
+query.rowRange(0, 10);
+```
+
+#### Value ranges
+
+The
+[`valueRange`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#valuerange)
+operator selects rows based on a given column's `start` .. `end` vaulue range
+(both inclusive) of row indices.
+
+```ts
+// select rows where ID is in closed [100,109] interval
+query.valueRange("id", 100, 109);
+```
+
 ### Custom operators
 
 Custom query operators can be registered via
@@ -412,7 +448,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.67 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 5.50 KB
 
 ## Dependencies
 
@@ -438,7 +474,13 @@ import {
     FLAG_UNIQUE,
 } from "@thi.ng/column-store";
 
-const table = new Table({
+interface Item {
+    id: number;
+    type: string;
+    tags: string[];
+}
+
+const table = new Table<Item>({
     // ID column stores 8bit ints (typed array)
     id: { type: "u8" },
     // Type column stores indexed strings

package/api.d.ts

@@ -1,8 +1,9 @@
-import type { FloatType, Fn3, IntType, Maybe, UintType } from "@thi.ng/api";
+import type { FloatType, Fn3, IntType, Maybe, Predicate, UintType } from "@thi.ng/api";
 import type { BitmapIndex } from "./bitmap.js";
 import type { QueryCtx } from "./query.js";
 import type { Table } from "./table.js";
-export type ColumnSchema = Record<string, ColumnSpec>;
+export type ColumnID<T extends Row> = Exclude<keyof T, number | symbol>;
+export type ColumnSchema<T extends Row> = Record<ColumnID<T>, ColumnSpec>;
 export type NumericType = IntType | UintType | FloatType;
 export type VectorType = `${NumericType}vec`;
 export type Cardinality = [number, number];
@@ -55,7 +56,7 @@ export interface ColumnTypeSpec {
      * Factory function to instantiate a colum type for given table, column name
      * & spec.
      */
-    impl: Fn3<Table, string, ColumnSpec, IColumn>;
+    impl: Fn3<Table<any>, string, ColumnSpec, IColumn>;
     /**
      * Bit mask of supported flags (default: 0, i.e. none allowed). During
      * column validation, the (inverted) mask will be applied to the user
@@ -91,14 +92,45 @@ export interface IColumn {
     validate(value: any): boolean;
     /**
      * Searches for `value` in the column data, optionally constrained to given
-     * `start`/`end` range. If found, returns row ID of first occurrence,
-     * otherwise -1.
+     * `start`/`end` range (end index is exclusive and defaults to current table
+     * length). If found, returns row ID of first occurrence, otherwise -1.
      *
      * @param value
      * @param start
      * @param end
      */
     indexOf(value: any, start?: number, end?: number): number;
+    /**
+     * Searches for `value` in the column data in reverse order (from the last
+     * row), optionally constrained to given `start`/`end` range (end index is
+     * exclusive and defaults to current table length). If found, returns row ID
+     * of first occurrence, otherwise -1.
+     *
+     * @param value
+     * @param start
+     * @param end
+     */
+    lastIndexOf(value: any, start?: number, end?: number): number;
+    /**
+     * Similar to {@link IColumn.indexOf}, but applies given predicate function
+     * to each row value. Returns index of first row for which the predicate is
+     * truthy, otherwise returns -1.
+     *
+     * @param pred
+     * @param start
+     * @param end
+     */
+    findIndex(pred: Predicate<any>, start?: number, end?: number): number;
+    /**
+     * Similar to {@link IColumn.lastIndexOf}, but applies given predicate
+     * function to each row value. Returns index of first row for which the
+     * predicate is truthy, otherwise returns -1.
+     *
+     * @param pred
+     * @param start
+     * @param end
+     */
+    findLastIndex(pred: Predicate<any>, start?: number, end?: number): number;
     /**
      * Returns value at given row.
      *
@@ -123,9 +155,9 @@ export interface IColumn {
     getRowKey(i: number): any;
     valueKey(value: any): any;
 }
-export interface SerializedTable {
-    schema: ColumnSchema;
-    columns: Record<string, SerializedColumn>;
+export interface SerializedTable<T extends Row> {
+    schema: ColumnSchema<T>;
+    columns: Record<ColumnID<T>, SerializedColumn>;
     length: number;
 }
 export interface SerializedColumn {
@@ -137,13 +169,16 @@ export interface SerializedIndex {
     next: number;
 }
 export type Row = Record<string, any>;
-export interface QueryTerm {
+export type RowWithMeta<T extends Row> = T & {
+    __row: number;
+};
+export interface QueryTerm<T extends Row> {
     type: string;
-    column?: string;
+    column?: ColumnID<T>;
     value: any;
     params?: any;
 }
-export type QueryTermOp = Fn3<QueryCtx, QueryTerm, Maybe<IColumn>, void>;
+export type QueryTermOp = Fn3<QueryCtx<any>, QueryTerm<any>, Maybe<IColumn>, void>;
 export interface QueryTermOpSpec {
     /**
      * Default mode is: "col";

package/bitmap.d.ts

@@ -28,8 +28,10 @@ export declare class Bitfield {
     constructor(buffer?: Uint32Array | undefined);
     ones(max?: number): Generator<number, void, unknown>;
     first(start?: number, end?: number): number;
+    last(start?: number, end?: number): number;
     setBit(id: number): void;
     clearBit(id: number): void;
+    fill(x: 0 | 1, start: number, end: number): this;
     ensure(size: number): Uint32Array<ArrayBufferLike>;
     removeBit(id: number): void;
 }

package/bitmap.js

@@ -1,3 +1,4 @@
+import { __clampRange } from "./internal/indexof.js";
 class BitmapIndex {
   index = /* @__PURE__ */ new Map();
   /**
@@ -62,9 +63,11 @@ class Bitfield {
       }
     }
   }
-  first(start = 0, end = (this.buffer?.length ?? -1) * 32) {
+  first(start = 0, end) {
     const { buffer } = this;
-    if (!buffer || start >= end) return -1;
+    if (!buffer) return -1;
+    [start, end] = __clampRange(buffer.length << 5, start, end);
+    if (start >= end) return -1;
     for (let i = start >>> 5, n = Math.min(Math.ceil(end / 32), buffer.length); i < n; i++) {
       let bits = buffer[i];
       while (bits) {
@@ -76,6 +79,22 @@ class Bitfield {
     }
     return -1;
   }
+  last(start = 0, end) {
+    const { buffer } = this;
+    if (!buffer) return -1;
+    [start, end] = __clampRange(buffer.length << 5, start, end);
+    if (start >= end) return -1;
+    for (let i = end >>> 5, n = start >>> 5; i >= n; i--) {
+      let bits = buffer[i];
+      while (bits) {
+        const msb = Math.clz32(bits) ^ 31;
+        const x = (i << 5) + msb;
+        if (x >= start && x < end) return x;
+        bits ^= 1 << msb;
+      }
+    }
+    return -1;
+  }
   setBit(id) {
     const w = id >>> 5;
     this.ensure(w)[w] |= 1 << (id & 31);
@@ -84,6 +103,22 @@ class Bitfield {
     const w = id >>> 5;
     this.ensure(w)[w] &= ~(1 << (id & 31));
   }
+  fill(x, start, end) {
+    const i = start >>> 5;
+    const j = end >>> 5;
+    let m = ~((1 << (start & 31)) - 1);
+    const buf = this.ensure(j);
+    if (i === j) {
+      m &= (1 << (end & 31)) - 1;
+      buf[i] = x ? buf[i] | m : buf[i] & ~m;
+    } else {
+      buf[i] = x ? buf[i] | m : buf[i] & ~m;
+      buf.fill(x ? -1 : 0, i + 1, j);
+      m = (1 << (end & 31)) - 1;
+      buf[j] = x ? buf[j] | m : buf[j] & ~m;
+    }
+    return this;
+  }
   ensure(size) {
     const b = this.buffer;
     if (!b || size >= b.length) {

package/columns/acolumn.d.ts

@@ -1,15 +1,16 @@
+import type { Predicate } from "@thi.ng/api";
 import type { BidirIndex } from "@thi.ng/bidir-index";
-import { type ColumnSpec, type IColumn, type SerializedColumn, type SerializedIndex } from "../api.js";
+import { type ColumnID, type ColumnSpec, type IColumn, type Row, type SerializedColumn, type SerializedIndex } from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
 import type { Table } from "../table.js";
-export declare abstract class AColumn implements IColumn {
-    readonly id: string;
-    table: Table;
+export declare abstract class AColumn<T extends Row = Row> implements IColumn {
+    readonly id: ColumnID<T>;
+    table: Table<T>;
     spec: ColumnSpec;
     bitmap?: BitmapIndex;
     dict?: BidirIndex<any>;
     abstract isArray: boolean;
-    constructor(id: string, table: Table);
+    constructor(id: ColumnID<T>, table: Table<T>);
     abstract load(spec: SerializedColumn): void;
     reindex(): void;
     abstract validate(value: any): boolean;
@@ -20,6 +21,9 @@ export declare abstract class AColumn implements IColumn {
     abstract getRow(i: number): any;
     abstract getRowKey(i: number): any;
     abstract indexOf(value: any, start?: number, end?: number): number;
+    abstract lastIndexOf(value: any, start?: number, end?: number): number;
+    findIndex(pred: Predicate<any>, start?: number, end?: number): number;
+    findLastIndex(pred: Predicate<any>, start?: number, end?: number): number;
     encode(value: any): any;
     decode(value: any): any;
     protected loadDict(serialized: SerializedIndex): void;

package/columns/acolumn.js

@@ -1,8 +1,9 @@
-import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
 import {
   FLAG_BITMAP
 } from "../api.js";
 import { BitmapIndex } from "../bitmap.js";
+import { __columnError } from "../internal/checks.js";
+import { __clampRange } from "../internal/indexof.js";
 class AColumn {
   constructor(id, table) {
     this.id = id;
@@ -16,6 +17,20 @@ class AColumn {
   reindex() {
     this.updateBitmap();
   }
+  findIndex(pred, start = 0, end) {
+    [start, end] = __clampRange(this.table.length, start, end);
+    for (let i = start; i < end; i++) {
+      if (pred(this.getRow(i))) return i;
+    }
+    return -1;
+  }
+  findLastIndex(pred, start = 0, end) {
+    [start, end] = __clampRange(this.table.length, start, end);
+    for (let i = end; i-- > start; ) {
+      if (pred(this.getRow(i))) return i;
+    }
+    return -1;
+  }
   encode(value) {
     return value;
   }
@@ -49,7 +64,7 @@ class AColumn {
     }
   }
   ensureValue(val) {
-    return val != null ? val : this.spec.cardinality[0] > 0 ? this.spec.default ?? illegalArgs(`missing value for column: ${this.id}`) : this.spec.default ?? null;
+    return val != null ? val : this.spec.cardinality[0] > 0 ? this.spec.default ?? __columnError(this.id, `missing value`) : this.spec.default ?? null;
   }
   ensureBitmap() {
     if (this.spec.flags & FLAG_BITMAP) this.bitmap = new BitmapIndex();

package/columns/dict-tuple.d.ts

@@ -1,7 +1,7 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
-import { type IColumn, type SerializedColumn } from "../api.js";
+import { type Row, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class DictTupleColumn extends AColumn implements IColumn {
+export declare class DictTupleColumn<T extends Row = Row> extends AColumn<T> {
     values: (number[] | null)[];
     dict: BidirIndex<any>;
     readonly isArray = true;
@@ -16,6 +16,7 @@ export declare class DictTupleColumn extends AColumn implements IColumn {
     valueKey(value: any): (number | null)[];
     removeRow(i: number): void;
     indexOf(value: any, start?: number, end?: number): number;
+    lastIndexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         dict: {

package/columns/dict-tuple.js

@@ -2,7 +2,7 @@ import { BidirIndex } from "@thi.ng/bidir-index";
 import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
-import { __indexOfTuple } from "../internal/indexof.js";
+import { __indexOfTuple, __lastIndexOfTuple } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
 class DictTupleColumn extends AColumn {
@@ -65,6 +65,15 @@ class DictTupleColumn extends AColumn {
       end
     );
   }
+  lastIndexOf(value, start = 0, end) {
+    return __lastIndexOfTuple(
+      value != null ? this.encode(value) : null,
+      this.values,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     const { dict, values, bitmap } = this;
     const res = dict.renameKey(currValue, newValue);

package/columns/dict.d.ts

@@ -1,7 +1,7 @@
 import { BidirIndex } from "@thi.ng/bidir-index";
-import { type IColumn, type SerializedColumn } from "../api.js";
+import { type Row, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class DictColumn extends AColumn implements IColumn {
+export declare class DictColumn<T extends Row = Row> extends AColumn<T> {
     values: (number | null)[];
     dict: BidirIndex<any>;
     readonly isArray = false;
@@ -16,6 +16,7 @@ export declare class DictColumn extends AColumn implements IColumn {
     valueKey(value: any): number | number[] | undefined;
     removeRow(i: number): void;
     indexOf(value: any, start?: number, end?: number): number;
+    lastIndexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         dict: {

package/columns/dict.js

@@ -4,7 +4,7 @@ import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
-import { __indexOfSingle } from "../internal/indexof.js";
+import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
 class DictColumn extends AColumn {
@@ -68,6 +68,16 @@ class DictColumn extends AColumn {
       end
     );
   }
+  lastIndexOf(value, start = 0, end) {
+    return __lastIndexOfSingle(
+      this.encode(value),
+      this.values,
+      this.bitmap,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     const { dict, values, bitmap } = this;
     const res = dict.renameKey(currValue, newValue);

package/columns/plain.d.ts

@@ -1,6 +1,6 @@
-import { type IColumn, type SerializedColumn } from "../api.js";
+import { type Row, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class PlainColumn extends AColumn implements IColumn {
+export declare class PlainColumn<T extends Row = Row> extends AColumn<T> {
     values: any[];
     readonly isArray = false;
     load({ values }: SerializedColumn): void;
@@ -11,6 +11,7 @@ export declare class PlainColumn extends AColumn implements IColumn {
     valueKey(x: any): any;
     removeRow(i: number): void;
     indexOf(value: any, start?: number, end?: number): number;
+    lastIndexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         values: any[];

package/columns/plain.js

@@ -1,7 +1,7 @@
 import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
-import { __indexOfSingle } from "../internal/indexof.js";
+import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
 import { __replaceValue } from "../internal/replace.js";
 import { AColumn } from "./acolumn.js";
 class PlainColumn extends AColumn {
@@ -46,6 +46,16 @@ class PlainColumn extends AColumn {
       end
     );
   }
+  lastIndexOf(value, start = 0, end) {
+    return __lastIndexOfSingle(
+      value,
+      this.values,
+      this.bitmap,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }

package/columns/tuple.d.ts

@@ -1,7 +1,7 @@
 import type { Nullable } from "@thi.ng/api";
-import { type IColumn, type SerializedColumn } from "../api.js";
+import { type Row, type SerializedColumn } from "../api.js";
 import { AColumn } from "./acolumn.js";
-export declare class TupleColumn extends AColumn implements IColumn {
+export declare class TupleColumn<T extends Row = Row> extends AColumn<T> {
     values: Nullable<number[]>[];
     readonly isArray = true;
     load(spec: SerializedColumn): void;
@@ -13,6 +13,7 @@ export declare class TupleColumn extends AColumn implements IColumn {
     valueKey(value: any): any[];
     removeRow(i: number): void;
     indexOf(value: any, start?: number, end?: number): number;
+    lastIndexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         values: Nullable<number[]>[];

package/columns/tuple.js

@@ -1,7 +1,7 @@
 import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
-import { __indexOfTuple } from "../internal/indexof.js";
+import { __indexOfTuple, __lastIndexOfTuple } from "../internal/indexof.js";
 import { AColumn } from "./acolumn.js";
 class TupleColumn extends AColumn {
   values = [];
@@ -48,6 +48,15 @@ class TupleColumn extends AColumn {
       end
     );
   }
+  lastIndexOf(value, start = 0, end) {
+    return __lastIndexOfTuple(
+      value,
+      this.values,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     const { values, bitmap } = this;
     const isUnique = this.spec.flags & FLAG_UNIQUE;

package/columns/typedarray.d.ts

@@ -1,14 +1,14 @@
 import { type TypedArray } from "@thi.ng/api/typedarray";
-import { type IColumn, type NumericType, type SerializedColumn } from "../api.js";
+import { type ColumnID, type NumericType, type Row, type SerializedColumn } from "../api.js";
 import type { Table } from "../table.js";
 import { AColumn } from "./acolumn.js";
-export declare class TypedArrayColumn extends AColumn implements IColumn {
+export declare class TypedArrayColumn<T extends Row = Row> extends AColumn<T> {
     values: TypedArray;
     type: NumericType;
     limit: [number, number];
     protected tmp: TypedArray;
     readonly isArray = false;
-    constructor(id: string, table: Table);
+    constructor(id: ColumnID<T>, table: Table<T>);
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
@@ -17,6 +17,7 @@ export declare class TypedArrayColumn extends AColumn implements IColumn {
     valueKey(value: any): number | number[];
     removeRow(i: number): void;
     indexOf(value: any, start?: number, end?: number): number;
+    lastIndexOf(value: any, start?: number, end?: number): number;
     replaceValue(currValue: any, newValue: any): boolean;
     toJSON(): {
         values: any[];

package/columns/typedarray.js

@@ -4,7 +4,7 @@ import { isNumber } from "@thi.ng/checks/is-number";
 import {
   LIMITS
 } from "../api.js";
-import { __indexOfSingle } from "../internal/indexof.js";
+import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
 import { __replaceValue } from "../internal/replace.js";
 import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
@@ -78,6 +78,16 @@ class TypedArrayColumn extends AColumn {
       end
     );
   }
+  lastIndexOf(value, start = 0, end) {
+    return __lastIndexOfSingle(
+      value,
+      this.values,
+      this.bitmap,
+      this.table.length,
+      start,
+      end
+    );
+  }
   replaceValue(currValue, newValue) {
     return __replaceValue(this.bitmap, this.values, currValue, newValue);
   }

package/columns/vector.d.ts

@@ -1,15 +1,15 @@
 import { type TypedArray } from "@thi.ng/api/typedarray";
-import { type IColumn, type NumericType, type SerializedColumn } from "../api.js";
+import { type ColumnID, type NumericType, type Row, type SerializedColumn } from "../api.js";
 import type { Table } from "../table.js";
 import { AColumn } from "./acolumn.js";
-export declare class VectorColumn extends AColumn implements IColumn {
+export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     values: TypedArray;
     type: NumericType;
     size: number;
     limit: [number, number];
     protected tmp: TypedArray;
     readonly isArray = false;
-    constructor(id: string, table: Table);
+    constructor(id: ColumnID<T>, table: Table<T>);
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
@@ -18,6 +18,7 @@ export declare class VectorColumn extends AColumn implements IColumn {
     valueKey(value: any): string | string[];
     removeRow(i: number): void;
     indexOf(value: any, start?: number, end?: number): number;
+    lastIndexOf(value: any, start?: number, end?: number): number;
     replaceValue(): boolean;
     toJSON(): {
         values: any[];

package/columns/vector.js

@@ -6,6 +6,7 @@ import { unsupportedOp } from "@thi.ng/errors/unsupported";
 import {
   LIMITS
 } from "../api.js";
+import { __clampRange } from "../internal/indexof.js";
 import { __deserializeTyped, __serializeTyped } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
 class VectorColumn extends AColumn {
@@ -76,10 +77,10 @@ class VectorColumn extends AColumn {
     this.values.fill(0, (length - 1) * size);
     this.bitmap?.removeBit(i);
   }
-  indexOf(value, start = 0, end = this.table.length) {
+  indexOf(value, start = 0, end) {
+    if (value == null) return -1;
     const { values, bitmap, size } = this;
-    start = Math.max(start, 0);
-    end = Math.min(end, this.table.length);
+    [start, end] = __clampRange(this.table.length, start, end);
     if (bitmap) {
       return bitmap.index.get(this.valueKey(value))?.first(start, end) ?? -1;
     }
@@ -93,6 +94,23 @@ class VectorColumn extends AColumn {
     }
     return -1;
   }
+  lastIndexOf(value, start = 0, end) {
+    if (value == null || value.length !== this.size) return -1;
+    const { values, bitmap, size } = this;
+    [start, end] = __clampRange(this.table.length, start, end);
+    if (bitmap) {
+      return bitmap.index.get(this.valueKey(value))?.last(start, end) ?? -1;
+    }
+    start *= size;
+    let i, j;
+    outer: for (i = end * size; (i -= size) >= start; ) {
+      for (j = 0; j < size; j++) {
+        if (values[i + j] !== value[j]) continue outer;
+      }
+      return i / size;
+    }
+    return -1;
+  }
   replaceValue() {
     unsupportedOp("TODO");
   }

package/internal/indexof.d.ts

@@ -2,5 +2,14 @@ import type { Maybe } from "@thi.ng/api";
 import type { BitmapIndex } from "../bitmap.js";
 /** @internal */
 export declare const __indexOfSingle: (needle: any, values: ArrayLike<any>, bitmap: Maybe<BitmapIndex>, max: number, start: number, end?: number) => number;
+/** @internal */
+export declare const __lastIndexOfSingle: (needle: any, values: ArrayLike<any>, bitmap: Maybe<BitmapIndex>, max: number, start: number, end?: number) => number;
+/** @internal */
 export declare const __indexOfTuple: (needle: ArrayLike<any> | null, values: ArrayLike<any>, max: number, start: number, end?: number) => number;
+/** @internal */
+export declare const __lastIndexOfTuple: (needle: ArrayLike<any> | null, values: ArrayLike<any>, max: number, start: number, end?: number) => number;
+/** @internal */
+export declare const __clamp: (x: number, a: number, b: number) => number;
+/** @internal */
+export declare const __clampRange: (max: number, start: number, end?: number) => number[];
 //# sourceMappingURL=indexof.d.ts.map

package/internal/indexof.js

@@ -1,19 +1,26 @@
 const __indexOfSingle = (needle, values, bitmap, max, start, end = max) => {
-  start = Math.max(start, 0);
-  end = Math.min(end, max);
+  [start, end] = __clampRange(max, start, end);
   if (bitmap) return bitmap.index.get(needle)?.first(start, end) ?? -1;
   for (let i = start; i < end; i++) {
     if (values[i] === needle) return i;
   }
   return -1;
 };
+const __lastIndexOfSingle = (needle, values, bitmap, max, start, end = max) => {
+  [start, end] = __clampRange(max, start, end);
+  if (bitmap) return bitmap.index.get(needle)?.last(start, end) ?? -1;
+  for (let i = end; i-- > start; ) {
+    if (values[i] === needle) return i;
+  }
+  return -1;
+};
 const __indexOfTuple = (needle, values, max, start, end = max) => {
-  start = Math.max(start, 0);
-  end = Math.min(end, max);
+  [start, end] = __clampRange(max, start, end);
   if (needle == null) {
     for (let i2 = start; i2 < end; i2++) {
       if (!values[i2]) return i2;
     }
+    return -1;
   }
   const n = needle.length;
   let i, j, row;
@@ -28,7 +35,38 @@ const __indexOfTuple = (needle, values, max, start, end = max) => {
   }
   return -1;
 };
+const __lastIndexOfTuple = (needle, values, max, start, end = max) => {
+  [start, end] = __clampRange(max, start, end);
+  if (needle == null) {
+    for (let i2 = end; i2-- > start; ) {
+      if (!values[i2]) return i2;
+    }
+    return -1;
+  }
+  const n = needle.length;
+  let i, j, row;
+  outer: for (i = end; i-- > start; ) {
+    row = values[i];
+    if (row?.length === n) {
+      for (j = 0; j < n; j++) {
+        if (row[j] !== needle[j]) continue outer;
+      }
+      return i;
+    }
+  }
+  return -1;
+};
+const __clamp = (x, a, b) => x < a ? a : x > b ? b : x;
+const __clampRange = (max, start, end) => {
+  start = __clamp(start, 0, max);
+  end = __clamp(end ?? max, start, max);
+  return [start, end];
+};
 export {
+  __clamp,
+  __clampRange,
   __indexOfSingle,
-  __indexOfTuple
+  __indexOfTuple,
+  __lastIndexOfSingle,
+  __lastIndexOfTuple
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.4.1",
+	"version": "0.6.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -126,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "3f0b151fb2283888314eb34206ba50c2dd5c7af6\n"
+	"gitHead": "f1d8da5644dd22f212fc313df2187380da968a7c\n"
 }

package/query.d.ts

@@ -1,30 +1,33 @@
 import type { Predicate } from "@thi.ng/api";
-import type { QueryTerm, QueryTermOpSpec } from "./api.js";
+import type { ColumnID, QueryTerm, QueryTermOpSpec, Row } from "./api.js";
 import type { Table } from "./table.js";
-export declare class Query {
-    table: Table;
-    terms: QueryTerm[];
-    constructor(table: Table, terms?: QueryTerm[]);
-    addTerm(term: QueryTerm): this;
+export declare class Query<T extends Row> {
+    readonly table: Table<T>;
+    terms: QueryTerm<T>[];
+    constructor(table: Table<T>, terms?: QueryTerm<T>[]);
+    addTerm(term: QueryTerm<T>): this;
     /** Alias for {@link Query.or} */
-    where: (column: string, value: any) => this;
+    where: (column: ColumnID<T>, value: any) => this;
     /** Alias for {@link Query.nor} */
-    whereNot: (column: string, value: any) => this;
-    or(column: string, value: any): this;
-    nor(column: string, value: any): this;
-    and(column: string, value: any): this;
-    nand(column: string, value: any): this;
-    matchColumn(column: string, pred: Predicate<any>): this;
-    matchPartialRow(columns: string[], pred: Predicate<Record<string, any>>): this;
-    matchRow(pred: Predicate<Record<string, any>>): this;
-    [Symbol.iterator](): Generator<import("./api.js").Row | undefined, void, unknown>;
+    whereNot: (column: ColumnID<T>, value: any) => this;
+    or(column: ColumnID<T>, value: any): this;
+    nor(column: ColumnID<T>, value: any): this;
+    and(column: ColumnID<T>, value: any): this;
+    nand(column: ColumnID<T>, value: any): this;
+    matchColumn(column: ColumnID<T>, pred: Predicate<any>): this;
+    matchPartialRow<K extends ColumnID<T>>(columns: K[], pred: Predicate<Pick<T, K>>): this;
+    matchRow(pred: Predicate<T>): this;
+    valueRange(column: ColumnID<T>, start: any, end?: any): this;
+    rowRange(start?: number, end?: number): this;
+    [Symbol.iterator](): Generator<import("./api.js").RowWithMeta<T>, void, unknown>;
 }
-export declare class QueryCtx {
-    readonly query: Query;
-    readonly table: Table;
+export declare class QueryCtx<T extends Row> {
+    readonly query: Query<T>;
+    readonly table: Table<T>;
     readonly size: number;
     bitmap?: Uint32Array;
-    constructor(query: Query);
+    constructor(query: Query<T>);
+    clear(): void;
     makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
     /**
      * Combines the `mask` with the context's mask (combined using bitwise AND).

package/query.js

@@ -2,6 +2,8 @@ import { isArray } from "@thi.ng/checks/is-array";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
 import { Bitfield } from "./bitmap.js";
+import { __columnError } from "./internal/checks.js";
+import { __clampRange } from "./internal/indexof.js";
 class Query {
   constructor(table, terms = []) {
     this.table = table;
@@ -49,6 +51,19 @@ class Query {
     this.terms.push({ type: "matchRow", value: pred });
     return this;
   }
+  valueRange(column, start, end) {
+    if (this.table.columns[column].isArray)
+      __columnError(
+        column,
+        `operator not supported with this column type`
+      );
+    this.terms.push({ type: "valueRange", column, value: { start, end } });
+    return this;
+  }
+  rowRange(start = 0, end) {
+    this.terms.push({ type: "rowRange", value: { start, end } });
+    return this;
+  }
   *[Symbol.iterator]() {
     const { table } = this;
     const ctx = new QueryCtx(this);
@@ -57,7 +72,7 @@ class Query {
       let column;
       if (term.column) {
         column = ctx.table.columns[term.column];
-        if (!column) illegalArgs(`column: ${term.column}`);
+        if (!column) illegalArgs(`column: ${String(term.column)}`);
       } else if (QUERY_OPS[term.type].mode !== "row") {
         illegalArgs(
           `query op: ${term.type} requires a column name given`
@@ -80,6 +95,9 @@ class QueryCtx {
   table;
   size;
   bitmap;
+  clear() {
+    this.bitmap = void 0;
+  }
   makeMask(seed) {
     const mask = new Uint32Array(this.size);
     if (seed) mask.set(seed);
@@ -174,7 +192,7 @@ const execBitAnd = (ctx, term, column) => {
     for (let k of key) {
       const b = bitmap.index.get(k)?.buffer;
       if (!b) {
-        if (term.type === "and") ctx.bitmap = void 0;
+        if (term.type === "and") ctx.clear();
         return;
       }
       colBitmaps.push(b);
@@ -191,9 +209,7 @@ const execBitAnd = (ctx, term, column) => {
   }
   if (mask) {
     term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-  } else {
-    ctx.bitmap = void 0;
-  }
+  } else ctx.clear();
 };
 const execAnd = (ctx, term, column) => {
   const n = ctx.table.length;
@@ -213,7 +229,7 @@ const execAnd = (ctx, term, column) => {
         for (let i = 0; i < n; i++) mask[i] &= m[i];
       } else mask = m;
     } else {
-      if (term.type === "and") ctx.bitmap = void 0;
+      if (term.type === "and") ctx.clear();
       return;
     }
   }
@@ -253,7 +269,7 @@ const QUERY_OPS = {
       const columns = term.params;
       const pred = term.value;
       let mask;
-      for (let i = 0, n = ctx.table.length; i < n; i++) {
+      for (let i = 0, n = table.length; i < n; i++) {
         if (pred(table.getPartialRow(i, columns, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
@@ -268,7 +284,7 @@ const QUERY_OPS = {
       const table = ctx.table;
       const pred = term.value;
       let mask;
-      for (let i = 0, n = ctx.table.length; i < n; i++) {
+      for (let i = 0, n = table.length; i < n; i++) {
         if (pred(table.getRow(i, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
@@ -276,8 +292,37 @@ const QUERY_OPS = {
       }
       if (mask) ctx.mergeMask(mask);
     }
+  },
+  valueRange: {
+    fn: (ctx, term, column) => {
+      const max = ctx.table.length;
+      const { start, end } = term.value;
+      let $start = 0;
+      let $end = max;
+      if (start != null) $start = column.findIndex((x) => x >= start);
+      if (end != null)
+        $end = column.findLastIndex((x) => x <= end, $start);
+      __fillMask(ctx, $start, $end >= 0 ? $end + 1 : $end);
+    }
+  },
+  rowRange: {
+    mode: "row",
+    fn: (ctx, term) => {
+      const max = ctx.table.length;
+      let { start = 0, end = max } = term.value;
+      [start, end] = __clampRange(max, start, end);
+      __fillMask(ctx, start, end);
+    }
   }
 };
+const __fillMask = (ctx, start, end, fill = 1) => {
+  if (start >= 0 && end >= 0) {
+    const mask = ctx.makeMask();
+    const bitmap = new Bitfield(mask);
+    bitmap.fill(fill, start, end);
+    ctx.mergeMask(mask);
+  } else ctx.clear();
+};
 const registerQueryOp = (type, spec) => {
   if (QUERY_OPS[type]) illegalArgs(`query op ${type} already registered`);
   QUERY_OPS[type] = spec;

package/table.d.ts

@@ -1,38 +1,44 @@
-import { type ColumnSchema, type ColumnSpec, type ColumnTypeSpec, type IColumn, type QueryTerm, type Row, type SerializedTable } from "./api.js";
+import type { Maybe } from "@thi.ng/api";
+import { type ColumnID, type ColumnSchema, type ColumnSpec, type ColumnTypeSpec, type IColumn, type QueryTerm, type Row, type RowWithMeta, type SerializedTable } from "./api.js";
 import { Query } from "./query.js";
 /**
  * Placeholder only. Unused so far.
  */
 export interface TableOpts {
 }
-export declare class Table {
+export declare class Table<T extends Row> {
     opts: TableOpts;
-    schema: ColumnSchema;
-    columns: Record<string, IColumn>;
+    schema: ColumnSchema<T>;
+    columns: Record<ColumnID<T>, IColumn>;
     length: number;
-    static load(serialized: SerializedTable, opts?: Partial<TableOpts>): Table;
-    constructor(schema: Record<string, Partial<ColumnSpec> & {
+    static load<T extends Row>(serialized: SerializedTable<T>, opts?: Partial<TableOpts>): Table<T>;
+    constructor(schema: Record<ColumnID<T>, Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
     }>, opts?: Partial<TableOpts>);
-    query(terms?: QueryTerm[]): Query;
-    addColumn(id: string, spec: Partial<ColumnSpec> & {
+    query(terms?: QueryTerm<T>[]): Query<T>;
+    addColumn(id: ColumnID<T>, spec: Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
     }): void;
-    removeColumn(id: string): boolean;
-    [Symbol.iterator](): Generator<Row | undefined, void, unknown>;
+    removeColumn(id: ColumnID<T>): boolean;
+    [Symbol.iterator](): Generator<Maybe<T>, void, unknown>;
     reindex(): void;
-    addRow(row: Row): void;
-    addRows(rows: Iterable<Row>): void;
-    updateRow(i: number, row: Row): void;
+    addRow(row: Partial<T>): void;
+    addRows(rows: Iterable<Partial<T>>): void;
+    updateRow(i: number, row: T): void;
     removeRow(i: number): void;
-    getRow(i: number, safe?: boolean, includeID?: boolean): Row | undefined;
-    getPartialRow(i: number, columns: string[], safe?: boolean, includeID?: boolean): Row | undefined;
-    indexOf(id: string, value: any, start?: number, end?: number): number;
-    validateRow(row: Row): void;
-    validateColumnSpec(id: string, spec: ColumnSpec): void;
+    getRow(i: number, safe?: boolean): Maybe<T>;
+    getRow(i: number, safe?: boolean, includeID?: false): Maybe<T>;
+    getRow(i: number, safe?: boolean, includeID?: true): Maybe<RowWithMeta<T>>;
+    getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean): Maybe<Pick<T, K>>;
+    getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: false): Maybe<Pick<T, K>>;
+    getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: true): Maybe<RowWithMeta<Pick<T, K>>>;
+    indexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
+    lastIndexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
+    validateRow(row: Partial<T>): void;
+    validateColumnSpec(id: ColumnID<T>, spec: ColumnSpec): void;
     toJSON(): {
-        schema: ColumnSchema;
-        columns: Record<string, IColumn>;
+        schema: ColumnSchema<T>;
+        columns: Record<ColumnID<T>, IColumn>;
         length: number;
     };
 }

package/table.js

@@ -43,7 +43,11 @@ class Table {
     };
     this.validateColumnSpec(id, $spec);
     this.schema[id] = $spec;
-    this.columns[id] = COLUMN_TYPES[spec.type].impl(this, id, $spec);
+    this.columns[id] = COLUMN_TYPES[spec.type].impl(
+      this,
+      String(id),
+      $spec
+    );
   }
   removeColumn(id) {
     if (this.columns[id]) return false;
@@ -101,6 +105,9 @@ class Table {
   indexOf(id, value, start, end) {
     return this.columns[id]?.indexOf(value, start, end) ?? -1;
   }
+  lastIndexOf(id, value, start, end) {
+    return this.columns[id]?.lastIndexOf(value, start, end) ?? -1;
+  }
   validateRow(row) {
     const { columns } = this;
     for (let id in columns) {