@thi.ng/column-store 0.8.0 → 0.10.0

package/README.md

@@ -30,6 +30,8 @@
     - [FLAG_RLE](#flag_rle)
     - [Custom flags](#custom-flags)
 - [Query engine](#query-engine)
+  - [Query execution](#query-execution)
+    - [Optimized row iteration](#optimized-row-iteration)
   - [Built-in operators](#built-in-operators)
     - [OR](#or)
     - [AND](#and)
@@ -315,7 +317,9 @@ types](#custom-column-types).
 ## Query engine
 
 The query engine is highly extensible and can be used for executing arbitrarily
-complex queries.
+complex queries via chaining of query operators.
+
+### Query execution
 
 The system allows predefining queries, which are then only evaluated and produce
 up-to-date results via the standard JS iterable mechanism (i.e. queries
@@ -328,19 +332,46 @@ const query = table.query().or("name", ["alice", "bob"]);
 // actually (re)execute query
 for(let result of query) { ... }
 
-// ..or using slice operator
+// ..or collect result into an array using slice operator
 const results = [...query];
 ```
 
-TODO see code examples below
-
-### Built-in operators
+#### Optimized row iteration
 
 The query engine works by applying a number of [query
 terms](https://docs.thi.ng/umbrella/column-store/interfaces/QueryTerm.html) in
 series, with each step intersecting (aka logical AND) its results with the
 results of the previous step(s), thereby narrowing down the result set.
 
+For each query term, only the rows already marked (aka pre-selected by
+predecessor query terms) are visited. When a query term does not manage to
+select any rows, the query is terminated. Internally, this selecting and
+intersecting of partial query results is done via bitfields only. There's no
+creation of interim result arrays, nor any full decoding/construction of interim
+row records. The latter only happens for the final result rows and/or when using
+the [`matchRow()` or `matchPartialRow()`](#predicate-based-matchers) query
+operators.
+
+When a column has an associated bitfield index (enabled via
+[`FLAG_BITMAP`](#flag_bitmap)), some query operators (see below) are optimized
+even further, entirely avoiding the need to visit any individual rows.
+
+The diagram below illustrates the application of the following 3-operator query
+and the resulting stepwise narrowing of the result set:
+
+```ts
+table.query()
+    .matchColumn("id", inRange(100, 110))
+    .matchColumn("age", inRange(20, 50))
+    .matchColumn("name", startsWith("a"))
+```
+
+![Diagram showing a list of rows with object values and three columns
+illustrating the narrowing effect of query operators with their partial
+results](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/column-store/query-narrowing.png)
+
+### Built-in operators
+
 By default, individual query terms operate on a single column, but can also can
 also apply to multiple. Terms are supplied either as array given to the
 [`Query`](https://docs.thi.ng/umbrella/column-store/classes/Query.html)
@@ -382,12 +413,18 @@ can be used, otherwise the behavior is:
 
 #### Predicate-based matchers
 
+> [!NOTE]
+> For best performance and to minimize/avoid potential decoding and construction
+> of interim row objects, prefer `matchColumn` or `matchPartialRow` over
+> `matchRow` if at all possible. Oftentimes, query predicates requiring multiple
+> column values can be easily refactored into separate query terms.
+
 - [`matchColumn`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchcolumn):
   apply predicate to column value
-- [`matchRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchrow):
-  apply predicate to full row
 - [`matchPartialRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchpartialrow):
   apply predicate to partial row (only selected columns)
+- [`matchRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchrow):
+  apply predicate to full row
 
 #### Row ranges
 
@@ -458,7 +495,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 5.71 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 5.78 KB
 
 ## Dependencies
 

package/api.d.ts

@@ -1,4 +1,4 @@
-import type { FloatType, Fn3, IntType, Maybe, Predicate, UintType } from "@thi.ng/api";
+import type { FloatType, Fn3, IClear, IntType, Maybe, Predicate, UintType } from "@thi.ng/api";
 import type { BitmapIndex } from "./bitmap.js";
 import type { QueryCtx } from "./query.js";
 import type { Table } from "./table.js";
@@ -84,7 +84,7 @@ export declare const FLAG_UNIQUE: number;
 export declare const FLAG_RLE: number;
 /** @internal */
 export declare const LIMITS: Record<NumericType, [number, number]>;
-export interface IColumn extends Iterable<any> {
+export interface IColumn extends Iterable<any>, IClear {
     bitmap?: BitmapIndex;
     readonly isArray: boolean;
     load(spec: SerializedColumn): void;
@@ -192,4 +192,10 @@ export interface QueryTermOpSpec {
      */
     fn: QueryTermOp;
 }
+/**
+ * Initial capacity for typedarray and vector columns
+ *
+ * @internal
+ */
+export declare const INITIAL_CAPACITY = 8;
 //# sourceMappingURL=api.d.ts.map

package/api.js

@@ -17,11 +17,13 @@ const LIMITS = {
   f32: [-Infinity, Infinity],
   f64: [-Infinity, Infinity]
 };
+const INITIAL_CAPACITY = 8;
 export {
   FLAG_BITMAP,
   FLAG_DICT,
   FLAG_RLE,
   FLAG_UNIQUE,
+  INITIAL_CAPACITY,
   LIMITS,
   ONE_PLUS,
   OPTIONAL,

package/columns/acolumn.d.ts

@@ -13,6 +13,7 @@ export declare abstract class AColumn<T extends Row = Row> implements IColumn {
     constructor(id: ColumnID<T>, table: Table<T>);
     [Symbol.iterator](): Generator<any, void, unknown>;
     reindex(): void;
+    abstract clear(): void;
     abstract load(spec: SerializedColumn): void;
     abstract validate(value: any): boolean;
     abstract setRow(i: number, value: any): void;

package/columns/dict-tuple.d.ts

@@ -5,6 +5,7 @@ export declare class DictTupleColumn<T extends Row = Row> extends AColumn<T> {
     values: (number[] | null)[];
     dict: BidirIndex<any>;
     readonly isArray = true;
+    clear(): void;
     load({ dict, values }: SerializedColumn): void;
     reindex(): void;
     encode(value: any): (number | null)[];

package/columns/dict-tuple.js

@@ -9,6 +9,11 @@ class DictTupleColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = true;
+  clear() {
+    this.values = [];
+    this.dict.clear();
+    this.bitmap?.clear();
+  }
   load({ dict, values }) {
     this.values = values;
     super.loadDict(dict);

package/columns/dict.d.ts

@@ -5,6 +5,7 @@ export declare class DictColumn<T extends Row = Row> extends AColumn<T> {
     values: (number | null)[];
     dict: BidirIndex<any>;
     readonly isArray = false;
+    clear(): void;
     load({ dict, values }: SerializedColumn): void;
     reindex(): void;
     encode(value: any): number | null;

package/columns/dict.js

@@ -11,6 +11,11 @@ class DictColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = false;
+  clear() {
+    this.values = [];
+    this.dict.clear();
+    this.bitmap?.clear();
+  }
   load({ dict, values }) {
     this.values = this.spec.flags & FLAG_RLE ? this.spec.cardinality[0] === 0 && this.spec.default == null ? decodeSimple(values) : Array.from(decodeBinary(values)) : values;
     super.loadDict(dict);

package/columns/plain.d.ts

@@ -3,6 +3,7 @@ import { AColumn } from "./acolumn.js";
 export declare class PlainColumn<T extends Row = Row> extends AColumn<T> {
     values: any[];
     readonly isArray = false;
+    clear(): void;
     load({ values }: SerializedColumn): void;
     ensureRows(): void;
     validate(value: any): boolean;

package/columns/plain.js

@@ -7,6 +7,10 @@ import { AColumn } from "./acolumn.js";
 class PlainColumn extends AColumn {
   values = [];
   isArray = false;
+  clear() {
+    this.values = [];
+    this.bitmap?.clear();
+  }
   load({ values }) {
     this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeSimple(values)) : values;
     this.reindex();

package/columns/tuple.d.ts

@@ -4,6 +4,7 @@ import { AColumn } from "./acolumn.js";
 export declare class TupleColumn<T extends Row = Row> extends AColumn<T> {
     values: Nullable<number[]>[];
     readonly isArray = true;
+    clear(): void;
     load(spec: SerializedColumn): void;
     encode(value: any): any[];
     validate(value: any): boolean;

package/columns/tuple.js

@@ -6,6 +6,10 @@ import { AColumn } from "./acolumn.js";
 class TupleColumn extends AColumn {
   values = [];
   isArray = true;
+  clear() {
+    this.values = [];
+    this.bitmap?.clear();
+  }
   load(spec) {
     this.values = spec.values;
     this.reindex();

package/columns/typedarray.d.ts

@@ -9,6 +9,7 @@ export declare class TypedArrayColumn<T extends Row = Row> extends AColumn<T> {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: ColumnID<T>, table: Table<T>);
+    clear(): void;
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     ensureRows(): void;

package/columns/typedarray.js

@@ -2,6 +2,7 @@ import { typedArray } from "@thi.ng/api/typedarray";
 import { isArray } from "@thi.ng/checks/is-array";
 import { isNumber } from "@thi.ng/checks/is-number";
 import {
+  INITIAL_CAPACITY,
   LIMITS
 } from "../api.js";
 import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
@@ -18,9 +19,13 @@ class TypedArrayColumn extends AColumn {
     super(id, table);
     this.type = table.schema[id].type;
     this.limit = LIMITS[this.type];
-    this.values = typedArray(this.type, 8);
+    this.values = typedArray(this.type, INITIAL_CAPACITY);
     this.tmp = typedArray(this.type, 1);
   }
+  clear() {
+    this.values = typedArray(this.type, INITIAL_CAPACITY);
+    this.bitmap?.clear();
+  }
   load({ values }) {
     this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();

package/columns/vector.d.ts

@@ -10,6 +10,7 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: ColumnID<T>, table: Table<T>);
+    clear(): void;
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     ensureRows(): void;

package/columns/vector.js

@@ -4,6 +4,7 @@ import { isArrayLike } from "@thi.ng/checks/is-arraylike";
 import { isNumber } from "@thi.ng/checks/is-number";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
 import {
+  INITIAL_CAPACITY,
   LIMITS
 } from "../api.js";
 import { __clampRange } from "../internal/indexof.js";
@@ -21,9 +22,13 @@ class VectorColumn extends AColumn {
     this.type = this.spec.type.split("v")[0];
     this.size = this.spec.cardinality[1];
     this.limit = LIMITS[this.type];
-    this.values = typedArray(this.type, 8 * this.size);
+    this.values = typedArray(this.type, INITIAL_CAPACITY * this.size);
     this.tmp = typedArray(this.type, this.size);
   }
+  clear() {
+    this.values = typedArray(this.type, INITIAL_CAPACITY * this.size);
+    this.bitmap?.clear();
+  }
   load({ values }) {
     this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();

package/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./api.js";
 export * from "./bitmap.js";
+export * from "./predicates.js";
 export * from "./query.js";
 export * from "./table.js";
 //# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,4 +1,5 @@
 export * from "./api.js";
 export * from "./bitmap.js";
+export * from "./predicates.js";
 export * from "./query.js";
 export * from "./table.js";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.8.0",
+	"version": "0.10.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -115,6 +115,9 @@
 		"./columns/vector": {
 			"default": "./columns/vector.js"
 		},
+		"./predicates": {
+			"default": "./predicates.js"
+		},
 		"./query": {
 			"default": "./query.js"
 		},
@@ -126,5 +129,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "63ca02b09367c07f6cb785642bc3f2b7df2e5804\n"
+	"gitHead": "f15a589f1695f67f2e90b8d221a89fc9960a2e83\n"
 }

package/predicates.d.ts

@@ -0,0 +1,25 @@
+import type { NumOrString, Predicate } from "@thi.ng/api";
+/**
+ * Higher-order query predicate for {@link Query.matchColumn}. The returned
+ * predicate returns true if a row value is in the semi-open interval defined by
+ * `[min,max)`.
+ *
+ * @param min
+ * @param max
+ */
+export declare const inRange: (min: NumOrString, max: NumOrString) => Predicate<NumOrString | null>;
+/**
+ * Higher-order query predicate for {@link Query.matchColumn}. The returned
+ * predicate returns true if a row value starts with given `prefix`.
+ *
+ * @param prefix
+ */
+export declare const startsWith: (prefix: string) => Predicate<string | null>;
+/**
+ * Higher-order query predicate for {@link Query.matchColumn}. The returned
+ * predicate returns true if a row value matches the given `regexp`.
+ *
+ * @param re
+ */
+export declare const matchRegExp: (re: RegExp) => Predicate<string | null>;
+//# sourceMappingURL=predicates.d.ts.map

package/predicates.js

@@ -0,0 +1,8 @@
+const inRange = (min, max) => (x) => x != null && x >= min && x < max;
+const startsWith = (prefix) => (x) => x?.startsWith(prefix) ?? false;
+const matchRegExp = (re) => (x) => x != null ? re.test(x) : false;
+export {
+  inRange,
+  matchRegExp,
+  startsWith
+};

package/table.d.ts

@@ -1,4 +1,4 @@
-import type { Maybe } from "@thi.ng/api";
+import type { IClear, ICopy, IEmpty, Maybe } from "@thi.ng/api";
 import { type ColumnID, type ColumnSchema, type ColumnSpec, type ColumnTypeSpec, type IColumn, type QueryTerm, type Row, type RowWithMeta, type SerializedTable } from "./api.js";
 import { Query } from "./query.js";
 /**
@@ -6,7 +6,7 @@ import { Query } from "./query.js";
  */
 export interface TableOpts {
 }
-export declare class Table<T extends Row> {
+export declare class Table<T extends Row> implements IClear, ICopy<Table<T>>, IEmpty<Table<T>> {
     opts: TableOpts;
     schema: ColumnSchema<T>;
     columns: Record<ColumnID<T>, IColumn>;
@@ -15,12 +15,15 @@ export declare class Table<T extends Row> {
     constructor(schema: Record<ColumnID<T>, Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
     }>, opts?: Partial<TableOpts>);
+    clear(): void;
+    copy(): Table<T>;
+    empty(): Table<T>;
     query(terms?: QueryTerm<T>[]): Query<T>;
     addColumn(id: ColumnID<T>, spec: Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
     }): IColumn;
     removeColumn(id: ColumnID<T>): boolean;
-    [Symbol.iterator](): Generator<Maybe<T>, void, unknown>;
+    [Symbol.iterator](): Generator<T, void, unknown>;
     reindex(): void;
     addRow(row: Partial<T>): void;
     addRows(rows: Iterable<Partial<T>>): void;
@@ -32,6 +35,7 @@ export declare class Table<T extends Row> {
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean): Maybe<Pick<T, K>>;
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: false): Maybe<Pick<T, K>>;
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: true): Maybe<RowWithMeta<Pick<T, K>>>;
+    slice(start?: number, end?: number): Table<T>;
     indexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
     lastIndexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
     validateRow(row: Partial<T>): void;

package/table.js

@@ -13,6 +13,7 @@ import { TupleColumn } from "./columns/tuple.js";
 import { TypedArrayColumn } from "./columns/typedarray.js";
 import { VectorColumn } from "./columns/vector.js";
 import { __columnError } from "./internal/checks.js";
+import { __clamp } from "./internal/indexof.js";
 import { Query } from "./query.js";
 class Table {
   opts;
@@ -31,6 +32,19 @@ class Table {
     this.opts = { ...opts };
     for (let id in schema) this.addColumn(id, schema[id]);
   }
+  clear() {
+    const { columns } = this;
+    for (let id in columns) columns[id].clear();
+    this.length = 0;
+  }
+  copy() {
+    const copy = new Table(this.schema, this.opts);
+    copy.addRows(this);
+    return copy;
+  }
+  empty() {
+    return new Table(this.schema, this.opts);
+  }
   query(terms) {
     return new Query(this, terms);
   }
@@ -104,6 +118,14 @@ class Table {
     }
     return row;
   }
+  slice(start = 0, end) {
+    const max = this.length;
+    start = __clamp(start, 0, max);
+    end = __clamp(end ?? max, start, max);
+    const copy = this.empty();
+    for (let i = start; i < end; i++) copy.addRow(this.getRow(i, true));
+    return copy;
+  }
   indexOf(id, value, start, end) {
     return this.columns[id]?.indexOf(value, start, end) ?? -1;
   }