@thi.ng/column-store 0.7.1 → 0.9.0

package/README.md

@@ -30,6 +30,8 @@
     - [FLAG_RLE](#flag_rle)
     - [Custom flags](#custom-flags)
 - [Query engine](#query-engine)
+  - [Query execution](#query-execution)
+    - [Optimized row iteration](#optimized-row-iteration)
   - [Built-in operators](#built-in-operators)
     - [OR](#or)
     - [AND](#and)
@@ -315,7 +317,9 @@ types](#custom-column-types).
 ## Query engine
 
 The query engine is highly extensible and can be used for executing arbitrarily
-complex queries.
+complex queries via chaining of query operators.
+
+### Query execution
 
 The system allows predefining queries, which are then only evaluated and produce
 up-to-date results via the standard JS iterable mechanism (i.e. queries
@@ -328,19 +332,46 @@ const query = table.query().or("name", ["alice", "bob"]);
 // actually (re)execute query
 for(let result of query) { ... }
 
-// ..or using slice operator
+// ..or collect result into an array using slice operator
 const results = [...query];
 ```
 
-TODO see code examples below
-
-### Built-in operators
+#### Optimized row iteration
 
 The query engine works by applying a number of [query
 terms](https://docs.thi.ng/umbrella/column-store/interfaces/QueryTerm.html) in
 series, with each step intersecting (aka logical AND) its results with the
 results of the previous step(s), thereby narrowing down the result set.
 
+For each query term, only the rows already marked (aka pre-selected by
+predecessor query terms) are visited. When a query term does not manage to
+select any rows, the query is terminated. Internally, this selecting and
+intersecting of partial query results is done via bitfields only. There's no
+creation of interim result arrays, nor any full decoding/construction of interim
+row records. The latter only happens for the final result rows and/or when using
+the [`matchRow()` or `matchPartialRow()`](#predicate-based-matchers) query
+operators.
+
+When a column has an associated bitfield index (enabled via
+[`FLAG_BITMAP`](#flag_bitmap)), some query operators (see below) are optimized
+even further, entirely avoiding the need to visit any individual rows.
+
+The diagram below illustrates the application of the following 3-operator query
+and the resulting stepwise narrowing of the result set:
+
+```ts
+table.query()
+    .matchColumn("id", inRange(100, 110))
+    .matchColumn("age", inRange(20, 50))
+    .matchColumn("name", startsWith("a"))
+```
+
+![Diagram showing a list of rows with object values and three columns
+illustrating the narrowing effect of query operators with their partial
+results](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/column-store/query-narrowing.png)
+
+### Built-in operators
+
 By default, individual query terms operate on a single column, but can also can
 also apply to multiple. Terms are supplied either as array given to the
 [`Query`](https://docs.thi.ng/umbrella/column-store/classes/Query.html)
@@ -382,12 +413,18 @@ can be used, otherwise the behavior is:
 
 #### Predicate-based matchers
 
+> [!NOTE]
+> For best performance and to minimize/avoid potential decoding and construction
+> of interim row objects, prefer `matchColumn` or `matchPartialRow` over
+> `matchRow` if at all possible. Oftentimes, query predicates requiring multiple
+> column values can be easily refactored into separate query terms.
+
 - [`matchColumn`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchcolumn):
   apply predicate to column value
-- [`matchRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchrow):
-  apply predicate to full row
 - [`matchPartialRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchpartialrow):
   apply predicate to partial row (only selected columns)
+- [`matchRow`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#matchrow):
+  apply predicate to full row
 
 #### Row ranges
 
@@ -458,7 +495,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 5.71 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 5.78 KB
 
 ## Dependencies
 

package/api.d.ts

@@ -84,7 +84,7 @@ export declare const FLAG_UNIQUE: number;
 export declare const FLAG_RLE: number;
 /** @internal */
 export declare const LIMITS: Record<NumericType, [number, number]>;
-export interface IColumn {
+export interface IColumn extends Iterable<any> {
     bitmap?: BitmapIndex;
     readonly isArray: boolean;
     load(spec: SerializedColumn): void;

package/columns/acolumn.d.ts

@@ -11,8 +11,9 @@ export declare abstract class AColumn<T extends Row = Row> implements IColumn {
     dict?: BidirIndex<any>;
     abstract isArray: boolean;
     constructor(id: ColumnID<T>, table: Table<T>);
-    abstract load(spec: SerializedColumn): void;
+    [Symbol.iterator](): Generator<any, void, unknown>;
     reindex(): void;
+    abstract load(spec: SerializedColumn): void;
     abstract validate(value: any): boolean;
     abstract setRow(i: number, value: any): void;
     abstract removeRow(i: number): void;

package/columns/acolumn.js

@@ -14,6 +14,9 @@ class AColumn {
   spec;
   bitmap;
   dict;
+  *[Symbol.iterator]() {
+    for (let i = 0, n = this.table.length; i < n; i++) yield this.getRow(i);
+  }
   reindex() {
     this.updateBitmap();
   }

package/columns/vector.d.ts

@@ -14,7 +14,7 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     validate(value: any): boolean;
     ensureRows(): void;
     setRow(i: number, value: any): void;
-    getRow(i: number): Uint32Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike>;
+    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;

package/index.d.ts

@@ -1,5 +1,6 @@
 export * from "./api.js";
 export * from "./bitmap.js";
+export * from "./predicates.js";
 export * from "./query.js";
 export * from "./table.js";
 //# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,4 +1,5 @@
 export * from "./api.js";
 export * from "./bitmap.js";
+export * from "./predicates.js";
 export * from "./query.js";
 export * from "./table.js";

package/internal/serialize.d.ts

@@ -11,5 +11,5 @@ export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec,
     values: any[];
 };
 /** @internal */
-export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Uint32Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike>;
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.7.1",
+	"version": "0.9.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -115,6 +115,9 @@
 		"./columns/vector": {
 			"default": "./columns/vector.js"
 		},
+		"./predicates": {
+			"default": "./predicates.js"
+		},
 		"./query": {
 			"default": "./query.js"
 		},
@@ -126,5 +129,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "3e53df924e859a37127070b16960c8c6242adeaa\n"
+	"gitHead": "1138bb64457a415733cf5c05d32c211ccfd6ee72\n"
 }

package/predicates.d.ts

@@ -0,0 +1,25 @@
+import type { NumOrString, Predicate } from "@thi.ng/api";
+/**
+ * Higher-order query predicate for {@link Query.matchColumn}. The returned
+ * predicate returns true if a row value is in the semi-open interval defined by
+ * `[min,max)`.
+ *
+ * @param min
+ * @param max
+ */
+export declare const inRange: (min: NumOrString, max: NumOrString) => Predicate<NumOrString | null>;
+/**
+ * Higher-order query predicate for {@link Query.matchColumn}. The returned
+ * predicate returns true if a row value starts with given `prefix`.
+ *
+ * @param prefix
+ */
+export declare const startsWith: (prefix: string) => Predicate<string | null>;
+/**
+ * Higher-order query predicate for {@link Query.matchColumn}. The returned
+ * predicate returns true if a row value matches the given `regexp`.
+ *
+ * @param re
+ */
+export declare const matchRegExp: (re: RegExp) => Predicate<string | null>;
+//# sourceMappingURL=predicates.d.ts.map

package/predicates.js

@@ -0,0 +1,8 @@
+const inRange = (min, max) => (x) => x != null && x >= min && x < max;
+const startsWith = (prefix) => (x) => x?.startsWith(prefix) ?? false;
+const matchRegExp = (re) => (x) => x != null ? re.test(x) : false;
+export {
+  inRange,
+  matchRegExp,
+  startsWith
+};