@thi.ng/column-store 0.10.0 → 0.11.1

package/README.md

@@ -39,6 +39,7 @@
     - [Predicate-based matchers](#predicate-based-matchers)
     - [Row ranges](#row-ranges)
     - [Value ranges](#value-ranges)
+  - [Result order and pagination](#result-order-and-pagination)
   - [Custom operators](#custom-operators)
   - [Result aggregation](#result-aggregation)
   - [Query ranges](#query-ranges)
@@ -450,6 +451,15 @@ operator selects rows based on a given column's `start` .. `end` vaulue range
 query.valueRange("id", 100, 109);
 ```
 
+### Result order and pagination
+
+- [`sortBy()`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#sortby)
+  allows query results to be ordered (ascending or descending) via an arbitrary
+  number of sort columns or criteria, applied in the given order.
+- [`limit()`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#limit)
+  provides basic pagination of query results by specifying a max. number of
+  results and start offset
+
 ### Custom operators
 
 Custom query operators can be registered via
@@ -495,13 +505,14 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 5.78 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 6.28 KB
 
 ## Dependencies
 
 - [@thi.ng/api](https://github.com/thi-ng/umbrella/tree/develop/packages/api)
 - [@thi.ng/bidir-index](https://github.com/thi-ng/umbrella/tree/develop/packages/bidir-index)
 - [@thi.ng/checks](https://github.com/thi-ng/umbrella/tree/develop/packages/checks)
+- [@thi.ng/compare](https://github.com/thi-ng/umbrella/tree/develop/packages/compare)
 - [@thi.ng/errors](https://github.com/thi-ng/umbrella/tree/develop/packages/errors)
 - [@thi.ng/rle-pack](https://github.com/thi-ng/umbrella/tree/develop/packages/rle-pack)
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.10.0",
+	"version": "0.11.1",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -40,11 +40,12 @@
 		"tool:tangle": "../../node_modules/.bin/tangle src/**/*.ts"
 	},
 	"dependencies": {
-		"@thi.ng/api": "^8.12.14",
-		"@thi.ng/bidir-index": "^1.5.0",
-		"@thi.ng/checks": "^3.8.4",
-		"@thi.ng/errors": "^2.6.3",
-		"@thi.ng/rle-pack": "^3.2.1"
+		"@thi.ng/api": "^8.12.15",
+		"@thi.ng/bidir-index": "^1.5.1",
+		"@thi.ng/checks": "^3.8.5",
+		"@thi.ng/compare": "^2.5.1",
+		"@thi.ng/errors": "^2.6.4",
+		"@thi.ng/rle-pack": "^3.2.2"
 	},
 	"devDependencies": {
 		"esbuild": "^0.27.2",
@@ -129,5 +130,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "f15a589f1695f67f2e90b8d221a89fc9960a2e83\n"
+	"gitHead": "8f50352caab9ec7757d645c0afa605dfb5427abe\n"
 }

package/query.d.ts

@@ -1,11 +1,27 @@
-import type { Predicate } from "@thi.ng/api";
-import type { ColumnID, QueryTerm, QueryTermOpSpec, Row } from "./api.js";
+import type { Comparator, Predicate } from "@thi.ng/api";
+import type { ColumnID, QueryTerm, QueryTermOpSpec, Row, RowWithMeta } from "./api.js";
 import type { Table } from "./table.js";
 export declare class Query<T extends Row> {
     readonly table: Table<T>;
     terms: QueryTerm<T>[];
+    protected _cmp?: Comparator<any>;
+    protected _limit: number;
+    protected _offset: number;
     constructor(table: Table<T>, terms?: QueryTerm<T>[]);
     addTerm(term: QueryTerm<T>): this;
+    limit(limit: number, offset?: number): void;
+    /**
+     * Constructs a comparator for query results based on given sort criteria,
+     * which are applied in given order. Each criteria can be on of:
+     *
+     * - comparator (applied to full rows)
+     * - columnID
+     * - tuple of `[columnID, boolean]` (if boolean is true, sorts in ascending order)
+     * - tuple of `[columnID, comparator]`
+     *
+     * @param order
+     */
+    sortBy(...order: (ColumnID<T> | Comparator<T> | [ColumnID<T>, boolean | Comparator<any>])[]): this;
     /** Alias for {@link Query.or} */
     where: (column: ColumnID<T>, value: any) => this;
     /** Alias for {@link Query.nor} */
@@ -19,7 +35,7 @@ export declare class Query<T extends Row> {
     matchRow(pred: Predicate<T>): this;
     valueRange(column: ColumnID<T>, start: any, end?: any): this;
     rowRange(start?: number, end?: number): this;
-    [Symbol.iterator](): Generator<import("./api.js").RowWithMeta<T>, void, unknown>;
+    [Symbol.iterator](): Generator<RowWithMeta<T>, void, unknown>;
 }
 export declare class QueryCtx<T extends Row> {
     readonly query: Query<T>;
@@ -33,7 +49,7 @@ export declare class QueryCtx<T extends Row> {
      */
     [Symbol.iterator](): Generator<number, void, unknown>;
     clear(): void;
-    makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
+    makeMask(seed?: number | Uint32Array): Uint32Array<ArrayBuffer>;
     /**
      * Combines the `mask` with the context's mask (combined using bitwise AND).
      * If the context mask is still undefined, assigns `mask` as the initial

package/query.js

@@ -1,4 +1,11 @@
 import { isArray } from "@thi.ng/checks/is-array";
+import { isFunction } from "@thi.ng/checks/is-function";
+import { isNumber } from "@thi.ng/checks/is-number";
+import { isString } from "@thi.ng/checks/is-string";
+import { compare } from "@thi.ng/compare/compare";
+import { composeComparators } from "@thi.ng/compare/compose";
+import { compareByKey } from "@thi.ng/compare/keys";
+import { reverse } from "@thi.ng/compare/reverse";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
 import { Bitfield } from "./bitmap.js";
@@ -10,11 +17,39 @@ class Query {
     for (let term of terms) this.addTerm(term);
   }
   terms = [];
+  _cmp;
+  _limit = Infinity;
+  _offset = 0;
   addTerm(term) {
     if (!QUERY_OPS[term.type]) unsupportedOp(`query type: ${term.type}`);
     this.terms.push(term);
     return this;
   }
+  limit(limit, offset = 0) {
+    this._limit = limit;
+    this._offset = offset;
+  }
+  /**
+   * Constructs a comparator for query results based on given sort criteria,
+   * which are applied in given order. Each criteria can be on of:
+   *
+   * - comparator (applied to full rows)
+   * - columnID
+   * - tuple of `[columnID, boolean]` (if boolean is true, sorts in ascending order)
+   * - tuple of `[columnID, comparator]`
+   *
+   * @param order
+   */
+  sortBy(...order) {
+    const fns = order.map(
+      (x) => isFunction(x) ? x : isString(x) ? compareByKey(x) : compareByKey(
+        x[0],
+        isFunction(x[1]) ? x[1] : x[1] ? compare : reverse(compare)
+      )
+    );
+    this._cmp = composeComparators(...fns);
+    return this;
+  }
   /** Alias for {@link Query.or} */
   where = (column, value) => this.or(column, value);
   /** Alias for {@link Query.nor} */
@@ -65,7 +100,7 @@ class Query {
     return this;
   }
   *[Symbol.iterator]() {
-    const { table } = this;
+    const { table, _limit, _offset } = this;
     const ctx = new QueryCtx(this);
     for (let term of this.terms) {
       const op = QUERY_OPS[term.type];
@@ -80,8 +115,25 @@ class Query {
       }
       if (!op.fn(ctx, term, column)) return;
     }
-    if (ctx.bitmap) {
-      for (let i of ctx) yield table.getRow(i, false, true);
+    if (!ctx.bitmap) return;
+    if (this._cmp) {
+      const rows = [];
+      for (let i of ctx) {
+        rows.push(table.getRow(i, false, true));
+      }
+      rows.sort(this._cmp);
+      for (let i = this._offset, n = Math.min(rows.length, i + this._limit); i < n; i++) {
+        yield rows[i];
+      }
+      return;
+    }
+    let j = 0;
+    for (let i of ctx) {
+      if (j >= _offset) {
+        if (j >= _limit) return;
+        yield table.getRow(i, false, true);
+      }
+      j++;
     }
   }
 }
@@ -111,7 +163,9 @@ class QueryCtx {
   }
   makeMask(seed) {
     const mask = new Uint32Array(this.size);
-    if (seed) mask.set(seed);
+    if (seed) {
+      isNumber(seed) ? mask.fill(seed) : mask.set(seed);
+    }
     return mask;
   }
   /**
@@ -166,11 +220,7 @@ const execBitOr = (ctx, term, column) => {
     const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
-  if (mask) {
-    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, term.type === "nor");
 };
 const execOr = (ctx, term, column) => {
   const key = column.valueKey(term.value);
@@ -184,11 +234,7 @@ const execOr = (ctx, term, column) => {
       }
     }
   }
-  if (mask) {
-    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, term.type === "nor");
 };
 const delegateOr = (ctx, term, column) => (term.value != null && column.bitmap ? execBitOr : execOr)(
   ctx,
@@ -198,15 +244,18 @@ const delegateOr = (ctx, term, column) => (term.value != null && column.bitmap ?
 const execBitAnd = (ctx, term, column) => {
   const bitmap = column.bitmap;
   const key = column.valueKey(term.value);
+  const isNeg = term.type === "nand";
   let mask;
   if (isArray(key)) {
-    const colBitmaps = [];
     for (let k of key) {
       const b = bitmap.index.get(k)?.buffer;
-      if (!b) return false;
-      colBitmaps.push(b);
-    }
-    for (let b of colBitmaps) {
+      if (!b) {
+        if (isNeg) {
+          if (mask) mask.fill(0);
+          else mask = ctx.makeMask();
+          continue;
+        } else return false;
+      }
       if (mask) {
         const n = b.length;
         for (let i = 0; i < n; i++) mask[i] &= b[i];
@@ -217,16 +266,13 @@ const execBitAnd = (ctx, term, column) => {
     const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
-  if (mask) {
-    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, isNeg);
 };
 const execAnd = (ctx, term, column) => {
   const n = ctx.table.length;
   const key = column.valueKey(term.value) ?? null;
   const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
+  const isNeg = term.type === "nand";
   let mask;
   for (let k of isArray(key) ? key : [key]) {
     let m;
@@ -241,20 +287,29 @@ const execAnd = (ctx, term, column) => {
         for (let i = 0; i < n; i++) mask[i] &= m[i];
       } else mask = m;
     } else {
-      return false;
+      if (isNeg) {
+        if (mask) mask.fill(0);
+        else mask = ctx.makeMask();
+      } else return false;
     }
   }
-  if (mask) {
-    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, isNeg);
 };
 const delegateAnd = (ctx, term, column) => (term.value != null && column.bitmap ? execBitAnd : execAnd)(
   ctx,
   term,
   column
 );
+const __finalizeMask = (ctx, mask, isNeg) => {
+  if (mask) {
+    isNeg ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
+    return true;
+  } else if (isNeg) {
+    if (!ctx.bitmap) ctx.bitmap = ctx.makeMask(-1);
+    return true;
+  }
+  return false;
+};
 const QUERY_OPS = {
   or: { fn: delegateOr },
   nor: { fn: delegateOr },

package/table.d.ts

@@ -35,6 +35,17 @@ export declare class Table<T extends Row> implements IClear, ICopy<Table<T>>, IE
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean): Maybe<Pick<T, K>>;
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: false): Maybe<Pick<T, K>>;
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: true): Maybe<RowWithMeta<Pick<T, K>>>;
+    getRows(start?: number, end?: number, includeID?: false): IterableIterator<T>;
+    getRows(start?: number, end?: number, includeID?: true): IterableIterator<RowWithMeta<T>>;
+    getPartialRows<K extends ColumnID<T>>(columns: K[], start?: number, end?: number, includeID?: false): IterableIterator<Pick<T, K>>;
+    getPartialRows<K extends ColumnID<T>>(columns: K[], start?: number, end?: number, includeID?: true): IterableIterator<RowWithMeta<Pick<T, K>>>;
+    /**
+     * Creates a new table with same schema and options, but only containing the
+     * subset of rows in the `[start,end)` interval.
+     *
+     * @param start
+     * @param end
+     */
     slice(start?: number, end?: number): Table<T>;
     indexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
     lastIndexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;

package/table.js

@@ -13,7 +13,7 @@ import { TupleColumn } from "./columns/tuple.js";
 import { TypedArrayColumn } from "./columns/typedarray.js";
 import { VectorColumn } from "./columns/vector.js";
 import { __columnError } from "./internal/checks.js";
-import { __clamp } from "./internal/indexof.js";
+import { __clampRange } from "./internal/indexof.js";
 import { Query } from "./query.js";
 class Table {
   opts;
@@ -118,10 +118,25 @@ class Table {
     }
     return row;
   }
+  *getRows(start = 0, end, includeID = false) {
+    [start, end] = __clampRange(this.length, start, end);
+    for (let i = start; i < end; i++)
+      yield this.getRow(i, false, includeID);
+  }
+  *getPartialRows(columns, start = 0, end, includeID = false) {
+    [start, end] = __clampRange(this.length, start, end);
+    for (let i = start; i < end; i++)
+      yield this.getPartialRow(i, columns, false, includeID);
+  }
+  /**
+   * Creates a new table with same schema and options, but only containing the
+   * subset of rows in the `[start,end)` interval.
+   *
+   * @param start
+   * @param end
+   */
   slice(start = 0, end) {
-    const max = this.length;
-    start = __clamp(start, 0, max);
-    end = __clamp(end ?? max, start, max);
+    [start, end] = __clampRange(this.length, start, end);
     const copy = this.empty();
     for (let i = start; i < end; i++) copy.addRow(this.getRow(i, true));
     return copy;