@thi.ng/column-store 0.11.4 → 0.13.0

package/README.md

@@ -337,6 +337,24 @@ for(let result of query) { ... }
 const results = [...query];
 ```
 
+Alternatively, queries can also be eagerly executed via
+[`.exec()`](https://docs.thi.ng/umbrella/column-store/classes/Query.html#exec).
+This will also include metadata alongside the results, useful for paging
+results.
+
+```ts
+// predefine query
+const query = table.query().or("name", ["alice", "bob"]).limit(10);
+
+const results = query.exec();
+// {
+//   results: [{...}, ...],
+//   total: 123,
+//   offset: 0,
+//   limit: 10
+// }
+```
+
 #### Optimized row iteration
 
 The query engine works by applying a number of [query
@@ -475,7 +493,7 @@ TODO
 
 ## Status
 
-**ALPHA** - bleeding edge / work-in-progress
+**BETA** - possibly breaking changes forthcoming
 
 [Search or submit any issues for this package](https://github.com/thi-ng/umbrella/issues?q=%5Bcolumn-store%5D+in%3Atitle)
 
@@ -505,7 +523,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 6.27 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 6.48 KB
 
 ## Dependencies
 

package/api.d.ts

@@ -192,6 +192,25 @@ export interface QueryTermOpSpec {
      */
     fn: QueryTermOp;
 }
+export interface QueryResult<T extends Row> {
+    /**
+     * Array of result rows (each incl. `__row` index)
+     */
+    results: RowWithMeta<T>[];
+    /**
+     * Total number of query results (might be more than in
+     * {@link QueryResult.results}).
+     */
+    total: number;
+    /**
+     * Page info. Configured result offset (via {@link Query.limit}).
+     */
+    offset: number;
+    /**
+     * Page info. Configured result limit (via {@link Query.limit}).
+     */
+    limit: number;
+}
 /**
  * Initial capacity for typedarray and vector columns
  *

package/bitmap.js

@@ -35,11 +35,11 @@ class BitmapIndex {
    * @param id
    */
   removeBit(rowID) {
-    for (let bitmap of this.index.values()) bitmap.removeBit(rowID);
+    for (const bitmap of this.index.values()) bitmap.removeBit(rowID);
   }
   toJSON() {
     const res = {};
-    for (let [k, bits] of this.index) {
+    for (const [k, bits] of this.index) {
       if (bits.buffer) res[k] = Array.from(bits.buffer);
     }
     return res;

package/columns/acolumn.js

@@ -62,7 +62,7 @@ class AColumn {
       const value = this.getRowKey(i);
       if (value == null) continue;
       if (isArray) {
-        for (let x of value) bitmap.setBit(x, i);
+        for (const x of value) bitmap.setBit(x, i);
       } else bitmap.setBit(value, i);
     }
   }

package/columns/dict-tuple.js

@@ -2,6 +2,10 @@ import { BidirIndex } from "@thi.ng/bidir-index";
 import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
+import {
+  __frequenciesTuples,
+  __frequencyIndex
+} from "../internal/frequencies.js";
 import { __indexOfTuple, __lastIndexOfTuple } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
@@ -21,9 +25,12 @@ class DictTupleColumn extends AColumn {
   }
   reindex() {
     const dict = this.dict;
-    const newDict = new BidirIndex();
+    const newDict = __frequencyIndex(
+      dict,
+      __frequenciesTuples(this.values)
+    );
     this.values = this.values.map(
-      (ids) => ids ? newDict.addAll(dict.getAllIDs(ids)) : null
+      (ids) => ids ? newDict.getAll(dict.getAllIDs(ids)) : null
     );
     this.dict = newDict;
     super.updateBitmap();
@@ -48,7 +55,7 @@ class DictTupleColumn extends AColumn {
     values.length = n;
     values.fill($value, 0, n);
     if (bitmap && $value) {
-      for (let x of $value) bitmap.ensure(x).fill(1, 0, n);
+      for (const x of $value) bitmap.ensure(x).fill(1, 0, n);
     }
   }
   setRow(i, value) {
@@ -57,8 +64,8 @@ class DictTupleColumn extends AColumn {
     const old = values[i];
     const encoded = values[i] = value != null ? this.table.schema[this.id].flags & FLAG_UNIQUE ? [...dict.addAllUnique(value)] : dict.addAll(value) : null;
     if (bitmap) {
-      if (old) for (let x of old) bitmap.clearBit(x, i);
-      if (encoded) for (let x of encoded) bitmap.setBit(x, i);
+      if (old) for (const x of old) bitmap.clearBit(x, i);
+      if (encoded) for (const x of encoded) bitmap.setBit(x, i);
     }
   }
   getRow(i) {

package/columns/dict.js

@@ -4,6 +4,7 @@ import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
+import { __frequencies, __frequencyIndex } from "../internal/frequencies.js";
 import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
@@ -23,9 +24,9 @@ class DictColumn extends AColumn {
   }
   reindex() {
     const dict = this.dict;
-    const newDict = new BidirIndex();
+    const newDict = __frequencyIndex(dict, __frequencies(this.values));
     this.values = this.values.map(
-      (x) => x != null ? newDict.add(dict.getID(x)) : null
+      (x) => x != null ? newDict.get(dict.getID(x)) : null
     );
     this.dict = newDict;
     super.updateBitmap();

package/columns/tuple.js

@@ -30,7 +30,7 @@ class TupleColumn extends AColumn {
     values.length = n;
     values.fill(value ?? null, 0, n);
     if (bitmap && value) {
-      for (let x of value) bitmap.ensure(x).fill(1, 0, n);
+      for (const x of value) bitmap.ensure(x).fill(1, 0, n);
     }
   }
   setRow(i, value) {
@@ -39,8 +39,8 @@ class TupleColumn extends AColumn {
     const old = values[i];
     const row = values[i] = value != null ? this.table.schema[this.id].flags & FLAG_UNIQUE ? [...new Set(value)] : value : null;
     if (bitmap) {
-      if (old) for (let x of old) bitmap.clearBit(x, i);
-      if (row) for (let x of row) bitmap.setBit(x, i);
+      if (old) for (const x of old) bitmap.clearBit(x, i);
+      if (row) for (const x of row) bitmap.setBit(x, i);
     }
   }
   getRow(i) {

package/columns/vector.d.ts

@@ -15,7 +15,7 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     validate(value: any): boolean;
     ensureRows(): void;
     setRow(i: number, value: any): void;
-    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRow(i: number): Uint32Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;

package/internal/frequencies.d.ts

@@ -0,0 +1,26 @@
+import type { TypedArray } from "@thi.ng/api";
+import { BidirIndex } from "@thi.ng/bidir-index";
+/**
+ * Constructs a new bidir index, sorted by key frequency, based on given index
+ * and pre-computed histogram.
+ *
+ * @param dict
+ * @param bins
+ */
+export declare const __frequencyIndex: (dict: BidirIndex<any>, bins: Map<number, number>) => BidirIndex<unknown>;
+/**
+ * Computes histogram of `values`, sorted frequency. Returns array of
+ * `[value,count]` bins. Nullish values in
+ *
+ * @internal
+ */
+export declare const __frequencies: (rows: TypedArray | (number | null)[]) => Map<number, number>;
+/**
+ * Same as {@link __frequencies}, but for tuple-based columns. Computes
+ * histogram for unique component values inside tuples, not for the tuples
+ * themselves.
+ *
+ * @internal
+ */
+export declare const __frequenciesTuples: (rows: (number[] | null)[]) => Map<number, number>;
+//# sourceMappingURL=frequencies.d.ts.map

package/internal/frequencies.js

@@ -0,0 +1,33 @@
+import { BidirIndex } from "@thi.ng/bidir-index";
+const __frequencyIndex = (dict, bins) => {
+  const newDict = new BidirIndex();
+  for (const bin of [...bins].sort((a, b) => b[1] - a[1])) {
+    newDict.add(dict.getID(bin[0]));
+  }
+  return newDict;
+};
+const __frequencies = (rows) => {
+  const bins = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    if (row == null) continue;
+    const n = bins.get(row);
+    bins.set(row, n != null ? n + 1 : 1);
+  }
+  return bins;
+};
+const __frequenciesTuples = (rows) => {
+  const bins = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    if (row == null) continue;
+    for (const v of row) {
+      const n = bins.get(v);
+      bins.set(v, n != null ? n + 1 : 1);
+    }
+  }
+  return bins;
+};
+export {
+  __frequencies,
+  __frequenciesTuples,
+  __frequencyIndex
+};

package/internal/indexof.d.ts

@@ -11,5 +11,5 @@ export declare const __lastIndexOfTuple: (needle: ArrayLike<any> | null, values:
 /** @internal */
 export declare const __clamp: (x: number, a: number, b: number) => number;
 /** @internal */
-export declare const __clampRange: (max: number, start: number, end?: number) => number[];
+export declare const __clampRange: (max: number, start: number, end?: number) => [number, number];
 //# sourceMappingURL=indexof.d.ts.map

package/internal/serialize.d.ts

@@ -11,5 +11,5 @@ export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec,
     values: any[];
 };
 /** @internal */
-export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Uint32Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.11.4",
+	"version": "0.13.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -127,8 +127,8 @@
 		}
 	},
 	"thi.ng": {
-		"status": "alpha",
+		"status": "beta",
 		"year": 2025
 	},
-	"gitHead": "7a21075f27ba00b28a6eb77e70918a4c7f4e6e68\n"
+	"gitHead": "7b9fc19fe963ac68bfe9d04c63f8a91cc3cfff5c\n"
 }

package/query.d.ts

@@ -1,5 +1,5 @@
 import type { Comparator, Predicate } from "@thi.ng/api";
-import type { ColumnID, QueryTerm, QueryTermOpSpec, Row, RowWithMeta } from "./api.js";
+import type { ColumnID, QueryResult, QueryTerm, QueryTermOpSpec, Row, RowWithMeta } from "./api.js";
 import type { Table } from "./table.js";
 export declare class Query<T extends Row> {
     readonly table: Table<T>;
@@ -7,8 +7,9 @@ export declare class Query<T extends Row> {
     protected _cmp?: Comparator<any>;
     protected _limit: number;
     protected _offset: number;
-    constructor(table: Table<T>, terms?: QueryTerm<T>[]);
+    constructor(table: Table<T>, terms?: Iterable<QueryTerm<T>>);
     addTerm(term: QueryTerm<T>): this;
+    addTerms(terms: Iterable<QueryTerm<T>>): void;
     limit(limit: number, offset?: number): this;
     /**
      * Constructs a comparator for query results based on given sort criteria,
@@ -35,6 +36,12 @@ export declare class Query<T extends Row> {
     matchRow(pred: Predicate<T>): this;
     valueRange(column: ColumnID<T>, start: any, end?: any): this;
     rowRange(start?: number, end?: number): this;
+    /**
+     * Eagerly executes query with current terms and returns results as array in
+     * an object including total number of results and selected paging details
+     * (i.e. offset & limit).
+     */
+    exec(): QueryResult<T>;
     [Symbol.iterator](): Generator<RowWithMeta<T>, void, unknown>;
 }
 export declare class QueryCtx<T extends Row> {
@@ -43,11 +50,13 @@ export declare class QueryCtx<T extends Row> {
     readonly size: number;
     bitmap?: Uint32Array;
     constructor(query: Query<T>);
+    exec(): this;
     /**
      * If a bitmap is already present, yield iterator of only currently selected
      * row IDs, otherwise yields row IDs in `[0,table.length)` range.
      */
     [Symbol.iterator](): Generator<number, void, unknown>;
+    rows(): Generator<RowWithMeta<T>, void, unknown>;
     clear(): void;
     makeMask(seed?: number | Uint32Array): Uint32Array<ArrayBuffer>;
     /**

package/query.js

@@ -12,9 +12,9 @@ import { Bitfield } from "./bitmap.js";
 import { __columnError } from "./internal/checks.js";
 import { __clampRange } from "./internal/indexof.js";
 class Query {
-  constructor(table, terms = []) {
+  constructor(table, terms) {
     this.table = table;
-    for (let term of terms) this.addTerm(term);
+    if (terms) this.addTerms(terms);
   }
   terms = [];
   _cmp;
@@ -25,6 +25,9 @@ class Query {
     this.terms.push(term);
     return this;
   }
+  addTerms(terms) {
+    for (const term of terms) this.addTerm(term);
+  }
   limit(limit, offset = 0) {
     this._limit = limit;
     this._offset = offset;
@@ -100,42 +103,60 @@ class Query {
     this.terms.push({ type: "rowRange", value: { start, end } });
     return this;
   }
-  *[Symbol.iterator]() {
+  /**
+   * Eagerly executes query with current terms and returns results as array in
+   * an object including total number of results and selected paging details
+   * (i.e. offset & limit).
+   */
+  exec() {
     const { table, _limit, _offset } = this;
-    const ctx = new QueryCtx(this);
-    for (let term of this.terms) {
-      const op = QUERY_OPS[term.type];
-      let column;
-      if (term.column) {
-        column = ctx.table.columns[term.column];
-        if (!column) illegalArgs(`column: ${String(term.column)}`);
-      } else if (QUERY_OPS[term.type].mode !== "row") {
-        illegalArgs(
-          `query op: ${term.type} requires a column name given`
-        );
+    const ctx = new QueryCtx(this).exec();
+    let rows = [];
+    const result = {
+      results: rows,
+      total: 0,
+      offset: _offset,
+      limit: _limit
+    };
+    if (!ctx.bitmap) return result;
+    if (this._cmp) {
+      rows = [...ctx.rows()].sort(this._cmp);
+      result.total = rows.length;
+      result.results = rows.slice(
+        ...__clampRange(rows.length, _offset, _offset + _limit)
+      );
+    } else {
+      let n = 0;
+      const max = _offset + _limit;
+      for (const i of ctx) {
+        if (n >= _offset && n < max) {
+          rows.push(table.getRow(i, false, true));
+        }
+        n++;
       }
-      if (!op.fn(ctx, term, column)) return;
+      result.total = n;
     }
+    return result;
+  }
+  *[Symbol.iterator]() {
+    const { table, _limit, _offset } = this;
+    const ctx = new QueryCtx(this).exec();
     if (!ctx.bitmap) return;
     if (this._cmp) {
-      const rows = [];
-      for (let i of ctx) {
-        rows.push(table.getRow(i, false, true));
-      }
-      rows.sort(this._cmp);
-      for (let i = this._offset, n2 = Math.min(rows.length, i + _limit); i < n2; i++) {
-        yield rows[i];
-      }
-      return;
-    }
-    let j = 0;
-    const n = _offset + _limit;
-    for (let i of ctx) {
-      if (j >= _offset) {
-        if (j >= n) return;
-        yield table.getRow(i, false, true);
+      const rows = [...ctx.rows()].sort(this._cmp);
+      yield* rows.slice(
+        ...__clampRange(rows.length, _offset, _offset + _limit)
+      );
+    } else {
+      let j = 0;
+      const n = _offset + _limit;
+      for (const i of ctx) {
+        if (j >= _offset) {
+          if (j >= n) return;
+          yield table.getRow(i, false, true);
+        }
+        j++;
       }
-      j++;
     }
   }
 }
@@ -148,6 +169,22 @@ class QueryCtx {
   table;
   size;
   bitmap;
+  exec() {
+    for (const term of this.query.terms) {
+      const op = QUERY_OPS[term.type];
+      let column;
+      if (term.column) {
+        column = this.table.columns[term.column];
+        if (!column) illegalArgs(`column: ${String(term.column)}`);
+      } else if (QUERY_OPS[term.type].mode !== "row") {
+        illegalArgs(
+          `query op: ${term.type} requires a column name given`
+        );
+      }
+      if (!op.fn(this, term, column)) break;
+    }
+    return this;
+  }
   /**
    * If a bitmap is already present, yield iterator of only currently selected
    * row IDs, otherwise yields row IDs in `[0,table.length)` range.
@@ -160,6 +197,10 @@ class QueryCtx {
       for (let i = 0; i < n; i++) yield i;
     }
   }
+  *rows() {
+    const table = this.query.table;
+    for (const i of this) yield table.getRow(i, false, true);
+  }
   clear() {
     this.bitmap = void 0;
   }
@@ -211,7 +252,7 @@ const execBitOr = (ctx, term, column) => {
   const key = column.valueKey(term.value);
   let mask;
   if (isArray(key)) {
-    for (let k of key) {
+    for (const k of key) {
       const b = bitmap.index.get(k)?.buffer;
       if (!b) continue;
       if (mask) {
@@ -228,8 +269,8 @@ const execOr = (ctx, term, column) => {
   const key = column.valueKey(term.value);
   const pred = column.isArray ? (row, k) => row.includes(k) : (row, k) => row === k;
   let mask;
-  for (let k of isArray(key) ? key : [key]) {
-    for (let i of ctx) {
+  for (const k of isArray(key) ? key : [key]) {
+    for (const i of ctx) {
       if (pred(column.getRowKey(i), k)) {
         if (!mask) mask = ctx.makeMask();
         mask[i >>> 5] |= 1 << (i & 31);
@@ -249,7 +290,7 @@ const execBitAnd = (ctx, term, column) => {
   const isNeg = term.type === "nand";
   let mask;
   if (isArray(key)) {
-    for (let k of key) {
+    for (const k of key) {
       const b = bitmap.index.get(k)?.buffer;
       if (!b) {
         if (isNeg) {
@@ -276,9 +317,9 @@ const execAnd = (ctx, term, column) => {
   const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
   const isNeg = term.type === "nand";
   let mask;
-  for (let k of isArray(key) ? key : [key]) {
+  for (const k of isArray(key) ? key : [key]) {
     let m;
-    for (let i of ctx) {
+    for (const i of ctx) {
       if (pred(column.getRowKey(i), k)) {
         if (!m) m = ctx.makeMask();
         m[i >>> 5] |= 1 << (i & 31);
@@ -321,7 +362,7 @@ const QUERY_OPS = {
     fn: (ctx, term, column) => {
       const pred = term.value;
       let mask;
-      for (let i of ctx) {
+      for (const i of ctx) {
         if (pred(column.getRow(i))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
@@ -338,7 +379,7 @@ const QUERY_OPS = {
       const columns = term.params;
       const pred = term.value;
       let mask;
-      for (let i of ctx) {
+      for (const i of ctx) {
         if (pred(table.getPartialRow(i, columns, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
@@ -354,7 +395,7 @@ const QUERY_OPS = {
       const table = ctx.table;
       const pred = term.value;
       let mask;
-      for (let i of ctx) {
+      for (const i of ctx) {
         if (pred(table.getRow(i, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);

package/table.js

@@ -86,7 +86,7 @@ class Table {
     this.length++;
   }
   addRows(rows) {
-    for (let row of rows) this.addRow(row);
+    for (const row of rows) this.addRow(row);
   }
   updateRow(i, row) {
     if (i < 0 || i >= this.length) illegalArgs(`row ID: ${i}`);
@@ -113,7 +113,7 @@ class Table {
   getPartialRow(i, columns, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
     const row = includeID ? { __row: i } : {};
-    for (let id of columns) {
+    for (const id of columns) {
       row[id] = this.columns[id]?.getRow(i);
     }
     return row;