@thi.ng/column-store 0.11.3 → 0.12.0

package/README.md

@@ -505,7 +505,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 6.28 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 6.27 KB
 
 ## Dependencies
 

package/bitmap.js

@@ -35,11 +35,11 @@ class BitmapIndex {
    * @param id
    */
   removeBit(rowID) {
-    for (let bitmap of this.index.values()) bitmap.removeBit(rowID);
+    for (const bitmap of this.index.values()) bitmap.removeBit(rowID);
   }
   toJSON() {
     const res = {};
-    for (let [k, bits] of this.index) {
+    for (const [k, bits] of this.index) {
       if (bits.buffer) res[k] = Array.from(bits.buffer);
     }
     return res;

package/columns/acolumn.js

@@ -62,7 +62,7 @@ class AColumn {
       const value = this.getRowKey(i);
       if (value == null) continue;
       if (isArray) {
-        for (let x of value) bitmap.setBit(x, i);
+        for (const x of value) bitmap.setBit(x, i);
       } else bitmap.setBit(value, i);
     }
   }

package/columns/dict-tuple.js

@@ -2,6 +2,10 @@ import { BidirIndex } from "@thi.ng/bidir-index";
 import { isArray } from "@thi.ng/checks/is-array";
 import { FLAG_UNIQUE } from "../api.js";
 import { __validateArrayValue } from "../internal/checks.js";
+import {
+  __frequenciesTuples,
+  __frequencyIndex
+} from "../internal/frequencies.js";
 import { __indexOfTuple, __lastIndexOfTuple } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
@@ -21,9 +25,12 @@ class DictTupleColumn extends AColumn {
   }
   reindex() {
     const dict = this.dict;
-    const newDict = new BidirIndex();
+    const newDict = __frequencyIndex(
+      dict,
+      __frequenciesTuples(this.values)
+    );
     this.values = this.values.map(
-      (ids) => ids ? newDict.addAll(dict.getAllIDs(ids)) : null
+      (ids) => ids ? newDict.getAll(dict.getAllIDs(ids)) : null
     );
     this.dict = newDict;
     super.updateBitmap();
@@ -48,7 +55,7 @@ class DictTupleColumn extends AColumn {
     values.length = n;
     values.fill($value, 0, n);
     if (bitmap && $value) {
-      for (let x of $value) bitmap.ensure(x).fill(1, 0, n);
+      for (const x of $value) bitmap.ensure(x).fill(1, 0, n);
     }
   }
   setRow(i, value) {
@@ -57,8 +64,8 @@ class DictTupleColumn extends AColumn {
     const old = values[i];
     const encoded = values[i] = value != null ? this.table.schema[this.id].flags & FLAG_UNIQUE ? [...dict.addAllUnique(value)] : dict.addAll(value) : null;
     if (bitmap) {
-      if (old) for (let x of old) bitmap.clearBit(x, i);
-      if (encoded) for (let x of encoded) bitmap.setBit(x, i);
+      if (old) for (const x of old) bitmap.clearBit(x, i);
+      if (encoded) for (const x of encoded) bitmap.setBit(x, i);
     }
   }
   getRow(i) {

package/columns/dict.js

@@ -4,6 +4,7 @@ import { decodeBinary, encodeBinary } from "@thi.ng/rle-pack/binary";
 import { decodeSimple, encodeSimple } from "@thi.ng/rle-pack/simple";
 import { FLAG_RLE } from "../api.js";
 import { __validateValue } from "../internal/checks.js";
+import { __frequencies, __frequencyIndex } from "../internal/frequencies.js";
 import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
 import { __serializeDict } from "../internal/serialize.js";
 import { AColumn } from "./acolumn.js";
@@ -23,9 +24,9 @@ class DictColumn extends AColumn {
   }
   reindex() {
     const dict = this.dict;
-    const newDict = new BidirIndex();
+    const newDict = __frequencyIndex(dict, __frequencies(this.values));
     this.values = this.values.map(
-      (x) => x != null ? newDict.add(dict.getID(x)) : null
+      (x) => x != null ? newDict.get(dict.getID(x)) : null
     );
     this.dict = newDict;
     super.updateBitmap();

package/columns/tuple.js

@@ -30,7 +30,7 @@ class TupleColumn extends AColumn {
     values.length = n;
     values.fill(value ?? null, 0, n);
     if (bitmap && value) {
-      for (let x of value) bitmap.ensure(x).fill(1, 0, n);
+      for (const x of value) bitmap.ensure(x).fill(1, 0, n);
     }
   }
   setRow(i, value) {
@@ -39,8 +39,8 @@ class TupleColumn extends AColumn {
     const old = values[i];
     const row = values[i] = value != null ? this.table.schema[this.id].flags & FLAG_UNIQUE ? [...new Set(value)] : value : null;
     if (bitmap) {
-      if (old) for (let x of old) bitmap.clearBit(x, i);
-      if (row) for (let x of row) bitmap.setBit(x, i);
+      if (old) for (const x of old) bitmap.clearBit(x, i);
+      if (row) for (const x of row) bitmap.setBit(x, i);
     }
   }
   getRow(i) {

package/internal/frequencies.d.ts

@@ -0,0 +1,26 @@
+import type { TypedArray } from "@thi.ng/api";
+import { BidirIndex } from "@thi.ng/bidir-index";
+/**
+ * Constructs a new bidir index, sorted by key frequency, based on given index
+ * and pre-computed histogram.
+ *
+ * @param dict
+ * @param bins
+ */
+export declare const __frequencyIndex: (dict: BidirIndex<any>, bins: Map<number, number>) => BidirIndex<unknown>;
+/**
+ * Computes histogram of `values`, sorted frequency. Returns array of
+ * `[value,count]` bins. Nullish values in
+ *
+ * @internal
+ */
+export declare const __frequencies: (rows: TypedArray | (number | null)[]) => Map<number, number>;
+/**
+ * Same as {@link __frequencies}, but for tuple-based columns. Computes
+ * histogram for unique component values inside tuples, not for the tuples
+ * themselves.
+ *
+ * @internal
+ */
+export declare const __frequenciesTuples: (rows: (number[] | null)[]) => Map<number, number>;
+//# sourceMappingURL=frequencies.d.ts.map

package/internal/frequencies.js

@@ -0,0 +1,33 @@
+import { BidirIndex } from "@thi.ng/bidir-index";
+const __frequencyIndex = (dict, bins) => {
+  const newDict = new BidirIndex();
+  for (const bin of [...bins].sort((a, b) => b[1] - a[1])) {
+    newDict.add(dict.getID(bin[0]));
+  }
+  return newDict;
+};
+const __frequencies = (rows) => {
+  const bins = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    if (row == null) continue;
+    const n = bins.get(row);
+    bins.set(row, n != null ? n + 1 : 1);
+  }
+  return bins;
+};
+const __frequenciesTuples = (rows) => {
+  const bins = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    if (row == null) continue;
+    for (const v of row) {
+      const n = bins.get(v);
+      bins.set(v, n != null ? n + 1 : 1);
+    }
+  }
+  return bins;
+};
+export {
+  __frequencies,
+  __frequenciesTuples,
+  __frequencyIndex
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.11.3",
+	"version": "0.12.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -43,7 +43,7 @@
 		"@thi.ng/api": "^8.12.16",
 		"@thi.ng/bidir-index": "^1.5.2",
 		"@thi.ng/checks": "^3.8.6",
-		"@thi.ng/compare": "^2.5.3",
+		"@thi.ng/compare": "^2.5.4",
 		"@thi.ng/errors": "^2.6.5",
 		"@thi.ng/rle-pack": "^3.2.3"
 	},
@@ -130,5 +130,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "869c807f9230120074f4ef5b9dce651ad0b846b9\n"
+	"gitHead": "82ec7cc54d0e89dad89cde87fa36c444512d71d7\n"
 }

package/query.d.ts

@@ -7,8 +7,9 @@ export declare class Query<T extends Row> {
     protected _cmp?: Comparator<any>;
     protected _limit: number;
     protected _offset: number;
-    constructor(table: Table<T>, terms?: QueryTerm<T>[]);
+    constructor(table: Table<T>, terms?: Iterable<QueryTerm<T>>);
     addTerm(term: QueryTerm<T>): this;
+    addTerms(terms: Iterable<QueryTerm<T>>): void;
     limit(limit: number, offset?: number): this;
     /**
      * Constructs a comparator for query results based on given sort criteria,

package/query.js

@@ -12,9 +12,9 @@ import { Bitfield } from "./bitmap.js";
 import { __columnError } from "./internal/checks.js";
 import { __clampRange } from "./internal/indexof.js";
 class Query {
-  constructor(table, terms = []) {
+  constructor(table, terms) {
     this.table = table;
-    for (let term of terms) this.addTerm(term);
+    if (terms) this.addTerms(terms);
   }
   terms = [];
   _cmp;
@@ -25,6 +25,9 @@ class Query {
     this.terms.push(term);
     return this;
   }
+  addTerms(terms) {
+    for (const term of terms) this.addTerm(term);
+  }
   limit(limit, offset = 0) {
     this._limit = limit;
     this._offset = offset;
@@ -103,7 +106,7 @@ class Query {
   *[Symbol.iterator]() {
     const { table, _limit, _offset } = this;
     const ctx = new QueryCtx(this);
-    for (let term of this.terms) {
+    for (const term of this.terms) {
       const op = QUERY_OPS[term.type];
       let column;
       if (term.column) {
@@ -119,7 +122,7 @@ class Query {
     if (!ctx.bitmap) return;
     if (this._cmp) {
       const rows = [];
-      for (let i of ctx) {
+      for (const i of ctx) {
         rows.push(table.getRow(i, false, true));
       }
       rows.sort(this._cmp);
@@ -130,7 +133,7 @@ class Query {
     }
     let j = 0;
     const n = _offset + _limit;
-    for (let i of ctx) {
+    for (const i of ctx) {
       if (j >= _offset) {
         if (j >= n) return;
         yield table.getRow(i, false, true);
@@ -211,7 +214,7 @@ const execBitOr = (ctx, term, column) => {
   const key = column.valueKey(term.value);
   let mask;
   if (isArray(key)) {
-    for (let k of key) {
+    for (const k of key) {
       const b = bitmap.index.get(k)?.buffer;
       if (!b) continue;
       if (mask) {
@@ -228,8 +231,8 @@ const execOr = (ctx, term, column) => {
   const key = column.valueKey(term.value);
   const pred = column.isArray ? (row, k) => row.includes(k) : (row, k) => row === k;
   let mask;
-  for (let k of isArray(key) ? key : [key]) {
-    for (let i of ctx) {
+  for (const k of isArray(key) ? key : [key]) {
+    for (const i of ctx) {
       if (pred(column.getRowKey(i), k)) {
         if (!mask) mask = ctx.makeMask();
         mask[i >>> 5] |= 1 << (i & 31);
@@ -249,7 +252,7 @@ const execBitAnd = (ctx, term, column) => {
   const isNeg = term.type === "nand";
   let mask;
   if (isArray(key)) {
-    for (let k of key) {
+    for (const k of key) {
       const b = bitmap.index.get(k)?.buffer;
       if (!b) {
         if (isNeg) {
@@ -276,9 +279,9 @@ const execAnd = (ctx, term, column) => {
   const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
   const isNeg = term.type === "nand";
   let mask;
-  for (let k of isArray(key) ? key : [key]) {
+  for (const k of isArray(key) ? key : [key]) {
     let m;
-    for (let i of ctx) {
+    for (const i of ctx) {
       if (pred(column.getRowKey(i), k)) {
         if (!m) m = ctx.makeMask();
         m[i >>> 5] |= 1 << (i & 31);
@@ -321,7 +324,7 @@ const QUERY_OPS = {
     fn: (ctx, term, column) => {
       const pred = term.value;
       let mask;
-      for (let i of ctx) {
+      for (const i of ctx) {
         if (pred(column.getRow(i))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
@@ -338,7 +341,7 @@ const QUERY_OPS = {
       const columns = term.params;
       const pred = term.value;
       let mask;
-      for (let i of ctx) {
+      for (const i of ctx) {
         if (pred(table.getPartialRow(i, columns, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
@@ -354,7 +357,7 @@ const QUERY_OPS = {
       const table = ctx.table;
       const pred = term.value;
       let mask;
-      for (let i of ctx) {
+      for (const i of ctx) {
         if (pred(table.getRow(i, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);

package/table.js

@@ -86,7 +86,7 @@ class Table {
     this.length++;
   }
   addRows(rows) {
-    for (let row of rows) this.addRow(row);
+    for (const row of rows) this.addRow(row);
   }
   updateRow(i, row) {
     if (i < 0 || i >= this.length) illegalArgs(`row ID: ${i}`);
@@ -113,7 +113,7 @@ class Table {
   getPartialRow(i, columns, safe = true, includeID = false) {
     if (safe && (i < 0 || i >= this.length)) return;
     const row = includeID ? { __row: i } : {};
-    for (let id of columns) {
+    for (const id of columns) {
       row[id] = this.columns[id]?.getRow(i);
     }
     return row;