@thi.ng/column-store 0.7.0 → 0.8.0

package/README.md

@@ -21,6 +21,7 @@
   - [Serialization options](#serialization-options)
   - [Custom column types](#custom-column-types)
   - [Cardinality](#cardinality)
+    - [Differences between tuple and vector column types](#differences-between-tuple-and-vector-column-types)
   - [Default values](#default-values)
   - [Flags](#flags)
     - [FLAG_BITMAP](#flag_bitmap)
@@ -143,10 +144,11 @@ Note: Booleans and `BigInt`s are still unsupported, but being worked on...
 | `i16`           | 16bit signed int    | ❌                    | ✅                     |
 | `u32`           | 32bit unsigned int  | ❌                    | ✅                     |
 | `i32`           | 32bit signed int    | ❌                    | ✅                     |
-| `f32`           | 32bit float         | ❌                    | ❌                     |
-| `f64`           | 64bit float         | ❌                    | ❌                     |
+| `f32`           | 32bit float         | ❌                    | ✅ <sup>(2)</sup>      |
+| `f64`           | 64bit float         | ❌                    | ✅ <sup>(2)</sup>      |
 
 - <sup>(1)</sup> only if max. cardinality is 1, [further information](#flag_rle)
+- <sup>(2)</sup> simple RLE only, [further information](#flag_rle)
 
 ### Vector column types
 
@@ -223,6 +225,14 @@ key of the column spec. The following presets are provided:
 | `ONE_PLUS`  | `[1, (2**32)-1]` | One or more values (always expects tuples)  |
 | `ZERO_PLUS` | `[0, (2**32)-1]` | Zero or more values (always expects tuples) |
 
+#### Differences between tuple and vector column types
+
+| **Feature**                      | **Tuples**            | **Vectors** |
+|----------------------------------|-----------------------|-------------|
+| Optional (without default value) | ✅                     | ❌           |
+| Flexible size                    | ✅                     | ❌           |
+| Query ops match...               | Individual components | Full vector |
+
 ### Default values
 
 Default values can be specified for columns with a minimum cardinality of 1 or
@@ -448,7 +458,7 @@ For Node.js REPL:
 const cs = await import("@thi.ng/column-store");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 5.50 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 5.71 KB
 
 ## Dependencies
 

package/api.d.ts

@@ -84,7 +84,7 @@ export declare const FLAG_UNIQUE: number;
 export declare const FLAG_RLE: number;
 /** @internal */
 export declare const LIMITS: Record<NumericType, [number, number]>;
-export interface IColumn {
+export interface IColumn extends Iterable<any> {
     bitmap?: BitmapIndex;
     readonly isArray: boolean;
     load(spec: SerializedColumn): void;
@@ -180,7 +180,7 @@ export interface QueryTerm<T extends Row> {
     value: any;
     params?: any;
 }
-export type QueryTermOp = Fn3<QueryCtx<any>, QueryTerm<any>, Maybe<IColumn>, void>;
+export type QueryTermOp = Fn3<QueryCtx<any>, QueryTerm<any>, Maybe<IColumn>, boolean>;
 export interface QueryTermOpSpec {
     /**
      * Default mode is: "col";

package/columns/acolumn.d.ts

@@ -11,8 +11,9 @@ export declare abstract class AColumn<T extends Row = Row> implements IColumn {
     dict?: BidirIndex<any>;
     abstract isArray: boolean;
     constructor(id: ColumnID<T>, table: Table<T>);
-    abstract load(spec: SerializedColumn): void;
+    [Symbol.iterator](): Generator<any, void, unknown>;
     reindex(): void;
+    abstract load(spec: SerializedColumn): void;
     abstract validate(value: any): boolean;
     abstract setRow(i: number, value: any): void;
     abstract removeRow(i: number): void;

package/columns/acolumn.js

@@ -14,6 +14,9 @@ class AColumn {
   spec;
   bitmap;
   dict;
+  *[Symbol.iterator]() {
+    for (let i = 0, n = this.table.length; i < n; i++) yield this.getRow(i);
+  }
   reindex() {
     this.updateBitmap();
   }

package/columns/vector.d.ts

@@ -14,7 +14,7 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     validate(value: any): boolean;
     ensureRows(): void;
     setRow(i: number, value: any): void;
-    getRow(i: number): Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;

package/internal/serialize.d.ts

@@ -11,5 +11,5 @@ export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec,
     values: any[];
 };
 /** @internal */
-export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.7.0",
+	"version": "0.8.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -126,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "3bb968f9005a82c2890510903aab375d2cc99fae\n"
+	"gitHead": "63ca02b09367c07f6cb785642bc3f2b7df2e5804\n"
 }

package/query.d.ts

@@ -27,6 +27,11 @@ export declare class QueryCtx<T extends Row> {
     readonly size: number;
     bitmap?: Uint32Array;
     constructor(query: Query<T>);
+    /**
+     * If a bitmap is already present, yield iterator of only currently selected
+     * row IDs, otherwise yields row IDs in `[0,table.length)` range.
+     */
+    [Symbol.iterator](): Generator<number, void, unknown>;
     clear(): void;
     makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
     /**

package/query.js

@@ -78,11 +78,10 @@ class Query {
           `query op: ${term.type} requires a column name given`
         );
       }
-      op.fn(ctx, term, column);
+      if (!op.fn(ctx, term, column)) return;
     }
     if (ctx.bitmap) {
-      for (let i of new Bitfield(ctx.bitmap).ones(table.length))
-        yield table.getRow(i, false, true);
+      for (let i of ctx) yield table.getRow(i, false, true);
     }
   }
 }
@@ -95,6 +94,18 @@ class QueryCtx {
   table;
   size;
   bitmap;
+  /**
+   * If a bitmap is already present, yield iterator of only currently selected
+   * row IDs, otherwise yields row IDs in `[0,table.length)` range.
+   */
+  *[Symbol.iterator]() {
+    const n = this.table.length;
+    if (this.bitmap) {
+      yield* new Bitfield(this.bitmap).ones(n);
+    } else {
+      for (let i = 0; i < n; i++) yield i;
+    }
+  }
   clear() {
     this.bitmap = void 0;
   }
@@ -148,7 +159,7 @@ const execBitOr = (ctx, term, column) => {
       const b = bitmap.index.get(k)?.buffer;
       if (!b) continue;
       if (mask) {
-        for (let i = 0; i < b.length; i++) mask[i] |= b[i];
+        for (let i = 0, n = b.length; i < n; i++) mask[i] |= b[i];
       } else mask = ctx.makeMask(b);
     }
   } else {
@@ -157,15 +168,16 @@ const execBitOr = (ctx, term, column) => {
   }
   if (mask) {
     term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
+    return true;
   }
+  return false;
 };
 const execOr = (ctx, term, column) => {
-  const n = ctx.table.length;
   const key = column.valueKey(term.value);
   const pred = column.isArray ? (row, k) => row.includes(k) : (row, k) => row === k;
   let mask;
   for (let k of isArray(key) ? key : [key]) {
-    for (let i = 0; i < n; i++) {
+    for (let i of ctx) {
       if (pred(column.getRowKey(i), k)) {
         if (!mask) mask = ctx.makeMask();
         mask[i >>> 5] |= 1 << (i & 31);
@@ -174,15 +186,15 @@ const execOr = (ctx, term, column) => {
   }
   if (mask) {
     term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
+    return true;
   }
+  return false;
 };
-const delegateOr = (ctx, term, column) => {
-  (term.value != null && column.bitmap ? execBitOr : execOr)(
-    ctx,
-    term,
-    column
-  );
-};
+const delegateOr = (ctx, term, column) => (term.value != null && column.bitmap ? execBitOr : execOr)(
+  ctx,
+  term,
+  column
+);
 const execBitAnd = (ctx, term, column) => {
   const bitmap = column.bitmap;
   const key = column.valueKey(term.value);
@@ -191,16 +203,14 @@ const execBitAnd = (ctx, term, column) => {
     const colBitmaps = [];
     for (let k of key) {
       const b = bitmap.index.get(k)?.buffer;
-      if (!b) {
-        if (term.type === "and") ctx.clear();
-        return;
-      }
+      if (!b) return false;
       colBitmaps.push(b);
     }
     for (let b of colBitmaps) {
       if (mask) {
-        for (let i = 0; i < b.length; i++) mask[i] &= b[i];
-        mask.fill(0, b.length);
+        const n = b.length;
+        for (let i = 0; i < n; i++) mask[i] &= b[i];
+        mask.fill(0, n);
       } else mask = ctx.makeMask(b);
     }
   } else {
@@ -209,7 +219,9 @@ const execBitAnd = (ctx, term, column) => {
   }
   if (mask) {
     term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-  } else ctx.clear();
+    return true;
+  }
+  return false;
 };
 const execAnd = (ctx, term, column) => {
   const n = ctx.table.length;
@@ -218,7 +230,7 @@ const execAnd = (ctx, term, column) => {
   let mask;
   for (let k of isArray(key) ? key : [key]) {
     let m;
-    for (let i = 0; i < n; i++) {
+    for (let i of ctx) {
       if (pred(column.getRowKey(i), k)) {
         if (!m) m = ctx.makeMask();
         m[i >>> 5] |= 1 << (i & 31);
@@ -229,21 +241,20 @@ const execAnd = (ctx, term, column) => {
         for (let i = 0; i < n; i++) mask[i] &= m[i];
       } else mask = m;
     } else {
-      if (term.type === "and") ctx.clear();
-      return;
+      return false;
     }
   }
   if (mask) {
     term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
+    return true;
   }
+  return false;
 };
-const delegateAnd = (ctx, term, column) => {
-  (term.value != null && column.bitmap ? execBitAnd : execAnd)(
-    ctx,
-    term,
-    column
-  );
-};
+const delegateAnd = (ctx, term, column) => (term.value != null && column.bitmap ? execBitAnd : execAnd)(
+  ctx,
+  term,
+  column
+);
 const QUERY_OPS = {
   or: { fn: delegateOr },
   nor: { fn: delegateOr },
@@ -253,13 +264,14 @@ const QUERY_OPS = {
     fn: (ctx, term, column) => {
       const pred = term.value;
       let mask;
-      for (let i = 0, n = ctx.table.length; i < n; i++) {
+      for (let i of ctx) {
         if (pred(column.getRow(i))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
         }
       }
       if (mask) ctx.mergeMask(mask);
+      return !!mask;
     }
   },
   matchPartialRow: {
@@ -269,13 +281,14 @@ const QUERY_OPS = {
       const columns = term.params;
       const pred = term.value;
       let mask;
-      for (let i = 0, n = table.length; i < n; i++) {
+      for (let i of ctx) {
         if (pred(table.getPartialRow(i, columns, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
         }
       }
       if (mask) ctx.mergeMask(mask);
+      return !!mask;
     }
   },
   matchRow: {
@@ -284,13 +297,14 @@ const QUERY_OPS = {
       const table = ctx.table;
       const pred = term.value;
       let mask;
-      for (let i = 0, n = table.length; i < n; i++) {
+      for (let i of ctx) {
         if (pred(table.getRow(i, false))) {
           if (!mask) mask = ctx.makeMask();
           mask[i >>> 5] |= 1 << (i & 31);
         }
       }
       if (mask) ctx.mergeMask(mask);
+      return !!mask;
     }
   },
   valueRange: {
@@ -302,7 +316,7 @@ const QUERY_OPS = {
       if (start != null) $start = column.findIndex((x) => x >= start);
       if (end != null)
         $end = column.findLastIndex((x) => x <= end, $start);
-      __fillMask(ctx, $start, $end >= 0 ? $end + 1 : $end);
+      return __fillMask(ctx, $start, $end >= 0 ? $end + 1 : $end, max);
     }
   },
   rowRange: {
@@ -311,17 +325,19 @@ const QUERY_OPS = {
       const max = ctx.table.length;
       let { start = 0, end = max } = term.value;
       [start, end] = __clampRange(max, start, end);
-      __fillMask(ctx, start, end);
+      return __fillMask(ctx, start, end, max);
     }
   }
 };
-const __fillMask = (ctx, start, end, fill = 1) => {
-  if (start >= 0 && end >= 0) {
+const __fillMask = (ctx, start, end, max, fill = 1) => {
+  if (start >= 0 && start < max && end >= 0) {
     const mask = ctx.makeMask();
     const bitmap = new Bitfield(mask);
     bitmap.fill(fill, start, end);
     ctx.mergeMask(mask);
-  } else ctx.clear();
+    return true;
+  }
+  return false;
 };
 const registerQueryOp = (type, spec) => {
   if (QUERY_OPS[type]) illegalArgs(`query op ${type} already registered`);