@thi.ng/column-store 0.9.0 → 0.10.1

package/api.d.ts

@@ -1,4 +1,4 @@
-import type { FloatType, Fn3, IntType, Maybe, Predicate, UintType } from "@thi.ng/api";
+import type { FloatType, Fn3, IClear, IntType, Maybe, Predicate, UintType } from "@thi.ng/api";
 import type { BitmapIndex } from "./bitmap.js";
 import type { QueryCtx } from "./query.js";
 import type { Table } from "./table.js";
@@ -84,7 +84,7 @@ export declare const FLAG_UNIQUE: number;
 export declare const FLAG_RLE: number;
 /** @internal */
 export declare const LIMITS: Record<NumericType, [number, number]>;
-export interface IColumn extends Iterable<any> {
+export interface IColumn extends Iterable<any>, IClear {
     bitmap?: BitmapIndex;
     readonly isArray: boolean;
     load(spec: SerializedColumn): void;
@@ -192,4 +192,10 @@ export interface QueryTermOpSpec {
      */
     fn: QueryTermOp;
 }
+/**
+ * Initial capacity for typedarray and vector columns
+ *
+ * @internal
+ */
+export declare const INITIAL_CAPACITY = 8;
 //# sourceMappingURL=api.d.ts.map

package/api.js

@@ -17,11 +17,13 @@ const LIMITS = {
   f32: [-Infinity, Infinity],
   f64: [-Infinity, Infinity]
 };
+const INITIAL_CAPACITY = 8;
 export {
   FLAG_BITMAP,
   FLAG_DICT,
   FLAG_RLE,
   FLAG_UNIQUE,
+  INITIAL_CAPACITY,
   LIMITS,
   ONE_PLUS,
   OPTIONAL,

package/columns/acolumn.d.ts

@@ -13,6 +13,7 @@ export declare abstract class AColumn<T extends Row = Row> implements IColumn {
     constructor(id: ColumnID<T>, table: Table<T>);
     [Symbol.iterator](): Generator<any, void, unknown>;
     reindex(): void;
+    abstract clear(): void;
     abstract load(spec: SerializedColumn): void;
     abstract validate(value: any): boolean;
     abstract setRow(i: number, value: any): void;

package/columns/dict-tuple.d.ts

@@ -5,6 +5,7 @@ export declare class DictTupleColumn<T extends Row = Row> extends AColumn<T> {
     values: (number[] | null)[];
     dict: BidirIndex<any>;
     readonly isArray = true;
+    clear(): void;
     load({ dict, values }: SerializedColumn): void;
     reindex(): void;
     encode(value: any): (number | null)[];

package/columns/dict-tuple.js

@@ -9,6 +9,11 @@ class DictTupleColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = true;
+  clear() {
+    this.values = [];
+    this.dict.clear();
+    this.bitmap?.clear();
+  }
   load({ dict, values }) {
     this.values = values;
     super.loadDict(dict);

package/columns/dict.d.ts

@@ -5,6 +5,7 @@ export declare class DictColumn<T extends Row = Row> extends AColumn<T> {
     values: (number | null)[];
     dict: BidirIndex<any>;
     readonly isArray = false;
+    clear(): void;
     load({ dict, values }: SerializedColumn): void;
     reindex(): void;
     encode(value: any): number | null;

package/columns/dict.js

@@ -11,6 +11,11 @@ class DictColumn extends AColumn {
   values = [];
   dict = new BidirIndex();
   isArray = false;
+  clear() {
+    this.values = [];
+    this.dict.clear();
+    this.bitmap?.clear();
+  }
   load({ dict, values }) {
     this.values = this.spec.flags & FLAG_RLE ? this.spec.cardinality[0] === 0 && this.spec.default == null ? decodeSimple(values) : Array.from(decodeBinary(values)) : values;
     super.loadDict(dict);

package/columns/plain.d.ts

@@ -3,6 +3,7 @@ import { AColumn } from "./acolumn.js";
 export declare class PlainColumn<T extends Row = Row> extends AColumn<T> {
     values: any[];
     readonly isArray = false;
+    clear(): void;
     load({ values }: SerializedColumn): void;
     ensureRows(): void;
     validate(value: any): boolean;

package/columns/plain.js

@@ -7,6 +7,10 @@ import { AColumn } from "./acolumn.js";
 class PlainColumn extends AColumn {
   values = [];
   isArray = false;
+  clear() {
+    this.values = [];
+    this.bitmap?.clear();
+  }
   load({ values }) {
     this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeSimple(values)) : values;
     this.reindex();

package/columns/tuple.d.ts

@@ -4,6 +4,7 @@ import { AColumn } from "./acolumn.js";
 export declare class TupleColumn<T extends Row = Row> extends AColumn<T> {
     values: Nullable<number[]>[];
     readonly isArray = true;
+    clear(): void;
     load(spec: SerializedColumn): void;
     encode(value: any): any[];
     validate(value: any): boolean;

package/columns/tuple.js

@@ -6,6 +6,10 @@ import { AColumn } from "./acolumn.js";
 class TupleColumn extends AColumn {
   values = [];
   isArray = true;
+  clear() {
+    this.values = [];
+    this.bitmap?.clear();
+  }
   load(spec) {
     this.values = spec.values;
     this.reindex();

package/columns/typedarray.d.ts

@@ -9,6 +9,7 @@ export declare class TypedArrayColumn<T extends Row = Row> extends AColumn<T> {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: ColumnID<T>, table: Table<T>);
+    clear(): void;
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     ensureRows(): void;

package/columns/typedarray.js

@@ -2,6 +2,7 @@ import { typedArray } from "@thi.ng/api/typedarray";
 import { isArray } from "@thi.ng/checks/is-array";
 import { isNumber } from "@thi.ng/checks/is-number";
 import {
+  INITIAL_CAPACITY,
   LIMITS
 } from "../api.js";
 import { __indexOfSingle, __lastIndexOfSingle } from "../internal/indexof.js";
@@ -18,9 +19,13 @@ class TypedArrayColumn extends AColumn {
     super(id, table);
     this.type = table.schema[id].type;
     this.limit = LIMITS[this.type];
-    this.values = typedArray(this.type, 8);
+    this.values = typedArray(this.type, INITIAL_CAPACITY);
     this.tmp = typedArray(this.type, 1);
   }
+  clear() {
+    this.values = typedArray(this.type, INITIAL_CAPACITY);
+    this.bitmap?.clear();
+  }
   load({ values }) {
     this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();

package/columns/vector.d.ts

@@ -10,11 +10,12 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     protected tmp: TypedArray;
     readonly isArray = false;
     constructor(id: ColumnID<T>, table: Table<T>);
+    clear(): void;
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
     ensureRows(): void;
     setRow(i: number, value: any): void;
-    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRow(i: number): Uint32Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;

package/columns/vector.js

@@ -4,6 +4,7 @@ import { isArrayLike } from "@thi.ng/checks/is-arraylike";
 import { isNumber } from "@thi.ng/checks/is-number";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
 import {
+  INITIAL_CAPACITY,
   LIMITS
 } from "../api.js";
 import { __clampRange } from "../internal/indexof.js";
@@ -21,9 +22,13 @@ class VectorColumn extends AColumn {
     this.type = this.spec.type.split("v")[0];
     this.size = this.spec.cardinality[1];
     this.limit = LIMITS[this.type];
-    this.values = typedArray(this.type, 8 * this.size);
+    this.values = typedArray(this.type, INITIAL_CAPACITY * this.size);
     this.tmp = typedArray(this.type, this.size);
   }
+  clear() {
+    this.values = typedArray(this.type, INITIAL_CAPACITY * this.size);
+    this.bitmap?.clear();
+  }
   load({ values }) {
     this.values = __deserializeTyped(this.type, this.spec.flags, values);
     this.reindex();

package/internal/serialize.d.ts

@@ -11,5 +11,5 @@ export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec,
     values: any[];
 };
 /** @internal */
-export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Uint32Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.9.0",
+	"version": "0.10.1",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -129,5 +129,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "1138bb64457a415733cf5c05d32c211ccfd6ee72\n"
+	"gitHead": "f2700429ff95ad80d2b4487f6bde4d7e220204c3\n"
 }

package/query.d.ts

@@ -33,7 +33,7 @@ export declare class QueryCtx<T extends Row> {
      */
     [Symbol.iterator](): Generator<number, void, unknown>;
     clear(): void;
-    makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
+    makeMask(seed?: number | Uint32Array): Uint32Array<ArrayBuffer>;
     /**
      * Combines the `mask` with the context's mask (combined using bitwise AND).
      * If the context mask is still undefined, assigns `mask` as the initial

package/query.js

@@ -1,4 +1,5 @@
 import { isArray } from "@thi.ng/checks/is-array";
+import { isNumber } from "@thi.ng/checks/is-number";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
 import { unsupportedOp } from "@thi.ng/errors/unsupported";
 import { Bitfield } from "./bitmap.js";
@@ -111,7 +112,9 @@ class QueryCtx {
   }
   makeMask(seed) {
     const mask = new Uint32Array(this.size);
-    if (seed) mask.set(seed);
+    if (seed) {
+      isNumber(seed) ? mask.fill(seed) : mask.set(seed);
+    }
     return mask;
   }
   /**
@@ -166,11 +169,7 @@ const execBitOr = (ctx, term, column) => {
     const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
-  if (mask) {
-    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, term.type === "nor");
 };
 const execOr = (ctx, term, column) => {
   const key = column.valueKey(term.value);
@@ -184,11 +183,7 @@ const execOr = (ctx, term, column) => {
       }
     }
   }
-  if (mask) {
-    term.type === "nor" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, term.type === "nor");
 };
 const delegateOr = (ctx, term, column) => (term.value != null && column.bitmap ? execBitOr : execOr)(
   ctx,
@@ -198,15 +193,18 @@ const delegateOr = (ctx, term, column) => (term.value != null && column.bitmap ?
 const execBitAnd = (ctx, term, column) => {
   const bitmap = column.bitmap;
   const key = column.valueKey(term.value);
+  const isNeg = term.type === "nand";
   let mask;
   if (isArray(key)) {
-    const colBitmaps = [];
     for (let k of key) {
       const b = bitmap.index.get(k)?.buffer;
-      if (!b) return false;
-      colBitmaps.push(b);
-    }
-    for (let b of colBitmaps) {
+      if (!b) {
+        if (isNeg) {
+          if (mask) mask.fill(0);
+          else mask = ctx.makeMask();
+          continue;
+        } else return false;
+      }
       if (mask) {
         const n = b.length;
         for (let i = 0; i < n; i++) mask[i] &= b[i];
@@ -217,16 +215,13 @@ const execBitAnd = (ctx, term, column) => {
     const b = bitmap.index.get(key)?.buffer;
     if (b) mask = ctx.makeMask(b);
   }
-  if (mask) {
-    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, isNeg);
 };
 const execAnd = (ctx, term, column) => {
   const n = ctx.table.length;
   const key = column.valueKey(term.value) ?? null;
   const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
+  const isNeg = term.type === "nand";
   let mask;
   for (let k of isArray(key) ? key : [key]) {
     let m;
@@ -241,20 +236,29 @@ const execAnd = (ctx, term, column) => {
         for (let i = 0; i < n; i++) mask[i] &= m[i];
       } else mask = m;
     } else {
-      return false;
+      if (isNeg) {
+        if (mask) mask.fill(0);
+        else mask = ctx.makeMask();
+      } else return false;
     }
   }
-  if (mask) {
-    term.type === "nand" ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
-    return true;
-  }
-  return false;
+  return __finalizeMask(ctx, mask, isNeg);
 };
 const delegateAnd = (ctx, term, column) => (term.value != null && column.bitmap ? execBitAnd : execAnd)(
   ctx,
   term,
   column
 );
+const __finalizeMask = (ctx, mask, isNeg) => {
+  if (mask) {
+    isNeg ? ctx.mergeInvMask(mask) : ctx.mergeMask(mask);
+    return true;
+  } else if (isNeg) {
+    if (!ctx.bitmap) ctx.bitmap = ctx.makeMask(-1);
+    return true;
+  }
+  return false;
+};
 const QUERY_OPS = {
   or: { fn: delegateOr },
   nor: { fn: delegateOr },

package/table.d.ts

@@ -1,4 +1,4 @@
-import type { Maybe } from "@thi.ng/api";
+import type { IClear, ICopy, IEmpty, Maybe } from "@thi.ng/api";
 import { type ColumnID, type ColumnSchema, type ColumnSpec, type ColumnTypeSpec, type IColumn, type QueryTerm, type Row, type RowWithMeta, type SerializedTable } from "./api.js";
 import { Query } from "./query.js";
 /**
@@ -6,7 +6,7 @@ import { Query } from "./query.js";
  */
 export interface TableOpts {
 }
-export declare class Table<T extends Row> {
+export declare class Table<T extends Row> implements IClear, ICopy<Table<T>>, IEmpty<Table<T>> {
     opts: TableOpts;
     schema: ColumnSchema<T>;
     columns: Record<ColumnID<T>, IColumn>;
@@ -15,12 +15,15 @@ export declare class Table<T extends Row> {
     constructor(schema: Record<ColumnID<T>, Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
     }>, opts?: Partial<TableOpts>);
+    clear(): void;
+    copy(): Table<T>;
+    empty(): Table<T>;
     query(terms?: QueryTerm<T>[]): Query<T>;
     addColumn(id: ColumnID<T>, spec: Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
     }): IColumn;
     removeColumn(id: ColumnID<T>): boolean;
-    [Symbol.iterator](): Generator<Maybe<T>, void, unknown>;
+    [Symbol.iterator](): Generator<T, void, unknown>;
     reindex(): void;
     addRow(row: Partial<T>): void;
     addRows(rows: Iterable<Partial<T>>): void;
@@ -32,6 +35,7 @@ export declare class Table<T extends Row> {
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean): Maybe<Pick<T, K>>;
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: false): Maybe<Pick<T, K>>;
     getPartialRow<K extends ColumnID<T>>(i: number, columns: K[], safe?: boolean, includeID?: true): Maybe<RowWithMeta<Pick<T, K>>>;
+    slice(start?: number, end?: number): Table<T>;
     indexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
     lastIndexOf(id: ColumnID<T>, value: any, start?: number, end?: number): number;
     validateRow(row: Partial<T>): void;

package/table.js

@@ -13,6 +13,7 @@ import { TupleColumn } from "./columns/tuple.js";
 import { TypedArrayColumn } from "./columns/typedarray.js";
 import { VectorColumn } from "./columns/vector.js";
 import { __columnError } from "./internal/checks.js";
+import { __clamp } from "./internal/indexof.js";
 import { Query } from "./query.js";
 class Table {
   opts;
@@ -31,6 +32,19 @@ class Table {
     this.opts = { ...opts };
     for (let id in schema) this.addColumn(id, schema[id]);
   }
+  clear() {
+    const { columns } = this;
+    for (let id in columns) columns[id].clear();
+    this.length = 0;
+  }
+  copy() {
+    const copy = new Table(this.schema, this.opts);
+    copy.addRows(this);
+    return copy;
+  }
+  empty() {
+    return new Table(this.schema, this.opts);
+  }
   query(terms) {
     return new Query(this, terms);
   }
@@ -104,6 +118,14 @@ class Table {
     }
     return row;
   }
+  slice(start = 0, end) {
+    const max = this.length;
+    start = __clamp(start, 0, max);
+    end = __clamp(end ?? max, start, max);
+    const copy = this.empty();
+    for (let i = start; i < end; i++) copy.addRow(this.getRow(i, true));
+    return copy;
+  }
   indexOf(id, value, start, end) {
     return this.columns[id]?.indexOf(value, start, end) ?? -1;
   }