@thi.ng/column-store 0.1.0

package/columns/typedarray.d.ts

@@ -0,0 +1,22 @@
+import { type TypedArray } from "@thi.ng/api/typedarray";
+import { type IColumn, type NumericType, type SerializedColumn } from "../api.js";
+import type { Table } from "../table.js";
+import { AColumn } from "./acolumn.js";
+export declare class TypedArrayColumn extends AColumn implements IColumn {
+    values: TypedArray;
+    type: NumericType;
+    limit: [number, number];
+    readonly isArray = false;
+    constructor(id: string, table: Table);
+    load(spec: SerializedColumn): void;
+    reindex(): void;
+    validate(value: any): boolean;
+    setRow(i: number, value: any): void;
+    getRow(i: number): number;
+    removeRow(i: number): void;
+    replaceValue(currValue: any, newValue: any): boolean;
+    toJSON(): {
+        values: number[];
+    };
+}
+//# sourceMappingURL=typedarray.d.ts.map

package/columns/typedarray.js

@@ -0,0 +1,84 @@
+import { SIZEOF, typedArray } from "@thi.ng/api/typedarray";
+import { isNumber } from "@thi.ng/checks/is-number";
+import { decode as decodeRLE, encode as encodeRLE } from "@thi.ng/rle-pack";
+import {
+  FLAG_RLE
+} from "../api.js";
+import { __replaceValue } from "../internal/replace.js";
+import { AColumn } from "./acolumn.js";
+const LIMITS = {
+  u8: [0, 255],
+  u8c: [0, 255],
+  u16: [0, 65535],
+  u32: [0, 4294967295],
+  i8: [-128, 127],
+  i16: [-32768, 32767],
+  i32: [-2147483648, 2147483647],
+  f32: [-Infinity, Infinity],
+  f64: [-Infinity, Infinity]
+};
+class TypedArrayColumn extends AColumn {
+  values;
+  type;
+  limit;
+  isArray = false;
+  constructor(id, table) {
+    super(id, table);
+    this.type = table.schema[id].type;
+    this.limit = LIMITS[this.type];
+    this.values = typedArray(this.type, 8);
+  }
+  load(spec) {
+    if (this.spec.flags & FLAG_RLE) {
+      const values = decodeRLE(spec.values);
+      this.values = typedArray(this.type, values.buffer);
+    } else {
+      this.values = typedArray(this.type, spec.values);
+    }
+    this.reindex();
+  }
+  reindex() {
+    super.updateBitmap(this.values.subarray(0, this.table.length));
+  }
+  validate(value) {
+    return isNumber(value) && (value >= this.limit[0] || value <= this.limit[1]) || value == null && this.spec.default != null;
+  }
+  setRow(i, value) {
+    value = this.ensureValue(value);
+    let len = this.values.length;
+    if (i >= len) {
+      while (i >= len) len <<= 1;
+      const tmp = typedArray(this.type, len);
+      tmp.set(this.values);
+      this.values = tmp;
+    }
+    const { values, bitmap } = this;
+    const old = values[i];
+    values[i] = value;
+    if (bitmap) {
+      bitmap.clearBit(old, i);
+      bitmap.setBit(value, i);
+    }
+  }
+  getRow(i) {
+    return this.values[i];
+  }
+  removeRow(i) {
+    this.values.copyWithin(i, i + 1, this.table.length);
+    this.values[this.table.length - 1] = 0;
+    this.bitmap?.removeBit(i);
+  }
+  replaceValue(currValue, newValue) {
+    return __replaceValue(this.bitmap, this.values, currValue, newValue);
+  }
+  toJSON() {
+    let values = this.values.subarray(0, this.table.length);
+    if (this.spec.flags & FLAG_RLE) {
+      values = encodeRLE(values, values.length, SIZEOF[this.type] * 8);
+    }
+    return { values: Array.from(values) };
+  }
+}
+export {
+  TypedArrayColumn
+};

package/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./api.js";
+export * from "./bitmap.js";
+export * from "./query.js";
+export * from "./table.js";
+//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -0,0 +1,4 @@
+export * from "./api.js";
+export * from "./bitmap.js";
+export * from "./query.js";
+export * from "./table.js";

package/internal/checks.d.ts

@@ -0,0 +1,8 @@
+import type { ColumnSpec } from "../api.js";
+/** @internal */
+export declare const __validateValue: (spec: ColumnSpec, x: any) => boolean;
+/** @internal */
+export declare const __validateArrayValue: (spec: ColumnSpec, x: any) => boolean;
+/** @internal */
+export declare const __columnError: (id: string, msg: string) => never;
+//# sourceMappingURL=checks.d.ts.map

package/internal/checks.js

@@ -0,0 +1,15 @@
+import { isArray } from "@thi.ng/checks/is-array";
+import { isNumber } from "@thi.ng/checks/is-number";
+import { isString } from "@thi.ng/checks/is-string";
+import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
+const __validateValue = (spec, x) => x == null && (spec.cardinality[0] === 0 || spec.default != null) || (spec.type === "str" ? isString(x) : isNumber(x));
+const __validateArrayValue = (spec, x) => {
+  const [min, max] = spec.cardinality;
+  return x == null && (min === 0 || spec.default != null) || isArray(x) && x.length >= min && x.length <= max && x.every(spec.type === "str" ? isString : isNumber);
+};
+const __columnError = (id, msg) => illegalArgs(msg + ` (column: ${id})`);
+export {
+  __columnError,
+  __validateArrayValue,
+  __validateValue
+};

package/internal/replace.d.ts

@@ -0,0 +1,5 @@
+import type { Maybe, TypedArray } from "@thi.ng/api";
+import type { BitmapIndex } from "../bitmap.js";
+/** @internal */
+export declare const __replaceValue: (bitmap: Maybe<BitmapIndex>, values: any[] | TypedArray, currVal: any, newVal: any) => boolean;
+//# sourceMappingURL=replace.d.ts.map

package/internal/replace.js

@@ -0,0 +1,16 @@
+const __replaceValue = (bitmap, values, currVal, newVal) => {
+  const bits = bitmap?.ensure(newVal);
+  bitmap?.index.delete(currVal);
+  let result = false;
+  for (let i = 0; i < values.length; i++) {
+    if (values[i] === currVal) {
+      values[i] = newVal;
+      bits?.setBit(i);
+      result = true;
+    }
+  }
+  return result;
+};
+export {
+  __replaceValue
+};

package/internal/serialize.d.ts

@@ -0,0 +1,7 @@
+import type { BidirIndex } from "@thi.ng/bidir-index";
+/** @internal */
+export declare const __serializeDict: (dict: BidirIndex<any>) => {
+    index: any;
+    next: number;
+};
+//# sourceMappingURL=serialize.d.ts.map

package/internal/serialize.js

@@ -0,0 +1,8 @@
+const __serializeDict = (dict) => {
+  const res = [];
+  for (let [val, id] of dict.entries()) res[id] = val;
+  return { index: res, next: dict.nextID };
+};
+export {
+  __serializeDict
+};

package/package.json

@@ -0,0 +1,127 @@
+{
+	"name": "@thi.ng/column-store",
+	"version": "0.1.0",
+	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
+	"type": "module",
+	"module": "./index.js",
+	"typings": "./index.d.ts",
+	"sideEffects": false,
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/thi-ng/umbrella.git",
+		"directory": "packages/column-store"
+	},
+	"homepage": "https://thi.ng/column-store",
+	"funding": [
+		{
+			"type": "github",
+			"url": "https://github.com/sponsors/postspectacular"
+		},
+		{
+			"type": "patreon",
+			"url": "https://patreon.com/thing_umbrella"
+		},
+		{
+			"type": "liberapay",
+			"url": "https://liberapay.com/thi.ng"
+		}
+	],
+	"author": "Karsten Schmidt (https://thi.ng)",
+	"license": "Apache-2.0",
+	"scripts": {
+		"build": "yarn build:esbuild && yarn build:decl",
+		"build:decl": "tsc --declaration --emitDeclarationOnly",
+		"build:esbuild": "esbuild --format=esm --platform=neutral --target=es2022 --tsconfig=tsconfig.json --outdir=. src/**/*.ts",
+		"clean": "bun ../../tools/src/clean-package.ts",
+		"doc": "typedoc --options ../../typedoc.json --out doc src/index.ts",
+		"doc:readme": "bun ../../tools/src/module-stats.ts && bun ../../tools/src/readme.ts",
+		"pub": "npm publish --access public",
+		"test": "bun test",
+		"tool:tangle": "../../node_modules/.bin/tangle src/**/*.ts"
+	},
+	"dependencies": {
+		"@thi.ng/api": "^8.12.14",
+		"@thi.ng/bidir-index": "^1.5.0",
+		"@thi.ng/checks": "^3.8.4",
+		"@thi.ng/errors": "^2.6.3",
+		"@thi.ng/rle-pack": "^3.1.118"
+	},
+	"devDependencies": {
+		"esbuild": "^0.27.2",
+		"typedoc": "^0.28.16",
+		"typescript": "^5.9.3"
+	},
+	"keywords": [
+		"acceleration",
+		"binary",
+		"bitmap",
+		"column",
+		"compression",
+		"database",
+		"indexed",
+		"json",
+		"memory",
+		"predicate",
+		"query",
+		"rle",
+		"set",
+		"typedarray",
+		"typescript"
+	],
+	"publishConfig": {
+		"access": "public"
+	},
+	"browser": {
+		"process": false,
+		"setTimeout": false
+	},
+	"engines": {
+		"node": ">=18"
+	},
+	"files": [
+		"./*.js",
+		"./*.d.ts",
+		"columns",
+		"internal"
+	],
+	"exports": {
+		".": {
+			"default": "./index.js"
+		},
+		"./api": {
+			"default": "./api.js"
+		},
+		"./bitmap": {
+			"default": "./bitmap.js"
+		},
+		"./columns/acolumn": {
+			"default": "./columns/acolumn.js"
+		},
+		"./columns/array": {
+			"default": "./columns/array.js"
+		},
+		"./columns/enum-array": {
+			"default": "./columns/enum-array.js"
+		},
+		"./columns/enum": {
+			"default": "./columns/enum.js"
+		},
+		"./columns/plain": {
+			"default": "./columns/plain.js"
+		},
+		"./columns/typedarray": {
+			"default": "./columns/typedarray.js"
+		},
+		"./query": {
+			"default": "./query.js"
+		},
+		"./table": {
+			"default": "./table.js"
+		}
+	},
+	"thi.ng": {
+		"status": "alpha",
+		"year": 2025
+	},
+	"gitHead": "4bd1b9d8ae52ba32b90a1b9a55d329c708ca7865\n"
+}

package/query.d.ts

@@ -0,0 +1,43 @@
+import type { Predicate } from "@thi.ng/api";
+import type { QueryTerm, QueryTermOpSpec } from "./api.js";
+import type { Table } from "./table.js";
+export declare class Query {
+    table: Table;
+    terms: QueryTerm[];
+    constructor(table: Table, terms?: QueryTerm[]);
+    addTerm(term: QueryTerm): this;
+    /** Alias for {@link Query.or} */
+    where: (column: string, value: any) => this;
+    /** Alias for {@link Query.nor} */
+    whereNot: (column: string, value: any) => this;
+    or(column: string, value: any): this;
+    nor(column: string, value: any): this;
+    and(column: string, value: any): this;
+    nand(column: string, value: any): this;
+    matchColumn(column: string, pred: Predicate<any>): this;
+    matchPartialRow(columns: string[], pred: Predicate<Record<string, any>>): this;
+    matchRow(pred: Predicate<Record<string, any>>): this;
+    [Symbol.iterator](): Generator<import("./api.js").Row | undefined, void, unknown>;
+}
+export declare class QueryCtx {
+    readonly query: Query;
+    readonly table: Table;
+    readonly size: number;
+    bitmap?: Uint32Array;
+    constructor(query: Query);
+    makeMask(seed?: Uint32Array): Uint32Array<ArrayBuffer>;
+    mergeMask(mask: Uint32Array): void;
+    invertMask(mask: Uint32Array): Uint32Array<ArrayBufferLike>;
+}
+/**
+ * Registers a custom query term operator for given `type` and later use with
+ * {@link Query}. Throws an error if `type` is already registered.
+ *
+ * @remarks
+ * See {@link QueryTerm} & {@link QueryTermOp} for further details.
+ *
+ * @param type
+ * @param spec
+ */
+export declare const registerQueryOp: (type: string, spec: QueryTermOpSpec) => void;
+//# sourceMappingURL=query.d.ts.map

package/query.js

@@ -0,0 +1,271 @@
+import { isArray } from "@thi.ng/checks/is-array";
+import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
+import { unsupportedOp } from "@thi.ng/errors/unsupported";
+import { Bitfield } from "./bitmap.js";
+class Query {
+  constructor(table, terms = []) {
+    this.table = table;
+    this.terms = terms;
+  }
+  terms = [];
+  addTerm(term) {
+    this.terms.push(term);
+    return this;
+  }
+  /** Alias for {@link Query.or} */
+  where = (column, value) => this.or(column, value);
+  /** Alias for {@link Query.nor} */
+  whereNot = (column, value) => this.nor(column, value);
+  or(column, value) {
+    this.terms.push({ type: "or", column, value });
+    return this;
+  }
+  nor(column, value) {
+    this.terms.push({ type: "nor", column, value });
+    return this;
+  }
+  and(column, value) {
+    this.terms.push({ type: "and", column, value });
+    return this;
+  }
+  nand(column, value) {
+    this.terms.push({ type: "nand", column, value });
+    return this;
+  }
+  matchColumn(column, pred) {
+    this.terms.push({ type: "matchCol", column, value: pred });
+    return this;
+  }
+  matchPartialRow(columns, pred) {
+    this.terms.push({
+      type: "matchPartialRow",
+      params: columns,
+      value: pred
+    });
+    return this;
+  }
+  matchRow(pred) {
+    this.terms.push({ type: "matchRow", value: pred });
+    return this;
+  }
+  *[Symbol.iterator]() {
+    const { table } = this;
+    const ctx = new QueryCtx(this);
+    for (let term of this.terms) {
+      const op = QUERY_OPS[term.type];
+      if (!op) unsupportedOp(`query type: ${term.type}`);
+      let column;
+      if (term.column) {
+        column = ctx.table.columns[term.column];
+        if (!column) illegalArgs(`column: ${term.column}`);
+      } else if (QUERY_OPS[term.type].mode !== "row") {
+        illegalArgs(
+          `query op: ${term.type} requires a column name given`
+        );
+      }
+      op.fn(ctx, term, column);
+    }
+    if (ctx.bitmap) {
+      for (let i of new Bitfield(ctx.bitmap).ones(table.length))
+        yield table.getRow(i);
+    }
+  }
+}
+class QueryCtx {
+  constructor(query) {
+    this.query = query;
+    this.table = query.table;
+    this.size = Math.ceil(this.table.length / 32);
+  }
+  table;
+  size;
+  bitmap;
+  makeMask(seed) {
+    const mask = new Uint32Array(this.size);
+    if (seed) mask.set(seed);
+    return mask;
+  }
+  mergeMask(mask) {
+    if (this.bitmap) {
+      for (let i = 0, n = this.size; i < n; i++)
+        this.bitmap[i] &= mask[i];
+    } else this.bitmap = mask;
+  }
+  invertMask(mask) {
+    for (let i = 0, n = this.size; i < n; i++) mask[i] ^= -1;
+    return mask;
+  }
+}
+const execBitOr = (ctx, term, column) => {
+  const bitmap = column.bitmap;
+  const value = column.encode(term.value);
+  let mask;
+  if (isArray(value)) {
+    for (let v of value) {
+      const b = bitmap.index.get(v)?.buffer;
+      if (!b) continue;
+      if (mask) {
+        for (let i = 0; i < b.length; i++) mask[i] |= b[i];
+      } else mask = ctx.makeMask(b);
+    }
+  } else {
+    const b = bitmap.index.get(value)?.buffer;
+    if (b) mask = ctx.makeMask(b);
+  }
+  if (mask) {
+    if (term.type === "nor") ctx.invertMask(mask);
+    ctx.mergeMask(mask);
+  }
+};
+const execOr = (ctx, term, column) => {
+  const n = ctx.table.length;
+  const encoded = column.encode(term.value);
+  const values = column.values;
+  const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
+  let mask;
+  for (let v of isArray(encoded) ? encoded : [encoded]) {
+    for (let i = 0; i < n; i++) {
+      if (pred(values[i], v)) {
+        if (!mask) mask = ctx.makeMask();
+        mask[i >>> 5] |= 1 << (i & 31);
+      }
+    }
+  }
+  if (mask) {
+    if (term.type === "nor") ctx.invertMask(mask);
+    ctx.mergeMask(mask);
+  }
+};
+const delegateOr = (ctx, term, column) => {
+  (term.value != null && column.bitmap ? execBitOr : execOr)(
+    ctx,
+    term,
+    column
+  );
+};
+const execBitAnd = (ctx, term, column) => {
+  const bitmap = column.bitmap;
+  const value = column.encode(term.value);
+  let mask;
+  if (isArray(value)) {
+    const colBitmaps = [];
+    for (let v of value) {
+      const b = bitmap.index.get(v)?.buffer;
+      if (!b) {
+        if (term.type === "and") ctx.bitmap = void 0;
+        return;
+      }
+      colBitmaps.push(b);
+    }
+    for (let b of colBitmaps) {
+      if (mask) {
+        for (let i = 0; i < b.length; i++) mask[i] &= b[i];
+        mask.fill(0, b.length);
+      } else mask = ctx.makeMask(b);
+    }
+  } else {
+    const b = bitmap.index.get(value)?.buffer;
+    if (b) mask = ctx.makeMask(b);
+  }
+  if (mask) {
+    if (term.type === "nand") ctx.invertMask(mask);
+    ctx.mergeMask(mask);
+  } else {
+    ctx.bitmap = void 0;
+  }
+};
+const execAnd = (ctx, term, column) => {
+  const n = ctx.table.length;
+  const encoded = column.encode(term.value) ?? null;
+  const values = column.values;
+  const pred = column.isArray ? (row, v) => row.includes(v) : (row, v) => row === v;
+  let mask;
+  for (let v of isArray(encoded) ? encoded : [encoded]) {
+    let m;
+    for (let i = 0; i < n; i++) {
+      if (pred(values[i], v)) {
+        if (!m) m = ctx.makeMask();
+        m[i >>> 5] |= 1 << (i & 31);
+      }
+    }
+    if (m) {
+      if (mask) {
+        for (let i = 0; i < n; i++) mask[i] &= m[i];
+      } else mask = m;
+    } else {
+      if (term.type === "and") ctx.bitmap = void 0;
+      return;
+    }
+  }
+  if (mask) {
+    if (term.type === "nand") ctx.invertMask(mask);
+    ctx.mergeMask(mask);
+  }
+};
+const delegateAnd = (ctx, term, column) => {
+  (term.value != null && column.bitmap ? execBitAnd : execAnd)(
+    ctx,
+    term,
+    column
+  );
+};
+const QUERY_OPS = {
+  or: { fn: delegateOr },
+  nor: { fn: delegateOr },
+  and: { fn: delegateAnd },
+  nand: { fn: delegateAnd },
+  matchCol: {
+    fn: (ctx, term, column) => {
+      const values = column.values;
+      const pred = term.value;
+      let mask;
+      for (let i = 0, n = ctx.table.length; i < n; i++) {
+        if (pred(column.decode(values[i]))) {
+          if (!mask) mask = ctx.makeMask();
+          mask[i >>> 5] |= 1 << (i & 31);
+        }
+      }
+      if (mask) ctx.mergeMask(mask);
+    }
+  },
+  matchPartialRow: {
+    mode: "row",
+    fn: (ctx, term) => {
+      const table = ctx.table;
+      const columns = term.params;
+      const pred = term.value;
+      let mask;
+      for (let i = 0, n = ctx.table.length; i < n; i++) {
+        if (pred(table.getPartialRow(i, columns, false))) {
+          if (!mask) mask = ctx.makeMask();
+          mask[i >>> 5] |= 1 << (i & 31);
+        }
+      }
+      if (mask) ctx.mergeMask(mask);
+    }
+  },
+  matchRow: {
+    mode: "row",
+    fn: (ctx, term) => {
+      const table = ctx.table;
+      const pred = term.value;
+      let mask;
+      for (let i = 0, n = ctx.table.length; i < n; i++) {
+        if (pred(table.getRow(i, false))) {
+          if (!mask) mask = ctx.makeMask();
+          mask[i >>> 5] |= 1 << (i & 31);
+        }
+      }
+      if (mask) ctx.mergeMask(mask);
+    }
+  }
+};
+const registerQueryOp = (type, spec) => {
+  if (QUERY_OPS[type]) illegalArgs(`query op ${type} already registered`);
+  QUERY_OPS[type] = spec;
+};
+export {
+  Query,
+  QueryCtx,
+  registerQueryOp
+};

package/table.d.ts

@@ -0,0 +1,51 @@
+import { type ColumnSchema, type ColumnSpec, type ColumnTypeSpec, type IColumn, type QueryTerm, type Row, type SerializedTable } from "./api.js";
+import { Query } from "./query.js";
+/**
+ * Placeholder only. Unused so far.
+ */
+export interface TableOpts {
+}
+export declare class Table {
+    opts: TableOpts;
+    schema: ColumnSchema;
+    columns: Record<string, IColumn>;
+    length: number;
+    static load(serialized: SerializedTable, opts?: Partial<TableOpts>): Table;
+    constructor(schema: Record<string, Partial<ColumnSpec> & {
+        type: ColumnSpec["type"];
+    }>, opts?: Partial<TableOpts>);
+    query(terms?: QueryTerm[]): Query;
+    addColumn(id: string, spec: Partial<ColumnSpec> & {
+        type: ColumnSpec["type"];
+    }): void;
+    removeColumn(id: string): boolean;
+    [Symbol.iterator](): Generator<Row | undefined, void, unknown>;
+    reindex(): void;
+    addRow(row: Row): void;
+    addRows(rows: Iterable<Row>): void;
+    updateRow(i: number, row: Row): void;
+    removeRow(i: number): void;
+    getRow(i: number, safe?: boolean): Row | undefined;
+    getPartialRow(i: number, columns: string[], safe?: boolean): Row | undefined;
+    validateRow(row: Row): void;
+    validateColumnSpec(id: string, spec: ColumnSpec): void;
+    toJSON(): {
+        schema: ColumnSchema;
+        columns: Record<string, IColumn>;
+        length: number;
+    };
+}
+/**
+ * Registry of column type definitions and their factory functions. See
+ * {@link registerColumnType}.
+ */
+export declare const COLUMN_TYPES: Record<string, ColumnTypeSpec>;
+/**
+ * Registers a custom column type for given `type` ID. Throws an error if `type`
+ * is already registered.
+ *
+ * @param type
+ * @param spec
+ */
+export declare const registerColumnType: (type: string, spec: ColumnTypeSpec) => void;
+//# sourceMappingURL=table.d.ts.map