@thi.ng/column-store 0.6.0 → 0.7.0

package/api.d.ts

@@ -149,6 +149,8 @@ export interface IColumn {
      */
     setRow(i: number, value: any): void;
     removeRow(i: number): void;
+    /** @internal */
+    ensureRows(): void;
     encode(value: any): any;
     decode(value: any): any;
     replaceValue(currValue: any, newValue: any): boolean;

package/columns/acolumn.d.ts

@@ -16,6 +16,7 @@ export declare abstract class AColumn<T extends Row = Row> implements IColumn {
     abstract validate(value: any): boolean;
     abstract setRow(i: number, value: any): void;
     abstract removeRow(i: number): void;
+    abstract ensureRows(): void;
     abstract replaceValue(currValue: any, newValue: any): boolean;
     abstract valueKey(x: any): any;
     abstract getRow(i: number): any;

package/columns/acolumn.js

@@ -56,11 +56,11 @@ class AColumn {
     if (!bitmap) return;
     bitmap.clear();
     for (let i = 0, n = this.table.length; i < n; i++) {
-      const value = this.getRow(i);
+      const value = this.getRowKey(i);
       if (value == null) continue;
       if (isArray) {
         for (let x of value) bitmap.setBit(x, i);
-      } else bitmap.setBit(this.getRowKey(i), i);
+      } else bitmap.setBit(value, i);
     }
   }
   ensureValue(val) {

package/columns/dict-tuple.d.ts

@@ -10,6 +10,7 @@ export declare class DictTupleColumn<T extends Row = Row> extends AColumn<T> {
     encode(value: any): (number | null)[];
     decode(value: any[]): any[];
     validate(value: any): boolean;
+    ensureRows(): void;
     setRow(i: number, value: any[]): void;
     getRow(i: number): any[] | null;
     getRowKey(i: number): number[] | null;

package/columns/dict-tuple.js

@@ -32,6 +32,20 @@ class DictTupleColumn extends AColumn {
   validate(value) {
     return __validateArrayValue(this.spec, value);
   }
+  ensureRows() {
+    const {
+      bitmap,
+      spec: { default: value },
+      table: { length: n },
+      values
+    } = this;
+    const $value = value != null ? this.dict.addAll(value) : null;
+    values.length = n;
+    values.fill($value, 0, n);
+    if (bitmap && $value) {
+      for (let x of $value) bitmap.ensure(x).fill(1, 0, n);
+    }
+  }
   setRow(i, value) {
     value = this.ensureValue(value);
     const { values, dict, bitmap } = this;

package/columns/dict.d.ts

@@ -10,6 +10,7 @@ export declare class DictColumn<T extends Row = Row> extends AColumn<T> {
     encode(value: any): number | null;
     decode(value: any): any;
     validate(value: any): boolean;
+    ensureRows(): void;
     setRow(i: number, value: any): void;
     getRow(i: number): any;
     getRowKey(i: number): number | null;

package/columns/dict.js

@@ -34,6 +34,18 @@ class DictColumn extends AColumn {
   validate(value) {
     return __validateValue(this.spec, value);
   }
+  ensureRows() {
+    const {
+      bitmap,
+      spec: { default: value },
+      table: { length: n },
+      values
+    } = this;
+    const $value = value != null ? this.dict.add(value) : null;
+    values.length = n;
+    values.fill($value, 0, n);
+    if (bitmap && $value != null) bitmap.ensure($value).fill(1, 0, n);
+  }
   setRow(i, value) {
     value = this.ensureValue(value);
     const { values, bitmap } = this;

package/columns/plain.d.ts

@@ -4,6 +4,7 @@ export declare class PlainColumn<T extends Row = Row> extends AColumn<T> {
     values: any[];
     readonly isArray = false;
     load({ values }: SerializedColumn): void;
+    ensureRows(): void;
     validate(value: any): boolean;
     setRow(i: number, value: any): void;
     getRow(i: number): any;

package/columns/plain.js

@@ -11,13 +11,25 @@ class PlainColumn extends AColumn {
     this.values = this.spec.flags & FLAG_RLE ? Array.from(decodeSimple(values)) : values;
     this.reindex();
   }
+  ensureRows() {
+    const {
+      bitmap,
+      spec: { default: value },
+      table: { length: n },
+      values
+    } = this;
+    values.length = n;
+    values.fill(value ?? null, 0, n);
+    if (bitmap && value != null) bitmap.ensure(value).fill(1, 0, n);
+  }
   validate(value) {
     return __validateValue(this.spec, value);
   }
   setRow(i, value) {
     const { values, bitmap } = this;
     const old = values[i];
-    values[i] = this.ensureValue(value);
+    value = this.ensureValue(value);
+    values[i] = value;
     if (bitmap) {
       if (old != null) bitmap.clearBit(old, i);
       if (value != null) bitmap.setBit(value, i);

package/columns/tuple.d.ts

@@ -7,6 +7,7 @@ export declare class TupleColumn<T extends Row = Row> extends AColumn<T> {
     load(spec: SerializedColumn): void;
     encode(value: any): any[];
     validate(value: any): boolean;
+    ensureRows(): void;
     setRow(i: number, value: any[]): void;
     getRow(i: number): Nullable<number[]>;
     getRowKey(i: number): Nullable<number[]>;

package/columns/tuple.js

@@ -16,6 +16,19 @@ class TupleColumn extends AColumn {
   validate(value) {
     return __validateArrayValue(this.spec, value);
   }
+  ensureRows() {
+    const {
+      bitmap,
+      spec: { default: value },
+      table: { length: n },
+      values
+    } = this;
+    values.length = n;
+    values.fill(value ?? null, 0, n);
+    if (bitmap && value) {
+      for (let x of value) bitmap.ensure(x).fill(1, 0, n);
+    }
+  }
   setRow(i, value) {
     value = this.ensureValue(value);
     const { values, bitmap } = this;

package/columns/typedarray.d.ts

@@ -11,6 +11,7 @@ export declare class TypedArrayColumn<T extends Row = Row> extends AColumn<T> {
     constructor(id: ColumnID<T>, table: Table<T>);
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
+    ensureRows(): void;
     setRow(i: number, value: any): void;
     getRow(i: number): number;
     getRowKey(i: number): number;
@@ -22,5 +23,6 @@ export declare class TypedArrayColumn<T extends Row = Row> extends AColumn<T> {
     toJSON(): {
         values: any[];
     };
+    protected ensureCapacity(capacity: number, copy?: boolean): void;
 }
 //# sourceMappingURL=typedarray.d.ts.map

package/columns/typedarray.js

@@ -28,15 +28,15 @@ class TypedArrayColumn extends AColumn {
   validate(value) {
     return isNumber(value) && value >= this.limit[0] && value <= this.limit[1] || value == null && this.spec.default != null;
   }
+  ensureRows() {
+    const n = this.table.length;
+    this.ensureCapacity(n, false);
+    this.values.fill(this.spec.default, 0, n);
+    this.bitmap?.ensure(this.spec.default).fill(1, 0, n);
+  }
   setRow(i, value) {
     value = this.ensureValue(value);
-    let len = this.values.length;
-    if (i >= len) {
-      while (i >= len) len <<= 1;
-      const tmp = typedArray(this.type, len);
-      tmp.set(this.values);
-      this.values = tmp;
-    }
+    this.ensureCapacity(i);
     const { values, bitmap } = this;
     const old = values[i];
     values[i] = value;
@@ -98,6 +98,15 @@ class TypedArrayColumn extends AColumn {
       this.type
     );
   }
+  ensureCapacity(capacity, copy = true) {
+    let len = this.values.length;
+    if (capacity >= len) {
+      while (capacity >= len) len <<= 1;
+      const tmp = typedArray(this.type, len);
+      copy && tmp.set(this.values);
+      this.values = tmp;
+    }
+  }
 }
 export {
   TypedArrayColumn

package/columns/vector.d.ts

@@ -12,8 +12,9 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     constructor(id: ColumnID<T>, table: Table<T>);
     load({ values }: SerializedColumn): void;
     validate(value: any): boolean;
+    ensureRows(): void;
     setRow(i: number, value: any): void;
-    getRow(i: number): Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+    getRow(i: number): Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
     getRowKey(i: number): string;
     valueKey(value: any): string | string[];
     removeRow(i: number): void;
@@ -23,5 +24,6 @@ export declare class VectorColumn<T extends Row = Row> extends AColumn<T> {
     toJSON(): {
         values: any[];
     };
+    protected ensureCapacity(capacity: number, copy?: boolean): void;
 }
 //# sourceMappingURL=vector.d.ts.map

package/columns/vector.js

@@ -31,22 +31,25 @@ class VectorColumn extends AColumn {
   validate(value) {
     return isArrayLike(value) && value.length == this.size || value == null && this.spec.default != null;
   }
+  ensureRows() {
+    const {
+      size,
+      spec: { default: value },
+      table: { length: n }
+    } = this;
+    this.ensureCapacity(n, false);
+    for (let i = 0; i < n; i++) this.values.set(value, i * size);
+    this.bitmap?.ensure(this.valueKey(value)).fill(1, 0, n);
+  }
   setRow(i, value) {
     value = this.ensureValue(value);
-    const j = i * this.size;
-    let len = this.values.length;
-    if (j >= len) {
-      while (j >= len) len <<= 1;
-      const tmp = typedArray(this.type, len);
-      tmp.set(this.values);
-      this.values = tmp;
-    }
+    this.ensureCapacity(i);
     const { values, bitmap } = this;
     if (bitmap) {
       bitmap.clearBit(this.getRowKey(i), i);
       bitmap.setBit(this.valueKey(value), i);
     }
-    values.set(value, j);
+    values.set(value, i * this.size);
   }
   getRow(i) {
     const { size } = this;
@@ -121,6 +124,16 @@ class VectorColumn extends AColumn {
       this.type
     );
   }
+  ensureCapacity(capacity, copy = true) {
+    capacity *= this.size;
+    let len = this.values.length;
+    if (capacity >= len) {
+      while (capacity >= len) len <<= 1;
+      const tmp = typedArray(this.type, len);
+      copy && tmp.set(this.values);
+      this.values = tmp;
+    }
+  }
 }
 export {
   VectorColumn

package/internal/serialize.d.ts

@@ -11,5 +11,5 @@ export declare const __serializeTyped: ($values: NumericArray, spec: ColumnSpec,
     values: any[];
 };
 /** @internal */
-export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
+export declare const __deserializeTyped: (type: Type, flags: number, values: number[]) => Uint8Array<ArrayBufferLike> | Float32Array<ArrayBufferLike> | Float64Array<ArrayBufferLike> | Int8Array<ArrayBufferLike> | Int16Array<ArrayBufferLike> | Int32Array<ArrayBufferLike> | Uint8ClampedArray<ArrayBufferLike> | Uint16Array<ArrayBufferLike> | Uint32Array<ArrayBufferLike>;
 //# sourceMappingURL=serialize.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@thi.ng/column-store",
-	"version": "0.6.0",
+	"version": "0.7.0",
 	"description": "In-memory column store database with customizable column types, extensible query engine, bitfield indexing for query acceleration, JSON serialization with optional RLE compression",
 	"type": "module",
 	"module": "./index.js",
@@ -126,5 +126,5 @@
 		"status": "alpha",
 		"year": 2025
 	},
-	"gitHead": "f1d8da5644dd22f212fc313df2187380da968a7c\n"
+	"gitHead": "3bb968f9005a82c2890510903aab375d2cc99fae\n"
 }

package/table.d.ts

@@ -18,7 +18,7 @@ export declare class Table<T extends Row> {
     query(terms?: QueryTerm<T>[]): Query<T>;
     addColumn(id: ColumnID<T>, spec: Partial<ColumnSpec> & {
         type: ColumnSpec["type"];
-    }): void;
+    }): IColumn;
     removeColumn(id: ColumnID<T>): boolean;
     [Symbol.iterator](): Generator<Maybe<T>, void, unknown>;
     reindex(): void;

package/table.js

@@ -43,11 +43,13 @@ class Table {
     };
     this.validateColumnSpec(id, $spec);
     this.schema[id] = $spec;
-    this.columns[id] = COLUMN_TYPES[spec.type].impl(
+    const column = this.columns[id] = COLUMN_TYPES[$spec.type].impl(
       this,
       String(id),
       $spec
     );
+    if (this.length) column.ensureRows();
+    return column;
   }
   removeColumn(id) {
     if (this.columns[id]) return false;
@@ -132,6 +134,8 @@ class Table {
       if (max > 1 !== isArray(spec.default))
         __columnError(id, `wrong default value`);
     }
+    if (this.length && spec.default == null && (def.required || min > 0))
+      __columnError(id, `missing default value to fill existing rows`);
   }
   toJSON() {
     return {