@heswell/vuu-table 1.0.0

package/README.md

@@ -0,0 +1,40 @@
+# `@heswell/vuu-table`
+
+Runtime-agnostic Vuu table store. It runs unchanged in the browser or on a
+server (Bun, Node), and is the data layer under `@heswell/vuu-viewport`.
+
+| Export                                          | Use                                                                                                                                                      |
+| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Table`                                         | Keyed row store built from a Vuu `TableSchema`: insert, upsert, update, delete, with row created/updated timestamps. Notifies listeners of every change. |
+| `JoinTable`                                     | A materialized join of two tables (left outer by default, or `joinType: "inner"`). Joins can be chained.                                                 |
+| `RowSource`, `TableListener`, `VuuDataRow`, ... | The types an engine uses to consume a table.                                                                                                             |
+
+```ts
+import { Table } from "@heswell/vuu-table";
+
+const prices = new Table({
+  table: { module: "SIMUL", table: "prices" },
+  key: "ric",
+  columns: [
+    { name: "ric", serverDataType: "string" },
+    { name: "bid", serverDataType: "double" },
+  ],
+});
+prices.upsert(["VOD.L", 101.5]);
+```
+
+Peer dependencies: `@vuu-ui/vuu-data-types` and `@vuu-ui/vuu-protocol-types`
+(type-only). See
+[docs/design/data-engine.md](https://github.com/heswell/vuu-websocket/blob/main/docs/design/data-engine.md)
+for the design.
+
+## Publishing
+
+```sh
+npm run build:packages -- --package @heswell/vuu-table
+npm run pub -- --package=@heswell/vuu-table --dry-run
+npm run pub -- --package=@heswell/vuu-table [--tag alpha]
+```
+
+The build is written to `dist/vuu-table`. Publish `@heswell/vuu-table` before
+`@heswell/vuu-viewport`, which depends on it.

package/index.js

@@ -0,0 +1,350 @@
+function buildColumnMap(columns) {
+    const map = {};
+    for(let i = 0; i < columns.length; i++){
+        const column = columns[i];
+        if ("string" == typeof column) map[column] = i;
+        else if ("number" == typeof column.key) map[column.name] = column.key;
+        else map[column.name] = i;
+    }
+    return map;
+}
+const NO_LISTENERS = [];
+class Table {
+    columnMap;
+    indexOfKeyField;
+    schema;
+    #index = new Map();
+    #listeners = NO_LISTENERS;
+    #nextSeq = 0;
+    #rows = [];
+    #seq = [];
+    #indexOfCreatedTs;
+    #indexOfUpdatedTs;
+    constructor(schemaOrDefinition){
+        const schema = "schema" in schemaOrDefinition ? schemaOrDefinition.schema : schemaOrDefinition;
+        this.schema = schema;
+        this.columnMap = buildColumnMap(schema.columns);
+        this.indexOfKeyField = this.columnMap[schema.key];
+        if (void 0 === this.indexOfKeyField) throw Error(`[Table] key column ${schema.key} not found in schema for ${schema.table.table}`);
+        this.#indexOfCreatedTs = this.columnMap.vuuCreatedTimestamp;
+        this.#indexOfUpdatedTs = this.columnMap.vuuUpdatedTimestamp;
+    }
+    get columns() {
+        return this.schema.columns;
+    }
+    get name() {
+        return this.schema.table.table;
+    }
+    get primaryKey() {
+        return this.schema.key;
+    }
+    get rowCount() {
+        return this.#rows.length;
+    }
+    get rows() {
+        return this.#rows;
+    }
+    get keys() {
+        return this.#index.keys();
+    }
+    addListener(listener) {
+        this.#listeners = this.#listeners === NO_LISTENERS ? [
+            listener
+        ] : this.#listeners.concat(listener);
+    }
+    removeListener(listener) {
+        const listeners = this.#listeners.filter((l)=>l !== listener);
+        this.#listeners = 0 === listeners.length ? NO_LISTENERS : listeners;
+    }
+    get listenerCount() {
+        return this.#listeners.length;
+    }
+    seqAt(rowIdx) {
+        return this.#seq[rowIdx];
+    }
+    get seq() {
+        return this.#seq;
+    }
+    rowAt(rowIdx) {
+        return this.#rows[rowIdx];
+    }
+    rowIndexAtKey = (key)=>this.#index.get(key) ?? -1;
+    hasKey(key) {
+        return this.#index.has(key);
+    }
+    getRowAtKey(key, throwIfMissing = true) {
+        const rowIdx = this.#index.get(key);
+        if (void 0 !== rowIdx) return this.#rows[rowIdx];
+        if (throwIfMissing) throw Error(`Table getRowAtKey, no row at key ${key}`);
+    }
+    insert(row, _emitEvent = true) {
+        const key = String(row[this.indexOfKeyField]);
+        const existingIdx = this.#index.get(key);
+        if (void 0 !== existingIdx) {
+            this.update(existingIdx, row);
+            return existingIdx;
+        }
+        const createdTs = this.#indexOfCreatedTs;
+        if (void 0 !== createdTs && !row[createdTs]) {
+            const ts = Date.now();
+            row[createdTs] = ts;
+            if (void 0 !== this.#indexOfUpdatedTs) row[this.#indexOfUpdatedTs] = ts;
+        }
+        const rowIdx = this.#rows.length;
+        this.#rows.push(row);
+        this.#seq.push(this.#nextSeq++);
+        this.#index.set(key, rowIdx);
+        const listeners = this.#listeners;
+        for(let i = 0; i < listeners.length; i++)listeners[i].onInsert?.(rowIdx, row);
+        return rowIdx;
+    }
+    update(rowIdx, row, _columnName) {
+        const previous = this.#rows[rowIdx];
+        if (void 0 === previous) throw Error(`[Table] update, no row at index ${rowIdx}`);
+        if (void 0 !== this.#indexOfUpdatedTs) row[this.#indexOfUpdatedTs] = Date.now();
+        this.#rows[rowIdx] = row;
+        const listeners = this.#listeners;
+        for(let i = 0; i < listeners.length; i++)listeners[i].onUpdate?.(rowIdx, row, previous);
+        return true;
+    }
+    updateByKey(key, values) {
+        const rowIdx = this.#index.get(key);
+        if (void 0 === rowIdx) throw Error(`[Table] updateByKey, no row at key ${key}`);
+        const row = this.#rows[rowIdx].slice();
+        for(const columnName in values){
+            const colIdx = this.columnMap[columnName];
+            if (void 0 !== colIdx) row[colIdx] = values[columnName];
+        }
+        return this.update(rowIdx, row);
+    }
+    upsert(row, emitEvent = true) {
+        const key = String(row[this.indexOfKeyField]);
+        const rowIdx = this.#index.get(key);
+        if (void 0 === rowIdx) this.insert(row, emitEvent);
+        else this.update(rowIdx, row);
+    }
+    delete(key) {
+        const rowIdx = this.#index.get(key);
+        if (void 0 === rowIdx) return false;
+        const rows = this.#rows;
+        const seq = this.#seq;
+        const row = rows[rowIdx];
+        const lastIdx = rows.length - 1;
+        let movedFromIdx = -1;
+        this.#index.delete(key);
+        if (rowIdx !== lastIdx) {
+            const lastRow = rows[lastIdx];
+            rows[rowIdx] = lastRow;
+            seq[rowIdx] = seq[lastIdx];
+            this.#index.set(String(lastRow[this.indexOfKeyField]), rowIdx);
+            movedFromIdx = lastIdx;
+        }
+        rows.pop();
+        seq.pop();
+        const listeners = this.#listeners;
+        for(let i = 0; i < listeners.length; i++)listeners[i].onDelete?.(rowIdx, row, movedFromIdx);
+        return true;
+    }
+    clear() {
+        this.#rows.length = 0;
+        this.#seq.length = 0;
+        this.#index.clear();
+        const listeners = this.#listeners;
+        for(let i = 0; i < listeners.length; i++)listeners[i].onClear?.();
+    }
+    pullRowAsArray(key, columns) {
+        const row = this.getRowAtKey(key, true);
+        return columns.map((column)=>row[this.columnMap[column.name]]);
+    }
+    getUniqueValuesForColumn(column, pattern) {
+        const colIdx = this.columnMap[column];
+        if (void 0 === colIdx) throw Error(`[Table] getUniqueValuesForColumn no column ${column}`);
+        const set = new Set();
+        const lowercasePattern = pattern?.toLowerCase();
+        for (const row of this.#rows){
+            const value = row[colIdx];
+            if (null == value) continue;
+            const strValue = String(value);
+            if (void 0 === lowercasePattern || strValue.toLowerCase().startsWith(lowercasePattern)) set.add(strValue);
+        }
+        return Array.from(set).sort();
+    }
+}
+const NO_KEYS = new Set();
+class JoinTable extends Table {
+    baseTable;
+    joinTable;
+    joinType;
+    #inner;
+    #leftIdx;
+    #rightIdx;
+    #rightIsKey;
+    #sources;
+    #sourceIndices;
+    #baseKeysByJoinValue = new Map();
+    #rightKeyByJoinValue = new Map();
+    #baseListener;
+    #rightListener;
+    constructor({ schema, baseTable, joinTable, leftColumn, rightColumn, joinType = "leftOuter" }){
+        super(schema);
+        this.baseTable = baseTable;
+        this.joinTable = joinTable;
+        this.joinType = joinType;
+        this.#inner = "inner" === joinType;
+        const leftIdx = baseTable.columnMap[leftColumn];
+        const rightIdx = joinTable.columnMap[rightColumn];
+        if (void 0 === leftIdx) throw Error(`[JoinTable] no column ${leftColumn} in ${baseTable.name}`);
+        if (void 0 === rightIdx) throw Error(`[JoinTable] no column ${rightColumn} in ${joinTable.name}`);
+        this.#leftIdx = leftIdx;
+        this.#rightIdx = rightIdx;
+        this.#rightIsKey = rightIdx === joinTable.indexOfKeyField;
+        this.#sources = [];
+        this.#sourceIndices = [];
+        for (const { name } of schema.columns){
+            const baseIdx = baseTable.columnMap[name];
+            if (void 0 !== baseIdx) {
+                this.#sources.push(0);
+                this.#sourceIndices.push(baseIdx);
+            } else {
+                const joinIdx = joinTable.columnMap[name];
+                this.#sources.push(1);
+                this.#sourceIndices.push(joinIdx ?? -1);
+            }
+        }
+        if (!this.#rightIsKey) for (const row of joinTable.rows)this.indexRightRow(row);
+        const baseKeyIdx = baseTable.indexOfKeyField;
+        for (const row of baseTable.rows){
+            this.indexBaseRow(row);
+            this.sync(String(row[baseKeyIdx]), row);
+        }
+        this.#baseListener = {
+            onInsert: (_, row)=>{
+                this.indexBaseRow(row);
+                this.sync(String(row[baseKeyIdx]), row);
+            },
+            onUpdate: (_, row, previous)=>{
+                const key = String(row[baseKeyIdx]);
+                if (previous !== row) {
+                    const prevJoinValue = previous[this.#leftIdx];
+                    if (prevJoinValue !== row[this.#leftIdx]) {
+                        this.unindexBaseRow(key, prevJoinValue);
+                        this.indexBaseRow(row);
+                    }
+                }
+                this.sync(key, row);
+            },
+            onDelete: (_, row)=>{
+                const key = String(row[baseKeyIdx]);
+                this.unindexBaseRow(key, row[this.#leftIdx]);
+                if (-1 !== this.rowIndexAtKey(key)) super.delete(key);
+            },
+            onClear: ()=>{
+                this.#baseKeysByJoinValue.clear();
+                super.clear();
+            }
+        };
+        this.#rightListener = {
+            onInsert: (_, row)=>{
+                if (!this.#rightIsKey) this.indexRightRow(row);
+                this.refreshForJoinValue(row[this.#rightIdx]);
+            },
+            onUpdate: (_, row, previous)=>{
+                const joinValue = row[this.#rightIdx];
+                if (previous !== row && previous[this.#rightIdx] !== joinValue) {
+                    if (!this.#rightIsKey) {
+                        this.unindexRightRow(previous);
+                        this.indexRightRow(row);
+                    }
+                    this.refreshForJoinValue(previous[this.#rightIdx]);
+                }
+                this.refreshForJoinValue(joinValue);
+            },
+            onDelete: (_, row)=>{
+                if (!this.#rightIsKey) this.unindexRightRow(row);
+                this.refreshForJoinValue(row[this.#rightIdx]);
+            },
+            onClear: ()=>{
+                this.#rightKeyByJoinValue.clear();
+                if (this.#inner) super.clear();
+                else for (const row of this.baseTable.rows)this.sync(String(row[baseKeyIdx]), row);
+            }
+        };
+        baseTable.addListener(this.#baseListener);
+        joinTable.addListener(this.#rightListener);
+    }
+    destroy() {
+        this.baseTable.removeListener(this.#baseListener);
+        this.joinTable.removeListener(this.#rightListener);
+    }
+    findRightRow(joinValue) {
+        if (null == joinValue) return;
+        if (this.#rightIsKey) return this.joinTable.getRowAtKey(String(joinValue), false);
+        {
+            const rightKey = this.#rightKeyByJoinValue.get(joinValue);
+            return void 0 === rightKey ? void 0 : this.joinTable.getRowAtKey(rightKey, false);
+        }
+    }
+    joinRow(baseRow, rightRow) {
+        const sources = this.#sources;
+        const indices = this.#sourceIndices;
+        const len = sources.length;
+        const row = new Array(len);
+        for(let i = 0; i < len; i++){
+            const idx = indices[i];
+            if (0 === sources[i]) row[i] = baseRow[idx];
+            else row[i] = void 0 === rightRow || -1 === idx ? null : rightRow[idx];
+        }
+        return row;
+    }
+    sync(key, baseRow) {
+        const rightRow = this.findRightRow(baseRow[this.#leftIdx]);
+        const rowIdx = this.rowIndexAtKey(key);
+        if (void 0 === rightRow && this.#inner) {
+            if (-1 !== rowIdx) super.delete(key);
+        } else if (-1 === rowIdx) super.insert(this.joinRow(baseRow, rightRow));
+        else super.update(rowIdx, this.joinRow(baseRow, rightRow));
+    }
+    refreshForJoinValue(joinValue) {
+        const keys = this.#baseKeysByJoinValue.get(joinValue) ?? NO_KEYS;
+        for (const key of keys){
+            const baseRow = this.baseTable.getRowAtKey(key, false);
+            if (baseRow) this.sync(key, baseRow);
+        }
+    }
+    indexBaseRow(row) {
+        const joinValue = row[this.#leftIdx];
+        const key = String(row[this.baseTable.indexOfKeyField]);
+        let keys = this.#baseKeysByJoinValue.get(joinValue);
+        if (void 0 === keys) this.#baseKeysByJoinValue.set(joinValue, keys = new Set());
+        keys.add(key);
+    }
+    unindexBaseRow(key, joinValue) {
+        const keys = this.#baseKeysByJoinValue.get(joinValue);
+        if (keys) {
+            keys.delete(key);
+            if (0 === keys.size) this.#baseKeysByJoinValue.delete(joinValue);
+        }
+    }
+    indexRightRow(row) {
+        const joinValue = row[this.#rightIdx];
+        if (!this.#rightKeyByJoinValue.has(joinValue)) this.#rightKeyByJoinValue.set(joinValue, String(row[this.joinTable.indexOfKeyField]));
+    }
+    unindexRightRow(row) {
+        const joinValue = row[this.#rightIdx];
+        const key = String(row[this.joinTable.indexOfKeyField]);
+        if (this.#rightKeyByJoinValue.get(joinValue) === key) {
+            this.#rightKeyByJoinValue.delete(joinValue);
+            for (const r of this.joinTable.rows)if (r[this.#rightIdx] === joinValue && String(r[this.joinTable.indexOfKeyField]) !== key) {
+                this.#rightKeyByJoinValue.set(joinValue, String(r[this.joinTable.indexOfKeyField]));
+                break;
+            }
+        }
+    }
+    insert(_row) {
+        throw Error(`[JoinTable] ${this.name} is read only`);
+    }
+    upsert(_row) {
+        throw Error(`[JoinTable] ${this.name} is read only`);
+    }
+}
+export { JoinTable, Table, buildColumnMap };

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@heswell/vuu-table",
+  "version": "1.0.0",
+  "description": "Runtime agnostic (browser or server) Vuu table store, including join tables",
+  "type": "module",
+  "main": "./index.js",
+  "module": "./index.js",
+  "types": "./src/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./src/index.d.ts",
+      "import": "./index.js"
+    }
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "author": "heswell",
+  "license": "MIT",
+  "peerDependencies": {
+    "@vuu-ui/vuu-data-types": "^3.4.0",
+    "@vuu-ui/vuu-protocol-types": "^3.4.0"
+  }
+}

package/src/JoinTable.d.ts

@@ -0,0 +1,54 @@
+import { Table } from "./Table.js";
+import type { TableSchema, VuuDataRow } from "./types.js";
+export interface JoinTableConfig {
+    /**
+     * schema of the join table. Key must be the key of the base table.
+     * Column order defines the layout of joined rows.
+     */
+    schema: TableSchema;
+    baseTable: Table;
+    joinTable: Table;
+    /** column in base table holding the join value */
+    leftColumn: string;
+    /** column in join (right) table matched against leftColumn */
+    rightColumn: string;
+    /**
+     * leftOuter (default): every base row is present, right columns are null
+     * when there is no matching right row.
+     * inner: base rows are present only while a matching right row exists.
+     */
+    joinType?: JoinType;
+}
+export type JoinType = "leftOuter" | "inner";
+/**
+ * Materialized left outer (or inner) join of two tables. Joined rows are maintained
+ * incrementally as either source table changes, so a JoinTable is itself a
+ * regular Table that viewports can sort, filter and group with no join cost
+ * at query time.
+ *
+ * Base table columns take precedence when a column name exists in both
+ * tables.
+ */
+export declare class JoinTable extends Table {
+    #private;
+    readonly baseTable: Table;
+    readonly joinTable: Table;
+    readonly joinType: JoinType;
+    constructor({ schema, baseTable, joinTable, leftColumn, rightColumn, joinType, }: JoinTableConfig);
+    destroy(): void;
+    private findRightRow;
+    private joinRow;
+    /**
+     * Bring the joined row for a base row up to date: insert, update or, for an
+     * inner join without a matching right row, delete it.
+     */
+    private sync;
+    private refreshForJoinValue;
+    private indexBaseRow;
+    private unindexBaseRow;
+    private indexRightRow;
+    private unindexRightRow;
+    /** Join tables are read-only, rows are derived from the source tables */
+    insert(_row: VuuDataRow): number;
+    upsert(_row: VuuDataRow): void;
+}

package/src/Table.d.ts

@@ -0,0 +1,72 @@
+import type { ColumnMap, RowSource, TableListener, TableSchema, VuuDataRow, VuuRowDataItemType } from "./types.js";
+export type ColumnDescriptor = string | {
+    name: string;
+    key?: number;
+};
+export declare function buildColumnMap(columns: readonly ColumnDescriptor[]): ColumnMap;
+export interface TableDefinition {
+    schema: TableSchema;
+}
+/**
+ * In-memory, runtime agnostic, keyed row store.
+ *
+ * - rows are stored densely in `rows`, index lookups by key are O(1)
+ * - delete is O(1), using swap-remove (last row moves into the vacated slot)
+ * - every row carries an insertion sequence number, providing a stable
+ *   natural order regardless of physical position
+ * - listeners are notified synchronously of every change
+ */
+export declare class Table implements RowSource {
+    #private;
+    readonly columnMap: ColumnMap;
+    readonly indexOfKeyField: number;
+    readonly schema: TableSchema;
+    constructor(schemaOrDefinition: TableSchema | TableDefinition);
+    get columns(): readonly import("@vuu-ui/vuu-data-types").SchemaColumn[];
+    get name(): string;
+    get primaryKey(): string;
+    get rowCount(): number;
+    /**
+     * Dense array of rows. Treat as read only, use the Table api to mutate.
+     */
+    get rows(): VuuDataRow[];
+    get keys(): IterableIterator<string>;
+    addListener(listener: TableListener): void;
+    removeListener(listener: TableListener): void;
+    get listenerCount(): number;
+    seqAt(rowIdx: number): number;
+    /**
+     * Insertion sequence numbers, parallel to rows. Treat as read only.
+     */
+    get seq(): readonly number[];
+    rowAt(rowIdx: number): VuuDataRow;
+    rowIndexAtKey: (key: string) => number;
+    hasKey(key: string): boolean;
+    getRowAtKey(key: string, throwIfMissing?: true): VuuDataRow;
+    getRowAtKey(key: string, throwIfMissing: false): VuuDataRow | undefined;
+    /**
+     * Insert a new row. If a row with the same key exists, it is replaced
+     * (treated as an update).
+     *
+     * @param _emitEvent retained for api compatibility. Listeners are always
+     * notified, a listener that missed an insert would be inconsistent.
+     */
+    insert(row: VuuDataRow, _emitEvent?: boolean): number;
+    /**
+     * Replace the row at rowIdx.
+     */
+    update(rowIdx: number, row: VuuDataRow, _columnName?: string): boolean;
+    /**
+     * Apply a partial update to the row with given key. Values are keyed by
+     * column name. A new row array is created so listeners can detect
+     * which columns have changed.
+     */
+    updateByKey(key: string, values: Record<string, VuuRowDataItemType>): boolean;
+    upsert(row: VuuDataRow, emitEvent?: boolean): void;
+    delete(key: string): boolean;
+    clear(): void;
+    pullRowAsArray(key: string, columns: readonly {
+        name: string;
+    }[]): VuuRowDataItemType[];
+    getUniqueValuesForColumn(column: string, pattern?: string): string[];
+}

package/src/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./types.js";
+export * from "./Table.js";
+export * from "./JoinTable.js";

package/src/types.d.ts

@@ -0,0 +1,57 @@
+import type { TableSchema } from "@vuu-ui/vuu-data-types";
+import type { VuuDataRow, VuuRowDataItemType } from "@vuu-ui/vuu-protocol-types";
+export type { TableSchema, VuuDataRow, VuuRowDataItemType };
+/**
+ * Tables may hold bigint values (e.g. long columns from vuu-ui test data),
+ * although rows are typed with the protocol value types. Viewport engines
+ * sort, filter, aggregate and join on bigint values and convert them to a
+ * protocol value on output. Hosts can cast such rows to VuuDataRow.
+ */
+export type VuuDataRowWithBigint = (VuuRowDataItemType | bigint)[];
+/**
+ * Map of column name to index position within a row array.
+ */
+export type ColumnMap = Record<string, number>;
+/**
+ * Receives notification of every change to a table. Listeners are invoked
+ * synchronously, in registration order. Rows are stored densely, a delete
+ * moves the last row into the vacated slot (swap-remove) so listeners that
+ * track row indices must handle `movedFromIdx`.
+ */
+export interface TableListener {
+    onInsert?(rowIdx: number, row: VuuDataRow): void;
+    /**
+     * `previous` is the row that was replaced. It may be the same array
+     * instance as `row` if the caller mutated the row in place, in which
+     * case it is not possible to determine which columns have changed.
+     */
+    onUpdate?(rowIdx: number, row: VuuDataRow, previous: VuuDataRow): void;
+    /**
+     * The row previously at `rowIdx` has been removed. If `movedFromIdx` is
+     * not -1, the row previously at `movedFromIdx` (always the last row) has
+     * been moved into `rowIdx`.
+     */
+    onDelete?(rowIdx: number, row: VuuDataRow, movedFromIdx: number): void;
+    onClear?(): void;
+}
+/**
+ * Minimal read-only view of a table, sufficient for a viewport engine.
+ */
+export interface RowSource {
+    readonly columnMap: ColumnMap;
+    readonly indexOfKeyField: number;
+    readonly name: string;
+    readonly rowCount: number;
+    readonly rows: readonly VuuDataRow[];
+    readonly schema: TableSchema;
+    /**
+     * Monotonic insertion sequence for the row at rowIdx. Used to provide a
+     * stable 'natural' (insertion) order independent of physical position.
+     */
+    seqAt(rowIdx: number): number;
+    /** insertion sequence numbers, parallel to rows */
+    readonly seq: readonly number[];
+    rowIndexAtKey(key: string): number;
+    addListener(listener: TableListener): void;
+    removeListener(listener: TableListener): void;
+}