@svgrid/enterprise 3.1.1 → 3.1.2

package/dist/worker/columnar.js

@@ -0,0 +1,410 @@
+import { planQuery } from '../sveltekit/query-plan.js';
+import { asString, createInMemoryDataSource, matchesPredicate, sortRows } from '../sveltekit/in-memory.js';
+const RESULT_CACHE_SIZE = 8;
+const collator = new Intl.Collator();
+export function createColumnarDataSource(initial, schema) {
+    const reference = createInMemoryDataSource(initial, schema, { cacheResults: true });
+    let store = reference.rows();
+    let columns = new Map();
+    const cache = new Map();
+    const fields = schema.fields.map((f) => f.field);
+    // Index of the next schema field `warm()` builds.
+    let warmed = 0;
+    function invalidate() {
+        store = reference.rows();
+        columns = new Map();
+        cache.clear();
+        warmed = 0;
+    }
+    // ---- columns ------------------------------------------------------------
+    function column(field) {
+        let col = columns.get(field);
+        if (col)
+            return col;
+        const n = store.length;
+        const codes = new Uint32Array(n);
+        const values = [];
+        const index = new Map();
+        for (let i = 0; i < n; i += 1) {
+            const v = store[i][field];
+            let code = index.get(v);
+            if (code === undefined) {
+                code = values.length;
+                index.set(v, code);
+                values.push(v);
+            }
+            codes[i] = code;
+        }
+        col = { codes, values };
+        columns.set(field, col);
+        return col;
+    }
+    function ranked(col) {
+        if (col.ranks !== undefined)
+            return col.ranks;
+        const { values } = col;
+        let numbers = 0;
+        let finite = 0;
+        for (const v of values) {
+            if (typeof v === 'number') {
+                numbers += 1;
+                if (Number.isFinite(v))
+                    finite += 1;
+            }
+        }
+        // The reference compares two numbers by subtraction and anything else as
+        // text. All finite numbers, or no numbers at all, is a total order a
+        // ranking can reproduce; a mix (or NaN / Infinity) is not.
+        if (numbers > 0 && (finite !== values.length)) {
+            col.ranks = null;
+            return null;
+        }
+        const byValue = values.map((_, code) => code);
+        const text = numbers === 0 ? values.map(asString) : null;
+        const cmp = text
+            ? (a, b) => collator.compare(text[a], text[b])
+            : (a, b) => values[a] - values[b];
+        byValue.sort(cmp);
+        const ranks = new Uint32Array(values.length);
+        let rank = 0;
+        for (let i = 0; i < byValue.length; i += 1) {
+            // Values the comparison calls equal share a rank, so the stable sort
+            // keeps them in row order, as the reference's index tiebreak does.
+            if (i > 0 && cmp(byValue[i - 1], byValue[i]) !== 0)
+                rank += 1;
+            ranks[byValue[i]] = rank;
+        }
+        col.ranks = ranks;
+        col.rankCount = values.length ? rank + 1 : 0;
+        return ranks;
+    }
+    function keyed(col) {
+        if (!col.keyOf) {
+            const keyOf = new Uint32Array(col.values.length);
+            const keys = [];
+            const index = new Map();
+            col.values.forEach((v, code) => {
+                const key = asString(v);
+                let k = index.get(key);
+                if (k === undefined) {
+                    k = keys.length;
+                    index.set(key, k);
+                    keys.push(key);
+                }
+                keyOf[code] = k;
+            });
+            col.keyOf = keyOf;
+            col.keys = keys;
+        }
+        return { keyOf: col.keyOf, keys: col.keys };
+    }
+    function numbers(col) {
+        if (!col.nums) {
+            const nums = new Float64Array(col.values.length);
+            col.values.forEach((v, code) => {
+                const n = Number(v);
+                nums[code] = Number.isFinite(n) ? n : NaN;
+            });
+            col.nums = nums;
+        }
+        return col.nums;
+    }
+    // ---- filtering ----------------------------------------------------------
+    /** Row indexes passing `where` and the search, in store order. */
+    function select(where, search) {
+        const n = store.length;
+        const tests = [];
+        for (const p of where) {
+            const col = column(p.field);
+            const pass = new Uint8Array(col.values.length);
+            col.values.forEach((v, code) => {
+                pass[code] = matchesPredicate({ [p.field]: v }, p) ? 1 : 0;
+            });
+            tests.push({ codes: col.codes, pass });
+        }
+        const searched = [];
+        if (search) {
+            const term = search.term.toLowerCase();
+            for (const field of search.fields) {
+                const col = column(field);
+                const pass = new Uint8Array(col.values.length);
+                col.values.forEach((v, code) => {
+                    pass[code] = asString(v).toLowerCase().includes(term) ? 1 : 0;
+                });
+                searched.push({ codes: col.codes, pass });
+            }
+        }
+        if (tests.length === 0 && !search) {
+            const all = new Uint32Array(n);
+            for (let i = 0; i < n; i += 1)
+                all[i] = i;
+            return all;
+        }
+        const out = new Uint32Array(n);
+        let m = 0;
+        rows: for (let i = 0; i < n; i += 1) {
+            if (search) {
+                let hit = false;
+                for (const s of searched) {
+                    if (s.pass[s.codes[i]]) {
+                        hit = true;
+                        break;
+                    }
+                }
+                if (!hit)
+                    continue;
+            }
+            for (const t of tests)
+                if (!t.pass[t.codes[i]])
+                    continue rows;
+            out[m++] = i;
+        }
+        return out.slice(0, m);
+    }
+    // ---- sorting ------------------------------------------------------------
+    /** Stable counting sort of `idx` by one column; LSD over several keys. */
+    function sortIndexes(idx, orderBy) {
+        const keys = [];
+        for (const o of orderBy) {
+            const col = column(o.field);
+            const ranks = ranked(col);
+            if (!ranks)
+                return null;
+            keys.push({ codes: col.codes, ranks, count: col.rankCount, desc: o.desc });
+        }
+        let cur = idx;
+        let next = new Uint32Array(idx.length);
+        // Least significant key first: each pass is stable, so earlier keys win.
+        for (let k = keys.length - 1; k >= 0; k -= 1) {
+            const { codes, ranks, count, desc } = keys[k];
+            const counts = new Uint32Array(count + 1);
+            for (let i = 0; i < cur.length; i += 1) {
+                const r = ranks[codes[cur[i]]];
+                counts[(desc ? count - 1 - r : r) + 1] += 1;
+            }
+            for (let r = 1; r <= count; r += 1)
+                counts[r] += counts[r - 1];
+            for (let i = 0; i < cur.length; i += 1) {
+                const row = cur[i];
+                const r = ranks[codes[row]];
+                next[counts[desc ? count - 1 - r : r]++] = row;
+            }
+            const swap = cur === idx ? new Uint32Array(idx.length) : cur;
+            cur = next;
+            next = swap;
+        }
+        return cur;
+    }
+    function accumulate(idx, 
+    /** The group of the i-th row of `idx`, or null for one group. */
+    rowGroup, groupCount, aggregations) {
+        const out = new Map();
+        for (const agg of aggregations) {
+            if (out.has(agg.field))
+                continue;
+            const col = column(agg.field);
+            const codes = col.codes;
+            const nums = numbers(col);
+            const acc = {
+                count: new Float64Array(groupCount),
+                sum: new Float64Array(groupCount),
+                min: new Float64Array(groupCount).fill(Infinity),
+                max: new Float64Array(groupCount).fill(-Infinity),
+                n: new Float64Array(groupCount),
+            };
+            for (let i = 0; i < idx.length; i += 1) {
+                const row = idx[i];
+                const g = rowGroup ? rowGroup[i] : 0;
+                acc.count[g] += 1;
+                const v = nums[codes[row]];
+                if (v !== v)
+                    continue; // NaN: not a finite number
+                acc.n[g] += 1;
+                acc.sum[g] += v;
+                if (v < acc.min[g])
+                    acc.min[g] = v;
+                if (v > acc.max[g])
+                    acc.max[g] = v;
+            }
+            out.set(agg.field, acc);
+        }
+        return out;
+    }
+    /** The reference's `aggregate`, read off the accumulators. */
+    function valueOf(acc, g, fn) {
+        if (fn === 'count')
+            return acc.count[g];
+        if (acc.n[g] === 0)
+            return null;
+        switch (fn) {
+            case 'sum':
+                return acc.sum[g];
+            case 'avg':
+                return acc.sum[g] / acc.n[g];
+            case 'min':
+                return acc.min[g];
+            case 'max':
+                return acc.max[g];
+            default:
+                return null;
+        }
+    }
+    function totals(idx, plan) {
+        const aggs = plan.aggregations ?? [];
+        const acc = accumulate(idx, null, 1, aggs);
+        const out = {};
+        for (const agg of aggs)
+            out[agg.field] = valueOf(acc.get(agg.field), 0, agg.fn);
+        return out;
+    }
+    // ---- answering ----------------------------------------------------------
+    function supported(plan) {
+        return !plan.expression && !plan.pivotBy?.length;
+    }
+    function prepare(plan) {
+        const key = JSON.stringify({ ...plan, limit: 0, offset: 0 });
+        const hit = cache.get(key);
+        if (hit) {
+            cache.delete(key);
+            cache.set(key, hit);
+            return hit;
+        }
+        const prepared = run(plan);
+        if (!prepared)
+            return null;
+        cache.set(key, prepared);
+        if (cache.size > RESULT_CACHE_SIZE)
+            cache.delete(cache.keys().next().value);
+        return prepared;
+    }
+    function run(plan) {
+        const idx = select(plan.where, plan.search);
+        const out = {};
+        if (plan.grandTotal) {
+            const filtersOnly = plan.where.slice(0, plan.where.length - (plan.pathPredicates ?? 0));
+            const all = filtersOnly.length === plan.where.length ? idx : select(filtersOnly, plan.search);
+            out.grandTotal = totals(all, plan);
+        }
+        if (!plan.groupBy) {
+            const order = plan.orderBy.length ? sortIndexes(idx, plan.orderBy) : idx;
+            if (!order)
+                return null;
+            out.order = order;
+            return out;
+        }
+        // Groups in order of first appearance, as the reference's Map buckets are.
+        const groupField = plan.groupBy;
+        const gcol = column(groupField);
+        const { keyOf, keys } = keyed(gcol);
+        const slot = new Int32Array(keys.length).fill(-1);
+        const groupKeys = [];
+        const gcodes = gcol.codes;
+        const rowGroup = new Uint32Array(idx.length);
+        for (let i = 0; i < idx.length; i += 1) {
+            const k = keyOf[gcodes[idx[i]]];
+            let g = slot[k];
+            if (g < 0) {
+                g = groupKeys.length;
+                slot[k] = g;
+                groupKeys.push(k);
+            }
+            rowGroup[i] = g;
+        }
+        const groupCount = groupKeys.length;
+        const acc = accumulate(idx, rowGroup, groupCount, plan.aggregations ?? []);
+        const childCounts = new Float64Array(groupCount);
+        if (plan.childGroupBy) {
+            const ccol = column(plan.childGroupBy);
+            const { keyOf: childKeyOf, keys: childKeys } = keyed(ccol);
+            const seen = new Set();
+            for (let i = 0; i < idx.length; i += 1) {
+                const g = rowGroup[i];
+                const c = childKeyOf[ccol.codes[idx[i]]];
+                const pair = g * childKeys.length + c;
+                if (!seen.has(pair)) {
+                    seen.add(pair);
+                    childCounts[g] += 1;
+                }
+            }
+        }
+        else {
+            for (let i = 0; i < idx.length; i += 1)
+                childCounts[rowGroup[i]] += 1;
+        }
+        const groups = groupKeys.map((k, g) => {
+            const row = { [groupField]: keys[k] };
+            for (const agg of plan.aggregations ?? [])
+                row[agg.field] = valueOf(acc.get(agg.field), g, agg.fn);
+            row.childCount = childCounts[g];
+            return row;
+        });
+        const produced = new Set([groupField, ...(plan.aggregations ?? []).map((a) => a.field)]);
+        const groupOrder = plan.orderBy.filter((o) => produced.has(o.field));
+        out.groups = groupOrder.length
+            ? sortRows(groups, groupOrder)
+            : sortRows(groups, [{ field: groupField, desc: false }]);
+        return out;
+    }
+    const counts = { columnar: 0, reference: 0 };
+    return {
+        async getRows(request) {
+            const plan = planQuery(schema, request);
+            const prepared = supported(plan) ? prepare(plan) : null;
+            if (!prepared) {
+                counts.reference += 1;
+                return reference.getRows(request);
+            }
+            counts.columnar += 1;
+            const from = plan.offset;
+            const to = plan.offset + plan.limit;
+            const total = prepared.groups ? prepared.groups.length : prepared.order.length;
+            const rows = prepared.groups
+                ? prepared.groups.slice(from, to)
+                : Array.from(prepared.order.subarray(from, Math.min(to, total)), (i) => store[i]);
+            return {
+                rows,
+                rowCount: total,
+                appliedExpression: false,
+                ...(prepared.grandTotal !== undefined ? { grandTotal: prepared.grandTotal } : {}),
+            };
+        },
+        async createRow(input) {
+            const row = await reference.createRow(input);
+            invalidate();
+            return row;
+        },
+        async updateRow(id, patch) {
+            const row = await reference.updateRow(id, patch);
+            invalidate();
+            return row;
+        },
+        async deleteRow(id) {
+            await reference.deleteRow(id);
+            invalidate();
+        },
+        async updateWhere(filterModel, patch, selection) {
+            const changed = await reference.updateWhere(filterModel, patch, selection);
+            invalidate();
+            return changed;
+        },
+        getAggregate: (request) => reference.getAggregate(request),
+        rows: () => store,
+        stats: () => ({ ...counts }),
+        warm() {
+            while (warmed < fields.length) {
+                const field = fields[warmed++];
+                const existing = columns.get(field);
+                if (existing?.ranks !== undefined && existing.keyOf && existing.nums)
+                    continue;
+                // Everything a sort, a grouping and an aggregate read, so none of
+                // them pays for it on the first click.
+                const col = column(field);
+                ranked(col);
+                keyed(col);
+                numbers(col);
+                return true;
+            }
+            return false;
+        },
+    };
+}

package/dist/worker/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * `@svgrid/enterprise/worker` - what runs INSIDE the worker behind
+ * `createWorkerDataSource`. Import it from your worker file and nowhere else:
+ * it has no Svelte and no DOM in its import graph, so the worker bundle stays
+ * the in-memory source and little more.
+ *
+ * The main-thread half, `createWorkerDataSource`, is on the package root and
+ * on `@svgrid/enterprise/server`.
+ */
+export { serveWorkerDataSource, type ServeWorkerDataSourceOptions, type WorkerScopeLike, } from './serve.js';
+export { inferSchemaFromRows, type InferSchemaOptions } from './infer-schema.js';
+export { createColumnarDataSource, type ColumnarDataSource } from './columnar.js';

package/dist/worker/index.js

@@ -0,0 +1,12 @@
+/**
+ * `@svgrid/enterprise/worker` - what runs INSIDE the worker behind
+ * `createWorkerDataSource`. Import it from your worker file and nowhere else:
+ * it has no Svelte and no DOM in its import graph, so the worker bundle stays
+ * the in-memory source and little more.
+ *
+ * The main-thread half, `createWorkerDataSource`, is on the package root and
+ * on `@svgrid/enterprise/server`.
+ */
+export { serveWorkerDataSource, } from './serve.js';
+export { inferSchemaFromRows } from './infer-schema.js';
+export { createColumnarDataSource } from './columnar.js';

package/dist/worker/infer-schema.d.ts

@@ -0,0 +1,21 @@
+/**
+ * An `EntitySchema` read off the rows themselves, for a worker data source
+ * that was handed plain objects. The in-memory source only filters, sorts and
+ * groups on fields its schema declares, and the type decides how a request's
+ * string values are compared: `'35'` is a number for a number field and text
+ * for a text one.
+ */
+import type { RowData } from '@svgrid/grid';
+import type { EntitySchema } from '../schema.js';
+export type InferSchemaOptions = {
+    /** The row id field. Default: `id` when the rows have one, else the first field. */
+    idField?: string;
+    /** The schema's name. Default `rows`. */
+    name?: string;
+};
+/**
+ * A field is `number` or `boolean` when every non-empty sampled value is one,
+ * and `text` otherwise. Dates come out as `text`: send them as ISO strings,
+ * which sort and compare correctly as text, or as epoch numbers.
+ */
+export declare function inferSchemaFromRows<TData extends RowData>(rows: ReadonlyArray<TData>, options?: InferSchemaOptions): EntitySchema<TData>;

package/dist/worker/infer-schema.js

@@ -0,0 +1,34 @@
+/** Rows read to decide each field's type. */
+const SAMPLE_SIZE = 200;
+/**
+ * A field is `number` or `boolean` when every non-empty sampled value is one,
+ * and `text` otherwise. Dates come out as `text`: send them as ISO strings,
+ * which sort and compare correctly as text, or as epoch numbers.
+ */
+export function inferSchemaFromRows(rows, options = {}) {
+    const sample = rows.slice(0, SAMPLE_SIZE);
+    const seen = new Map();
+    for (const row of sample) {
+        for (const [field, value] of Object.entries(row)) {
+            let kinds = seen.get(field);
+            if (!kinds)
+                seen.set(field, (kinds = new Set()));
+            if (value != null && value !== '')
+                kinds.add(typeof value);
+        }
+    }
+    const fields = [...seen].map(([field, kinds]) => {
+        let type = 'text';
+        if (kinds.size === 1 && kinds.has('number'))
+            type = 'number';
+        else if (kinds.size === 1 && kinds.has('boolean'))
+            type = 'boolean';
+        return { field: field, type };
+    });
+    const idField = options.idField ?? (seen.has('id') ? 'id' : fields[0]?.field) ?? 'id';
+    return {
+        name: options.name ?? 'rows',
+        idField: idField,
+        fields,
+    };
+}

package/dist/worker/protocol.d.ts

@@ -0,0 +1,73 @@
+/**
+ * The messages between `createWorkerDataSource` (main thread) and
+ * `serveWorkerDataSource` (inside the worker). Internal: both ends ship in
+ * the same package version, so the shape can change without a migration.
+ *
+ * Every message carries `channel`, so a worker that also answers messages of
+ * its own can host the data source beside them; each end ignores anything
+ * without it.
+ */
+import type { EntitySchema } from '../schema.js';
+export declare const CHANNEL: "svgrid-worker-data-source";
+/** The data source methods the client may call across the boundary. */
+export type WorkerMethod = 'getRows' | 'createRow' | 'updateRow' | 'deleteRow' | 'updateWhere' | 'getAggregate';
+/** Main thread to worker. */
+export type ToWorker = 
+/** Replace the dataset. Answered with a `result` carrying the row count. */
+{
+    channel: typeof CHANNEL;
+    kind: 'load';
+    id: number;
+    rows: ReadonlyArray<unknown>;
+    schema?: EntitySchema;
+    idField?: string;
+} | {
+    channel: typeof CHANNEL;
+    kind: 'call';
+    id: number;
+    method: WorkerMethod;
+    args: unknown[];
+}
+/** Drop a queued call: its block was evicted, or the sort or filter moved on. */
+ | {
+    channel: typeof CHANNEL;
+    kind: 'cancel';
+    id: number;
+};
+/** An error as it crosses the boundary: an `Error` does not survive cloning intact everywhere. */
+export type WorkerError = {
+    name: string;
+    message: string;
+};
+/** Worker to main thread. */
+export type FromWorker = 
+/** The worker holds data and answers calls. Sent after every load. */
+{
+    channel: typeof CHANNEL;
+    kind: 'ready';
+    rowCount: number;
+}
+/** The worker's own `rows` loader threw: nothing will ever be answered. */
+ | {
+    channel: typeof CHANNEL;
+    kind: 'failed';
+    error: WorkerError;
+} | {
+    channel: typeof CHANNEL;
+    kind: 'result';
+    id: number;
+    ok: true;
+    value: unknown;
+} | {
+    channel: typeof CHANNEL;
+    kind: 'result';
+    id: number;
+    ok: false;
+    error: WorkerError;
+};
+export declare function toWorkerError(error: unknown): WorkerError;
+export declare function fromWorkerError(error: WorkerError): Error;
+export declare function isChannelMessage(data: unknown): data is {
+    channel: typeof CHANNEL;
+    kind: string;
+};

package/dist/worker/protocol.js

@@ -0,0 +1,14 @@
+export const CHANNEL = 'svgrid-worker-data-source';
+export function toWorkerError(error) {
+    if (error instanceof Error)
+        return { name: error.name, message: error.message };
+    return { name: 'Error', message: String(error) };
+}
+export function fromWorkerError(error) {
+    const out = new Error(error.message);
+    out.name = error.name;
+    return out;
+}
+export function isChannelMessage(data) {
+    return typeof data === 'object' && data !== null && data.channel === CHANNEL;
+}

package/dist/worker/serve.d.ts

@@ -0,0 +1,44 @@
+/**
+ * The worker half of `createWorkerDataSource`: holds the rows, answers the
+ * grid's requests with the columnar engine (`createColumnarDataSource`, which
+ * answers exactly as the in-memory reference source does), and sends back only
+ * the rows each request asks for. While no request is waiting, it builds the
+ * next column's index, so the first sort on any column finds it ready.
+ *
+ * ```ts
+ * // grid.worker.ts
+ * import { serveWorkerDataSource } from '@svgrid/enterprise/worker'
+ * serveWorkerDataSource()
+ * ```
+ *
+ * Calls run one at a time, each in its own task, so a `cancel` that arrives
+ * while earlier calls are running still reaches the queue before the call it
+ * names: a fast scroll skips the blocks it scrolled past instead of computing
+ * every one of them.
+ */
+import type { RowData } from '@svgrid/grid';
+import type { EntitySchema } from '../schema.js';
+/** The part of a worker's global scope (or a `MessagePort`) this needs. */
+export type WorkerScopeLike = {
+    postMessage(message: unknown): void;
+    addEventListener(type: 'message', listener: (event: MessageEvent) => void): void;
+    removeEventListener(type: 'message', listener: (event: MessageEvent) => void): void;
+};
+export type ServeWorkerDataSourceOptions<TData extends RowData> = {
+    /**
+     * The rows, loaded inside the worker: an array, or a function that fetches
+     * or builds one. Loading here means a large dataset never passes through
+     * the main thread. Omit it to wait for the rows the main thread sends
+     * (`createWorkerDataSource(worker, { rows })`).
+     */
+    rows?: ReadonlyArray<TData> | (() => ReadonlyArray<TData> | Promise<ReadonlyArray<TData>>);
+    /** Which fields can be filtered, sorted and grouped, and their types. Default: read off the rows. */
+    schema?: EntitySchema<TData>;
+    /** The row id field, when the schema is inferred. See `inferSchemaFromRows`. */
+    idField?: string;
+    /** Where to listen. Default: the worker's global scope. */
+    scope?: WorkerScopeLike;
+};
+export declare function serveWorkerDataSource<TData extends RowData>(options?: ServeWorkerDataSourceOptions<TData>): {
+    dispose(): void;
+};