@yydb/light-db 0.0.0

package/dist/performance/ColumnarStorage.js

@@ -0,0 +1,875 @@
+"use strict";
+/**
+ * 列式存储实现
+ * 使用平行数组存储数据，降低 GC 压力，提高聚合和范围扫描性能
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ColumnarStorage = void 0;
+const types_1 = require("./types");
+/**
+ * 类型化数组列实现
+ */
+class TypedArrayColumn {
+    constructor(name, type, capacity, createArray) {
+        this.name = name;
+        this.type = type;
+        this._capacity = capacity;
+        this._length = 0;
+        this.data = createArray(capacity);
+        this.nullBitmap = new Uint8Array(Math.ceil(capacity / 8));
+    }
+    get length() {
+        return this._length;
+    }
+    getData() {
+        return this.data;
+    }
+    get(index) {
+        if (index < 0 || index >= this._length) {
+            return undefined;
+        }
+        if (this.isNull(index)) {
+            return null;
+        }
+        return this.getValue(index);
+    }
+    set(index, value) {
+        if (index < 0 || index >= this._capacity) {
+            throw new RangeError(`Index ${index} out of bounds`);
+        }
+        if (value === null || value === undefined) {
+            this.setNull(index, true);
+        } else {
+            this.setNull(index, false);
+            this.setValue(index, value);
+        }
+        if (index >= this._length) {
+            this._length = index + 1;
+        }
+    }
+    push(value) {
+        if (this._length >= this._capacity) {
+            throw new Error("Column is full, call grow() first");
+        }
+        this.set(this._length, value);
+    }
+    fill(value, start, end) {
+        for (let i = start; i < end; i++) {
+            this.set(i, value);
+        }
+    }
+    isNull(index) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        return (this.nullBitmap[byteIndex] & (1 << bitIndex)) !== 0;
+    }
+    setNull(index, isNull) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        if (isNull) {
+            this.nullBitmap[byteIndex] |= 1 << bitIndex;
+        } else {
+            this.nullBitmap[byteIndex] &= ~(1 << bitIndex);
+        }
+    }
+    getMemoryUsage() {
+        return this.data.byteLength + this.nullBitmap.byteLength + 8;
+    }
+    getNonNullCount() {
+        let count = 0;
+        for (let i = 0; i < this._length; i++) {
+            if (!this.isNull(i)) {
+                count++;
+            }
+        }
+        return count;
+    }
+    grow(newCapacity) {
+        if (newCapacity <= this._capacity) {
+            return;
+        }
+        const newData = this.createArray(newCapacity);
+        newData.set(this.data);
+        this.data = newData;
+        const newNullBitmap = new Uint8Array(Math.ceil(newCapacity / 8));
+        newNullBitmap.set(this.nullBitmap);
+        this.nullBitmap = newNullBitmap;
+        this._capacity = newCapacity;
+    }
+}
+/**
+ * Int8 列
+ */
+class Int8Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.Int8, capacity, (s) => new Int8Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Int8Array(size);
+    }
+}
+/**
+ * Int16 列
+ */
+class Int16Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.Int16, capacity, (s) => new Int16Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Int16Array(size);
+    }
+}
+/**
+ * Int32 列
+ */
+class Int32Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.Int32, capacity, (s) => new Int32Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Int32Array(size);
+    }
+}
+/**
+ * Int64 列（使用 BigInt64Array）
+ */
+class Int64Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.Int64, capacity, (s) => new BigInt64Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = BigInt(value);
+    }
+    createArray(size) {
+        return new BigInt64Array(size);
+    }
+}
+/**
+ * UInt8 列
+ */
+class UInt8Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.UInt8, capacity, (s) => new Uint8Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Uint8Array(size);
+    }
+}
+/**
+ * UInt16 列
+ */
+class UInt16Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.UInt16, capacity, (s) => new Uint16Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Uint16Array(size);
+    }
+}
+/**
+ * UInt32 列
+ */
+class UInt32Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.UInt32, capacity, (s) => new Uint32Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Uint32Array(size);
+    }
+}
+/**
+ * UInt64 列
+ */
+class UInt64Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.UInt64, capacity, (s) => new BigUint64Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = BigInt(value);
+    }
+    createArray(size) {
+        return new BigUint64Array(size);
+    }
+}
+/**
+ * Float32 列
+ */
+class Float32Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.Float32, capacity, (s) => new Float32Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Float32Array(size);
+    }
+}
+/**
+ * Float64 列
+ */
+class Float64Column extends TypedArrayColumn {
+    constructor(name, capacity) {
+        super(name, types_1.ColumnType.Float64, capacity, (s) => new Float64Array(s));
+    }
+    getValue(index) {
+        return this.data[index];
+    }
+    setValue(index, value) {
+        this.data[index] = value;
+    }
+    createArray(size) {
+        return new Float64Array(size);
+    }
+}
+/**
+ * 字符串列
+ */
+class StringColumn {
+    constructor(name, capacity) {
+        this.type = types_1.ColumnType.String;
+        this.name = name;
+        this._capacity = capacity;
+        this._length = 0;
+        this.data = new Array(capacity);
+        this.nullBitmap = new Uint8Array(Math.ceil(capacity / 8));
+    }
+    get length() {
+        return this._length;
+    }
+    get(index) {
+        if (index < 0 || index >= this._length) {
+            return undefined;
+        }
+        if (this.isNull(index)) {
+            return null;
+        }
+        return this.data[index];
+    }
+    set(index, value) {
+        if (index < 0 || index >= this._capacity) {
+            throw new RangeError(`Index ${index} out of bounds`);
+        }
+        if (value === null || value === undefined) {
+            this.setNull(index, true);
+            this.data[index] = "";
+        } else {
+            this.setNull(index, false);
+            this.data[index] = String(value);
+        }
+        if (index >= this._length) {
+            this._length = index + 1;
+        }
+    }
+    push(value) {
+        if (this._length >= this._capacity) {
+            throw new Error("Column is full, call grow() first");
+        }
+        this.set(this._length, value);
+    }
+    fill(value, start, end) {
+        for (let i = start; i < end; i++) {
+            this.set(i, value);
+        }
+    }
+    getMemoryUsage() {
+        let total = this.nullBitmap.byteLength + 8;
+        for (let i = 0; i < this._length; i++) {
+            if (!this.isNull(i) && this.data[i]) {
+                total += this.data[i].length * 2;
+            }
+        }
+        return total;
+    }
+    getNonNullCount() {
+        let count = 0;
+        for (let i = 0; i < this._length; i++) {
+            if (!this.isNull(i)) {
+                count++;
+            }
+        }
+        return count;
+    }
+    grow(newCapacity) {
+        if (newCapacity <= this._capacity) {
+            return;
+        }
+        this.data.length = newCapacity;
+        const newNullBitmap = new Uint8Array(Math.ceil(newCapacity / 8));
+        newNullBitmap.set(this.nullBitmap);
+        this.nullBitmap = newNullBitmap;
+        this._capacity = newCapacity;
+    }
+    isNull(index) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        return (this.nullBitmap[byteIndex] & (1 << bitIndex)) !== 0;
+    }
+    setNull(index, isNull) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        if (isNull) {
+            this.nullBitmap[byteIndex] |= 1 << bitIndex;
+        } else {
+            this.nullBitmap[byteIndex] &= ~(1 << bitIndex);
+        }
+    }
+}
+/**
+ * 布尔列
+ */
+class BooleanColumn {
+    constructor(name, capacity) {
+        this.type = types_1.ColumnType.Boolean;
+        this.name = name;
+        this._capacity = capacity;
+        this._length = 0;
+        this.data = new Uint8Array(capacity);
+        this.nullBitmap = new Uint8Array(Math.ceil(capacity / 8));
+    }
+    get length() {
+        return this._length;
+    }
+    get(index) {
+        if (index < 0 || index >= this._length) {
+            return undefined;
+        }
+        if (this.isNull(index)) {
+            return null;
+        }
+        return this.data[index] === 1;
+    }
+    set(index, value) {
+        if (index < 0 || index >= this._capacity) {
+            throw new RangeError(`Index ${index} out of bounds`);
+        }
+        if (value === null || value === undefined) {
+            this.setNull(index, true);
+            this.data[index] = 0;
+        } else {
+            this.setNull(index, false);
+            this.data[index] = value ? 1 : 0;
+        }
+        if (index >= this._length) {
+            this._length = index + 1;
+        }
+    }
+    push(value) {
+        if (this._length >= this._capacity) {
+            throw new Error("Column is full, call grow() first");
+        }
+        this.set(this._length, value);
+    }
+    fill(value, start, end) {
+        for (let i = start; i < end; i++) {
+            this.set(i, value);
+        }
+    }
+    getMemoryUsage() {
+        return this.data.byteLength + this.nullBitmap.byteLength + 8;
+    }
+    getNonNullCount() {
+        let count = 0;
+        for (let i = 0; i < this._length; i++) {
+            if (!this.isNull(i)) {
+                count++;
+            }
+        }
+        return count;
+    }
+    grow(newCapacity) {
+        if (newCapacity <= this._capacity) {
+            return;
+        }
+        const newData = new Uint8Array(newCapacity);
+        newData.set(this.data);
+        this.data = newData;
+        const newNullBitmap = new Uint8Array(Math.ceil(newCapacity / 8));
+        newNullBitmap.set(this.nullBitmap);
+        this.nullBitmap = newNullBitmap;
+        this._capacity = newCapacity;
+    }
+    isNull(index) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        return (this.nullBitmap[byteIndex] & (1 << bitIndex)) !== 0;
+    }
+    setNull(index, isNull) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        if (isNull) {
+            this.nullBitmap[byteIndex] |= 1 << bitIndex;
+        } else {
+            this.nullBitmap[byteIndex] &= ~(1 << bitIndex);
+        }
+    }
+}
+/**
+ * 列式存储
+ * 使用平行数组存储数据，降低 GC 压力
+ */
+class ColumnarStorage {
+    /**
+     * 创建列式存储实例
+     * @param options 配置选项
+     */
+    constructor(options) {
+        this.columns = new Map();
+        this.columnOrder = [];
+        this.capacity = options.initialCapacity ?? 1024;
+        this.autoGrow = options.autoGrow ?? true;
+        for (const colDef of options.columns) {
+            this.addColumn(colDef);
+        }
+    }
+    /**
+     * 添加列
+     * @param definition 列定义
+     */
+    addColumn(definition) {
+        if (this.columns.has(definition.name)) {
+            throw new Error(`Column "${definition.name}" already exists`);
+        }
+        const column = this.createColumn(definition);
+        this.columns.set(definition.name, column);
+        this.columnOrder.push(definition.name);
+    }
+    /**
+     * 创建列实例
+     */
+    createColumn(definition) {
+        const { name, type } = definition;
+        switch (type) {
+            case types_1.ColumnType.Int8:
+                return new Int8Column(name, this.capacity);
+            case types_1.ColumnType.Int16:
+                return new Int16Column(name, this.capacity);
+            case types_1.ColumnType.Int32:
+                return new Int32Column(name, this.capacity);
+            case types_1.ColumnType.Int64:
+                return new Int64Column(name, this.capacity);
+            case types_1.ColumnType.UInt8:
+                return new UInt8Column(name, this.capacity);
+            case types_1.ColumnType.UInt16:
+                return new UInt16Column(name, this.capacity);
+            case types_1.ColumnType.UInt32:
+                return new UInt32Column(name, this.capacity);
+            case types_1.ColumnType.UInt64:
+                return new UInt64Column(name, this.capacity);
+            case types_1.ColumnType.Float32:
+                return new Float32Column(name, this.capacity);
+            case types_1.ColumnType.Float64:
+                return new Float64Column(name, this.capacity);
+            case types_1.ColumnType.String:
+                return new StringColumn(name, this.capacity);
+            case types_1.ColumnType.Boolean:
+                return new BooleanColumn(name, this.capacity);
+            default:
+                throw new Error(`Unsupported column type: ${type}`);
+        }
+    }
+    /**
+     * 获取行数
+     */
+    get rowCount() {
+        if (this.columnOrder.length === 0) {
+            return 0;
+        }
+        const firstColumn = this.columns.get(this.columnOrder[0]);
+        return firstColumn ? firstColumn.length : 0;
+    }
+    /**
+     * 获取列数
+     */
+    get columnCount() {
+        return this.columns.size;
+    }
+    /**
+     * 获取列名称列表
+     */
+    getColumnNames() {
+        return [...this.columnOrder];
+    }
+    /**
+     * 获取列类型
+     * @param columnName 列名称
+     */
+    getColumnType(columnName) {
+        const column = this.columns.get(columnName);
+        return column?.type;
+    }
+    /**
+     * 插入一行数据
+     * @param row 行数据
+     * @returns 插入的行索引
+     */
+    insert(row) {
+        const rowIndex = this.rowCount;
+        if (rowIndex >= this.capacity && this.autoGrow) {
+            this.grow(this.capacity * 2);
+        }
+        for (const [colName, value] of Object.entries(row)) {
+            const column = this.columns.get(colName);
+            if (column) {
+                column.push(value);
+            }
+        }
+        for (const colName of this.columnOrder) {
+            const column = this.columns.get(colName);
+            if (column.length <= rowIndex) {
+                column.push(null);
+            }
+        }
+        return rowIndex;
+    }
+    /**
+     * 批量插入数据
+     * @param rows 行数据数组
+     * @returns 插入的行索引范围
+     */
+    insertMany(rows) {
+        const start = this.rowCount;
+        for (const row of rows) {
+            this.insert(row);
+        }
+        return { start, end: this.rowCount };
+    }
+    /**
+     * 获取单行数据
+     * @param rowIndex 行索引
+     */
+    getRow(rowIndex) {
+        if (rowIndex < 0 || rowIndex >= this.rowCount) {
+            return null;
+        }
+        const row = {};
+        for (const colName of this.columnOrder) {
+            const column = this.columns.get(colName);
+            row[colName] = column.get(rowIndex);
+        }
+        return row;
+    }
+    /**
+     * 更新单行数据
+     * @param rowIndex 行索引
+     * @param row 新数据
+     */
+    updateRow(rowIndex, row) {
+        if (rowIndex < 0 || rowIndex >= this.rowCount) {
+            return false;
+        }
+        for (const [colName, value] of Object.entries(row)) {
+            const column = this.columns.get(colName);
+            if (column) {
+                column.set(rowIndex, value);
+            }
+        }
+        return true;
+    }
+    /**
+     * 获取单列数据
+     * @param columnName 列名称
+     */
+    getColumn(columnName) {
+        const column = this.columns.get(columnName);
+        if (!column) {
+            return [];
+        }
+        const result = [];
+        for (let i = 0; i < column.length; i++) {
+            result.push(column.get(i));
+        }
+        return result;
+    }
+    /**
+     * 获取类型化数组列（仅数值类型）
+     * @param columnName 列名称
+     */
+    getTypedColumn(columnName) {
+        const column = this.columns.get(columnName);
+        if (!column) {
+            return null;
+        }
+        if (column instanceof TypedArrayColumn) {
+            return column.getData();
+        }
+        return null;
+    }
+    /**
+     * 聚合计算
+     * @param columnName 列名称
+     * @param type 聚合类型
+     */
+    aggregate(columnName, type) {
+        const column = this.columns.get(columnName);
+        if (!column) {
+            return {
+                type,
+                field: columnName,
+                value: null,
+                recordCount: 0,
+            };
+        }
+        const values = [];
+        for (let i = 0; i < column.length; i++) {
+            const val = column.get(i);
+            if (val !== null && val !== undefined) {
+                if (typeof val === "number" || typeof val === "bigint") {
+                    values.push(val);
+                }
+            }
+        }
+        if (values.length === 0) {
+            return {
+                type,
+                field: columnName,
+                value: null,
+                recordCount: 0,
+            };
+        }
+        let value = null;
+        switch (type) {
+            case "sum": {
+                if (typeof values[0] === "bigint") {
+                    value = values.reduce((a, b) => a + b, 0n);
+                } else {
+                    value = values.reduce((a, b) => a + b, 0);
+                }
+                break;
+            }
+            case "avg": {
+                if (typeof values[0] === "bigint") {
+                    const sum = values.reduce((a, b) => a + b, 0n);
+                    value = Number(sum) / values.length;
+                } else {
+                    value = values.reduce((a, b) => a + b, 0) / values.length;
+                }
+                break;
+            }
+            case "min": {
+                value = values.reduce((a, b) => (a < b ? a : b));
+                break;
+            }
+            case "max": {
+                value = values.reduce((a, b) => (a > b ? a : b));
+                break;
+            }
+            case "count": {
+                value = values.length;
+                break;
+            }
+            case "countDistinct": {
+                const unique = new Set(values);
+                value = unique.size;
+                break;
+            }
+        }
+        return {
+            type,
+            field: columnName,
+            value,
+            recordCount: values.length,
+        };
+    }
+    /**
+     * 范围扫描
+     * @param columnName 列名称
+     * @param condition 范围条件
+     */
+    rangeScan(columnName, condition) {
+        const column = this.columns.get(columnName);
+        if (!column) {
+            return [];
+        }
+        const result = [];
+        for (let i = 0; i < column.length; i++) {
+            const value = column.get(i);
+            if (value === null || value === undefined) {
+                continue;
+            }
+            if (this.matchRange(value, condition)) {
+                result.push(i);
+            }
+        }
+        return result;
+    }
+    /**
+     * 匹配范围条件
+     */
+    matchRange(value, condition) {
+        if (typeof value !== "number" && typeof value !== "bigint" && typeof value !== "string") {
+            return false;
+        }
+        const numValue = value;
+        if (condition.gt !== undefined && condition.gt !== null) {
+            if (!(numValue > condition.gt)) {
+                return false;
+            }
+        }
+        if (condition.gte !== undefined && condition.gte !== null) {
+            if (!(numValue >= condition.gte)) {
+                return false;
+            }
+        }
+        if (condition.lt !== undefined && condition.lt !== null) {
+            if (!(numValue < condition.lt)) {
+                return false;
+            }
+        }
+        if (condition.lte !== undefined && condition.lte !== null) {
+            if (!(numValue <= condition.lte)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * 创建行迭代器
+     * @param start 起始索引
+     * @param end 结束索引
+     */
+    *iterateRows(start = 0, end) {
+        const stop = end ?? this.rowCount;
+        for (let i = start; i < stop; i++) {
+            yield this.getRow(i);
+        }
+    }
+    /**
+     * 创建过滤迭代器
+     * @param predicate 过滤条件函数
+     */
+    *filter(predicate) {
+        for (let i = 0; i < this.rowCount; i++) {
+            const row = this.getRow(i);
+            if (predicate(row, i)) {
+                yield row;
+            }
+        }
+    }
+    /**
+     * 扩容
+     * @param newCapacity 新容量
+     */
+    grow(newCapacity) {
+        if (newCapacity <= this.capacity) {
+            return;
+        }
+        for (const column of this.columns.values()) {
+            column.grow(newCapacity);
+        }
+        this.capacity = newCapacity;
+    }
+    /**
+     * 清空数据
+     */
+    clear() {
+        for (const colName of this.columnOrder) {
+            const column = this.columns.get(colName);
+            column.fill(null, 0, column.length);
+        }
+    }
+    /**
+     * 获取统计信息
+     */
+    getStats() {
+        const columnStats = [];
+        for (const colName of this.columnOrder) {
+            const column = this.columns.get(colName);
+            columnStats.push({
+                name: colName,
+                type: column.type,
+                nonNullCount: column.getNonNullCount(),
+                memoryUsage: column.getMemoryUsage(),
+            });
+        }
+        const totalMemory = columnStats.reduce((sum, stat) => sum + stat.memoryUsage, 0);
+        return {
+            columnCount: this.columns.size,
+            rowCount: this.rowCount,
+            memoryUsage: totalMemory,
+            columns: columnStats,
+        };
+    }
+    /**
+     * 获取容量
+     */
+    getCapacity() {
+        return this.capacity;
+    }
+    /**
+     * 删除行（标记为 null，不实际删除）
+     * @param rowIndex 行索引
+     */
+    deleteRow(rowIndex) {
+        if (rowIndex < 0 || rowIndex >= this.rowCount) {
+            return false;
+        }
+        for (const column of this.columns.values()) {
+            column.set(rowIndex, null);
+        }
+        return true;
+    }
+    /**
+     * 按条件删除行
+     * @param predicate 条件函数
+     * @returns 删除的行数
+     */
+    deleteWhere(predicate) {
+        let count = 0;
+        for (let i = 0; i < this.rowCount; i++) {
+            const row = this.getRow(i);
+            if (predicate(row, i)) {
+                this.deleteRow(i);
+                count++;
+            }
+        }
+        return count;
+    }
+}
+exports.ColumnarStorage = ColumnarStorage;
+//# sourceMappingURL=ColumnarStorage.js.map