flatsql 0.4.2 → 1.0.0

package/wasm/index.js

@@ -5,34 +5,72 @@ import FlatSQLModule from './flatsql.js';
 
 let Module = null;
 let api = null;
+const textEncoder = new TextEncoder();
+
+const PARAM_NULL = 0;
+const PARAM_BOOL = 1;
+const PARAM_INT64 = 2;
+const PARAM_FLOAT64 = 3;
+const PARAM_STRING = 4;
+const PARAM_BYTES = 5;
 
 // Security: Track if integrity was verified
 let integrityVerified = false;
 
+function toBase64(bytes) {
+    if (typeof Buffer !== 'undefined') {
+        return Buffer.from(bytes).toString('base64');
+    }
+    let binary = '';
+    for (let offset = 0; offset < bytes.length; offset += 0x8000) {
+        binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000));
+    }
+    return btoa(binary);
+}
+
+function normalizeHashResult(value) {
+    if (typeof value === 'string') {
+        return value;
+    }
+    if (value instanceof Uint8Array) {
+        return toBase64(value);
+    }
+    if (ArrayBuffer.isView(value)) {
+        return toBase64(new Uint8Array(value.buffer, value.byteOffset, value.byteLength));
+    }
+    if (value instanceof ArrayBuffer) {
+        return toBase64(new Uint8Array(value));
+    }
+    throw new Error('SHA-384 provider must return a base64 string or binary digest');
+}
+
+function hasNodeProcess() {
+    return typeof process !== 'undefined' &&
+        process.versions !== undefined &&
+        typeof process.versions.node === 'string';
+}
+
 /**
- * Compute SHA-384 hash of data and return as base64
- * Works in both Node.js and browser environments
+ * Compute SHA-384 hash of data and return as base64.
+ * Browser WebCrypto is intentionally not used; browser callers that require
+ * integrity verification must pass computeSHA384 backed by WASM/native crypto.
  * @param {ArrayBuffer} data
+ * @param {(data: ArrayBuffer) => Promise<string|Uint8Array|ArrayBuffer>|string|Uint8Array|ArrayBuffer} [provider]
  * @returns {Promise<string>}
  */
-async function computeSHA384(data) {
-    if (typeof crypto !== 'undefined' && crypto.subtle) {
-        // Browser or Node.js 18+
-        const hashBuffer = await crypto.subtle.digest('SHA-384', data);
-        const hashArray = new Uint8Array(hashBuffer);
-        // Convert to base64
-        if (typeof btoa !== 'undefined') {
-            return btoa(String.fromCharCode(...hashArray));
-        } else {
-            return Buffer.from(hashArray).toString('base64');
-        }
-    } else {
-        // Node.js fallback using dynamic import
-        const cryptoModule = await import('crypto');
+async function computeSHA384(data, provider) {
+    if (typeof provider === 'function') {
+        return normalizeHashResult(await provider(data));
+    }
+    if (hasNodeProcess()) {
+        const cryptoModule = await import('node:crypto');
         const hash = cryptoModule.createHash('sha384');
         hash.update(Buffer.from(data));
         return hash.digest('base64');
     }
+    throw new Error(
+        'WASM integrity verification requires a computeSHA384 option backed by WASM/native crypto; browser WebCrypto is intentionally not used.'
+    );
 }
 
 /**
@@ -41,8 +79,8 @@ async function computeSHA384(data) {
  * @param {string} expectedHash - Expected SHA-384 hash (base64)
  * @returns {Promise<boolean>}
  */
-async function verifyWASMIntegrity(wasmBinary, expectedHash) {
-    const computedHash = await computeSHA384(wasmBinary);
+async function verifyWASMIntegrity(wasmBinary, expectedHash, options = {}) {
+    const computedHash = await computeSHA384(wasmBinary, options.computeSHA384);
     return computedHash === expectedHash;
 }
 
@@ -177,7 +215,7 @@ export async function initFlatSQL(moduleFactoryOrOptions) {
                 }
 
                 // Verify integrity
-                const isValid = await verifyWASMIntegrity(wasmBinary, expectedIntegrity);
+                const isValid = await verifyWASMIntegrity(wasmBinary, expectedIntegrity, options);
                 if (!isValid) {
                     throw new Error(
                         'WASM integrity check failed: hash mismatch. ' +
@@ -223,6 +261,22 @@ export async function initFlatSQL(moduleFactoryOrOptions) {
 
         // Query execution
         query: Module.cwrap('flatsql_query', 'number', ['number', 'string']),
+        queryParams: Module.cwrap('flatsql_query_params', 'number', ['number', 'string', 'number', 'number', 'number']),
+        queryMany: Module.cwrap('flatsql_query_many', 'number', ['number', 'number', 'number', 'number']),
+        buildQueryCacheKey: Module.cwrap('flatsql_build_query_cache_key', 'string', ['string', 'string', 'string', 'number', 'number', 'number']),
+        buildResponseArtifactCacheKey: Module.cwrap('flatsql_build_response_artifact_cache_key', 'string', ['string', 'string', 'string', 'string', 'string', 'string', 'number', 'number', 'number']),
+        registerQueryTemplate: Module.cwrap('flatsql_register_query_template', 'number', ['number', 'string', 'string', 'number']),
+        queryTemplate: Module.cwrap('flatsql_query_template', 'number', ['number', 'string', 'number', 'number', 'number']),
+        clearQueryCache: Module.cwrap('flatsql_clear_query_cache', null, ['number']),
+        configureQueryCache: Module.cwrap('flatsql_configure_query_cache', 'number', ['number', 'number', 'number']),
+        queryCacheHits: Module.cwrap('flatsql_query_cache_hits', 'number', ['number']),
+        queryCacheMisses: Module.cwrap('flatsql_query_cache_misses', 'number', ['number']),
+        queryCacheSize: Module.cwrap('flatsql_query_cache_size', 'number', ['number']),
+        queryCacheGeneration: Module.cwrap('flatsql_query_cache_generation', 'number', ['number']),
+        queryCacheMaxEntries: Module.cwrap('flatsql_query_cache_max_entries', 'number', ['number']),
+        queryCacheMaxRows: Module.cwrap('flatsql_query_cache_max_rows', 'number', ['number']),
+        batchResultCount: Module.cwrap('flatsql_batch_result_count', 'number', []),
+        selectBatchResult: Module.cwrap('flatsql_select_batch_result', 'number', ['number']),
         getError: Module.cwrap('flatsql_get_error', 'string', []),
 
         // Result access
@@ -234,17 +288,25 @@ export async function initFlatSQL(moduleFactoryOrOptions) {
         resultCellString: Module.cwrap('flatsql_result_cell_string', 'string', ['number', 'number']),
         resultCellBlob: Module.cwrap('flatsql_result_cell_blob', 'number', ['number', 'number']),
         resultCellBlobSize: Module.cwrap('flatsql_result_cell_blob_size', 'number', ['number', 'number']),
+        queryRawFlatBufferStream: Module.cwrap('flatsql_query_raw_flatbuffer_stream', 'number', ['number', 'string', 'number', 'number', 'number']),
+        responseArtifactData: Module.cwrap('flatsql_response_artifact_data', 'number', []),
+        responseArtifactSize: Module.cwrap('flatsql_response_artifact_size', 'number', []),
+        responseArtifactRowCount: Module.cwrap('flatsql_response_artifact_row_count', 'number', []),
+        responseArtifactColumnCount: Module.cwrap('flatsql_response_artifact_column_count', 'number', []),
 
         // Export/Import
         exportData: Module.cwrap('flatsql_export_data', 'number', ['number']),
         exportSize: Module.cwrap('flatsql_export_size', 'number', []),
         loadAndRebuild: Module.cwrap('flatsql_load_and_rebuild', null, ['number', 'number', 'number']),
+        reserveStorage: Module.cwrap('flatsql_reserve_storage', null, ['number', 'number']),
+        loadFromDb: Module.cwrap('flatsql_load_from_db', null, ['number', 'number']),
 
         // Test helpers
         createTestUser: Module.cwrap('flatsql_create_test_user', 'number', ['number', 'string', 'string', 'number']),
         createTestPost: Module.cwrap('flatsql_create_test_post', 'number', ['number', 'number', 'string']),
         createTestMPE: Module.cwrap('flatsql_create_test_mpe', 'number', ['string', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number']),
         createTestTelemetry: Module.cwrap('flatsql_create_test_telemetry', 'number', ['number', 'string', 'string', 'string', 'number', 'number', 'number']),
+        createTestPublishEvent: Module.cwrap('flatsql_create_test_publish_event', 'number', ['string', 'string', 'number', 'number']),
         testBufferSize: Module.cwrap('flatsql_test_buffer_size', 'number', []),
 
         // Stats
@@ -264,6 +326,15 @@ export async function initFlatSQL(moduleFactoryOrOptions) {
         getDeletedCount: Module.cwrap('flatsql_get_deleted_count', 'number', ['number', 'string']),
         clearTombstones: Module.cwrap('flatsql_clear_tombstones', null, ['number', 'string']),
 
+        // Raw FlatBuffer access
+        getFlatBufferById: Module.cwrap('flatsql_get_flatbuffer_by_id', 'number', ['number', 'string', 'number']),
+        getFlatBufferByEmail: Module.cwrap('flatsql_get_flatbuffer_by_email', 'number', ['number', 'string', 'string']),
+        getFlatBufferByIndex: Module.cwrap('flatsql_get_flatbuffer_by_index', 'number', ['number', 'string', 'string', 'number', 'number', 'number']),
+        getRawFlatBufferSize: Module.cwrap('flatsql_get_raw_flatbuffer_size', 'number', []),
+        getRawFlatBufferSequence: Module.cwrap('flatsql_get_raw_flatbuffer_sequence', 'number', []),
+        getStorageBuffer: Module.cwrap('flatsql_get_storage_buffer', 'number', ['number']),
+        getStorageSize: Module.cwrap('flatsql_get_storage_size', 'number', ['number']),
+
         // Encryption
         setEncryptionKey: Module.cwrap('flatsql_set_encryption_key', 'number', ['number', 'number', 'number']),
         isEncrypted: Module.cwrap('flatsql_is_encrypted', 'number', ['number']),
@@ -298,6 +369,134 @@ function withHeapBytes(data, callback) {
     }
 }
 
+function encodeQueryParams(params) {
+    const parts = [];
+    let total = 0;
+
+    for (const value of params) {
+        let tag;
+        let payload;
+
+        if (value === null) {
+            tag = PARAM_NULL;
+            payload = new Uint8Array(0);
+        } else if (typeof value === 'boolean') {
+            tag = PARAM_BOOL;
+            payload = Uint8Array.of(value ? 1 : 0);
+        } else if (typeof value === 'number' && Number.isInteger(value)) {
+            if (!Number.isSafeInteger(value)) {
+                throw new TypeError('Integer query parameters must be safe integers');
+            }
+            tag = PARAM_INT64;
+            payload = new Uint8Array(8);
+            new DataView(payload.buffer, payload.byteOffset, payload.byteLength).setBigInt64(0, BigInt(value), true);
+        } else if (typeof value === 'number') {
+            tag = PARAM_FLOAT64;
+            payload = new Uint8Array(8);
+            new DataView(payload.buffer, payload.byteOffset, payload.byteLength).setFloat64(0, value, true);
+        } else if (typeof value === 'string') {
+            tag = PARAM_STRING;
+            payload = textEncoder.encode(value);
+        } else if (value instanceof Uint8Array) {
+            tag = PARAM_BYTES;
+            payload = value;
+        } else {
+            throw new TypeError(`Unsupported query parameter type: ${typeof value}`);
+        }
+
+        const header = new Uint8Array(5);
+        const view = new DataView(header.buffer);
+        view.setUint8(0, tag);
+        view.setUint32(1, payload.length, true);
+
+        parts.push(header, payload);
+        total += header.length + payload.length;
+    }
+
+    const encoded = new Uint8Array(total);
+    let offset = 0;
+    for (const part of parts) {
+        encoded.set(part, offset);
+        offset += part.length;
+    }
+
+    return encoded;
+}
+
+function encodeQueryRequests(queries) {
+    const parts = [];
+    let total = 0;
+
+    for (const query of queries) {
+        const sqlBytes = textEncoder.encode(query.sql);
+        const paramList = query.params ?? [];
+        const paramBytes = encodeQueryParams(paramList);
+        const header = new Uint8Array(12);
+        const view = new DataView(header.buffer);
+        view.setUint32(0, sqlBytes.length, true);
+        view.setUint32(4, paramList.length, true);
+        view.setUint32(8, paramBytes.length, true);
+
+        parts.push(header, sqlBytes, paramBytes);
+        total += header.length + sqlBytes.length + paramBytes.length;
+    }
+
+    const encoded = new Uint8Array(total);
+    let offset = 0;
+    for (const part of parts) {
+        encoded.set(part, offset);
+        offset += part.length;
+    }
+
+    return encoded;
+}
+
+function readCellValue(row, col) {
+    const type = api.resultCellType(row, col);
+    switch (type) {
+        case 0:
+            return null;
+        case 1:
+            return api.resultCellNumber(row, col) !== 0;
+        case 2:
+        case 3:
+        case 4:
+            return api.resultCellNumber(row, col);
+        case 5:
+            return api.resultCellString(row, col);
+        case 6: {
+            const blobPtr = api.resultCellBlob(row, col);
+            const blobSize = api.resultCellBlobSize(row, col);
+            return blobPtr && blobSize > 0
+                ? Array.from(new Uint8Array(Module.HEAPU8.buffer, blobPtr, blobSize))
+                : [];
+        }
+        default:
+            return null;
+    }
+}
+
+function readQueryResult() {
+    const colCount = api.resultColumnCount();
+    const rowCount = api.resultRowCount();
+
+    const columns = [];
+    for (let i = 0; i < colCount; i++) {
+        columns.push(api.resultColumnName(i));
+    }
+
+    const rows = [];
+    for (let r = 0; r < rowCount; r++) {
+        const row = [];
+        for (let c = 0; c < colCount; c++) {
+            row.push(readCellValue(r, c));
+        }
+        rows.push(row);
+    }
+
+    return { columns, rows };
+}
+
 function buildSizePrefixedStream(buffers) {
     let total = 0;
     for (const buffer of buffers) {
@@ -335,6 +534,12 @@ export class FlatSQL {
         return new Uint8Array(Module.HEAPU8.buffer, ptr, size).slice();
     }
 
+    createTestPublishEvent(fileId, recordId, eventIndex, payloadSize) {
+        const ptr = api.createTestPublishEvent(fileId, recordId, eventIndex, payloadSize);
+        const size = api.testBufferSize();
+        return new Uint8Array(Module.HEAPU8.buffer, ptr, size).slice();
+    }
+
     createTestMPE(entityId, epoch, meanMotion, eccentricity, inclination,
                   raOfAscNode, argOfPericenter, meanAnomaly, bstar, meanElementTheory = 0) {
         const ptr = api.createTestMPE(
@@ -366,6 +571,58 @@ export class FlatSQL {
     wasIntegrityVerified() {
         return integrityVerified;
     }
+
+    buildQueryCacheKey(dataset, artifactVersion, queryId, params = []) {
+        const encodedParams = encodeQueryParams(params);
+        const key = encodedParams.length === 0
+            ? api.buildQueryCacheKey(dataset, artifactVersion, queryId, 0, 0, 0)
+            : withHeapBytes(
+                encodedParams,
+                (ptr) => api.buildQueryCacheKey(dataset, artifactVersion, queryId, ptr, encodedParams.length, params.length)
+            );
+        if (!key) {
+            throw new Error(api.getError());
+        }
+        return key;
+    }
+
+    buildResponseArtifactCacheKey(schemaName, schemaVersion, sql, options = {}) {
+        const format = options.format ?? 'json';
+        const publishEventKey = options.publishEventKey ?? '';
+        const projectionList = (options.projection ?? []).join('\n');
+        const params = options.params ?? [];
+        const encodedParams = encodeQueryParams(params);
+        const key = encodedParams.length === 0
+            ? api.buildResponseArtifactCacheKey(
+                schemaName,
+                schemaVersion,
+                sql,
+                format,
+                publishEventKey,
+                projectionList,
+                0,
+                0,
+                0
+            )
+            : withHeapBytes(
+                encodedParams,
+                (ptr) => api.buildResponseArtifactCacheKey(
+                    schemaName,
+                    schemaVersion,
+                    sql,
+                    format,
+                    publishEventKey,
+                    projectionList,
+                    ptr,
+                    encodedParams.length,
+                    params.length
+                )
+            );
+        if (!key) {
+            throw new Error(api.getError());
+        }
+        return key;
+    }
 }
 
 // Database wrapper class
@@ -435,58 +692,114 @@ export class FlatSQLDatabase {
         return sources;
     }
 
-    query(sql) {
-        const success = api.query(this._handle, sql);
+    query(sql, params = undefined) {
+        const success = params === undefined
+            ? api.query(this._handle, sql)
+            : (() => {
+                const encodedParams = encodeQueryParams(params);
+                if (encodedParams.length === 0) {
+                    return api.queryParams(this._handle, sql, 0, 0, 0);
+                }
+                return withHeapBytes(
+                    encodedParams,
+                    (ptr) => api.queryParams(this._handle, sql, ptr, encodedParams.length, params.length)
+                );
+            })();
         if (!success) {
             throw new Error(api.getError());
         }
+        return readQueryResult();
+    }
 
-        // Read results
-        const colCount = api.resultColumnCount();
-        const rowCount = api.resultRowCount();
+    registerQueryTemplate(queryId, sql, cacheable = true) {
+        const success = api.registerQueryTemplate(this._handle, queryId, sql, cacheable ? 1 : 0);
+        if (!success) {
+            throw new Error(api.getError());
+        }
+    }
 
-        const columns = [];
-        for (let i = 0; i < colCount; i++) {
-            columns.push(api.resultColumnName(i));
+    queryTemplate(queryId, params = []) {
+        const encodedParams = encodeQueryParams(params);
+        const success = encodedParams.length === 0
+            ? api.queryTemplate(this._handle, queryId, 0, 0, 0)
+            : withHeapBytes(
+                encodedParams,
+                (ptr) => api.queryTemplate(this._handle, queryId, ptr, encodedParams.length, params.length)
+            );
+        if (!success) {
+            throw new Error(api.getError());
         }
+        return readQueryResult();
+    }
 
-        const rows = [];
-        for (let r = 0; r < rowCount; r++) {
-            const row = [];
-            for (let c = 0; c < colCount; c++) {
-                const type = api.resultCellType(r, c);
-                switch (type) {
-                    case 0: // null
-                        row.push(null);
-                        break;
-                    case 1: // bool
-                        row.push(api.resultCellNumber(r, c) !== 0);
-                        break;
-                    case 2: // int32
-                    case 3: // int64
-                    case 4: // double
-                        row.push(api.resultCellNumber(r, c));
-                        break;
-                    case 5: // string
-                        row.push(api.resultCellString(r, c));
-                        break;
-                    case 6: // blob
-                        const blobPtr = api.resultCellBlob(r, c);
-                        const blobSize = api.resultCellBlobSize(r, c);
-                        if (blobPtr && blobSize > 0) {
-                            row.push(Array.from(new Uint8Array(Module.HEAPU8.buffer, blobPtr, blobSize)));
-                        } else {
-                            row.push([]);
-                        }
-                        break;
-                    default:
-                        row.push(null);
-                }
+    clearQueryCache() {
+        api.clearQueryCache(this._handle);
+    }
+
+    configureQueryCache({ maxEntries, maxRows }) {
+        if (!Number.isSafeInteger(maxEntries) || maxEntries <= 0) {
+            throw new TypeError(`maxEntries must be a positive safe integer, received: ${maxEntries}`);
+        }
+        if (!Number.isSafeInteger(maxRows) || maxRows <= 0) {
+            throw new TypeError(`maxRows must be a positive safe integer, received: ${maxRows}`);
+        }
+        const success = api.configureQueryCache(this._handle, maxEntries, maxRows);
+        if (!success) {
+            throw new Error(api.getError());
+        }
+    }
+
+    getQueryCacheStats() {
+        return {
+            hits: api.queryCacheHits(this._handle),
+            misses: api.queryCacheMisses(this._handle),
+            size: api.queryCacheSize(this._handle),
+            generation: api.queryCacheGeneration(this._handle),
+            maxEntries: api.queryCacheMaxEntries(this._handle),
+            maxRows: api.queryCacheMaxRows(this._handle)
+        };
+    }
+
+    queryMany(queries) {
+        if (queries.length === 0) {
+            return [];
+        }
+
+        const encodedRequests = encodeQueryRequests(queries);
+        const success = withHeapBytes(
+            encodedRequests,
+            (ptr) => api.queryMany(this._handle, ptr, encodedRequests.length, queries.length)
+        );
+        if (!success) {
+            throw new Error(api.getError());
+        }
+
+        const resultCount = api.batchResultCount();
+        const results = [];
+        for (let index = 0; index < resultCount; index++) {
+            if (!api.selectBatchResult(index)) {
+                throw new Error(`Failed to select batch result ${index}`);
             }
-            rows.push(row);
+            results.push(readQueryResult());
+        }
+        return results;
+    }
+
+    queryRawFlatBufferStream(sql, params = []) {
+        const encodedParams = encodeQueryParams(params);
+        const success = encodedParams.length === 0
+            ? api.queryRawFlatBufferStream(this._handle, sql, 0, 0, 0)
+            : withHeapBytes(
+                encodedParams,
+                (ptr) => api.queryRawFlatBufferStream(this._handle, sql, ptr, encodedParams.length, params.length)
+            );
+        if (!success) {
+            throw new Error(api.getError());
         }
 
-        return { columns, rows };
+        const ptr = api.responseArtifactData();
+        const size = api.responseArtifactSize();
+        return ptr && size > 0 ? new Uint8Array(Module.HEAPU8.buffer, ptr, size).slice() : new Uint8Array();
     }
 
     exportData() {
@@ -502,6 +815,17 @@ export class FlatSQLDatabase {
         Module._free(ptr);
     }
 
+    reserveStorageBytes(bytes) {
+        api.reserveStorage(this._handle, bytes);
+    }
+
+    loadAndRebuildFrom(sourceDb) {
+        if (!sourceDb?._handle) {
+            throw new TypeError('loadAndRebuildFrom requires a FlatSQL WASM database.');
+        }
+        api.loadFromDb(this._handle, sourceDb._handle);
+    }
+
     getStats() {
         const count = api.getStatsCount(this._handle);
         const stats = [];
@@ -541,6 +865,80 @@ export class FlatSQLDatabase {
         api.clearTombstones(this._handle, tableName);
     }
 
+    getFlatBufferById(tableName, id) {
+        const ptr = api.getFlatBufferById(this._handle, tableName, id);
+        if (!ptr) {
+            return null;
+        }
+        return {
+            ptr,
+            size: api.getRawFlatBufferSize(),
+            sequence: api.getRawFlatBufferSequence()
+        };
+    }
+
+    getFlatBufferByEmail(tableName, email) {
+        const ptr = api.getFlatBufferByEmail(this._handle, tableName, email);
+        if (!ptr) {
+            return null;
+        }
+        return {
+            ptr,
+            size: api.getRawFlatBufferSize(),
+            sequence: api.getRawFlatBufferSequence()
+        };
+    }
+
+    getFlatBufferByIndex(tableName, columnName, value) {
+        const encodedParams = encodeQueryParams([value]);
+        const ptr = withHeapBytes(
+            encodedParams,
+            (paramPtr) => api.getFlatBufferByIndex(
+                this._handle,
+                tableName,
+                columnName,
+                paramPtr,
+                encodedParams.length,
+                1
+            )
+        );
+        if (!ptr) {
+            const error = api.getError();
+            if (error) {
+                throw new Error(error);
+            }
+            return null;
+        }
+        return {
+            ptr,
+            size: api.getRawFlatBufferSize(),
+            sequence: api.getRawFlatBufferSequence()
+        };
+    }
+
+    getFlatBufferDataById(tableName, id) {
+        const record = this.getFlatBufferById(tableName, id);
+        if (!record) {
+            return null;
+        }
+        return new Uint8Array(Module.HEAPU8.buffer, record.ptr, record.size).slice();
+    }
+
+    getFlatBufferDataByIndex(tableName, columnName, value) {
+        const record = this.getFlatBufferByIndex(tableName, columnName, value);
+        if (!record) {
+            return null;
+        }
+        return new Uint8Array(Module.HEAPU8.buffer, record.ptr, record.size).slice();
+    }
+
+    getStorageInfo() {
+        return {
+            ptr: api.getStorageBuffer(this._handle),
+            size: api.getStorageSize(this._handle)
+        };
+    }
+
     // ==================== Encryption API ====================
 
     /**