flatsql 0.2.0

package/dist/storage/stacked-flatbuffers.js

@@ -0,0 +1,253 @@
+// Stacked FlatBuffers storage implementation
+// Files are append-only sequences of FlatBuffer records with headers
+// Fully readable by standard FlatBuffer tools
+import { MAGIC, VERSION, crc32 } from './types.js';
+// Header sizes (fixed layout for easy seeking)
+const FILE_HEADER_SIZE = 64; // Padded for alignment
+const RECORD_HEADER_SIZE = 48; // Fixed size record header
+// Security: Default maximum storage size (1GB)
+const DEFAULT_MAX_STORAGE_SIZE = 1024 * 1024 * 1024;
+// Security: Warning threshold (80% of max)
+const WARNING_THRESHOLD = 0.8;
+export class StackedFlatBufferStore {
+    data;
+    dataView;
+    writeOffset;
+    recordCount = 0n;
+    sequence = 0n;
+    schemaName;
+    // Index of records by offset for quick lookup
+    recordIndex = new Map();
+    // Security: Storage limits
+    maxSize;
+    warningEmitted = false;
+    onStorageWarning;
+    constructor(schemaName, options = {}) {
+        // Support legacy signature: constructor(schemaName, initialCapacity)
+        const opts = typeof options === 'number'
+            ? { initialCapacity: options }
+            : options;
+        this.schemaName = schemaName;
+        this.maxSize = opts.maxSize ?? DEFAULT_MAX_STORAGE_SIZE;
+        this.onStorageWarning = opts.onStorageWarning;
+        const initialCapacity = opts.initialCapacity ?? 1024 * 1024;
+        this.data = new Uint8Array(initialCapacity);
+        this.dataView = new DataView(this.data.buffer);
+        this.writeOffset = BigInt(FILE_HEADER_SIZE);
+        this.writeFileHeader();
+    }
+    writeFileHeader() {
+        const view = this.dataView;
+        // Magic number
+        view.setUint32(0, MAGIC, true);
+        // Version
+        view.setUint32(4, VERSION, true);
+        // Data start offset
+        view.setBigUint64(8, BigInt(FILE_HEADER_SIZE), true);
+        // Record count (updated on each append)
+        view.setBigUint64(16, 0n, true);
+        // Schema name (up to 40 bytes, null-terminated)
+        const nameBytes = new TextEncoder().encode(this.schemaName);
+        const nameLen = Math.min(nameBytes.length, 39);
+        this.data.set(nameBytes.slice(0, nameLen), 24);
+        this.data[24 + nameLen] = 0;
+    }
+    updateFileHeader() {
+        this.dataView.setBigUint64(16, this.recordCount, true);
+    }
+    ensureCapacity(needed) {
+        const totalNeeded = Number(this.writeOffset) + needed;
+        if (totalNeeded <= this.data.length)
+            return;
+        // Security: Check if we would exceed max size
+        if (this.maxSize > 0 && totalNeeded > this.maxSize) {
+            throw new Error(`Storage size limit exceeded: requested ${totalNeeded} bytes, ` +
+                `maximum is ${this.maxSize} bytes`);
+        }
+        // Grow by doubling
+        let newSize = this.data.length * 2;
+        while (newSize < totalNeeded) {
+            newSize *= 2;
+        }
+        // Security: Cap at max size
+        if (this.maxSize > 0 && newSize > this.maxSize) {
+            newSize = this.maxSize;
+        }
+        // Security: Emit warning when approaching limit
+        if (this.maxSize > 0 && !this.warningEmitted) {
+            const threshold = this.maxSize * WARNING_THRESHOLD;
+            if (newSize >= threshold) {
+                this.warningEmitted = true;
+                if (this.onStorageWarning) {
+                    this.onStorageWarning(newSize, this.maxSize);
+                }
+            }
+        }
+        const newData = new Uint8Array(newSize);
+        newData.set(this.data);
+        this.data = newData;
+        this.dataView = new DataView(this.data.buffer);
+    }
+    // Append a FlatBuffer record
+    // Returns the offset where the record was stored
+    append(tableName, flatbufferData) {
+        const checksum = crc32(flatbufferData);
+        const timestamp = BigInt(Date.now());
+        const seq = this.sequence++;
+        const header = {
+            sequence: seq,
+            tableName,
+            timestamp,
+            dataLength: flatbufferData.length,
+            checksum,
+        };
+        const totalSize = RECORD_HEADER_SIZE + flatbufferData.length;
+        this.ensureCapacity(totalSize);
+        const recordOffset = this.writeOffset;
+        // Write record header
+        const offset = Number(this.writeOffset);
+        this.dataView.setBigUint64(offset, seq, true);
+        // Table name (16 bytes, null-padded)
+        const tableBytes = new TextEncoder().encode(tableName);
+        const tableLen = Math.min(tableBytes.length, 15);
+        this.data.set(tableBytes.slice(0, tableLen), offset + 8);
+        this.data[offset + 8 + tableLen] = 0;
+        this.dataView.setBigUint64(offset + 24, timestamp, true);
+        this.dataView.setUint32(offset + 32, flatbufferData.length, true);
+        this.dataView.setUint32(offset + 36, checksum, true);
+        // Reserved bytes 40-47
+        // Write FlatBuffer data
+        this.data.set(flatbufferData, offset + RECORD_HEADER_SIZE);
+        this.writeOffset += BigInt(totalSize);
+        this.recordCount++;
+        this.recordIndex.set(recordOffset, header);
+        this.updateFileHeader();
+        return recordOffset;
+    }
+    // Read a record at a given offset
+    readRecord(offset) {
+        const off = Number(offset);
+        // Read header
+        const sequence = this.dataView.getBigUint64(off, true);
+        const tableNameBytes = this.data.slice(off + 8, off + 24);
+        const nullIdx = tableNameBytes.indexOf(0);
+        const tableName = new TextDecoder().decode(tableNameBytes.slice(0, nullIdx >= 0 ? nullIdx : tableNameBytes.length));
+        const timestamp = this.dataView.getBigUint64(off + 24, true);
+        const dataLength = this.dataView.getUint32(off + 32, true);
+        const checksum = this.dataView.getUint32(off + 36, true);
+        // Read data
+        const dataStart = off + RECORD_HEADER_SIZE;
+        const data = this.data.slice(dataStart, dataStart + dataLength);
+        // Verify checksum
+        const computedChecksum = crc32(data);
+        if (computedChecksum !== checksum) {
+            throw new Error(`Checksum mismatch at offset ${offset}: expected ${checksum}, got ${computedChecksum}`);
+        }
+        return {
+            header: {
+                sequence,
+                tableName,
+                timestamp,
+                dataLength,
+                checksum,
+            },
+            offset,
+            data,
+        };
+    }
+    // Iterate all records
+    *iterateRecords() {
+        let offset = BigInt(FILE_HEADER_SIZE);
+        const end = this.writeOffset;
+        while (offset < end) {
+            const record = this.readRecord(offset);
+            yield record;
+            offset += BigInt(RECORD_HEADER_SIZE + record.header.dataLength);
+        }
+    }
+    // Get records for a specific table
+    *iterateTableRecords(tableName) {
+        for (const record of this.iterateRecords()) {
+            if (record.header.tableName === tableName) {
+                yield record;
+            }
+        }
+    }
+    // Get the raw data (for export/persistence)
+    getData() {
+        return this.data.slice(0, Number(this.writeOffset));
+    }
+    // Load from existing data
+    static fromData(data, options = {}) {
+        const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+        // Verify magic
+        const magic = view.getUint32(0, true);
+        if (magic !== MAGIC) {
+            throw new Error(`Invalid file magic: expected ${MAGIC.toString(16)}, got ${magic.toString(16)}`);
+        }
+        const version = view.getUint32(4, true);
+        if (version !== VERSION) {
+            throw new Error(`Unsupported version: ${version}`);
+        }
+        // Read schema name
+        const nameBytes = data.slice(24, 64);
+        const nullIdx = nameBytes.indexOf(0);
+        const schemaName = new TextDecoder().decode(nameBytes.slice(0, nullIdx >= 0 ? nullIdx : nameBytes.length));
+        const store = new StackedFlatBufferStore(schemaName, {
+            ...options,
+            initialCapacity: data.length,
+        });
+        store.data.set(data);
+        store.dataView = new DataView(store.data.buffer);
+        // Rebuild index by scanning records
+        store.writeOffset = BigInt(FILE_HEADER_SIZE);
+        store.recordCount = 0n;
+        store.sequence = 0n;
+        // Find end and rebuild index
+        let offset = BigInt(FILE_HEADER_SIZE);
+        while (offset < BigInt(data.length)) {
+            try {
+                const record = store.readRecord(offset);
+                store.recordIndex.set(offset, record.header);
+                store.recordCount++;
+                if (record.header.sequence >= store.sequence) {
+                    store.sequence = record.header.sequence + 1n;
+                }
+                offset += BigInt(RECORD_HEADER_SIZE + record.header.dataLength);
+                store.writeOffset = offset;
+            }
+            catch {
+                // End of valid data
+                break;
+            }
+        }
+        return store;
+    }
+    getRecordCount() {
+        return this.recordCount;
+    }
+    getSchemaName() {
+        return this.schemaName;
+    }
+    // Security: Get current storage size
+    getCurrentSize() {
+        return Number(this.writeOffset);
+    }
+    // Security: Get maximum storage size (0 = unlimited)
+    getMaxSize() {
+        return this.maxSize;
+    }
+    // Security: Get storage usage as a percentage (0-100)
+    getUsagePercent() {
+        if (this.maxSize === 0)
+            return 0;
+        return (Number(this.writeOffset) / this.maxSize) * 100;
+    }
+    // Security: Check if storage is near capacity (>80%)
+    isNearCapacity() {
+        if (this.maxSize === 0)
+            return false;
+        return Number(this.writeOffset) >= this.maxSize * WARNING_THRESHOLD;
+    }
+}
+//# sourceMappingURL=stacked-flatbuffers.js.map

package/dist/storage/stacked-flatbuffers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stacked-flatbuffers.js","sourceRoot":"","sources":["../../src/storage/stacked-flatbuffers.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qEAAqE;AACrE,8CAA8C;AAE9C,OAAO,EAA8C,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAE/F,+CAA+C;AAC/C,MAAM,gBAAgB,GAAG,EAAE,CAAC,CAAC,uBAAuB;AACpD,MAAM,kBAAkB,GAAG,EAAE,CAAC,CAAC,2BAA2B;AAE1D,+CAA+C;AAC/C,MAAM,wBAAwB,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAEpD,2CAA2C;AAC3C,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAW9B,MAAM,OAAO,sBAAsB;IACzB,IAAI,CAAa;IACjB,QAAQ,CAAW;IACnB,WAAW,CAAS;IACpB,WAAW,GAAW,EAAE,CAAC;IACzB,QAAQ,GAAW,EAAE,CAAC;IACtB,UAAU,CAAS;IAE3B,8CAA8C;IACtC,WAAW,GAA8B,IAAI,GAAG,EAAE,CAAC;IAE3D,2BAA2B;IACnB,OAAO,CAAS;IAChB,cAAc,GAAY,KAAK,CAAC;IAChC,gBAAgB,CAAkD;IAE1E,YAAY,UAAkB,EAAE,UAAmC,EAAE;QACnE,qEAAqE;QACrE,MAAM,IAAI,GAAmB,OAAO,OAAO,KAAK,QAAQ;YACtD,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE;YAC9B,CAAC,CAAC,OAAO,CAAC;QAEZ,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,wBAAwB,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAE9C,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAE5C,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC;IAEO,eAAe;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE3B,eAAe;QACf,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QAC/B,UAAU;QACV,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QACjC,oBAAoB;QACpB,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,CAAC,CAAC;QACrD,wCAAwC;QACxC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAChC,gDAAgD;QAChD,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAEO,cAAc,CAAC,MAAc;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;QACtD,IAAI,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QAE5C,8CAA8C;QAC9C,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACb,0CAA0C,WAAW,UAAU;gBAC/D,cAAc,IAAI,CAAC,OAAO,QAAQ,CACnC,CAAC;QACJ,CAAC;QAED,mBAAmB;QACnB,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACnC,OAAO,OAAO,GAAG,WAAW,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QAED,4BAA4B;QAC5B,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YAC/C,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QACzB,CAAC;QAED,gDAAgD;QAChD,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,GAAG,iBAAiB,CAAC;YACnD,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAC1B,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,6BAA6B;IAC7B,iDAAiD;IACjD,MAAM,CAAC,SAAiB,EAAE,cAA0B;QAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAE5B,MAAM,MAAM,GAAiB;YAC3B,QAAQ,EAAE,GAAG;YACb,SAAS;YACT,SAAS;YACT,UAAU,EAAE,cAAc,CAAC,MAAM;YACjC,QAAQ;SACT,CAAC;QAEF,MAAM,SAAS,GAAG,kBAAkB,GAAG,cAAc,CAAC,MAAM,CAAC;QAC7D,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAE/B,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;QAEtC,sBAAsB;QACtB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAE9C,qCAAqC;QACrC,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAErC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,EAAE,EAAE,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAClE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrD,uBAAuB;QAEvB,wBAAwB;QACxB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,GAAG,kBAAkB,CAAC,CAAC;QAE3D,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAExB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,kCAAkC;IAClC,UAAU,CAAC,MAAc;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAE3B,cAAc;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACvD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CACxC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CACxE,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;QAEzD,YAAY;QACZ,MAAM,SAAS,GAAG,GAAG,GAAG,kBAAkB,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,GAAG,UAAU,CAAC,CAAC;QAEhE,kBAAkB;QAClB,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,gBAAgB,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,cAAc,QAAQ,SAAS,gBAAgB,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,OAAO;YACL,MAAM,EAAE;gBACN,QAAQ;gBACR,SAAS;gBACT,SAAS;gBACT,UAAU;gBACV,QAAQ;aACT;YACD,MAAM;YACN,IAAI;SACL,CAAC;IACJ,CAAC;IAED,sBAAsB;IACtB,CAAC,cAAc;QACb,IAAI,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC;QAE7B,OAAO,MAAM,GAAG,GAAG,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,MAAM,CAAC;YACb,MAAM,IAAI,MAAM,CAAC,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,CAAC,mBAAmB,CAAC,SAAiB;QACpC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC1C,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,0BAA0B;IAC1B,MAAM,CAAC,QAAQ,CAAC,IAAgB,EAAE,UAA0B,EAAE;QAC5D,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzE,eAAe;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,gCAAgC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,mBAAmB;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CACzC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAC9D,CAAC;QAEF,MAAM,KAAK,GAAG,IAAI,sBAAsB,CAAC,UAAU,EAAE;YACnD,GAAG,OAAO;YACV,eAAe,EAAE,IAAI,CAAC,MAAM;SAC7B,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,KAAK,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjD,oCAAoC;QACpC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,CAAC,WAAW,GAAG,EAAE,CAAC;QACvB,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC;QAEpB,6BAA6B;QAC7B,IAAI,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACtC,OAAO,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBACxC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC7C,KAAK,CAAC,WAAW,EAAE,CAAC;gBACpB,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAC7C,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC;gBAC/C,CAAC;gBACD,MAAM,IAAI,MAAM,CAAC,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAChE,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;gBACpB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,cAAc;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,qCAAqC;IACrC,cAAc;QACZ,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAClC,CAAC;IAED,qDAAqD;IACrD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,sDAAsD;IACtD,eAAe;QACb,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,qDAAqD;IACrD,cAAc;QACZ,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACrC,OAAO,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,iBAAiB,CAAC;IACtE,CAAC;CACF"}

package/dist/storage/types.d.ts

@@ -0,0 +1,23 @@
+export interface RecordHeader {
+    sequence: bigint;
+    tableName: string;
+    timestamp: bigint;
+    dataLength: number;
+    checksum: number;
+}
+export interface DataFileHeader {
+    magic: number;
+    version: number;
+    dataStartOffset: bigint;
+    recordCount: bigint;
+    schemaName: string;
+}
+export interface StoredRecord {
+    header: RecordHeader;
+    offset: bigint;
+    data: Uint8Array;
+}
+export declare const MAGIC = 1179407441;
+export declare const VERSION = 1;
+export declare function crc32(data: Uint8Array): number;
+//# sourceMappingURL=types.d.ts.map

package/dist/storage/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;CAClB;AAED,eAAO,MAAM,KAAK,aAAa,CAAC;AAChC,eAAO,MAAM,OAAO,IAAI,CAAC;AAGzB,wBAAgB,KAAK,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAS9C"}

package/dist/storage/types.js

@@ -0,0 +1,27 @@
+// Storage types for stacked FlatBuffers
+export const MAGIC = 0x464c5451; // "FLTQ"
+export const VERSION = 1;
+// Simple CRC32 implementation
+export function crc32(data) {
+    let crc = 0xffffffff;
+    const table = getCRC32Table();
+    for (let i = 0; i < data.length; i++) {
+        crc = (crc >>> 8) ^ table[(crc ^ data[i]) & 0xff];
+    }
+    return (crc ^ 0xffffffff) >>> 0;
+}
+let crc32Table = null;
+function getCRC32Table() {
+    if (crc32Table)
+        return crc32Table;
+    crc32Table = new Uint32Array(256);
+    for (let i = 0; i < 256; i++) {
+        let c = i;
+        for (let j = 0; j < 8; j++) {
+            c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+        }
+        crc32Table[i] = c;
+    }
+    return crc32Table;
+}
+//# sourceMappingURL=types.js.map

package/dist/storage/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA,wCAAwC;AAwBxC,MAAM,CAAC,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAC1C,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC;AAEzB,8BAA8B;AAC9B,MAAM,UAAU,KAAK,CAAC,IAAgB;IACpC,IAAI,GAAG,GAAG,UAAU,CAAC;IACrB,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,CAAC,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,IAAI,UAAU,GAAuB,IAAI,CAAC;AAE1C,SAAS,aAAa;IACpB,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,UAAU,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC;QACD,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/package.json

@@ -0,0 +1,79 @@
+{
+  "name": "flatsql",
+  "version": "0.2.0",
+  "description": "SQL queries over raw FlatBuffer storage - zero conversion, real-time indexing, pure streaming",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./wasm": {
+      "types": "./wasm/index.d.ts",
+      "import": "./wasm/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "wasm/*.js",
+    "wasm/*.wasm",
+    "wasm/*.d.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "build:wasm": "bash scripts/build-wasm.sh",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test:wasm": "node wasm/test-node.mjs",
+    "dev": "tsc --watch",
+    "serve": "vite docs --port 8081",
+    "prepublishOnly": "npm run build",
+    "postinstall": "bash scripts/install-emsdk.sh",
+    "emsdk:install": "bash scripts/install-emsdk.sh",
+    "emsdk:env": "echo 'Run: source packages/emsdk/emsdk_env.sh'"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/DigitalArsenal/flatsql.git"
+  },
+  "homepage": "https://digitalarsenal.github.io/flatsql/",
+  "bugs": {
+    "url": "https://github.com/DigitalArsenal/flatsql/issues"
+  },
+  "keywords": [
+    "flatbuffers",
+    "sqlite",
+    "sql",
+    "database",
+    "streaming",
+    "binary",
+    "zero-copy",
+    "wasm",
+    "webassembly",
+    "query",
+    "virtual-table"
+  ],
+  "author": "Digital Arsenal <tj@digitalarsenal.io>",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "flatc-wasm": "^26.1.14",
+    "sql.js": "^1.10.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.3.0",
+    "jest": "^29.7.0",
+    "@types/jest": "^29.5.0",
+    "ts-jest": "^29.1.0",
+    "vite": "^5.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/wasm/flatsql-client.js

@@ -0,0 +1,280 @@
+/**
+ * FlatSQL Worker Client
+ * Provides a clean async API for communicating with the FlatSQL worker.
+ * Works in: Browser, Node.js, Deno
+ *
+ * Usage:
+ *   import { FlatSQLClient, FlatSQLWorkerDatabase } from './flatsql-client.js';
+ *
+ *   const client = new FlatSQLClient();
+ *   await client.init();
+ *
+ *   const db = await client.createDatabase(schema, 'mydb');
+ *   await db.registerFileId('USER', 'User');
+ *   const result = await db.query('SELECT * FROM User');
+ */
+
+// Environment detection
+const isBrowser = typeof window !== 'undefined' || (typeof self !== 'undefined' && typeof importScripts === 'undefined');
+const isNode = typeof process !== 'undefined' && process.versions && process.versions.node;
+const isDeno = typeof Deno !== 'undefined';
+
+/**
+ * Create a worker based on the environment
+ */
+async function createWorker(workerPath) {
+    if (isNode) {
+        const { Worker } = await import('worker_threads');
+        return new Worker(workerPath);
+    } else if (isDeno) {
+        return new Worker(new URL(workerPath, import.meta.url), { type: 'module' });
+    } else {
+        // Browser
+        return new Worker(workerPath, { type: 'module' });
+    }
+}
+
+/**
+ * Setup message handlers for the worker
+ */
+function setupWorkerHandlers(worker, handlers) {
+    if (isNode) {
+        worker.on('message', handlers.onMessage);
+        worker.on('error', handlers.onError);
+        worker.on('exit', handlers.onExit);
+    } else {
+        worker.onmessage = (e) => handlers.onMessage(e.data);
+        worker.onerror = handlers.onError;
+    }
+}
+
+/**
+ * FlatSQL Worker Client
+ * Manages communication with the FlatSQL worker.
+ */
+export class FlatSQLClient {
+    constructor(workerPath = './flatsql.worker.js') {
+        this.workerPath = workerPath;
+        this.worker = null;
+        this.pending = new Map();
+        this.nextId = 1;
+        this.ready = false;
+        this.readyPromise = null;
+        this.databases = new Map();
+    }
+
+    /**
+     * Initialize the client and worker
+     */
+    async init() {
+        if (this.worker) return this;
+
+        this.worker = await createWorker(this.workerPath);
+
+        this.readyPromise = new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                reject(new Error('Worker initialization timeout'));
+            }, 30000);
+
+            const handlers = {
+                onMessage: (data) => {
+                    if (data.type === 'ready') {
+                        clearTimeout(timeout);
+                        this.ready = true;
+                        resolve();
+                        return;
+                    }
+
+                    const { id, success, result, error } = data;
+                    const pending = this.pending.get(id);
+                    if (pending) {
+                        this.pending.delete(id);
+                        if (success) {
+                            pending.resolve(result);
+                        } else {
+                            pending.reject(new Error(error));
+                        }
+                    }
+                },
+                onError: (err) => {
+                    clearTimeout(timeout);
+                    reject(err);
+                },
+                onExit: (code) => {
+                    if (code !== 0) {
+                        reject(new Error(`Worker exited with code ${code}`));
+                    }
+                }
+            };
+
+            setupWorkerHandlers(this.worker, handlers);
+        });
+
+        await this.readyPromise;
+
+        // Initialize WASM in worker
+        await this.call('init');
+
+        return this;
+    }
+
+    /**
+     * Call a method on the worker
+     */
+    async call(method, params = {}) {
+        if (!this.worker) {
+            throw new Error('Client not initialized. Call init() first.');
+        }
+
+        const id = this.nextId++;
+
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+            this.worker.postMessage({ id, method, params });
+        });
+    }
+
+    /**
+     * Create a new database
+     */
+    async createDatabase(schema, name = 'default') {
+        const dbId = `db_${this.nextId++}`;
+        await this.call('createDatabase', { id: dbId, schema, name });
+        const db = new FlatSQLWorkerDatabase(this, dbId);
+        this.databases.set(dbId, db);
+        return db;
+    }
+
+    /**
+     * Create a test User FlatBuffer (for demos)
+     */
+    async createTestUser(id, name, email, age) {
+        const result = await this.call('createTestUser', { id, name, email, age });
+        return new Uint8Array(result.data);
+    }
+
+    /**
+     * Create a test Post FlatBuffer (for demos)
+     */
+    async createTestPost(id, userId, title) {
+        const result = await this.call('createTestPost', { id, userId, title });
+        return new Uint8Array(result.data);
+    }
+
+    /**
+     * Terminate the worker
+     */
+    terminate() {
+        if (this.worker) {
+            if (isNode) {
+                this.worker.terminate();
+            } else {
+                this.worker.terminate();
+            }
+            this.worker = null;
+            this.ready = false;
+        }
+    }
+}
+
+/**
+ * FlatSQL Database accessed through a worker
+ */
+export class FlatSQLWorkerDatabase {
+    constructor(client, dbId) {
+        this._client = client;
+        this._dbId = dbId;
+    }
+
+    /**
+     * Register a file ID for routing FlatBuffers to tables
+     */
+    async registerFileId(fileId, tableName) {
+        await this._client.call('registerFileId', {
+            dbId: this._dbId,
+            fileId,
+            tableName
+        });
+    }
+
+    /**
+     * Enable built-in demo extractors (User/Post)
+     */
+    async enableDemoExtractors() {
+        await this._client.call('enableDemoExtractors', { dbId: this._dbId });
+    }
+
+    /**
+     * Execute a SQL query
+     */
+    async query(sql) {
+        return await this._client.call('query', { dbId: this._dbId, sql });
+    }
+
+    /**
+     * Ingest a size-prefixed FlatBuffer stream
+     */
+    async ingest(data) {
+        const result = await this._client.call('ingest', {
+            dbId: this._dbId,
+            data: Array.from(data)
+        });
+        return result.bytesConsumed;
+    }
+
+    /**
+     * Ingest a single FlatBuffer (without size prefix)
+     */
+    async ingestOne(data) {
+        const result = await this._client.call('ingestOne', {
+            dbId: this._dbId,
+            data: Array.from(data)
+        });
+        return result.rowId;
+    }
+
+    /**
+     * Ingest multiple FlatBuffers (builds stream internally)
+     */
+    async ingestStream(buffers) {
+        const result = await this._client.call('ingestStream', {
+            dbId: this._dbId,
+            buffers: buffers.map(b => Array.from(b))
+        });
+        return result;
+    }
+
+    /**
+     * Export database as binary blob
+     */
+    async exportData() {
+        const result = await this._client.call('exportData', { dbId: this._dbId });
+        return new Uint8Array(result.data);
+    }
+
+    /**
+     * List all tables
+     */
+    async listTables() {
+        const result = await this._client.call('listTables', { dbId: this._dbId });
+        return result.tables;
+    }
+
+    /**
+     * Get database statistics
+     */
+    async getStats() {
+        const result = await this._client.call('getStats', { dbId: this._dbId });
+        return result.stats;
+    }
+
+    /**
+     * Delete the database and free resources
+     */
+    async delete() {
+        await this._client.call('deleteDatabase', { dbId: this._dbId });
+        this._client.databases.delete(this._dbId);
+    }
+}
+
+export default FlatSQLClient;