corebasic 1.0.196 → 1.0.199

package/libs/cpp.js

@@ -0,0 +1,216 @@
+
+import crypto from 'node:crypto';
+
+
+function readContent(buffer, valueRef, bytesRef) {
+    const MASK = 0x7F;
+    const MSB = 0x80;
+    const MAX_VLQ_BYTES = 10;
+
+    // Ensure we have a valid Uint8Array view
+    if (!buffer) return false;
+    const view = ArrayBuffer.isView(buffer) ? buffer : new Uint8Array(buffer);
+    const bufferSize = view.length;
+
+    let length = 0n;
+    let shift = 0n;
+    let lenBytes = 0;
+
+    while (true) {
+        if (lenBytes >= MAX_VLQ_BYTES || lenBytes >= bufferSize) {
+            return false;
+        }
+
+        const byte = view[lenBytes];
+        lenBytes++;
+
+        length |= BigInt(byte & MASK) << shift;
+        shift += 7n;
+
+        if ((byte & MSB) === 0) {
+            break;
+        }
+    }
+
+    // Emulate C++ pointer assignment by modifying the wrapper objects
+    valueRef.value = length;
+    bytesRef.value = lenBytes;
+
+    return true;
+}
+
+
+
+export function vlqToUint64(arrayBuffer) {
+    // Objects act as containers to simulate C++ pointers/references
+    const valueRef = { value: 0n };
+    const bytesRef = { value: 0 };
+
+    const success = readContent(arrayBuffer, valueRef, bytesRef);
+
+    // If read_content returned false, handle or return empty/default values
+    if (!success) {
+        return { value: Number(0n), bytes: 0 };
+    }
+
+    // Returns a plain object, the equivalent of QVariantMap
+    return {
+        value: Number(valueRef.value), // BigInt to Number
+        bytes: bytesRef.value  // Number
+    };
+}
+
+
+
+
+function writeVlqcontent(buffer, value) {
+    const MASK = 0x7Fn;
+    const MSB = 0x80;
+    let bufferPtr = 0;
+    let tempValue = BigInt(value);
+
+    do {
+        let byte = Number(tempValue & MASK);
+        tempValue >>= 7n;
+        if (tempValue > 0n) {
+            byte |= MSB;
+        }
+        buffer[bufferPtr++] = byte;
+    } while (tempValue > 0n);
+
+    return bufferPtr;
+}
+
+export function uint64ToVlq(value) {
+    // 10 bytes is the maximum VLQ size for a uint64
+    const out = Buffer.allocUnsafe(10);
+    const size = writeVlqcontent(out, value);
+
+    // Returns a fresh, correctly sized buffer copy
+    return Buffer.from(out.subarray(0, size));
+}
+
+
+
+
+export function sliceArrayBuffer(buffer, offset, length) {
+    // Ensure we are working with a Node.js Buffer
+    const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+    const totalSize = buf.length;
+
+    // Boundary check matching your C++ logic
+    if (offset < 0 || offset >= totalSize) {
+        return Buffer.alloc(0); // Returns an empty Buffer (like QByteArray())
+    }
+
+    // FIX: If length is -1, automatically calculate the remaining size to the end
+    if (length <= 0 || offset + length > totalSize) {
+        length = totalSize - offset;
+    }
+
+    // subarray() creates a zero-copy/shallow view over the original memory.
+    // Note: second argument is end index instead of cpp length
+    return buf.subarray(offset, offset + length);  // COW/Shallow copy I hope or else what is the use of this function
+}
+
+export function stringToUtf8ArrayBuffer(arg) {
+    // Converts the string into a Node.js Buffer encoded in UTF-8
+    return Buffer.from(arg, 'utf8');
+}
+
+
+export function numberToSortableBytes(value) {
+    // 1. Create an 8-byte buffer and a data view to read/write raw bits
+    const buffer = Buffer.alloc(8);
+    const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+
+    // 2. Write the double value into the buffer to extract its raw 64-bit bits
+    view.setFloat64(0, value, true); // Use little-endian for direct memory mapping
+    let bits = view.getBigUint64(0, true);
+
+    const SIGN_MASK = 0x8000000000000000n; // Use BigInt literal for 64-bit mask
+
+    // 3. Apply the sorting transformation logic
+    if ((bits & SIGN_MASK) !== 0n) {
+        // Negative: invert all bits
+        bits = ~bits;
+    } else {
+        // Positive: flip the sign bit
+        bits ^= SIGN_MASK;
+    }
+
+    // 4. Write the transformed bits back as Big Endian (equivalent to qToBigEndian)
+    view.setBigUint64(0, bits, false); // false = Big Endian
+
+    return buffer;
+}
+
+
+
+
+export function defaultHash(buffer) {
+    const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+
+    // Uses the high-speed, non-cryptographic xxhash64 algorithm
+    // supported natively in both Node.js and Bun's crypto module layers.
+    const hashBytes = crypto.createHash('md5').update(buf).digest(); // Earlier used xxhash64 but not available in my openssl. so md5
+
+    // Reads the 8-byte hash output into a 64-bit BigInt (quint64)
+    return hashBytes.readBigUInt64BE(0);
+}
+
+
+export function memcmpEqual(buffer1, buffer2) {
+    const b1 = Buffer.isBuffer(buffer1) ? buffer1 : Buffer.from(buffer1);
+    const b2 = Buffer.isBuffer(buffer2) ? buffer2 : Buffer.from(buffer2);
+
+    return b1.equals(b2);
+}
+
+
+export function arrayBufferToString(buffer) {
+    const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer);
+
+    // Decodes the buffer bytes as a UTF-8 JavaScript string
+    return buf.toString('utf8');
+}
+
+
+
+
+
+// Matches QString Response::getSliceAsText
+export function getSliceAsText(rawTextBuffer, offset, length) {
+    // Reuses the identical slice boundary logic from above
+    try {
+    const slice = getSliceAsArrayBuffer(rawTextBuffer, offset, length);
+
+    // Returns an empty string if slice was invalid, or the decoded UTF-8 string
+    return slice.length === 0 ? "" : slice.toString('utf8');
+    } catch (e) { console.log(e); throw e}
+}
+
+// Matches QByteArray Response::getSliceAsArrayBuffer
+export function getSliceAsArrayBuffer(rawTextBuffer, offset, length) {
+    const totalSize = rawTextBuffer.length;
+
+    if (offset < 0 || offset >= totalSize) {
+        return Buffer.alloc(0);
+    }
+
+    if (length === -1) {
+        length = totalSize - offset;
+    } else if (length <= 0 || (offset + length) > totalSize) {
+        return Buffer.alloc(0); // Strict failure rule matching your new C++ code
+    }
+
+    // Creates a zero-copy view of the sliced buffer
+    return rawTextBuffer.subarray(offset, offset + length);
+}
+
+
+
+
+
+
+

package/libs/dip.js

@@ -8,6 +8,7 @@ export const query = Elabase.query
 export const update = Elabase.update
 export const remove = Elabase.remove
 
+export const newRaw = Elabase.newRaw
 export const batchBegin = Elabase.batchBegin
 export const batchAbort = Elabase.batchAbort
 export const batchSubmit = Elabase.batchSubmit

package/libs/elabase.js

@@ -1,7 +1,7 @@
 import axios from 'axios'
 import {default as dipper, shard_stats as ShardStats} from './dipper.js'
 import * as Utils from './utils.js'
-
+import * as Cpp from './cpp.js'
 
 
 
@@ -29,6 +29,9 @@ let Elabase = {
         Lmdb: 1,
         Sled: 2,
         Sanakirja: 3,
+        Singularity: 4,
+        Memory: 5,
+        Analytics: 6,
     }
 }
 
@@ -48,9 +51,26 @@ export function start(dbName) {
     }
 }
 
+function deepFreeze(obj) {
+    if (obj && typeof obj === "object" && !Object.isFrozen(obj)) {
+        Object.freeze(obj);
+
+        for (const key of Object.keys(obj)) {
+            deepFreeze(obj[key]);
+        }
+    }
+    return obj;
+}
+class DipMeta {
+    constructor(obj) {
+        Object.assign(this, deepFreeze(obj));
+        Object.freeze(this);
+    }
+}
+
 
 export const globalMeta = args => {
-    return { ...args, company: "GLOBAL", outlet: "GLOBAL" }
+    return new DipMeta({ ...args, company: "GLOBAL", outlet: "GLOBAL" })
 }
 export const companyMeta = (company, args) => {
     if (typeof company !== "string" && !Array.isArray(company))
@@ -58,7 +78,7 @@ export const companyMeta = (company, args) => {
     if (!company || (typeof company === "string" && company.trim() === "") ||  company === "GLOBAL" || (Array.isArray(company) && (!company.length || company.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
         throw  {message: "Error: Invalid company provided in Dip.companyMeta()"}
 
-    return {...args, outlet: "GLOBAL", company}
+    return new DipMeta({...args, outlet: "GLOBAL", company})
 }
 export const outletMeta = (outlet, args) => {
     if (typeof outlet !== "string" && !Array.isArray(outlet))
@@ -66,7 +86,7 @@ export const outletMeta = (outlet, args) => {
     if (!outlet || (typeof outlet === "string" && outlet.trim() === "") ||  outlet === "GLOBAL" || (Array.isArray(outlet) && (!outlet.length || outlet.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
         throw  {message: "Error: Invalid outlet provided in Dip.outletMeta()"}
 
-    return {...args, company: "GLOBAL", outlet}
+    return new DipMeta({...args, company: "GLOBAL", outlet})
 }
 
 export const customMeta = (company, outlet, args) => {
@@ -81,7 +101,7 @@ export const customMeta = (company, outlet, args) => {
     if (!outlet || (typeof outlet === "string" && outlet.trim() === "") ||  outlet === "GLOBAL" || (Array.isArray(outlet) && (!outlet.length || outlet.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL")) ))
         throw  {message: "Error: Invalid outlet provided in Dip.customMeta()"}
 
-    return {...args, company, outlet}
+    return new DipMeta({...args, company, outlet})
 }
 
 function validate_collection(collection, fn_name) {
@@ -142,19 +162,23 @@ function generate_suffix(meta) {
     return suffixes
 }
 
+// Ported
 let BATCH_COUNTER = 0n;
 let batches = {}
 
+// Ported
 export function batchBegin() { // Manual Lifecycle. If you forget to batchSubmit() or batchAbort(), then memory leak due to dangling batch
     const uid = ++BATCH_COUNTER; // Safe. No skew in async ++BATCH_COUNTER as single thread and ++COUNTER is atomic in event loop sense. ⚠️ Only breaks on worker_threads or cluster (multi-process)
     batches[uid] = []
     return uid
 }
-export function batchAbort(batch) { // Mandatory  to prevent meory leak due to dangling batch if batchBegin() is called without batchSubmit()
-    delete batches[batch]
+// Ported
+export function batchAbort(batch_id) { // Mandatory  to prevent meory leak due to dangling batch if batchBegin() is called without batchSubmit()
+    delete batches[batch_id]
 }
 
-export const batch = async (callback) => { // Convenience function with auto lifecyle management and auto submit when callback returns. abort is called appropriately.
+// Ported
+export const batch = async (callback, arg) => { // Convenience function with auto lifecyle management and auto submit when callback returns. abort is called appropriately.
     /**
      * Usage:
      * const result = await Dip.batch(async batch => {
@@ -168,23 +192,29 @@ export const batch = async (callback) => { // Convenience function with auto lif
 
     try {
         await callback(batch)
-        return await batchSubmit(batch)
+        return await batchSubmit(batch, arg)
     } catch (err) {
         batchAbort(batch)
         throw err
     }
 }
 
-export function batchSubmit(batch, arg) {
-    if (!batch || !batches[batch] || !batches[batch].length)
+// Ported
+export function batchSubmit(batch_id, arg) {
+    if (!batch_id || !batches[batch_id] || !batches[batch_id].length)
         throw {message: `Error: Invalid batch in Dip.batchSubmit()`}
-    let txns = batches[batch]
-    let mode = batches[batch].some(txn => txn.insert || txn.update || txn.delete) ? "command" : "query"
-    delete batches[batch]
+    let txns = batches[batch_id]
+    const executor = arg?.executor ?? batches[batch_id].find(txn => txn.executor)?.executor
+    const syncMode = arg?.syncMode ?? batches[batch_id].find(txn => txn.syncMode)?.syncMode
+    const raw = arg?.raw
+    const engine = arg?.engine
+    const port = arg?.port
+    const timeout = arg?.timeout
+    delete batches[batch_id]
     return new Promise((resolve, reject) => resolve())
-    .then(() => execute({ "batch": txns, mode, version: arg?.version, compression: arg?.compression }))
+    .then(() => execute({ "batch": txns, version: arg?.version, raw, engine, compression: arg?.compression, executor, syncMode, port, timeout }))
     .then (result => {
-        Events.send("Elabase.batchSubmit", { response: result })
+        Events.send("Dip.batchSubmit", { response: result })
         return result
     })
 }
@@ -244,69 +274,74 @@ function validate_topology(meta, collection, fn_name) {
     return new_meta
 }
 
+// Ported
 export const operation = async (name, extras) => {
     return execute({ operation: name, ...extras })
 }
 
 
-export const insert = async (meta, collection, value, _id, extras) => {
+// Ported
+export const insert = async (meta, collection, value, _id, options, extras) => {
+
+    if (!(meta instanceof DipMeta)) console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.insert()') // TODO: throw error once review completes
 
     validate_collection(collection, 'in Dip.insert()')
 
     validate_meta(meta, 'in Dip.insert()')
 
-    const topology_meta = validate_topology(meta, collection, 'in Dip.insert()')
+    const topology_meta = validate_topology(meta, collection, 'in Dip.insert()') // TODO: loop for arrays
 
-    var arg = {
+    let arg = {
         collection: collection,
-        suffix: generate_suffix(meta),
         insert: value,
+        options: options ? options : {},
+        syncMode: meta.syncMode,
         ...(meta.DIP_URL ? {DIP_URL: meta.DIP_URL} : {}),
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
+    arg.suffix = generate_suffix(meta)
+    arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (_id)
         arg._id = _id
 
-    if (!(["string", "number"].includes(typeof value?._id)))
-        throw {message: "Error: invalid _id in Dip.insert()"}
-    if (typeof value?._id === "string" && value._id.trim().length === 0)
-        throw {message: "Error: invalid _id in Dip.insert()"}
-
-
     if (meta.batch) {
         if (!batches[meta.batch])
             throw {message: 'Error: Invalid batch in Dip.insert()'}
         batches[meta.batch].push(arg)
         return
     }
-    arg.mode = "command"
     return new Promise((resolve, reject) => resolve())
     .then(() => execute(arg))
     .then(result => {
-        Events.send("Elabase.insert", { collection: collectionArray(collection), response: result })
+        Events.send("Dip.insert", { collection: collectionArray(collection), response: result })
         return result
     })
 }
 
+// Ported
 export const query = async (meta, collection, query, options, extras) => {
 
+    if (!(meta instanceof DipMeta)) console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.query()') // TODO: throw error once review completes
+
     validate_collection(collection, 'in Dip.query()')
 
     validate_meta(meta, 'in Dip.query()')
 
-    const topology_meta = validate_topology(meta, collection, 'in Dip.query()')
+    const topology_meta = validate_topology(meta, collection, 'in Dip.query()') // TODO: loop for arrays
 
-    var arg = {
+    let arg = {
         collection: collection,
-        suffix: generate_suffix(meta),
         query: query,
         options: options ? options : {},
+        syncMode: meta.syncMode,
         ...(meta.DIP_URL ? {DIP_URL: meta.DIP_URL} : {}),
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
+    arg.suffix = generate_suffix(meta)
+    arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (meta.batch) {
         if (!batches[meta.batch])
@@ -314,17 +349,16 @@ export const query = async (meta, collection, query, options, extras) => {
         batches[meta.batch].push(arg)
         return
     }
-    arg.mode = "query"
     return new Promise((resolve, reject) => resolve())
     .then(() => execute(arg))
 }
 
 
 function collectionArray(col) {
-    var type = Object.prototype.toString.call(col)
+    let type = Object.prototype.toString.call(col)
     if (type === '[object Array]') {
-        var newcol = []
-        for (var i in col) {
+        let newcol = []
+        for (let i in col) {
             if (!Utils.isEmpty(col[i]))
                 newcol.push(col[i])
         }
@@ -336,25 +370,29 @@ function collectionArray(col) {
     return []
 }
 
-
+// Ported
 export const update = async (meta, collection, query, update, options, extras) => {
 
+    if (!(meta instanceof DipMeta)) console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.update()') // TODO: throw error once review completes
+
     validate_collection(collection, 'in Dip.update()')
 
     validate_meta(meta, 'in Dip.update()')
 
-    const topology_meta = validate_topology(meta, collection, 'in Dip.update()')
+    const topology_meta = validate_topology(meta, collection, 'in Dip.update()') // TODO: loop for arrays
 
-    var arg = {
+    let arg = {
         collection: collection,
-        suffix: generate_suffix(meta),
         query: query,
         update: update,
         options: options ? options : {},
+        syncMode: meta.syncMode,
         ...(meta.DIP_URL ? {DIP_URL: meta.DIP_URL} : {}),
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
+    arg.suffix = generate_suffix(meta)
+    arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (meta.batch) {
         if (!batches[meta.batch])
@@ -362,17 +400,19 @@ export const update = async (meta, collection, query, update, options, extras) =
         batches[meta.batch].push(arg)
         return
     }
-    arg.mode = "command"
     return new Promise((resolve, reject) => resolve())
     .then(() => execute(arg))
     .then (result => {
-        Events.send("Elabase.update", { collection: collectionArray(collection), response: result })
+        Events.send("Dip.update", { collection: collectionArray(collection), response: result })
         return result
     })
 }
 
+// Ported
 export const remove = async (meta, collection, query, options, extras) => {
 
+    if (!(meta instanceof DipMeta)) console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.remove()') // TODO: throw error once review completes
+
     validate_collection(collection, 'in Dip.remove()')
 
     if (Utils.isEmpty(query._id) && !meta.allowDangerousRemove)
@@ -380,18 +420,20 @@ export const remove = async (meta, collection, query, options, extras) => {
 
     validate_meta(meta, 'in Dip.remove()')
 
-    const topology_meta = validate_topology(meta, collection, 'in Dip.remove()')
+    const topology_meta = validate_topology(meta, collection, 'in Dip.remove()') // TODO: loop for arrays
 
-    var arg = {
+    let arg = {
         collection: collection,
-        suffix: generate_suffix(meta),
         query: query,
-        delete: true,
         options: options ? options : {},
+        delete: true,
+        syncMode: meta.syncMode,
         ...(meta.DIP_URL ? {DIP_URL: meta.DIP_URL} : {}),
         ...(meta.DIP_DB ? {db: meta.DIP_DB} : {}),
     }
     arg = Object.assign(arg, extras)
+    arg.suffix = generate_suffix(meta)
+    arg.executor = meta.executor ?? "native"
     arg = Object.assign({}, topology_meta, arg)
     if (meta.batch) {
         if (!batches[meta.batch])
@@ -399,18 +441,14 @@ export const remove = async (meta, collection, query, options, extras) => {
         batches[meta.batch].push(arg)
         return
     }
-    arg.mode = "command"
     return new Promise((resolve, reject) => resolve())
     .then(() => execute(arg))
     .then(result => {
-        Events.send("Elabase.remove", { collection: collectionArray(collection), response: result })
+        Events.send("Dip.remove", { collection: collectionArray(collection), response: result })
         return result
     })
 }
 
-export const close = async (meta, db) => {
-    return query(meta, "-", {}, {}, {db: db ?? root.db, close: true})
-}
 
 
 export let config = {
@@ -430,29 +468,304 @@ function execute(arg) {
 //            _id: 1 // Optional. Used with insert
 //        }
         arg.db = arg.db ?? db
+        arg.executor = arg.executor ?? "native"
         arg.engine = arg.engine ?? internal.engineName
-        arg.compression = true // Compress all for now
+        if (arg.compression === undefined)
+            arg.compression = true // Compress all for now by default
+
+        let fullurl = `${url}:${arg?.port ?? port}/${api}`
+
+        delete arg.port
+        const timeout = arg.timeout
+        delete arg.timeout
 
 
         return config.useDipper  ? dipper(arg)
         : axios
-        .post(arg.DIP_URL ?? fullurl, {...arg, DIP_URL: undefined})
-        // .timeout(5000)
+        .post(arg.DIP_URL ?? fullurl, buildHybridRequest({...arg, DIP_URL: undefined}), { responseType: 'arraybuffer', timeout: timeout ?? 5000, headers: {"Content-Type": "application/dip"} })
         .then(result => {
-//                  console.log(res.status);
-//                  console.log(JSON.stringify(res.header, null, 4));
-//                  console.log(JSON.stringify(res.body, null, 4));
-
-                    let res = result.data.mode === "query" ? result.data.items : result.data
-                    res.stats = result.data.mode === "query" ? _ => result.data.stats : res.stats
-                    return res
-                })
+            // console.log(res.status);
+            // console.log(JSON.stringify(res.header, null, 4));
+            // console.log(JSON.stringify(res.body, null, 4));
+
+            result.getSliceAsText = (offset, length) => Cpp.getSliceAsText(result.data, offset, length)
+            result.getSliceAsArrayBuffer = (offset, length) => Cpp.getSliceAsArrayBuffer(result.data, offset, length)
+            result["Content-Type"] = result.headers["content-type"]
+            result.body = result.data
+            if (result.headers["content-type"].includes("application/json"))
+                result.body = JSON.parse(result.data)
+
+            let parsed = parseBinaryResponse(result)
+            let response = prepareResult(parsed)
+
+            return response
+
+        })
 //       .catch(err => { console.log(err.message); console.log(err.response); return err});
 }
 
 
 
 
+let Essentials = {
+    vlqToUint64: Cpp.vlqToUint64,
+    sliceArrayBuffer: Cpp.sliceArrayBuffer,
+    stringToUtf8ArrayBuffer: Cpp.stringToUtf8ArrayBuffer,
+    numberToSortableBytes: Cpp.numberToSortableBytes,
+    defaultHash: Cpp.defaultHash,
+    memcmpEqual: Cpp.memcmpEqual,
+    uint64ToVlq: Cpp.uint64ToVlq,
+    arrayBufferToString: Cpp.arrayBufferToString,
+}
+
+
+
+function prepareResult(result) {
+    let res = result.content?.length === 1  ? result.content[0].items : result.content // For raw, this resolves to result.content which is ArrayBuffer
+    res.metadata = _ => result.metadata
+    if (result.metadata.executor === "raw")
+        setRawBatchIterator(res)
+        return res
+}
+
+function parseBinaryResponse(result) {
+    const contentType = (result["Content-Type"] || "").toLowerCase()
+    if (contentType.includes("application/json")) {
+        return {metadata: result.body, content: result.body.items}
+    }
+    if (!contentType.includes("application/dip")) {
+        throw {message: "Error: Dip unsupported media. Invalid content-type."}
+    }
+
+    const arrayBuffer = result.body
+
+    // let wire = parseWireFormat(arrayBuffer)
+    // let meta = wireFormat.metadata
+    // let content = wireFormat.content
+
+    // try {
+    //     meta = JSON.parse(meta);
+    // } catch (_) {
+    //     throw {contentType, meta, content, message: "Error: Dip failed to parse metadata. Unsupported response format. Response format does not match content-type."}
+    // }
+
+    // parseWireFormat() equivalent
+    // =============================
+    let dataView = new DataView(arrayBuffer.buffer,  arrayBuffer.byteOffset,  arrayBuffer.byteLength);
+    let offset = 0;
+
+    // 1. Read the metadata size header
+    let vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
+    let metaSize = vlq.value;
+    offset += vlq.bytes;
+
+    // 2. Extract and parse metadata string via C++ instantly
+    let metaText = result.getSliceAsText(offset, metaSize);
+    let nativeResponse = JSON.parse(metaText);
+    offset += metaSize;
+
+    // Extract everything remaining automatically by passing -1
+    let content = result.getSliceAsArrayBuffer(offset, -1); // content is already ArrayBuffer but QByteArray to ArrayBuffer cannot be Uint8Array() wrapped
+    // content = new Uint8Array(content ?? new ArrayBuffer(0))
+
+    return { metadata: nativeResponse, content: content }
+}
+
+
+function setRawBatchIterator(arrayBuffer) {
+    arrayBuffer.metadata().items.forEach(batch => {
+        batch.kv = _ => {
+            const batchItem = {
+                count: batch.count,
+                slice: Essentials.sliceArrayBuffer(arrayBuffer, batch.offset, batch.size),
+                                         [Symbol.iterator]() {
+                                             let index = 0;
+                                             let offset = 0;
+                                             return {
+                                                 next: () => {
+                                                     if (index < this.count) {
+                                                         let kv = getRawKeyValue(this.slice, offset)
+                                                         offset = kv.next
+                                                         index++
+                                                         return { value: kv, done: false };
+                                                     }
+                                                     return { done: true };
+                                                 }
+                                             }
+                                         }
+            }
+            return batchItem
+        }
+    })
+    arrayBuffer.items = arrayBuffer.metadata().items
+}
+
+
+
+// ===============
+
+
+
+
+function convert_to_array_buffer(value) {
+    if (typeof value === "string")
+        return Essentials.stringToUtf8ArrayBuffer(value)
+    else if (typeof value === "number")
+        return Essentials.numberToSortableBytes(value)
+    return Essentials.stringToUtf8ArrayBuffer(JSON.stringify(value))
+}
+
+export function newRaw() {
+    let obj = {buffers: [], size: 0, hashes:{}}
+    obj.add = value => {
+        if (value === undefined)
+            return undefined
+            const buf = (value instanceof ArrayBuffer) ? value : convert_to_array_buffer(value);
+
+        const hash = Essentials.defaultHash(buf)
+        for (let item of obj.hashes[hash] ?? []) { // Conflict. Fallback to verify
+            if (Essentials.memcmpEqual(buf, item.buf))
+                return item.offset
+        }
+
+        const vlq = new Uint8Array(Essentials.uint64ToVlq(buf.byteLength))
+        obj.buffers.push({buf, vlq})
+        let offset = obj.size
+        obj.size += vlq.byteLength + buf.byteLength;
+
+        obj.hashes[hash] = obj.hashes[hash] ?? []
+        obj.hashes[hash].push({offset, buf})
+
+        return offset
+    }
+    return obj
+}
+
+function buildHybridRequest(request) {
+    const batch = request.batch ?? [request];
+
+    let raw = request.raw ?? newRaw()
+    request.raw = undefined
+
+    // Pass 1: Direct offset assignment & exact size calculation (Zero allocation)
+    for (let i = 0; i < batch.length; i++) {
+        const item = batch[i];
+        delete item.batch
+        if (item.update) {
+            if ( item._id === undefined && item.options?.upsert && item.query?._id !== undefined)
+                item._id = item.query._id
+                item.update = raw.add(item.update)
+        } else if (item.insert) {
+            if (item._id === undefined && item.insert?._id !== undefined)
+                item._id = item.insert._id
+                item.insert = raw.add(item.insert)
+        }
+        item._id = raw.add(item._id)
+
+        if (item.options?.$rangeFrom !== undefined)
+            item.options.$rangeFrom = raw.add(item.options.$rangeFrom)
+            if (item.options?.$rangeTo !== undefined)
+                item.options.$rangeTo = raw.add(item.options.$rangeTo)
+    }
+
+    // Pass 2: Generate the final JSON payload string
+    const jsonBuffer = convert_to_array_buffer(request)
+    const jsonByteLen = jsonBuffer.byteLength;
+
+    // Pass 3: Allocate the exact Master Buffer ONCE
+    let vlq = new Uint8Array(Essentials.uint64ToVlq(jsonByteLen))
+    const totalSize = vlq.byteLength + jsonByteLen + raw.size;
+    const masterBuffer = new ArrayBuffer(totalSize);
+    const masterView = new Uint8Array(masterBuffer);
+
+    // Write the JSON size prefix
+    masterView.set(vlq, 0);
+
+    // Direct copy of JSON string bytes into their final destination
+    masterView.set(new Uint8Array(jsonBuffer), vlq.byteLength);
+
+    // Copy binary payloads directly into their final destination window
+    let writeOffset = vlq.byteLength + jsonByteLen;
+    for (const buffer of  raw.buffers) {
+        const buf = buffer.buf
+        const vlq = buffer.vlq
+
+        // Write length prefix directly into final master buffer
+        masterView.set(vlq, writeOffset);
+        writeOffset += vlq.byteLength
+
+        // Blit bytes directly from source into final master buffer destination
+        masterView.set(new Uint8Array(buf), writeOffset);
+        writeOffset += buf.byteLength;
+    }
+
+    return masterBuffer;
+}
+
+// Usage getRawKeyValue()
+// Dip.query()
+// .then(r => {
+//           let count = r.metadata().count
+//           let offset = 0
+//           for (let i = 0; i < count; i++) {
+//               let item = getKeyValue(r, offset)
+//               offset = item.next
+//               let res = Utils.arrayBufferToFile(item.data, '/home/khayaali/Downloads/zdip.png')
+//               console.log(item._id,res)
+//           }
+//       })
+
+
+
+function bufferToString(arrayBuffer) {
+    return Essentials.arrayBufferToString(arrayBuffer)
+}
+
+function getRawKeyValue(arrayBuffer, offset) {
+    let dataView = new DataView(arrayBuffer.buffer,  arrayBuffer.byteOffset,  arrayBuffer.byteLength);
+    let masterView = new Uint8Array(arrayBuffer);
+
+    // 1. Extract Key Size and Key
+    let vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
+    let keySize = vlq.value;
+    offset += vlq.bytes;
+
+    // Zero-copy view window bounded strictly to the key bytes
+    let _id = masterView.subarray(offset, offset + keySize);
+    let _id_string
+    _id.toText = _ => {
+        // Fix: Pass the typed array view itself, NOT the underlying root buffer
+        _id_string = _id_string === undefined ? _id.buffer.slice(_id.byteOffset, _id.byteOffset + _id.byteLength) : _id_string
+        return bufferToString(_id_string)
+    }
+    offset += keySize;
+
+    // 2. Extract Data Size and Data Binary Payload
+    vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
+    let dataSize = vlq.value;
+    offset += vlq.bytes;
+
+    // Optimization: Use subarray() instead of .slice() to make the data payload 100% zero-copy too
+    let _val = masterView.subarray(offset, offset + dataSize);
+    offset += dataSize;
+
+    _val = Essentials.sliceArrayBuffer(arrayBuffer, _val.byteOffset, _val.byteLength)
+    // _val = _val.buffer.slice(_val.byteOffset, _val.byteOffset + _val.byteLength);
+    _val.toText = _ => bufferToString(_val)
+    return { _id, _val, next: offset };
+}
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "corebasic",
   "type": "module",
-  "version": "1.0.196",
+  "version": "1.0.199",
   "description": "",
   "main": "index.js",
   "scripts": {