corebasic 1.0.211 → 1.0.213

package/dist/libs/elabase.js

@@ -0,0 +1,656 @@
+import axios from 'axios';
+import { default as dipper, shard_stats as ShardStats } from './dipper.js';
+import * as Utils from './utils.js';
+import * as Cpp from './cpp.js';
+import { applySuffixPolicy } from './dip/suffix/policy.js';
+export const shard_stats = ShardStats;
+let Events = {
+    send: function () { }
+};
+const url = process.env.DIP_URL || "http://127.0.0.1"; // Set DIP_URL to "http://dip" (Prod) or http://dip-dev (Dev) in CI/CD
+let port = process.env.DIP_PORT || 9401;
+let api = 'execute';
+let fullurl = `${url}:${port}/${api}`;
+let Elabase = {
+    Engine: {
+        Lmdb: 1,
+        Sled: 2,
+        Sanakirja: 3,
+        Singularity: 4,
+        Memory: 5,
+        Analytics: 6,
+    }
+};
+let engine = Elabase.Engine.Lmdb;
+let internal = {
+    db: undefined,
+    engineName: engine === Elabase.Engine.Lmdb ? "lmdb" : (engine === Elabase.Engine.Sled ? "sled" : engine === Elabase.Engine.Sanakirja ? "sanakirja" : "lmdb")
+};
+let db = internal.db; // default database
+export function start(dbName) {
+    if (!Utils.isEmpty(dbName) && Utils.isEmpty(db)) {
+        internal.db = "data/" + dbName;
+        db = "data/" + dbName;
+    }
+}
+function deepFreeze(obj) {
+    if (obj && typeof obj === "object" && !Object.isFrozen(obj)) {
+        Object.freeze(obj);
+        for (const key of Object.keys(obj)) {
+            deepFreeze(obj[key]);
+        }
+    }
+    return obj;
+}
+class DipMeta {
+    constructor(obj) {
+        Object.assign(this, deepFreeze(obj));
+        Object.freeze(this);
+    }
+}
+export const isDipMeta = meta => meta instanceof DipMeta;
+export const globalMeta = args => {
+    return new DipMeta({ ...args, company: "GLOBAL", outlet: "GLOBAL" });
+};
+export const companyMeta = (company, args) => {
+    if (typeof company !== "string" && !Array.isArray(company))
+        throw new Error("Error: Invalid company provided in Dip.companyMeta()");
+    if (!company || (typeof company === "string" && company.trim() === "") || company === "GLOBAL" || (Array.isArray(company) && (!company.length || company.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL"))))
+        throw new Error("Error: Invalid company provided in Dip.companyMeta()");
+    return new DipMeta({ ...args, outlet: "GLOBAL", company });
+};
+export const outletMeta = (outlet, args) => {
+    if (typeof outlet !== "string" && !Array.isArray(outlet))
+        throw new Error("Error: Invalid outlet provided in Dip.outletMeta()");
+    if (!outlet || (typeof outlet === "string" && outlet.trim() === "") || outlet === "GLOBAL" || (Array.isArray(outlet) && (!outlet.length || outlet.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL"))))
+        throw new Error("Error: Invalid outlet provided in Dip.outletMeta()");
+    return new DipMeta({ ...args, company: "GLOBAL", outlet });
+};
+export const customMeta = (company, outlet, args) => {
+    if (typeof company !== "string" && !Array.isArray(company))
+        throw new Error("Error: Invalid company provided in Dip.customMeta()");
+    if (typeof outlet !== "string" && !Array.isArray(outlet))
+        throw new Error("Error: Invalid outlet provided in Dip.customMeta()");
+    if (!company || (typeof company === "string" && company.trim() === "") || company === "GLOBAL" || (Array.isArray(company) && (!company.length || company.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL"))))
+        throw new Error("Error: Invalid company provided in Dip.customMeta()");
+    if (!outlet || (typeof outlet === "string" && outlet.trim() === "") || outlet === "GLOBAL" || (Array.isArray(outlet) && (!outlet.length || outlet.some(item => typeof item !== "string" || item.trim() === "" || item === "GLOBAL"))))
+        throw new Error("Error: Invalid outlet provided in Dip.customMeta()");
+    return new DipMeta({ ...args, company, outlet });
+};
+function validate_collection(collection, fn_name) {
+    if ((typeof collection === 'string' && collection.trim() === "") || (typeof collection !== 'string' && !Array.isArray(collection)) || (Array.isArray(collection) && (!collection.length || collection.some(item => typeof item !== "string" || item.trim() === "")))) {
+        throw new Error(`Error: Invalid collection ${fn_name}`);
+    }
+}
+function validate_meta(meta, fn_name) {
+    if (!meta || typeof meta !== 'object') {
+        throw new Error(`Error: Invalid meta ${fn_name}`);
+    }
+    if ((typeof meta.company === 'string' && meta.company.trim() === "") || (typeof meta.company !== 'string' && !Array.isArray(meta.company)) || (Array.isArray(meta.company) && (!meta.company.length || meta.company.some(item => typeof item !== "string" || item.trim() === "")))) {
+        throw new Error(`Error: Invalid meta.company ${fn_name}`);
+    }
+    if ((typeof meta.outlet === 'string' && meta.outlet.trim() === "") || (typeof meta.outlet !== 'string' && !Array.isArray(meta.outlet)) || (Array.isArray(meta.outlet) && (!meta.outlet.length || meta.outlet.some(item => typeof item !== "string" || item.trim() === "")))) {
+        throw new Error(`Error: Invalid meta.outlet ${fn_name}`);
+    }
+    if ((meta.suffix !== undefined && typeof meta.suffix !== 'string' && !Array.isArray(meta.suffix)) || (Array.isArray(meta.suffix) && meta.suffix.some(item => typeof item !== "string" || item.trim() === ""))) {
+        throw new Error(`Error: Invalid meta.suffix ${fn_name}`);
+    }
+    else if (typeof meta.suffix === 'string' && meta.suffix.trim() === "")
+        throw new Error(`Error: Invalid meta.suffix ${fn_name}`);
+}
+function generate_suffix(meta) {
+    // Assuming validate_meta() is called before reaching here. If not then this is intentional validation bypass
+    let company = typeof meta.company == 'string' ? [meta.company] : (meta.company?.length ? meta.company : []);
+    let outlet = typeof meta.outlet == 'string' ? [meta.outlet] : (meta.outlet?.length ? meta.outlet : []);
+    let suffix = typeof meta.suffix == 'string' ? [meta.suffix] : (meta.suffix?.length ? meta.suffix : []);
+    company = company.filter(item => item.trim() !== "");
+    outlet = outlet.filter(item => item.trim() !== "");
+    suffix = suffix.filter(item => item.trim() !== "");
+    company = company.length ? company : [""];
+    outlet = outlet.length ? outlet : [""];
+    suffix = suffix.length ? suffix : [""];
+    company = new Set(company);
+    outlet = new Set(outlet);
+    suffix = new Set(suffix);
+    // console.log(company, outlet, suffix)
+    let suffixes = [];
+    for (let c of company) {
+        for (let o of outlet) {
+            for (let s of suffix) {
+                let pathSegments = [c, o, s].filter(segment => segment !== "");
+                if (pathSegments.length > 0) {
+                    suffixes.push(pathSegments.join('/'));
+                }
+            }
+        }
+    }
+    return suffixes;
+}
+// Ported
+let BATCH_COUNTER = 0n;
+let batches = {};
+// Ported
+export function batchBegin() {
+    const uid = ++BATCH_COUNTER; // Safe. No skew in async ++BATCH_COUNTER as single thread and ++COUNTER is atomic in event loop sense. ⚠️ Only breaks on worker_threads or cluster (multi-process)
+    batches[uid] = [];
+    return uid;
+}
+// Ported
+export function batchAbort(batch_id) {
+    delete batches[batch_id];
+}
+// Ported
+export const batch = async (callback, arg) => {
+    /**
+     * Usage:
+     * const result = await Dip.batch(async batch => {
+     *      for (let i = 0; i < 45000; i++)
+     *          Dip.insert({...meta, batch}, collection, {_id: i})
+     * })
+     *
+     * Note: Nested batch is supported as they are just independant batches.
+     */
+    const batch = batchBegin();
+    try {
+        await callback(batch);
+        return await batchSubmit(batch, arg);
+    }
+    catch (err) {
+        batchAbort(batch);
+        throw err;
+    }
+};
+// Ported
+export function batchSubmit(batch_id, arg) {
+    if (!batch_id || !batches[batch_id] || !batches[batch_id].length)
+        throw new Error(`Error: Invalid batch in Dip.batchSubmit()`);
+    let txns = batches[batch_id];
+    const executor = arg?.executor ?? batches[batch_id].find(txn => txn.executor)?.executor;
+    const syncMode = arg?.syncMode ?? batches[batch_id].find(txn => txn.syncMode)?.syncMode;
+    const raw = arg?.raw;
+    const engine = arg?.engine;
+    const port = arg?.port;
+    const timeout = arg?.timeout;
+    delete batches[batch_id];
+    return new Promise((resolve, reject) => resolve())
+        .then(() => execute({ "batch": txns, version: arg?.version, raw, engine, compression: arg?.compression, executor, syncMode, port, timeout }))
+        .then(result => {
+        Events.send("Dip.batchSubmit", { response: result });
+        return result;
+    });
+}
+const SCHEMAS_JSON_PROMISES = {};
+export const schema = async (name) => {
+    // Cache the loading promise right away so parallel requests share it
+    if (!SCHEMAS_JSON_PROMISES[name])
+        SCHEMAS_JSON_PROMISES[name] = Utils.fileToJson(new URL('../../..', import.meta.url), `schemas/${name}.json`);
+    try {
+        const schema = await SCHEMAS_JSON_PROMISES[name];
+        return structuredClone(schema);
+    }
+    catch (e) {
+        // Delete the broken promise from cache so a retry can occur later
+        delete SCHEMAS_JSON_PROMISES[name];
+        throw new Error(`Error: Failed to load schema ${name} in Dip`);
+    }
+};
+const COLLECTIONS_JSON = await (async () => {
+    try {
+        return await Utils.fileToJson(new URL('../../..', import.meta.url), 'collections.json');
+    }
+    catch (e) {
+        console.error(e);
+        console.warn("Warn: Dip Failed to load collections.json. Semantics, topology and legality enforcement is not active");
+    }
+})();
+function init_collections_config() {
+    for (let collection in COLLECTIONS_JSON) {
+        let config = COLLECTIONS_JSON[collection];
+        if (config.suffix !== undefined && (typeof config.suffix !== "object" || Array.isArray(config.suffix))) {
+            throw new Error(`Error: Initiating collection configs failed. Invalid suffix in collection: ${collection}`);
+        }
+    }
+}
+if (COLLECTIONS_JSON)
+    init_collections_config();
+const EXCLUDED_COLLECTIONS_CONFIG = {};
+export function excludeCollectionConfig(collection) {
+    EXCLUDED_COLLECTIONS_CONFIG[collection] = true;
+}
+export function includeCollectionConfig(collection) {
+    delete EXCLUDED_COLLECTIONS_CONFIG[collection];
+}
+function validate_topology(meta, collection, fn_name) {
+    let new_meta = meta;
+    let config = COLLECTIONS_JSON?.[collection];
+    if (!config) {
+        if (!EXCLUDED_COLLECTIONS_CONFIG[collection])
+            console.warn(`Warn: Missing config for collection: ${collection}${fn_name ? ` ${fn_name}` : ''}`);
+        return new_meta;
+    }
+    function check_core_meta(config, meta, suffix) {
+        if (config.company !== meta.company && typeof config.company !== "object")
+            throw new Error(`Error: Incompatible meta.company config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`);
+        if (config.outlet !== meta.outlet && typeof config.outlet !== "object")
+            throw new Error(`Error: Incompatible meta.outlet config for collection: ${collection}${suffix ? ' suffix: ' + suffix : ''}${fn_name ? ` ${fn_name}` : ''}`);
+    }
+    check_core_meta(config, meta);
+    if (config.suffix && meta.suffix) {
+        if (!config.suffix[meta.suffix]) {
+            console.warn(`Warn: Missing suffix config for collection: ${collection} suffix:${meta.suffix}${fn_name ? ` ${fn_name}` : ''}`);
+            return new_meta;
+        }
+        check_core_meta(config.suffix[meta.suffix], meta, meta.suffix);
+        new_meta = Object.assign({}, { ...config, suffix: undefined, company: undefined, outlet: undefined }, { ...config.suffix[meta.suffix], company: undefined, outlet: undefined }, meta);
+    }
+    else {
+        new_meta = Object.assign({}, { ...config, suffix: undefined, company: undefined, outlet: undefined }, { company: undefined, outlet: undefined }, meta);
+    }
+    return new_meta;
+}
+// Ported
+export const operation = async (name, extras) => {
+    return execute({ operation: name, ...extras });
+};
+// Ported
+export const insert = async (meta, collection, value, options, extras) => {
+    if (!(meta instanceof DipMeta))
+        console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.insert()'); // TODO: throw error once review completes
+    validate_collection(collection, 'in Dip.insert()');
+    validate_meta(meta, 'in Dip.insert()');
+    const topology_meta = validate_topology(meta, collection, 'in Dip.insert()'); // TODO: loop for arrays
+    let arg = {
+        collection: collection,
+        insert: value,
+        options: options ? options : {},
+        syncMode: meta.syncMode,
+        ...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
+        ...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
+    };
+    arg = Object.assign(arg, extras);
+    let suffix_from_policy = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.insert, true, arg);
+    let suffix_from_meta = generate_suffix(meta);
+    arg.suffix = (suffix_from_meta.length ? suffix_from_meta : [""]).flatMap(meta => (suffix_from_policy.length ? suffix_from_policy : [""]).map(policy => meta + policy));
+    arg.executor = meta.executor ?? "native";
+    arg = Object.assign({}, topology_meta, arg);
+    if (options?._id) {
+        arg._id = options._id;
+        arg.options = { ...options, _id: undefined };
+    }
+    if (meta.batch) {
+        if (!batches[meta.batch])
+            throw new Error('Error: Invalid batch in Dip.insert()');
+        batches[meta.batch].push(arg);
+        return;
+    }
+    return new Promise((resolve, reject) => resolve())
+        .then(() => execute(arg))
+        .then(result => {
+        Events.send("Dip.insert", { collection: collectionArray(collection), response: result });
+        return result;
+    });
+};
+// Ported
+export const query = async (meta, collection, query, options, extras) => {
+    if (!(meta instanceof DipMeta))
+        console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.query()'); // TODO: throw error once review completes
+    validate_collection(collection, 'in Dip.query()');
+    validate_meta(meta, 'in Dip.query()');
+    const topology_meta = validate_topology(meta, collection, 'in Dip.query()'); // TODO: loop for arrays
+    let arg = {
+        collection: collection,
+        query: query,
+        options: options ? options : {},
+        syncMode: meta.syncMode,
+        ...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
+        ...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
+    };
+    arg = Object.assign(arg, extras);
+    let suffix_from_policy = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.query, false, arg);
+    let suffix_from_meta = generate_suffix(meta);
+    arg.suffix = (suffix_from_meta.length ? suffix_from_meta : [""]).flatMap(meta => (suffix_from_policy.length ? suffix_from_policy : [""]).map(policy => meta + policy));
+    arg.executor = meta.executor ?? "native";
+    arg = Object.assign({}, topology_meta, arg);
+    if (meta.batch) {
+        if (!batches[meta.batch])
+            throw new Error('Error: Invalid batch in Dip.query()');
+        batches[meta.batch].push(arg);
+        return;
+    }
+    return new Promise((resolve, reject) => resolve())
+        .then(() => execute(arg));
+};
+function collectionArray(col) {
+    let type = Object.prototype.toString.call(col);
+    if (type === '[object Array]') {
+        let newcol = [];
+        for (let i in col) {
+            if (!Utils.isEmpty(col[i]))
+                newcol.push(col[i]);
+        }
+        return newcol;
+    }
+    else if (type === '[object String]') {
+        if (!Utils.isEmpty(col))
+            return [col];
+    }
+    return [];
+}
+// Ported
+export const update = async (meta, collection, query, update, options, extras) => {
+    if (!(meta instanceof DipMeta))
+        console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.update()'); // TODO: throw error once review completes
+    validate_collection(collection, 'in Dip.update()');
+    validate_meta(meta, 'in Dip.update()');
+    const topology_meta = validate_topology(meta, collection, 'in Dip.update()'); // TODO: loop for arrays
+    let arg = {
+        collection: collection,
+        query: query,
+        update: update,
+        options: options ? options : {},
+        syncMode: meta.syncMode,
+        ...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
+        ...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
+    };
+    arg = Object.assign(arg, extras);
+    let suffix_from_policy = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.query, false, arg);
+    let suffix_from_meta = generate_suffix(meta);
+    arg.suffix = (suffix_from_meta.length ? suffix_from_meta : [""]).flatMap(meta => (suffix_from_policy.length ? suffix_from_policy : [""]).map(policy => meta + policy));
+    arg.executor = meta.executor ?? "native";
+    arg = Object.assign({}, topology_meta, arg);
+    if (meta.batch) {
+        if (!batches[meta.batch])
+            throw new Error('Error: Invalid batch in Dip.update()');
+        batches[meta.batch].push(arg);
+        return;
+    }
+    return new Promise((resolve, reject) => resolve())
+        .then(() => execute(arg))
+        .then(result => {
+        Events.send("Dip.update", { collection: collectionArray(collection), response: result });
+        return result;
+    });
+};
+// Ported
+export const remove = async (meta, collection, query, options, extras) => {
+    if (!(meta instanceof DipMeta))
+        console.warn('Warn: Provided meta is not an instance of DipMeta in Dip.remove()'); // TODO: throw error once review completes
+    validate_collection(collection, 'in Dip.remove()');
+    if (Utils.isEmpty(query._id) && !meta.allowDangerousRemove)
+        throw new Error("Error: Dip.remove() without _id blocked by default (Dangerous Operation). Set meta.allowDangerousRemove if intentional.");
+    validate_meta(meta, 'in Dip.remove()');
+    const topology_meta = validate_topology(meta, collection, 'in Dip.remove()'); // TODO: loop for arrays
+    let arg = {
+        collection: collection,
+        query: query,
+        options: options ? options : {},
+        delete: true,
+        syncMode: meta.syncMode,
+        ...(meta.DIP_URL ? { DIP_URL: meta.DIP_URL } : {}),
+        ...(meta.DIP_DB ? { db: meta.DIP_DB } : {}),
+    };
+    arg = Object.assign(arg, extras);
+    let suffix_from_policy = await applySuffixPolicy(COLLECTIONS_JSON, collection, arg.query, false, arg);
+    let suffix_from_meta = generate_suffix(meta);
+    arg.suffix = (suffix_from_meta.length ? suffix_from_meta : [""]).flatMap(meta => (suffix_from_policy.length ? suffix_from_policy : [""]).map(policy => meta + policy));
+    arg.executor = meta.executor ?? "native";
+    arg = Object.assign({}, topology_meta, arg);
+    if (meta.batch) {
+        if (!batches[meta.batch])
+            throw new Error('Error: Invalid batch in Dip.query()');
+        batches[meta.batch].push(arg);
+        return;
+    }
+    return new Promise((resolve, reject) => resolve())
+        .then(() => execute(arg))
+        .then(result => {
+        Events.send("Dip.remove", { collection: collectionArray(collection), response: result });
+        return result;
+    });
+};
+export let config = {
+    useDipper: false
+};
+function execute(arg) {
+    //         arg example
+    //        {
+    //            db: "middle-earth",
+    //            collection: "staff",
+    //            query: "[@][?name == 'Bilbo'] | [0]",
+    //            insert: { "name": "Bilbo" },
+    //            query: { "name": "Frodo"},
+    //            update: { "$set": { "age": 21 } }
+    //            _id: 1 // Optional. Used with insert
+    //        }
+    arg.db = arg.db ?? db;
+    arg.executor = arg.executor ?? "native";
+    arg.engine = arg.engine ?? internal.engineName;
+    if (arg.compression === undefined)
+        arg.compression = true; // Compress all for now by default
+    let fullurl = `${url}:${arg?.port ?? port}/${api}`;
+    delete arg.port;
+    const timeout = arg.timeout;
+    delete arg.timeout;
+    return config.useDipper ? dipper(arg)
+        : axios
+            .post(arg.DIP_URL ?? fullurl, buildHybridRequest({ ...arg, DIP_URL: undefined }), { responseType: 'arraybuffer', timeout: timeout ?? 5000, headers: { "Content-Type": "application/dip" } })
+            .then(result => {
+            // console.log(res.status);
+            // console.log(JSON.stringify(res.header, null, 4));
+            // console.log(JSON.stringify(res.body, null, 4));
+            result.getSliceAsText = (offset, length) => Cpp.getSliceAsText(result.data, offset, length);
+            result.getSliceAsArrayBuffer = (offset, length) => Cpp.getSliceAsArrayBuffer(result.data, offset, length);
+            result["Content-Type"] = result.headers["content-type"];
+            result.body = result.data;
+            if (result.headers["content-type"].includes("application/json"))
+                result.body = JSON.parse(result.data);
+            let parsed = parseBinaryResponse(result);
+            let response = prepareResult(parsed);
+            return response;
+        });
+    //       .catch(err => { console.log(err.message); console.log(err.response); return err});
+}
+let Essentials = {
+    vlqToUint64: Cpp.vlqToUint64,
+    sliceArrayBuffer: Cpp.sliceArrayBuffer,
+    stringToUtf8ArrayBuffer: Cpp.stringToUtf8ArrayBuffer,
+    numberToSortableBytes: Cpp.numberToSortableBytes,
+    defaultHash: Cpp.defaultHash,
+    memcmpEqual: Cpp.memcmpEqual,
+    uint64ToVlq: Cpp.uint64ToVlq,
+    arrayBufferToString: Cpp.arrayBufferToString,
+};
+function prepareResult(result) {
+    let res = result.content?.length === 1 ? result.content[0].items : result.content; // For raw, this resolves to result.content which is ArrayBuffer
+    res.metadata = _ => result.metadata;
+    if (result.metadata.executor === "raw")
+        setRawBatchIterator(res);
+    return res;
+}
+function parseBinaryResponse(result) {
+    const contentType = (result["Content-Type"] || "").toLowerCase();
+    if (contentType.includes("application/json")) {
+        return { metadata: result.body, content: result.body.items };
+    }
+    if (!contentType.includes("application/dip")) {
+        throw new Error("Error: Dip unsupported media. Invalid content-type.");
+    }
+    const arrayBuffer = result.body;
+    // let wire = parseWireFormat(arrayBuffer)
+    // let meta = wireFormat.metadata
+    // let content = wireFormat.content
+    // try {
+    //     meta = JSON.parse(meta);
+    // } catch (_) {
+    //     throw {contentType, meta, content, message: "Error: Dip failed to parse metadata. Unsupported response format. Response format does not match content-type."}
+    // }
+    // parseWireFormat() equivalent
+    // =============================
+    let dataView = new DataView(arrayBuffer.buffer, arrayBuffer.byteOffset, arrayBuffer.byteLength);
+    let offset = 0;
+    // 1. Read the metadata size header
+    let vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
+    let metaSize = vlq.value;
+    offset += vlq.bytes;
+    // 2. Extract and parse metadata string via C++ instantly
+    let metaText = result.getSliceAsText(offset, metaSize);
+    let nativeResponse = JSON.parse(metaText);
+    offset += metaSize;
+    // Extract everything remaining automatically by passing -1
+    let content = result.getSliceAsArrayBuffer(offset, -1); // content is already ArrayBuffer but QByteArray to ArrayBuffer cannot be Uint8Array() wrapped
+    // content = new Uint8Array(content ?? new ArrayBuffer(0))
+    return { metadata: nativeResponse, content: content };
+}
+function setRawBatchIterator(arrayBuffer) {
+    arrayBuffer.metadata().items.forEach(batch => {
+        batch.kv = _ => {
+            const batchItem = {
+                count: batch.count,
+                slice: Essentials.sliceArrayBuffer(arrayBuffer, batch.offset, batch.size),
+                [Symbol.iterator]() {
+                    let index = 0;
+                    let offset = 0;
+                    return {
+                        next: () => {
+                            if (index < this.count) {
+                                let kv = getRawKeyValue(this.slice, offset);
+                                offset = kv.next;
+                                index++;
+                                return { value: kv, done: false };
+                            }
+                            return { done: true };
+                        }
+                    };
+                }
+            };
+            return batchItem;
+        };
+    });
+    arrayBuffer.items = arrayBuffer.metadata().items;
+}
+// ===============
+function convert_to_array_buffer(value) {
+    if (typeof value === "string")
+        return Essentials.stringToUtf8ArrayBuffer(value);
+    else if (typeof value === "number")
+        return Essentials.numberToSortableBytes(value);
+    return Essentials.stringToUtf8ArrayBuffer(JSON.stringify(value));
+}
+export function newRaw() {
+    let obj = { buffers: [], size: 0, hashes: {} };
+    obj.add = value => {
+        if (value === undefined)
+            return undefined;
+        const buf = (value instanceof ArrayBuffer) ? value : convert_to_array_buffer(value);
+        const hash = Essentials.defaultHash(buf);
+        for (let item of obj.hashes[hash] ?? []) { // Conflict. Fallback to verify
+            if (Essentials.memcmpEqual(buf, item.buf))
+                return item.offset;
+        }
+        const vlq = new Uint8Array(Essentials.uint64ToVlq(buf.byteLength));
+        obj.buffers.push({ buf, vlq });
+        let offset = obj.size;
+        obj.size += vlq.byteLength + buf.byteLength;
+        obj.hashes[hash] = obj.hashes[hash] ?? [];
+        obj.hashes[hash].push({ offset, buf });
+        return offset;
+    };
+    return obj;
+}
+function buildHybridRequest(request) {
+    const batch = request.batch ?? [request];
+    let raw = request.raw ?? newRaw();
+    request.raw = undefined;
+    // Pass 1: Direct offset assignment & exact size calculation (Zero allocation)
+    for (let i = 0; i < batch.length; i++) {
+        const item = batch[i];
+        delete item.batch;
+        if (item.update) {
+            if (item._id === undefined && item.options?.upsert && item.query?._id !== undefined)
+                item._id = item.query._id;
+            item.update = raw.add(item.update);
+        }
+        else if (item.insert) {
+            if (item._id === undefined && item.insert?._id !== undefined)
+                item._id = item.insert._id;
+            item.insert = raw.add(item.insert);
+        }
+        item._id = raw.add(item._id);
+        if (item.options?.$rangeFrom !== undefined)
+            item.options.$rangeFrom = raw.add(item.options.$rangeFrom);
+        if (item.options?.$rangeTo !== undefined)
+            item.options.$rangeTo = raw.add(item.options.$rangeTo);
+    }
+    // Pass 2: Generate the final JSON payload string
+    const jsonBuffer = convert_to_array_buffer(request);
+    const jsonByteLen = jsonBuffer.byteLength;
+    // Pass 3: Allocate the exact Master Buffer ONCE
+    let vlq = new Uint8Array(Essentials.uint64ToVlq(jsonByteLen));
+    const totalSize = vlq.byteLength + jsonByteLen + raw.size;
+    const masterBuffer = new ArrayBuffer(totalSize);
+    const masterView = new Uint8Array(masterBuffer);
+    // Write the JSON size prefix
+    masterView.set(vlq, 0);
+    // Direct copy of JSON string bytes into their final destination
+    masterView.set(new Uint8Array(jsonBuffer), vlq.byteLength);
+    // Copy binary payloads directly into their final destination window
+    let writeOffset = vlq.byteLength + jsonByteLen;
+    for (const buffer of raw.buffers) {
+        const buf = buffer.buf;
+        const vlq = buffer.vlq;
+        // Write length prefix directly into final master buffer
+        masterView.set(vlq, writeOffset);
+        writeOffset += vlq.byteLength;
+        // Blit bytes directly from source into final master buffer destination
+        masterView.set(new Uint8Array(buf), writeOffset);
+        writeOffset += buf.byteLength;
+    }
+    return masterBuffer;
+}
+// Usage getRawKeyValue()
+// Dip.query()
+// .then(r => {
+//           let count = r.metadata().count
+//           let offset = 0
+//           for (let i = 0; i < count; i++) {
+//               let item = getKeyValue(r, offset)
+//               offset = item.next
+//               let res = Utils.arrayBufferToFile(item.data, '/home/khayaali/Downloads/zdip.png')
+//               console.log(item._id,res)
+//           }
+//       })
+function bufferToString(arrayBuffer) {
+    return Essentials.arrayBufferToString(arrayBuffer);
+}
+function getRawKeyValue(arrayBuffer, offset) {
+    let dataView = new DataView(arrayBuffer.buffer, arrayBuffer.byteOffset, arrayBuffer.byteLength);
+    let masterView = new Uint8Array(arrayBuffer);
+    // 1. Extract Key Size and Key
+    let vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
+    let keySize = vlq.value;
+    offset += vlq.bytes;
+    // Zero-copy view window bounded strictly to the key bytes
+    let _id = masterView.subarray(offset, offset + keySize);
+    let _id_string;
+    _id.toText = _ => {
+        // Fix: Pass the typed array view itself, NOT the underlying root buffer
+        _id_string = _id_string === undefined ? _id.buffer.slice(_id.byteOffset, _id.byteOffset + _id.byteLength) : _id_string;
+        return bufferToString(_id_string);
+    };
+    offset += keySize;
+    // 2. Extract Data Size and Data Binary Payload
+    vlq = Essentials.vlqToUint64(Essentials.sliceArrayBuffer(arrayBuffer, offset, 10));
+    let dataSize = vlq.value;
+    offset += vlq.bytes;
+    // Optimization: Use subarray() instead of .slice() to make the data payload 100% zero-copy too
+    let _val = masterView.subarray(offset, offset + dataSize);
+    offset += dataSize;
+    _val = Essentials.sliceArrayBuffer(arrayBuffer, _val.byteOffset, _val.byteLength);
+    // _val = _val.buffer.slice(_val.byteOffset, _val.byteOffset + _val.byteLength);
+    _val.toText = _ => bufferToString(_val);
+    return { _id, _val, next: offset };
+}
+// Axios usage
+//--------------
+// const axios = require('axios')
+// axios.get('http://www.floatrates.com/daily/usd.json').then(data => console.log(data)).catch(error=> console.log(error))