min-mphash 0.5.0

package/dist/index.js

@@ -0,0 +1,937 @@
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function writeVarInt(val, buffer, offset) {
+    let bytes = 0;
+    while(val >= 0x80){
+        const b = 0x7f & val | 0x80;
+        if (Array.isArray(buffer)) buffer.push(b);
+        else if (void 0 !== offset) buffer[offset + bytes] = b;
+        val >>>= 7;
+        bytes++;
+    }
+    if (Array.isArray(buffer)) buffer.push(val);
+    else if (void 0 !== offset) buffer[offset + bytes] = val;
+    return bytes + 1;
+}
+function readVarInt(buffer, offset) {
+    let val = 0;
+    let shift = 0;
+    let bytes = 0;
+    while(true){
+        const b = buffer[offset + bytes];
+        val |= (0x7f & b) << shift;
+        bytes++;
+        if ((0x80 & b) === 0) break;
+        shift += 7;
+    }
+    return {
+        value: val,
+        bytes
+    };
+}
+const CBOR = {
+    encodeInt (val, buffer) {
+        const major = 0x00;
+        if (val < 24) buffer.push(major | val);
+        else if (val <= 0xff) buffer.push(24 | major, val);
+        else if (val <= 0xffff) buffer.push(25 | major, val >> 8, 0xff & val);
+        else buffer.push(26 | major, val >>> 24 & 0xff, val >>> 16 & 0xff, val >>> 8 & 0xff, 0xff & val);
+    },
+    encodeBytes (bytes, buffer) {
+        const major = 0x40;
+        const len = bytes.byteLength;
+        if (len < 24) buffer.push(major | len);
+        else if (len <= 0xff) buffer.push(24 | major, len);
+        else if (len <= 0xffff) buffer.push(25 | major, len >> 8, 0xff & len);
+        else buffer.push(26 | major, len >>> 24 & 0xff, len >>> 16 & 0xff, len >>> 8 & 0xff, 0xff & len);
+        for(let i = 0; i < len; i++)buffer.push(bytes[i]);
+    },
+    encodeNull (buffer) {
+        buffer.push(0xf6);
+    },
+    encodeArrayHead (len, buffer) {
+        const major = 0x80;
+        if (len < 24) buffer.push(major | len);
+    },
+    decode (view, offsetRef) {
+        const byte = view.getUint8(offsetRef.current++);
+        const major = 0xe0 & byte;
+        const additional = 0x1f & byte;
+        let val = 0;
+        if (additional < 24) val = additional;
+        else if (24 === additional) {
+            val = view.getUint8(offsetRef.current);
+            offsetRef.current += 1;
+        } else if (25 === additional) {
+            val = view.getUint16(offsetRef.current, false);
+            offsetRef.current += 2;
+        } else if (26 === additional) {
+            val = view.getUint32(offsetRef.current, false);
+            offsetRef.current += 4;
+        } else throw new Error("Unsupported CBOR size");
+        if (0x00 === major) return val;
+        if (0x40 === major) {
+            const len = val;
+            const buf = new Uint8Array(view.buffer.slice(view.byteOffset + offsetRef.current, view.byteOffset + offsetRef.current + len));
+            offsetRef.current += len;
+            return buf;
+        }
+        if (0x80 === major) {
+            const len = val;
+            const arr = [];
+            for(let i = 0; i < len; i++)arr.push(CBOR.decode(view, offsetRef));
+            return arr;
+        }
+        if (0xf6 === byte) return null;
+        throw new Error(`Unknown CBOR type: ${byte.toString(16)}`);
+    }
+};
+const MODE_TO_INT = {
+    none: 0,
+    4: 1,
+    8: 2,
+    16: 3,
+    32: 4,
+    2: 5
+};
+const INT_TO_MODE = [
+    "none",
+    "4",
+    "8",
+    "16",
+    "32",
+    "2"
+];
+function dictToCBOR(dict) {
+    const buffer = [];
+    CBOR.encodeArrayHead(9, buffer);
+    CBOR.encodeInt(dict.n, buffer);
+    CBOR.encodeInt(dict.m, buffer);
+    CBOR.encodeInt(dict.seed0, buffer);
+    CBOR.encodeBytes(dict.bucketSizes, buffer);
+    CBOR.encodeBytes(dict.seedStream, buffer);
+    CBOR.encodeInt(MODE_TO_INT[dict.validationMode], buffer);
+    if (dict.fingerprints && "none" !== dict.validationMode) {
+        let fpBytes;
+        if (dict.fingerprints instanceof Uint8Array) fpBytes = dict.fingerprints;
+        else if (dict.fingerprints instanceof Uint16Array || dict.fingerprints instanceof Uint32Array) fpBytes = new Uint8Array(dict.fingerprints.buffer, dict.fingerprints.byteOffset, dict.fingerprints.byteLength);
+        else {
+            const mode = dict.validationMode;
+            let typed;
+            typed = "2" === mode || "4" === mode || "8" === mode ? new Uint8Array(dict.fingerprints) : "16" === mode ? new Uint16Array(dict.fingerprints) : new Uint32Array(dict.fingerprints);
+            fpBytes = new Uint8Array(typed.buffer, typed.byteOffset, typed.byteLength);
+        }
+        CBOR.encodeBytes(fpBytes, buffer);
+    } else CBOR.encodeNull(buffer);
+    if (dict.seedZeroBitmap) CBOR.encodeBytes(dict.seedZeroBitmap, buffer);
+    else CBOR.encodeNull(buffer);
+    CBOR.encodeInt(dict.hashSeed || 0, buffer);
+    return new Uint8Array(buffer);
+}
+function dictFromCBOR(bin) {
+    const view = new DataView(bin.buffer, bin.byteOffset, bin.byteLength);
+    const offsetRef = {
+        current: 0
+    };
+    const arr = CBOR.decode(view, offsetRef);
+    if (!Array.isArray(arr) || arr.length < 7) throw new Error("Invalid CBOR format");
+    const [n, m, seed0, bucketSizes, seedStream, modeInt, fpRaw, seedZeroBitmap, hashSeed] = arr;
+    const validationMode = INT_TO_MODE[modeInt] || "none";
+    let fingerprints;
+    if (fpRaw && "none" !== validationMode) {
+        if ("2" === validationMode || "4" === validationMode || "8" === validationMode) fingerprints = fpRaw;
+        else if ("16" === validationMode) fingerprints = new Uint16Array(fpRaw.buffer, fpRaw.byteOffset, fpRaw.byteLength / 2);
+        else if ("32" === validationMode) fingerprints = new Uint32Array(fpRaw.buffer, fpRaw.byteOffset, fpRaw.byteLength / 4);
+    }
+    return {
+        n,
+        m,
+        seed0,
+        hashSeed: hashSeed || 0,
+        bucketSizes,
+        seedStream,
+        validationMode,
+        fingerprints,
+        seedZeroBitmap: seedZeroBitmap || void 0
+    };
+}
+async function compressIBinary(data) {
+    const stream = new Blob([
+        data
+    ]).stream().pipeThrough(new CompressionStream("gzip"));
+    return new Uint8Array(await new Response(stream).arrayBuffer());
+}
+async function decompressIBinary(data) {
+    const stream = new Blob([
+        data
+    ]).stream().pipeThrough(new DecompressionStream("gzip"));
+    return new Uint8Array(await new Response(stream).arrayBuffer());
+}
+class BitWriter {
+    write(value, bits) {
+        for(let i = 0; i < bits; i++){
+            const bit = value >> i & 1;
+            this.currentByte |= bit << this.bitCount;
+            this.bitCount++;
+            if (8 === this.bitCount) {
+                this.buffer.push(this.currentByte);
+                this.currentByte = 0;
+                this.bitCount = 0;
+            }
+        }
+    }
+    flush() {
+        if (this.bitCount > 0) {
+            this.buffer.push(this.currentByte);
+            this.currentByte = 0;
+            this.bitCount = 0;
+        }
+    }
+    getData() {
+        this.flush();
+        return new Uint8Array(this.buffer);
+    }
+    constructor(){
+        _define_property(this, "buffer", []);
+        _define_property(this, "currentByte", 0);
+        _define_property(this, "bitCount", 0);
+    }
+}
+class BitReader {
+    read(bits) {
+        let value = 0;
+        for(let i = 0; i < bits; i++){
+            if (this.byteOffset >= this.buffer.length) return 0;
+            const bit = this.buffer[this.byteOffset] >> this.bitOffset & 1;
+            value |= bit << i;
+            this.bitOffset++;
+            if (8 === this.bitOffset) {
+                this.byteOffset++;
+                this.bitOffset = 0;
+            }
+        }
+        return value;
+    }
+    constructor(buffer){
+        _define_property(this, "buffer", void 0);
+        _define_property(this, "byteOffset", 0);
+        _define_property(this, "bitOffset", 0);
+        this.buffer = buffer;
+    }
+}
+function readBitsAt(buffer, bitOffset, bitLength) {
+    let value = 0;
+    let currentBit = bitOffset;
+    for(let i = 0; i < bitLength; i++){
+        const byteIdx = currentBit >>> 3;
+        const bitIdx = 7 & currentBit;
+        if (byteIdx >= buffer.length) return 0;
+        const bit = buffer[byteIdx] >> bitIdx & 1;
+        value |= bit << i;
+        currentBit++;
+    }
+    return value;
+}
+function MinMPHash_define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function createMinMPHashDict(dataSet, options) {
+    const n = dataSet.length;
+    if (0 === n) {
+        const emptyDict = {
+            n: 0,
+            m: 0,
+            seed0: 0,
+            seedStream: new Uint8Array(0),
+            bucketSizes: new Uint8Array(0),
+            validationMode: "none"
+        };
+        return options?.outputBinary ? dictToCBOR(emptyDict) : emptyDict;
+    }
+    const targetRate = options?.level ?? 5.0;
+    let validationMode = "none";
+    if (options?.onlySet === true) validationMode = "8";
+    else if ("string" == typeof options?.onlySet) validationMode = options.onlySet;
+    const adjustedRate = n > 500000 ? Math.max(1, 0.90 * targetRate) : targetRate;
+    const m = Math.max(1, Math.ceil(n / adjustedRate));
+    const hashesL = new Uint32Array(n);
+    const hashesH = new Uint32Array(n);
+    let hashSeed = 0;
+    while(true){
+        const seen = new Map();
+        const complexSeen = new Map();
+        let collision = false;
+        for(let i = 0; i < n; i++){
+            const h1 = murmurHash3_32(dataSet[i], hashSeed);
+            const h2 = murmurHash3_32(dataSet[i], ~hashSeed);
+            let isDuplicate = false;
+            if (complexSeen.has(h1)) {
+                const set = complexSeen.get(h1);
+                if (set.has(h2)) isDuplicate = true;
+                else set.add(h2);
+            } else if (seen.has(h1)) {
+                const existingH2 = seen.get(h1);
+                if (existingH2 === h2) isDuplicate = true;
+                else {
+                    const set = new Set();
+                    set.add(existingH2);
+                    set.add(h2);
+                    complexSeen.set(h1, set);
+                    seen.delete(h1);
+                }
+            } else seen.set(h1, h2);
+            if (isDuplicate) {
+                collision = true;
+                break;
+            }
+            hashesL[i] = h1;
+            hashesH[i] = h2;
+        }
+        if (!collision) break;
+        hashSeed++;
+        if (hashSeed > 100) throw new Error(`Could not find a collision-free hash seed after ${hashSeed} attempts.`);
+    }
+    let bestHead = new Int32Array(m).fill(-1);
+    let bestNext = new Int32Array(n).fill(-1);
+    let bestSeed0 = 0;
+    let minMaxLen = 1 / 0;
+    const currentHead = new Int32Array(m);
+    const currentNext = new Int32Array(n);
+    const bucketCounts = new Int32Array(m);
+    const maxAttempts = 2000;
+    for(let attempt = 0; attempt < maxAttempts; attempt++){
+        const currentSeed = Math.floor(0xffffffff * Math.random());
+        currentHead.fill(-1);
+        bucketCounts.fill(0);
+        let currentMaxLen = 0;
+        for(let i = 0; i < n; i++){
+            const h = (scramble(hashesL[i], currentSeed) ^ hashesH[i]) >>> 0;
+            const bIdx = Math.floor(h / 4294967296 * m);
+            currentNext[i] = currentHead[bIdx];
+            currentHead[bIdx] = i;
+            bucketCounts[bIdx]++;
+            if (bucketCounts[bIdx] > currentMaxLen) currentMaxLen = bucketCounts[bIdx];
+        }
+        if (currentMaxLen < 13) {
+            bestSeed0 = currentSeed;
+            bestHead.set(currentHead);
+            bestNext.set(currentNext);
+            minMaxLen = currentMaxLen;
+            break;
+        }
+        if (currentMaxLen < minMaxLen) {
+            minMaxLen = currentMaxLen;
+            bestSeed0 = currentSeed;
+            bestHead.set(currentHead);
+            bestNext.set(currentNext);
+        }
+        if (minMaxLen < 16 && attempt > 50) break;
+    }
+    if (minMaxLen >= 16) throw new Error(`MPHF Build Failed: Could not find a bucket distribution with max size < 16 (best: ${minMaxLen}). Try reducing the optimization level (current: ${options?.level ?? 5}).`);
+    const bucketSizes = new Uint8Array(Math.ceil(m / 2));
+    for(let i = 0; i < m; i++){
+        let count = 0;
+        let ptr = bestHead[i];
+        while(-1 !== ptr){
+            count++;
+            ptr = bestNext[ptr];
+        }
+        const byteIdx = i >>> 1;
+        if ((1 & i) === 0) bucketSizes[byteIdx] |= count;
+        else bucketSizes[byteIdx] |= count << 4;
+    }
+    const seedWriter = new VarIntBuffer();
+    const seedZeroBitmap = new Uint8Array(Math.ceil(m / 8));
+    for(let i = 0; i < m; i++){
+        let k = 0;
+        let p = bestHead[i];
+        while(-1 !== p){
+            k++;
+            p = bestNext[p];
+        }
+        if (k <= 1) {
+            seedZeroBitmap[i >>> 3] |= 1 << (7 & i);
+            continue;
+        }
+        let s = 0;
+        let found = false;
+        const MAX_TRIALS = k > 14 ? 50000000 : 5000000;
+        while(!found){
+            let visited = 0;
+            let collision = false;
+            let ptr = bestHead[i];
+            while(-1 !== ptr){
+                const h = (scramble(hashesL[ptr], s) ^ hashesH[ptr]) >>> 0;
+                const pos = h % k;
+                if ((visited & 1 << pos) !== 0) {
+                    collision = true;
+                    break;
+                }
+                visited |= 1 << pos;
+                ptr = bestNext[ptr];
+            }
+            if (collision) {
+                s++;
+                if (s > MAX_TRIALS) throw new Error(`MPHF Failed: Bucket ${i} (size ${k}) is too hard.`);
+            } else {
+                if (0 === s) seedZeroBitmap[i >>> 3] |= 1 << (7 & i);
+                else seedWriter.write(s);
+                found = true;
+            }
+        }
+    }
+    const dict = {
+        n,
+        m,
+        seed0: bestSeed0,
+        hashSeed,
+        seedStream: seedWriter.toUint8Array(),
+        bucketSizes,
+        seedZeroBitmap,
+        validationMode
+    };
+    if ("none" !== validationMode) {
+        let fingerprints;
+        fingerprints = "2" === validationMode ? new Uint8Array(Math.ceil(n / 4)) : "4" === validationMode ? new Uint8Array(Math.ceil(n / 2)) : "8" === validationMode ? new Uint8Array(n) : "16" === validationMode ? new Uint16Array(n) : new Uint32Array(n);
+        const tempHasher = new MinMPHash({
+            ...dict,
+            validationMode: "none"
+        });
+        const FP_SEED = 0x1234abcd;
+        for(let i = 0; i < n; i++){
+            const key = dataSet[i];
+            const idx = tempHasher.hash(key);
+            if (idx >= 0 && idx < n) {
+                const fullHash = murmurHash3_32(key, FP_SEED);
+                if ("2" === validationMode) {
+                    const fp2 = 0x03 & fullHash;
+                    const byteIdx = idx >>> 2;
+                    const shift = (3 & idx) << 1;
+                    fingerprints[byteIdx] |= fp2 << shift;
+                } else if ("4" === validationMode) {
+                    const fp4 = 0x0f & fullHash;
+                    const byteIdx = idx >>> 1;
+                    if ((1 & idx) === 0) fingerprints[byteIdx] |= fp4;
+                    else fingerprints[byteIdx] |= fp4 << 4;
+                } else if ("8" === validationMode) fingerprints[idx] = 0xff & fullHash;
+                else if ("16" === validationMode) fingerprints[idx] = 0xffff & fullHash;
+                else fingerprints[idx] = fullHash >>> 0;
+            }
+        }
+        dict.fingerprints = fingerprints;
+    }
+    if (options?.outputBinary) {
+        const binary = dictToCBOR(dict);
+        if (options.enableCompression) return compressIBinary(binary);
+        return binary;
+    }
+    return dict;
+}
+class MinMPHash {
+    static async fromCompressed(data) {
+        const decompressed = await decompressIBinary(data);
+        return new MinMPHash(decompressed);
+    }
+    hash(input) {
+        if (0 === this.n) return -1;
+        const h1 = murmurHash3_32(input, this.hashSeed);
+        const h2 = murmurHash3_32(input, ~this.hashSeed);
+        const h0 = (scramble(h1, this.seed0) ^ h2) >>> 0;
+        const bIdx = Math.floor(h0 / 4294967296 * this.m);
+        const offset = this.offsets[bIdx];
+        const nextOffset = this.offsets[bIdx + 1];
+        const bucketSize = nextOffset - offset;
+        if (0 === bucketSize) return -1;
+        let resultIdx = 0;
+        if (1 === bucketSize) resultIdx = offset;
+        else {
+            const s = this.seeds[bIdx];
+            const h = (scramble(h1, s) ^ h2) >>> 0;
+            resultIdx = offset + h % bucketSize;
+        }
+        if ("none" !== this.validationMode && this.fingerprints) {
+            const fpHash = murmurHash3_32(input, MinMPHash.FP_SEED);
+            if ("2" === this.validationMode) {
+                const expectedFp2 = 0x03 & fpHash;
+                const byteIdx = resultIdx >>> 2;
+                const shift = (3 & resultIdx) << 1;
+                if ((this.fingerprints[byteIdx] >>> shift & 0x03) !== expectedFp2) return -1;
+            } else if ("4" === this.validationMode) {
+                const expectedFp4 = 0x0f & fpHash;
+                const byteIdx = resultIdx >>> 1;
+                const storedByte = this.fingerprints[byteIdx];
+                const storedFp4 = (1 & resultIdx) === 0 ? 0x0f & storedByte : storedByte >>> 4 & 0x0f;
+                if (storedFp4 !== expectedFp4) return -1;
+            } else if ("8" === this.validationMode) {
+                if (this.fingerprints[resultIdx] !== (0xff & fpHash)) return -1;
+            } else if ("16" === this.validationMode) {
+                if (this.fingerprints[resultIdx] !== (0xffff & fpHash)) return -1;
+            } else if (this.fingerprints[resultIdx] !== fpHash >>> 0) return -1;
+        }
+        return resultIdx;
+    }
+    constructor(dict){
+        MinMPHash_define_property(this, "n", void 0);
+        MinMPHash_define_property(this, "m", void 0);
+        MinMPHash_define_property(this, "seed0", void 0);
+        MinMPHash_define_property(this, "hashSeed", void 0);
+        MinMPHash_define_property(this, "offsets", void 0);
+        MinMPHash_define_property(this, "seeds", void 0);
+        MinMPHash_define_property(this, "validationMode", void 0);
+        MinMPHash_define_property(this, "fingerprints", null);
+        if (dict instanceof Uint8Array) dict = dictFromCBOR(dict);
+        this.n = dict.n;
+        this.m = dict.m;
+        this.seed0 = dict.seed0;
+        this.hashSeed = dict.hashSeed || 0;
+        this.validationMode = dict.validationMode || "none";
+        if (0 === this.n) {
+            this.offsets = new Uint32Array(0);
+            this.seeds = new Int32Array(0);
+            return;
+        }
+        this.offsets = new Uint32Array(this.m + 1);
+        let currentOffset = 0;
+        for(let i = 0; i < this.m; i++){
+            this.offsets[i] = currentOffset;
+            const byte = dict.bucketSizes[i >>> 1];
+            const len = 1 & i ? byte >>> 4 : 0x0f & byte;
+            currentOffset += len;
+        }
+        this.offsets[this.m] = currentOffset;
+        this.seeds = new Int32Array(this.m);
+        let ptr = 0;
+        const buf = dict.seedStream;
+        const bitmap = dict.seedZeroBitmap;
+        for(let i = 0; i < this.m; i++){
+            let isZero = false;
+            if (bitmap) {
+                if ((bitmap[i >>> 3] & 1 << (7 & i)) !== 0) isZero = true;
+            }
+            if (isZero) this.seeds[i] = 0;
+            else {
+                let result = 0;
+                let shift = 0;
+                while(true){
+                    const byte = buf[ptr++];
+                    result |= (0x7f & byte) << shift;
+                    if ((0x80 & byte) === 0) break;
+                    shift += 7;
+                }
+                this.seeds[i] = result;
+            }
+        }
+        if ("none" !== this.validationMode && dict.fingerprints) {
+            const raw = dict.fingerprints;
+            if ("2" === this.validationMode || "4" === this.validationMode || "8" === this.validationMode) this.fingerprints = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
+            else if ("16" === this.validationMode) this.fingerprints = raw instanceof Uint16Array ? raw : new Uint16Array(raw);
+            else this.fingerprints = raw instanceof Uint32Array ? raw : new Uint32Array(raw);
+        }
+    }
+}
+MinMPHash_define_property(MinMPHash, "FP_SEED", 0x1234abcd);
+function scramble(k, seed) {
+    k ^= seed;
+    k = Math.imul(k, 0x85ebca6b);
+    k ^= k >>> 13;
+    k = Math.imul(k, 0xc2b2ae35);
+    k ^= k >>> 16;
+    return k >>> 0;
+}
+function murmurHash3_32(key, seed) {
+    let h1 = seed;
+    const c1 = 0xcc9e2d51;
+    const c2 = 0x1b873593;
+    for(let i = 0; i < key.length; i++){
+        let k1 = key.charCodeAt(i);
+        k1 = Math.imul(k1, c1);
+        k1 = k1 << 15 | k1 >>> 17;
+        k1 = Math.imul(k1, c2);
+        h1 ^= k1;
+        h1 = h1 << 13 | h1 >>> 19;
+        h1 = Math.imul(h1, 5) + 0xe6546b64;
+    }
+    h1 ^= key.length;
+    h1 ^= h1 >>> 16;
+    h1 = Math.imul(h1, 0x85ebca6b);
+    h1 ^= h1 >>> 13;
+    h1 = Math.imul(h1, 0xc2b2ae35);
+    h1 ^= h1 >>> 16;
+    return h1 >>> 0;
+}
+class VarIntBuffer {
+    write(value) {
+        while(value >= 0x80){
+            this.buffer.push(0x7f & value | 0x80);
+            value >>>= 7;
+        }
+        this.buffer.push(value);
+    }
+    toUint8Array() {
+        return new Uint8Array(this.buffer);
+    }
+    constructor(){
+        MinMPHash_define_property(this, "buffer", []);
+    }
+}
+function MinMPLookup_define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function createMinMPLookupDict(lookupMap, options) {
+    const keys = Object.keys(lookupMap);
+    const uniqueValuesSet = new Set();
+    for (const key of keys){
+        const values = lookupMap[key];
+        for (const v of values)uniqueValuesSet.add(v);
+    }
+    const allValues = Array.from(uniqueValuesSet);
+    const mphBin = createMinMPHashDict(allValues, {
+        level: options?.level,
+        outputBinary: true,
+        onlySet: options?.onlySet ?? "8"
+    });
+    const mph = new MinMPHash(mphBin);
+    const valueToKeys = new Map();
+    for(let i = 0; i < keys.length; i++){
+        const key = keys[i];
+        const values = lookupMap[key];
+        for (const v of values){
+            if (!valueToKeys.has(v)) valueToKeys.set(v, []);
+            valueToKeys.get(v).push(i);
+        }
+    }
+    let collisionCount = 0;
+    for (const [_, kIndices] of valueToKeys)if (kIndices.length > 1) collisionCount++;
+    const isMostlyOneToOne = collisionCount < 0.1 * allValues.length;
+    let dict;
+    if (isMostlyOneToOne) {
+        const bitsPerKey = Math.ceil(Math.log2(keys.length + 1));
+        const bw = new BitWriter();
+        const collisionMap = new Map();
+        const valueToKeyMap = new Int32Array(mph.n).fill(-1);
+        for (const [v, kIndices] of valueToKeys){
+            const h = mph.hash(v);
+            if (h >= 0) if (1 === kIndices.length) valueToKeyMap[h] = kIndices[0];
+            else {
+                valueToKeyMap[h] = keys.length;
+                collisionMap.set(h, kIndices);
+            }
+        }
+        for(let i = 0; i < mph.n; i++){
+            const keyIdx = valueToKeyMap[i];
+            bw.write(keyIdx >= 0 ? keyIdx : 0, bitsPerKey);
+        }
+        dict = {
+            mmpHashDictBin: mphBin,
+            keys,
+            valueToKeyIndexes: bw.getData(),
+            bitsPerKey,
+            collisionMap: collisionMap.size > 0 ? collisionMap : void 0
+        };
+    } else {
+        const keyToHashes = [];
+        for (const key of keys){
+            const values = lookupMap[key];
+            const hashes = [];
+            for (const v of values){
+                const h = mph.hash(v);
+                if (h >= 0) hashes.push(h);
+            }
+            hashes.sort((a, b)=>a - b);
+            keyToHashes.push(new Uint32Array(hashes));
+        }
+        dict = {
+            mmpHashDictBin: mphBin,
+            keys,
+            keyToHashes
+        };
+    }
+    if (options?.outputBinary) {
+        const serialized = serializeMinMPLookupDict(dict);
+        if (options?.enableCompression) return compressIBinary(serialized);
+        return serialized;
+    }
+    if (options?.enableCompression) return Promise.resolve(dict);
+    return dict;
+}
+function serializeMinMPLookupDict(dict) {
+    const parts = [];
+    const encoder = new TextEncoder();
+    const writeU32 = (val)=>{
+        const b = new Uint8Array(4);
+        new DataView(b.buffer).setUint32(0, val, false);
+        return b;
+    };
+    parts.push(writeU32(dict.mmpHashDictBin.length));
+    parts.push(dict.mmpHashDictBin);
+    parts.push(writeU32(dict.keys.length));
+    for (const key of dict.keys){
+        const keyBytes = encoder.encode(key);
+        parts.push(writeU32(keyBytes.length));
+        parts.push(keyBytes);
+    }
+    if (dict.valueToKeyIndexes && void 0 !== dict.bitsPerKey) {
+        parts.push(writeU32(0xFFFFFFFF));
+        parts.push(writeU32(dict.bitsPerKey));
+        parts.push(writeU32(dict.valueToKeyIndexes.length));
+        parts.push(dict.valueToKeyIndexes);
+        if (dict.collisionMap && dict.collisionMap.size > 0) {
+            const colBuffer = [];
+            writeVarInt(dict.collisionMap.size, colBuffer);
+            const sortedHashes = Array.from(dict.collisionMap.keys()).sort((a, b)=>a - b);
+            let prevHash = 0;
+            for (const h of sortedHashes){
+                writeVarInt(h - prevHash, colBuffer);
+                prevHash = h;
+                const kIndices = dict.collisionMap.get(h);
+                writeVarInt(kIndices.length, colBuffer);
+                kIndices.sort((a, b)=>a - b);
+                let prevKey = 0;
+                for (const k of kIndices){
+                    writeVarInt(k - prevKey, colBuffer);
+                    prevKey = k;
+                }
+            }
+            const colBytes = new Uint8Array(colBuffer);
+            parts.push(writeU32(colBytes.length));
+            parts.push(colBytes);
+        } else parts.push(writeU32(0));
+    } else if (dict.keyToHashes) {
+        const hashBuffer = [];
+        for (const hashes of dict.keyToHashes){
+            writeVarInt(hashes.length, hashBuffer);
+            if (0 === hashes.length) continue;
+            let maxDelta = 0;
+            let prev = 0;
+            const deltas = [];
+            for(let i = 0; i < hashes.length; i++){
+                const h = hashes[i];
+                const delta = h - prev;
+                deltas.push(delta);
+                if (delta > maxDelta) maxDelta = delta;
+                prev = h;
+            }
+            let bits = 0;
+            if (maxDelta > 0) bits = Math.ceil(Math.log2(maxDelta + 1));
+            hashBuffer.push(bits);
+            const bw = new BitWriter();
+            for (const d of deltas)bw.write(d, bits);
+            const packed = bw.getData();
+            for(let i = 0; i < packed.length; i++)hashBuffer.push(packed[i]);
+        }
+        const hashBytes = new Uint8Array(hashBuffer);
+        parts.push(writeU32(hashBytes.length));
+        parts.push(hashBytes);
+    }
+    const totalLen = parts.reduce((sum, b)=>sum + b.length, 0);
+    const res = new Uint8Array(totalLen);
+    let offset = 0;
+    for (const b of parts){
+        res.set(b, offset);
+        offset += b.length;
+    }
+    return res;
+}
+function deserializeLookupDict(data) {
+    const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+    let offset = 0;
+    const decoder = new TextDecoder();
+    const readU32 = ()=>{
+        const val = view.getUint32(offset, false);
+        offset += 4;
+        return val;
+    };
+    const mphLen = readU32();
+    const mmpHashDictBin = data.subarray(offset, offset + mphLen);
+    offset += mphLen;
+    const keysCount = readU32();
+    const keys = [];
+    for(let i = 0; i < keysCount; i++){
+        const kLen = readU32();
+        const kBytes = data.subarray(offset, offset + kLen);
+        offset += kLen;
+        keys.push(decoder.decode(kBytes));
+    }
+    const sectionLen = readU32();
+    if (0xFFFFFFFF === sectionLen) {
+        const bitsPerKey = readU32();
+        const dataLen = readU32();
+        const valueToKeyIndexes = data.subarray(offset, offset + dataLen);
+        offset += dataLen;
+        const colMapLen = readU32();
+        let collisionMap;
+        if (colMapLen > 0) {
+            const colBytes = data.subarray(offset, offset + colMapLen);
+            offset += colMapLen;
+            collisionMap = new Map();
+            let cOffset = 0;
+            const { value: count, bytes: b1 } = readVarInt(colBytes, cOffset);
+            cOffset += b1;
+            let prevHash = 0;
+            for(let i = 0; i < count; i++){
+                const { value: deltaHash, bytes: b2 } = readVarInt(colBytes, cOffset);
+                cOffset += b2;
+                const h = prevHash + deltaHash;
+                prevHash = h;
+                const { value: kCount, bytes: b3 } = readVarInt(colBytes, cOffset);
+                cOffset += b3;
+                const kIndices = [];
+                let prevKey = 0;
+                for(let j = 0; j < kCount; j++){
+                    const { value: deltaKey, bytes: b4 } = readVarInt(colBytes, cOffset);
+                    cOffset += b4;
+                    const k = prevKey + deltaKey;
+                    prevKey = k;
+                    kIndices.push(k);
+                }
+                collisionMap.set(h, kIndices);
+            }
+        }
+        return {
+            mmpHashDictBin,
+            keys,
+            valueToKeyIndexes,
+            bitsPerKey,
+            collisionMap
+        };
+    }
+    {
+        const hashBytesLen = sectionLen;
+        const hashBytes = data.subarray(offset, offset + hashBytesLen);
+        offset += hashBytesLen;
+        const keyToHashes = [];
+        let hOffset = 0;
+        for(let i = 0; i < keysCount; i++){
+            const { value: count, bytes: b1 } = readVarInt(hashBytes, hOffset);
+            hOffset += b1;
+            if (0 === count) {
+                keyToHashes.push(new Uint32Array(0));
+                continue;
+            }
+            const bits = hashBytes[hOffset];
+            hOffset += 1;
+            const totalBits = bits * count;
+            const packedBytesLen = Math.ceil(totalBits / 8);
+            const packedData = hashBytes.subarray(hOffset, hOffset + packedBytesLen);
+            hOffset += packedBytesLen;
+            const br = new BitReader(packedData);
+            const hashes = new Uint32Array(count);
+            let prev = 0;
+            for(let j = 0; j < count; j++){
+                const delta = br.read(bits);
+                prev += delta;
+                hashes[j] = prev;
+            }
+            keyToHashes.push(hashes);
+        }
+        return {
+            mmpHashDictBin,
+            keys,
+            keyToHashes
+        };
+    }
+}
+class MinMPLookup {
+    static async fromCompressed(data) {
+        const decompressed = await decompressIBinary(data);
+        const dict = deserializeLookupDict(decompressed);
+        return new MinMPLookup(dict);
+    }
+    static fromBinary(data) {
+        const dict = deserializeLookupDict(data);
+        return new MinMPLookup(dict);
+    }
+    buildInvertedIndex() {
+        if (!this.dict.keyToHashes) return;
+        const n = this.mph.n;
+        this._invertedIndex = Array.from({
+            length: n
+        }, ()=>[]);
+        for(let i = 0; i < this.dict.keys.length; i++){
+            const hashes = this.dict.keyToHashes[i];
+            for(let j = 0; j < hashes.length; j++){
+                const h = hashes[j];
+                if (h < n) this._invertedIndex[h].push(i);
+            }
+        }
+    }
+    query(value) {
+        if (this.dict.valueToKeyIndexes && this.dict.bitsPerKey) {
+            const h = this.mph.hash(value);
+            if (h < 0 || h >= this.mph.n) return null;
+            const keyIdx = readBitsAt(this.dict.valueToKeyIndexes, h * this.dict.bitsPerKey, this.dict.bitsPerKey);
+            if (keyIdx === this.dict.keys.length) {
+                if (this.dict.collisionMap && this.dict.collisionMap.has(h)) {
+                    const indices = this.dict.collisionMap.get(h);
+                    return indices.length > 0 ? this.dict.keys[indices[0]] : null;
+                }
+                return null;
+            }
+            if (keyIdx >= this.dict.keys.length) return null;
+            return this.dict.keys[keyIdx];
+        }
+        const keys = this.queryAll(value);
+        return keys && keys.length > 0 ? keys[0] : null;
+    }
+    queryAll(value) {
+        if (this.dict.valueToKeyIndexes && this.dict.bitsPerKey) {
+            const h = this.mph.hash(value);
+            if (h < 0 || h >= this.mph.n) return null;
+            const keyIdx = readBitsAt(this.dict.valueToKeyIndexes, h * this.dict.bitsPerKey, this.dict.bitsPerKey);
+            if (keyIdx === this.dict.keys.length) {
+                if (this.dict.collisionMap && this.dict.collisionMap.has(h)) {
+                    const indices = this.dict.collisionMap.get(h);
+                    return indices.map((i)=>this.dict.keys[i]);
+                }
+                return null;
+            }
+            if (keyIdx >= this.dict.keys.length) return null;
+            return [
+                this.dict.keys[keyIdx]
+            ];
+        }
+        const idx = this.mph.hash(value);
+        if (idx < 0 || !this._invertedIndex) return null;
+        if (idx >= this._invertedIndex.length) return null;
+        const keyIndices = this._invertedIndex[idx];
+        if (0 === keyIndices.length) return null;
+        const results = [];
+        for (const keyIdx of keyIndices)results.push(this.dict.keys[keyIdx]);
+        return results.length > 0 ? results : null;
+    }
+    keys() {
+        return this.dict.keys;
+    }
+    constructor(dict){
+        MinMPLookup_define_property(this, "dict", void 0);
+        MinMPLookup_define_property(this, "mph", void 0);
+        MinMPLookup_define_property(this, "_invertedIndex", void 0);
+        this.dict = dict;
+        this._invertedIndex = null;
+        this.mph = new MinMPHash(dict.mmpHashDictBin);
+        if (dict.keyToHashes) this.buildInvertedIndex();
+    }
+}
+export { MinMPHash, MinMPLookup, createMinMPHashDict, createMinMPLookupDict };