flatpack-json 9.7.0 → 9.8.0

package/dist/storageV1.mjs

@@ -0,0 +1,445 @@
+import assert from 'node:assert';
+import { CompactStorage } from './CompactStorage.mjs';
+import { optimizeFlatpacked } from './optimizeFlatpacked.mjs';
+import { stringifyFlatpacked } from './stringify.mjs';
+import { TrieOfStrings } from './Trie.mjs';
+import { blockSplitRegex, dataHeaderV1_0, ElementType } from './types.mjs';
+const collator = new Intl.Collator('en', {
+    usage: 'sort',
+    numeric: true,
+    sensitivity: 'variant',
+    caseFirst: 'upper',
+    ignorePunctuation: false,
+});
+const compare = collator.compare;
+const forceStringPrimitives = false;
+const minSubStringLen = 4;
+export class CompactStorageV1 extends CompactStorage {
+    data = [dataHeaderV1_0];
+    dedupe = true;
+    sortKeys = true;
+    emptyObjIdx = 0;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    cache = new Map([[undefined, 0]]);
+    /**
+     * Set of indexes that have been referenced by other indexes.
+     */
+    referenced = new Set();
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    cachedArrays = new Map();
+    /**
+     * Cache of strings used for prefix matching.
+     */
+    knownStrings = new TrieOfStrings();
+    /**
+     * Cache of reversed strings used for suffix matching.
+     */
+    knownStringsRev = new TrieOfStrings();
+    cachedElements = new Map();
+    constructor(options) {
+        super(options);
+        this.dedupe = options?.dedupe ?? true;
+        this.sortKeys = options?.sortKeys || this.dedupe;
+    }
+    primitiveToIdx(value) {
+        if (typeof value === 'string')
+            return this.stringToIdx(value);
+        if (typeof value === 'bigint')
+            return this.bigintToIdx(value);
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(value) - 1;
+        this.cache.set(value, idx);
+        return idx;
+    }
+    addSubStringRef(idxString, value, offset) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const sub = offset
+            ? [ElementType.SubString, idxString, value.length, offset]
+            : [ElementType.SubString, idxString, value.length];
+        const idx = this.data.push(sub) - 1;
+        this.cache.set(value, idx);
+        return idx;
+    }
+    estimateSubStringCost(idxString, value, offset) {
+        if (this.cache.has(value)) {
+            return 0;
+        }
+        let cost = 5;
+        cost += Math.ceil(Math.log10(idxString));
+        cost += Math.ceil(Math.log10(value.length));
+        if (offset) {
+            cost += Math.ceil(Math.log10(offset)) + 1;
+        }
+        return cost;
+    }
+    addKnownString(idx, value) {
+        if (value.length >= minSubStringLen) {
+            const data = { idx, value };
+            this.knownStrings.add(value.length > 256 ? [...value].slice(0, 256) : value, data);
+            this.knownStringsRev.add(revStr(value, 256), data);
+        }
+    }
+    addStringPrimitive(value) {
+        const idx = this.data.push(value) - 1;
+        this.addKnownString(idx, value);
+        this.cache.set(value, idx);
+        return idx;
+    }
+    duplicateIndex(idx) {
+        const element = this.data[idx];
+        const duplicate = this.data.push(element) - 1;
+        return duplicate;
+    }
+    stringToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        if (forceStringPrimitives || value.length < minSubStringLen || blockSplitRegex.test(value)) {
+            return this.addStringPrimitive(value);
+        }
+        const foundPrefix = this.findPrefix(value);
+        const foundSuffix = this.findSuffix(value);
+        if (!foundPrefix && !foundSuffix) {
+            return this.addStringPrimitive(value);
+        }
+        if (foundPrefix &&
+            (!foundSuffix ||
+                foundPrefix.subStr.length - foundPrefix.cost >= foundSuffix.subStr.length - foundSuffix.cost)) {
+            const { idx: pfIdx, subStr, offset } = foundPrefix;
+            const sIdx = this.addSubStringRef(pfIdx, subStr, offset);
+            if (subStr === value)
+                return sIdx;
+            const v = [sIdx, this.stringToIdx(value.slice(subStr.length))];
+            const idx = this.data.push([ElementType.String, ...v]) - 1;
+            this.cache.set(value, idx);
+            this.addKnownString(idx, value);
+            return idx;
+        }
+        if (!foundSuffix) {
+            return this.addStringPrimitive(value);
+        }
+        const { idx: pfIdx, subStr, offset } = foundSuffix;
+        const sIdx = this.addSubStringRef(pfIdx, subStr, offset);
+        const v = [this.stringToIdx(value.slice(0, -subStr.length)), sIdx];
+        const idx = this.data.push([ElementType.String, ...v]) - 1;
+        this.cache.set(value, idx);
+        this.addKnownString(idx, value);
+        return idx;
+    }
+    findPrefix(value) {
+        const trieFound = this.knownStrings.find(value);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+            return undefined;
+        }
+        const { data: tData, found: subStr } = trieFound;
+        const cost = this.estimateSubStringCost(tData.idx, subStr, undefined);
+        if (cost > subStr.length) {
+            return undefined;
+        }
+        return { idx: tData.idx, subStr, offset: undefined, cost };
+    }
+    findSuffix(value) {
+        const rev = revStr(value, 256);
+        const trieFound = this.knownStringsRev.find(rev);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+            return undefined;
+        }
+        const { data: tData, found: subStrRev } = trieFound;
+        const offset = tData.value.length - subStrRev.length;
+        const cost = this.estimateSubStringCost(tData.idx, subStrRev, offset);
+        if (cost > subStrRev.length) {
+            return undefined;
+        }
+        const srcStr = tData.value;
+        const subStr = srcStr.slice(-subStrRev.length);
+        return { idx: tData.idx, subStr, offset, cost };
+    }
+    objSetToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const keys = [...value];
+        const k = this.createUniqueKeys(keys, false);
+        const element = [ElementType.Set, k];
+        return this.storeElement(value, idx, element);
+    }
+    createUniqueKeys(keys, cacheValue = true) {
+        let k = this.arrToIdx(keys, cacheValue);
+        const elementKeys = this.data[k];
+        const uniqueKeys = new Set(elementKeys.slice(1));
+        if (uniqueKeys.size !== keys.length) {
+            // one or more of the keys got deduped. We need to duplicate it.
+            uniqueKeys.clear();
+            const indexes = elementKeys.slice(1).map((idx) => {
+                if (uniqueKeys.has(idx)) {
+                    return this.duplicateIndex(idx);
+                }
+                uniqueKeys.add(idx);
+                return idx;
+            });
+            k = this.createArrayElementFromIndexValues(this.data.length, indexes);
+        }
+        return k;
+    }
+    objMapToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const entries = [...value.entries()];
+        const k = this.createUniqueKeys(entries.map(([key]) => key), false);
+        const v = this.arrToIdx(entries.map(([, value]) => value), false);
+        const element = [ElementType.Map, k, v];
+        return this.storeElement(value, idx, element);
+    }
+    objRegExpToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const element = [
+            ElementType.RegExp,
+            this.stringToIdx(value.source),
+            this.stringToIdx(value.flags),
+        ];
+        return this.storeElement(value, idx, element);
+    }
+    objDateToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const element = [ElementType.Date, value.getTime()];
+        return this.storeElement(value, idx, element);
+    }
+    bigintToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const element = [
+            ElementType.BigInt,
+            this.primitiveToIdx(value <= Number.MAX_SAFE_INTEGER && value >= -Number.MAX_SAFE_INTEGER
+                ? Number(value)
+                : value.toString()),
+        ];
+        return this.storeElement(value, idx, element);
+    }
+    objToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        if (isObjectWrapper(value)) {
+            const idx = this.data.push({}) - 1;
+            this.cache.set(value, idx);
+            const element = [ElementType.Object, 0, this.valueToIdx(value.valueOf())];
+            return this.storeElement(value, idx, element);
+        }
+        const entries = Object.entries(value);
+        if (!entries.length) {
+            if (this.emptyObjIdx) {
+                return this.emptyObjIdx;
+            }
+            const idx = this.data.push({}) - 1;
+            this.emptyObjIdx = idx;
+            return idx;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        if (this.sortKeys) {
+            entries.sort(([a], [b]) => compare(a, b));
+        }
+        const k = this.arrToIdx(entries.map(([key]) => key));
+        const v = this.arrToIdx(entries.map(([, value]) => value));
+        const element = [ElementType.Object, k, v];
+        return this.storeElement(value, idx, element);
+    }
+    storeElement(value, idx, element) {
+        const useIdx = this.dedupe ? this.cacheElement(idx, element) : idx;
+        if (useIdx !== idx && idx === this.data.length - 1) {
+            this.data.length = idx;
+            this.cache.set(value, useIdx);
+            return useIdx;
+        }
+        this.data[idx] = element;
+        return idx;
+    }
+    cacheElement(elemIdx, element) {
+        let map = this.cachedElements;
+        for (let i = 0; i < element.length - 1; i++) {
+            const idx = element[i];
+            let found = map.get(idx);
+            if (!found) {
+                found = new Map();
+                map.set(idx, found);
+            }
+            assert(found instanceof Map);
+            map = found;
+        }
+        const idx = element[element.length - 1];
+        const foundIdx = map.get(idx);
+        if (typeof foundIdx === 'number') {
+            return this.referenced.has(elemIdx) ? elemIdx : foundIdx;
+        }
+        map.set(idx, elemIdx);
+        return elemIdx;
+    }
+    stashArray(idx, element) {
+        const indexHash = simpleHash(element);
+        let found = this.cachedArrays.get(indexHash);
+        if (!found) {
+            found = [];
+            this.cachedArrays.set(indexHash, found);
+        }
+        const foundIdx = found.find((entry) => isEqual(entry.v, element));
+        if (foundIdx) {
+            return this.referenced.has(idx) ? idx : foundIdx.idx;
+        }
+        found.push({ idx, v: element });
+        return idx;
+    }
+    createArrayElementFromIndexValues(idx, indexValues) {
+        const element = [ElementType.Array, ...indexValues];
+        const useIdx = this.dedupe ? this.stashArray(idx, element) : idx;
+        if (useIdx !== idx) {
+            assert(this.data.length === idx + 1, `Expected ${idx + 1} but got ${this.data.length}`);
+            this.data.length = idx;
+            return useIdx;
+        }
+        this.data[idx] = element;
+        return idx;
+    }
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    arrToIdx(value, cacheValue = true) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const useIdx = this.createArrayElementFromIndexValues(idx, value.map((idx) => this.valueToIdx(idx)));
+        if (cacheValue) {
+            this.cache.set(value, useIdx);
+        }
+        return useIdx;
+    }
+    valueToIdx(value) {
+        if (value === null) {
+            return this.primitiveToIdx(value);
+        }
+        if (typeof value === 'object') {
+            if (value instanceof Set) {
+                return this.objSetToIdx(value);
+            }
+            if (value instanceof Map) {
+                return this.objMapToIdx(value);
+            }
+            if (value instanceof RegExp) {
+                return this.objRegExpToIdx(value);
+            }
+            if (Array.isArray(value)) {
+                return this.arrToIdx(value);
+            }
+            if (value instanceof Date) {
+                return this.objDateToIdx(value);
+            }
+            return this.objToIdx(value);
+        }
+        return this.primitiveToIdx(value);
+    }
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    softReset() {
+        this.cache.clear();
+        this.cache.set(undefined, 0);
+    }
+    toJSON(json) {
+        this.softReset();
+        this.valueToIdx(json);
+        return this.options?.optimize ? optimizeFlatpacked(this.data) : this.data;
+    }
+}
+function* revStr(str, limit) {
+    let i = str.length - 1;
+    let n = 0;
+    while (i >= 0 && n < limit) {
+        // eslint-disable-next-line unicorn/prefer-code-point
+        const code = str.charCodeAt(i);
+        if (code & 0xfc00) {
+            // surrogate pair
+            i -= 1;
+            yield str.slice(i, i + 2);
+        }
+        else {
+            yield str[i];
+        }
+        i -= 1;
+        n += 1;
+    }
+}
+function isEqual(a, b) {
+    if (a === b)
+        return true;
+    if (a.length !== b.length)
+        return false;
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] !== b[i])
+            return false;
+    }
+    return true;
+}
+function simpleHash(values) {
+    let hash = Math.sqrt(values.length);
+    for (const value of values) {
+        hash += value * value;
+    }
+    return hash;
+}
+function isObjectWrapper(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.valueOf === 'function' &&
+        value.valueOf() !== value);
+}
+export function toJSON(json, options) {
+    return new CompactStorageV1(options).toJSON(json);
+}
+export function stringify(data, pretty = true, options) {
+    const json = toJSON(data, options);
+    return pretty ? stringifyFlatpacked(json) : JSON.stringify(json);
+}
+//# sourceMappingURL=storageV1.mjs.map

package/dist/storageV2.d.mts

@@ -0,0 +1,70 @@
+import { CompactStorage } from './CompactStorage.mjs';
+import type { Flatpacked, FlatpackOptions, Serializable, UnpackMetaData } from './types.mjs';
+export declare class CompactStorageV2 extends CompactStorage {
+    #private;
+    private data;
+    private stringTable;
+    private dedupe;
+    private sortKeys;
+    private emptyObjIdx;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    private cache;
+    /**
+     * Set of indexes that have been referenced by other indexes.
+     */
+    private referencedFromCache;
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    private cachedArrays;
+    private cachedElementsTrie;
+    private unpackMetaData;
+    private used;
+    constructor(options?: FlatpackOptions | undefined);
+    private primitiveToIdx;
+    private addStringPrimitive;
+    private stringToIdx;
+    private objSetToIdx;
+    private createUniqueKeys;
+    private objMapToIdx;
+    private objRegExpToIdx;
+    private objDateToIdx;
+    private bigintToIdx;
+    private objToIdx;
+    private objectKeysOrValuesToIdx;
+    private mapValuesToIndexes;
+    /**
+     * Store an element in the data array, using the cache if deduplication is enabled.
+     * @param value - the value being stored, used for caching purposes.
+     * @param idx - the index at which to store the element if it is not found in the cache.
+     * @param element - the element to store if it is not found in the cache.
+     * @returns the index at which the element is stored.
+     */
+    private storeElement;
+    private cacheElement;
+    private stashArray;
+    private createArrayElementFromIndexValues;
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    private arrToIdx;
+    private valueToIdx;
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    private softReset;
+    useFlatpackMetaData(data: UnpackMetaData | undefined): void;
+    /**
+     * Cache the primitives from the unpacked data to improve performance when re-packing the same data.
+     */
+    private initFromFlatpackData;
+    private useFlattenedElement;
+    toJSON<V extends Serializable>(json: V): Flatpacked;
+}
+//# sourceMappingURL=storageV2.d.mts.map