flatpack-json 9.7.0 → 10.0.0

package/dist/Flatpack.mjs

@@ -1,598 +1,19 @@
-import assert from 'node:assert';
 import { FlatpackedWrapper } from './flatpackUtil.mjs';
-import { proxyDate, proxyObject, proxySet } from './proxy.mjs';
-import { ArrayRefElement, BigIntRefElement, DateRefElement, isStringRefElements, MapRefElement, ObjectRefElement, ObjectWrapperRefElement, PrimitiveRefElement, PrimitiveRefElementBase, RegExpRefElement, SetRefElement, StringConcatRefElement, StringPrimitiveRefElement, SubStringRefElement, } from './RefElements.mjs';
+import { FlatpackStoreV2 } from './FlatpackV2.mjs';
 import { stringifyFlatpacked } from './stringify.mjs';
-import { Trie } from './Trie.mjs';
-import { blockSplitRegex, dataHeader } from './types.mjs';
 import { fromJSON } from './unpack.mjs';
-const collator = new Intl.Collator('en', {
-    usage: 'sort',
-    numeric: true,
-    sensitivity: 'variant',
-    caseFirst: 'upper',
-    ignorePunctuation: false,
-});
-const compare = collator.compare;
-const forceStringPrimitives = false;
-const minSubStringLen = 8;
-const minSubStringSuffixLen = 16;
-const useSuffix = true;
-const maxCachedStringLen = 256;
 export class FlatpackStore {
     options;
-    knownElements = new Set();
-    assignedElements = new Map();
-    elements = [undefined];
-    root = undefined;
-    dedupe = true;
-    sortKeys = true;
-    emptyObjIdx = undefined;
-    ids = 0;
-    /**
-     * Cache of primitives and objects that have been added to the data.
-     */
-    cache = new Map();
-    /**
-     * Set of elements that have been referenced by other indexes.
-     */
-    referenced = new Set();
-    /**
-     * Cache of arrays that have been deduped.
-     * The key is a hash of the array elements as a function of the index of the element.
-     */
-    cachedArrays = new Map();
-    cachedObjects = new Map();
-    cachedSets = new Map();
-    cachedMaps = new Map();
-    cachedProxies = new WeakMap();
-    /**
-     * Cache of strings that have been deduped and stored in the data array.
-     */
-    knownStrings = new Trie();
-    /**
-     * Cache of reversed strings that have been deduped and stored in the data array.
-     * This is used to find matching suffixes.
-     */
-    knownStringsRev = new Trie();
-    refUndefined;
+    #flatpackApi;
     constructor(value, options) {
         this.options = options;
-        this.dedupe = options?.dedupe ?? true;
-        this.sortKeys = options?.sortKeys || this.dedupe;
-        this.refUndefined = this.addValueAndElement(undefined, new PrimitiveRefElement(undefined));
-        if (value instanceof FlatpackedWrapper) {
-            this.#fromWrapper(value);
-        }
-        else {
-            this.#setValue(value);
-        }
-    }
-    #fromWrapper(wrapper) {
-        this.elements = wrapper.toRefElements();
-        this.root = this.elements[1];
-        for (let i = 1; i < this.elements.length; i++) {
-            const element = this.elements[i];
-            if (!element)
-                continue;
-            this.knownElements.add(element);
-            if (element instanceof PrimitiveRefElement) {
-                this.addValueAndElement(element.value, element);
-            }
-            if (isStringRefElements(element)) {
-                this.addStringElement(element.value, element);
-            }
-        }
-        this.ids = this.elements.length;
-        this.#resolveRefs();
+        this.#flatpackApi = new FlatpackStoreV2(value, options);
     }
     setValue(value) {
-        this.#setValue(value);
-    }
-    #setValue(value) {
-        this.softReset();
-        this.root = this.valueToRef(value);
-        this.#resolveRefs();
-        return this.root;
-    }
-    nextId() {
-        return this.ids++;
-    }
-    addElement(element) {
-        if (this.knownElements.has(element))
-            return element;
-        element.setId(this.nextId());
-        this.knownElements.add(element);
-        return element;
-    }
-    addValueAndElement(value, element, cache = true) {
-        this.addElement(element);
-        if (cache) {
-            this.cache.set(value, element);
-        }
-        return element;
-    }
-    primitiveToRef(value) {
-        if (typeof value === 'string')
-            return this.stringToRef(value);
-        if (typeof value === 'bigint')
-            return this.cvtBigintToRef(value);
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        return this.addValueAndElement(value, new PrimitiveRefElement(value));
-    }
-    createSubStringRef(baseString, value, offset) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        return this.addStringElement(value, new SubStringRefElement(baseString, value.length, offset));
-    }
-    addKnownString(ref, value) {
-        if (value.length >= minSubStringLen) {
-            this.knownStrings.add(prefix(value, maxCachedStringLen), ref);
-            const rev = reverse(suffix(value, maxCachedStringLen));
-            this.knownStringsRev.add(rev, ref);
-        }
-    }
-    addStringPrimitive(value) {
-        return this.addStringElement(value, new StringPrimitiveRefElement(value));
-    }
-    addStringElement(value, element) {
-        this.addKnownString(element, value);
-        return this.addValueAndElement(value, element);
-    }
-    stringPrefix(value) {
-        // if (value.length < maxCachedStringLen * 2) return undefined;
-        const trieFound = this.knownStrings.find(value);
-        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
-            return undefined;
-        }
-        const { data: tData, found: subStr } = trieFound;
-        // assert(subStr === value.slice(0, subStr.length));
-        return this.createSubStringRef(tData, subStr);
-    }
-    stringSuffix(value) {
-        if (!useSuffix)
-            return undefined;
-        const rev = reverse(value);
-        const trieFound = this.knownStringsRev.find(rev);
-        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringSuffixLen) {
-            return undefined;
-        }
-        const { data: tData, found: subStr } = trieFound;
-        // assert(subStr === value.slice(0, subStr.length));
-        return this.createSubStringRef(tData, value.slice(-subStr.length), tData.length - subStr.length);
-    }
-    stringToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        if (forceStringPrimitives || value.length < minSubStringLen || blockSplitRegex.test(value)) {
-            return this.addStringPrimitive(value);
-        }
-        const partialsPfx = [];
-        const partialsSfx = [];
-        let subStr = value;
-        while (subStr.length) {
-            const prefix = this.stringPrefix(subStr);
-            const suffix = this.stringSuffix(subStr);
-            if (!prefix && !suffix)
-                break;
-            if (prefix && prefix.length >= (suffix?.length || 0)) {
-                partialsPfx.push(prefix);
-                subStr = subStr.slice(prefix.length);
-            }
-            else {
-                const sfx = suffix;
-                partialsSfx.push(sfx);
-                subStr = subStr.slice(0, -sfx.length);
-            }
-        }
-        partialsSfx.reverse();
-        if (!partialsPfx.length && !partialsSfx.length) {
-            return this.addStringPrimitive(value);
-        }
-        if (subStr.length) {
-            partialsPfx.push(this.stringToRef(subStr));
-        }
-        const partials = [...partialsPfx, ...partialsSfx];
-        return this.addStringElement(value, partials.length === 1 ? partials[0] : new StringConcatRefElement(partials));
-    }
-    cvtSetToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        const element = this.addValueAndElement(value, new SetRefElement());
-        element.setValues(this.createUniqueKeys([...value]));
-        return this.dedupeSetRefs(value, element);
-    }
-    dedupeSetRefs(value, element) {
-        if (!this.dedupe)
-            return element;
-        const values = element.valueRefs();
-        const found = this.cachedSets.get(values);
-        if (!found) {
-            this.cachedSets.set(values, element);
-            return element;
-        }
-        if (this.referenced.has(element))
-            return element;
-        this.knownElements.delete(element);
-        this.cache.set(value, found);
-        return found;
-    }
-    proxySetRef(ref) {
-        return proxySet(new Set(this.#toValue(ref.valueRefs())), () => { });
-    }
-    createUniqueKeys(keys) {
-        const cacheValue = false;
-        let k = this.arrToRef(keys, cacheValue);
-        const uniqueKeys = new Set(k.valueRefs());
-        if (uniqueKeys.size !== keys.length) {
-            // one or more of the keys got deduped. We need to duplicate it.
-            uniqueKeys.clear();
-            const values = k.valueRefs().map((ref) => {
-                if (uniqueKeys.has(ref)) {
-                    return this.addElement(ref.clone());
-                }
-                uniqueKeys.add(ref);
-                return ref;
-            });
-            k = this.addValueAndElement(keys, new ArrayRefElement(values), cacheValue);
-        }
-        return k;
-    }
-    cvtMapToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        const element = this.addValueAndElement(value, new MapRefElement());
-        element.setKeysAndValues(this.createUniqueKeys([...value.keys()]), this.arrToRef([...value.values()], false));
-        return this.dedupeMapRefs(value, element);
-    }
-    dedupeMapRefs(value, element) {
-        if (!this.dedupe)
-            return element;
-        const keys = element.keyRefs();
-        const values = element.valueRefs();
-        let found = this.cachedMaps.get(keys);
-        if (!found) {
-            found = new Map();
-            found.set(values, element);
-            this.cachedMaps.set(keys, found);
-            return element;
-        }
-        const foundValue = found.get(values);
-        if (foundValue) {
-            if (this.referenced.has(element))
-                return element;
-            this.knownElements.delete(element);
-            this.cache.set(value, foundValue);
-            return foundValue;
-        }
-        found.set(values, element);
-        return element;
-    }
-    proxyMapRef(_ref) {
-        return new Map();
-    }
-    cvtRegExpToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        return this.addValueAndElement(value, new RegExpRefElement(this.stringToRef(value.source), this.stringToRef(value.flags)));
-    }
-    cvtDateToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        return this.addValueAndElement(value, new DateRefElement(value.getTime()));
-    }
-    proxyDateRef(ref) {
-        return proxyDate(ref.value, (date) => ref.setTime(date.getTime()));
-    }
-    cvtBigintToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        const valElement = this.primitiveToRef(value <= Number.MAX_SAFE_INTEGER && value >= -Number.MAX_SAFE_INTEGER ? Number(value) : value.toString());
-        return this.addValueAndElement(value, new BigIntRefElement(valElement));
-    }
-    cvtObjToRef(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        if (isObjectWrapper(value)) {
-            const element = this.addValueAndElement(value, new ObjectWrapperRefElement());
-            element.setValue(this.valueToRef(value.valueOf()));
-            return element;
-        }
-        const entries = Object.entries(value);
-        if (!entries.length) {
-            if (this.emptyObjIdx) {
-                return this.emptyObjIdx;
-            }
-            this.emptyObjIdx = this.addValueAndElement(value, new ObjectRefElement());
-            return this.emptyObjIdx;
-        }
-        if (this.sortKeys) {
-            entries.sort((a, b) => compare(a[0], b[0]));
-        }
-        const element = this.addValueAndElement(value, new ObjectRefElement());
-        const k = this.arrToRef(entries.map(([key]) => key), false);
-        const v = this.arrToRef(entries.map(([, value]) => value), false);
-        element.setKeysAndValues(k, v);
-        return this.dedupeObject(value, element);
-    }
-    dedupeObject(value, element) {
-        if (!this.dedupe)
-            return element;
-        const keys = element.keyRefs();
-        const values = element.valueRefs();
-        let found = this.cachedObjects.get(keys);
-        if (!found) {
-            found = new Map();
-            found.set(values, element);
-            this.cachedObjects.set(keys, found);
-            return element;
-        }
-        const foundValue = found.get(values);
-        if (foundValue) {
-            if (this.referenced.has(element))
-                return element;
-            this.knownElements.delete(element);
-            this.cache.set(value, foundValue);
-            return foundValue;
-        }
-        found.set(values, element);
-        return element;
-    }
-    proxyObjectRef(ref) {
-        const keys = this.#toValue(ref.keyRefs());
-        const values = this.#toValue(ref.valueRefs());
-        const obj = keys && values ? Object.fromEntries(keys.map((key, i) => [key, values[i]])) : {};
-        return proxyObject(obj, (_value) => { });
-    }
-    proxyObjectWrapperRef(ref) {
-        const value = Object(this.#toValue(ref.valueRef()));
-        return proxyObject(value, (_value) => { });
-    }
-    /**
-     *
-     * @param value - The array converted to an ArrayRefElement.
-     * @param element - the element to dedupe.
-     * @param cacheValue - Whether to cache the value. It is false when it is a dynamic array, like object keys,
-     *      in that case, we want to dedupe the keys and values.
-     * @returns the element to use.
-     */
-    dedupeArray(value, element, cacheValue) {
-        if (cacheValue && !this.dedupe)
-            return element;
-        const indexHash = element.hash;
-        let cached = this.cachedArrays.get(indexHash);
-        if (!cached) {
-            cached = [];
-            this.cachedArrays.set(indexHash, cached);
-        }
-        const found = cached.find((entry) => element.isEqual(entry));
-        if (found) {
-            if (this.referenced.has(element))
-                return element;
-            this.knownElements.delete(element);
-            if (cacheValue || this.cache.has(value)) {
-                this.cache.set(value, found);
-            }
-            return found;
-        }
-        cached.push(element);
-        return element;
-    }
-    /**
-     * Convert an array to an index.
-     * @param value - The array to convert to an index.
-     * @param cacheValue - Whether to cache the value.
-     * @returns the index of the array.
-     */
-    arrToRef(value, cacheValue = true) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        const element = this.addValueAndElement(value, new ArrayRefElement(), cacheValue);
-        element.setValues(value.map((v) => this.valueToRef(v)));
-        return this.dedupeArray(value, element, cacheValue);
-    }
-    proxyArrayRef(ref) {
-        const arr = ref.valueRefs().map((v) => this.#toValue(v));
-        return proxyObject(arr, (_value) => { });
-    }
-    valueToRef(value) {
-        if (value === null) {
-            return this.primitiveToRef(value);
-        }
-        if (typeof value === 'object') {
-            if (value instanceof Set) {
-                return this.cvtSetToRef(value);
-            }
-            if (value instanceof Map) {
-                return this.cvtMapToRef(value);
-            }
-            if (value instanceof RegExp) {
-                return this.cvtRegExpToRef(value);
-            }
-            if (Array.isArray(value)) {
-                return this.arrToRef(value);
-            }
-            if (value instanceof Date) {
-                return this.cvtDateToRef(value);
-            }
-            return this.cvtObjToRef(value);
-        }
-        return this.primitiveToRef(value);
-    }
-    /**
-     * Reset things in a way that allows for reuse.
-     */
-    softReset() {
-        this.referenced.clear();
-        if (this.root) {
-            const idx = this.assignedElements.get(this.root);
-            if (idx) {
-                this.elements[idx] = undefined;
-                this.assignedElements.delete(this.root);
-            }
-        }
-        this.root = undefined;
-    }
-    #resolveRefs() {
-        if (!this.root)
-            return;
-        const elements = this.elements;
-        const assigned = this.assignedElements;
-        const referenced = this.referenced;
-        const availableIndexes = [];
-        function addElement(ref) {
-            if (assigned.has(ref))
-                return false;
-            const emptyCell = availableIndexes.pop();
-            if (emptyCell) {
-                assigned.set(ref, emptyCell);
-                elements[emptyCell] = ref;
-                return true;
-            }
-            const i = elements.push(ref) - 1;
-            assigned.set(ref, i);
-            return true;
-        }
-        function walk(ref, action) {
-            function _walk(ref) {
-                if (!action(ref))
-                    return;
-                const deps = ref.getDependencies();
-                if (!deps)
-                    return;
-                for (const dep of deps) {
-                    _walk(dep);
-                }
-            }
-            _walk(ref);
-        }
-        function calcReferences(root) {
-            referenced.clear();
-            walk(root, (ref) => {
-                if (referenced.has(ref))
-                    return false;
-                referenced.add(ref);
-                return true;
-            });
-        }
-        function calcAvailableIndexes() {
-            availableIndexes.length = 0;
-            for (let i = 1; i < elements.length; i++) {
-                const ref = elements[i];
-                if (!ref || !referenced.has(ref)) {
-                    availableIndexes.push(i);
-                    if (ref) {
-                        assigned.delete(ref);
-                    }
-                    elements[i] = undefined;
-                }
-                else {
-                    assigned.set(ref, i);
-                }
-            }
-            availableIndexes.reverse();
-        }
-        function addElements(root) {
-            walk(root, addElement);
-            let i = elements.length - 1;
-            for (; i > 0 && elements[i] === undefined; i--) {
-                // empty
-            }
-            elements.length = i + 1;
-        }
-        calcReferences(this.root);
-        elements[0] = this.refUndefined;
-        assigned.set(this.refUndefined, 0);
-        calcAvailableIndexes();
-        addElements(this.root);
-        this.#cleanCache();
-    }
-    /**
-     * Remove objects from the cache after the FlatpackStore has been built.
-     */
-    #cleanCache() {
-        const toRemove = new Set();
-        for (const key of this.cache.keys()) {
-            if (key && typeof key === 'object') {
-                toRemove.add(key);
-            }
-        }
-        for (const key of toRemove) {
-            this.cache.delete(key);
-        }
-    }
-    #resolveToValueProxy(ref) {
-        if (!ref)
-            return undefined;
-        if (ref instanceof ArrayRefElement)
-            return this.proxyArrayRef(ref);
-        if (ref instanceof ObjectRefElement)
-            return this.proxyObjectRef(ref);
-        if (ref instanceof PrimitiveRefElementBase)
-            return ref.value;
-        if (isStringRefElements(ref))
-            return ref.value;
-        if (ref instanceof MapRefElement)
-            return this.proxyMapRef(ref);
-        if (ref instanceof SetRefElement)
-            return this.proxySetRef(ref);
-        if (ref instanceof BigIntRefElement)
-            return ref.value;
-        if (ref instanceof RegExpRefElement)
-            return ref.value;
-        if (ref instanceof DateRefElement)
-            return this.proxyDateRef(ref);
-        if (ref instanceof ObjectWrapperRefElement)
-            return this.proxyObjectWrapperRef(ref);
-        assert(false, 'Unknown ref type');
-    }
-    #toValue(ref) {
-        if (!ref)
-            return undefined;
-        return getOrResolve(this.cachedProxies, ref, (ref) => this.#resolveToValueProxy(ref));
+        this.#flatpackApi.setValue(value);
     }
     toJSON() {
-        const data = [dataHeader];
-        const idxLookup = this.assignedElements;
-        const lookup = (ref) => (ref && idxLookup.get(ref)) || 0;
-        const elements = this.elements;
-        for (let i = 1; i < elements.length; i++) {
-            const element = elements[i];
-            if (!element) {
-                data.push(0);
-                continue;
-            }
-            const value = element.toElement(lookup);
-            if (value === undefined)
-                continue;
-            data.push(value);
-        }
-        return data;
+        return this.#flatpackApi.toJSON();
     }
     static fromJSON(data) {
         return new FlatpackStore(new FlatpackedWrapper(data));
@@ -606,24 +27,6 @@ export class FlatpackStore {
     toValue() {
         return fromJSON(this.toJSON());
     }
-    _toValueProxy() {
-        return this.#toValue(this.root);
-    }
-}
-function isObjectWrapper(value) {
-    return (typeof value === 'object' &&
-        value !== null &&
-        typeof value.valueOf === 'function' &&
-        value.valueOf() !== value);
-}
-function prefix(value, len) {
-    return value.length > len ? [...value].slice(0, len) : value;
-}
-function suffix(value, len) {
-    return value.length > len ? [...value].slice(-len) : value;
-}
-function reverse(value) {
-    return [...value].reverse();
 }
 export function toJSON(json, options) {
     return new FlatpackStore(json, options).toJSON();
@@ -631,12 +34,4 @@ export function toJSON(json, options) {
 export function stringify(data, pretty = true) {
     return pretty ? stringifyFlatpacked(toJSON(data)) : JSON.stringify(toJSON(data));
 }
-function getOrResolve(map, key, resolver) {
-    let value = map.get(key);
-    if (value === undefined && !map.has(key)) {
-        value = resolver(key);
-        map.set(key, value);
-    }
-    return value;
-}
 //# sourceMappingURL=Flatpack.mjs.map

package/dist/FlatpackData.d.mts

@@ -0,0 +1,24 @@
+import { StringTableBuilder } from './stringTable.mjs';
+import type { Flatpacked, FlatpackIndex, FlattenedElement } from './types.mjs';
+export declare class FlatpackData {
+    #private;
+    flatpack: Flatpacked;
+    stringTable: StringTableBuilder;
+    rootIndex: FlatpackIndex;
+    used: Set<FlatpackIndex>;
+    ownedBy: Map<FlatpackIndex, FlatpackIndex>;
+    available: FlatpackIndex[];
+    constructor(flatpack: Flatpacked | undefined);
+    add(element: FlattenedElement): FlatpackIndex;
+    reserve(): FlatpackIndex;
+    set(idx: FlatpackIndex, element: FlattenedElement): void;
+    get(idx: FlatpackIndex): FlattenedElement;
+    markUsed(idx: FlatpackIndex): void;
+    isUsed(idx: FlatpackIndex): boolean;
+    duplicateIndex(idx: FlatpackIndex): FlatpackIndex;
+    delete(idx: FlatpackIndex): void;
+    claimOwnership(idx: FlatpackIndex, ownerIdx: FlatpackIndex): FlatpackIndex;
+    markUnusedAsAvailable(): void;
+    finalize(): Flatpacked;
+}
+//# sourceMappingURL=FlatpackData.d.mts.map