flatpack-json 8.14.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jason Dent
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,17 @@
+# Flatpack JSON
+
+A library to reduce the size of JSON data by flattening and normalizing the values.
+
+It is similar to [flatted](https://www.npmjs.com/package/flatted).
+
+## Install
+
+```sh
+npm install -S flatpack-json
+```
+
+## Usage
+
+```ts
+import { parse, stringify } from 'flatpack-json';
+```

package/dist/Flatpack.d.mts

@@ -0,0 +1,92 @@
+import { FlatpackedWrapper } from './flatpackUtil.mjs';
+import type { Flatpacked, FlatpackOptions, Serializable, Unpacked } from './types.mjs';
+export declare class FlatpackStore {
+    #private;
+    readonly options?: FlatpackOptions | undefined;
+    private knownElements;
+    private assignedElements;
+    private elements;
+    private root;
+    private dedupe;
+    private sortKeys;
+    private emptyObjIdx;
+    private ids;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    private cache;
+    /**
+     * Set of elements that have been referenced by other indexes.
+     */
+    private referenced;
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    private cachedArrays;
+    private cachedObjects;
+    private cachedSets;
+    private cachedMaps;
+    /**
+     * Cache of strings that have been deduped and stored in the data array.
+     */
+    private knownStrings;
+    /**
+     * Cache of reversed strings that have been deduped and stored in the data array.
+     * This is used to find matching suffixes.
+     */
+    private knownStringsRev;
+    private refUndefined;
+    constructor(value: Serializable | FlatpackedWrapper, options?: FlatpackOptions | undefined);
+    setValue(value: Serializable): void;
+    private nextId;
+    private addElement;
+    private addValueAndElement;
+    private primitiveToRef;
+    private createSubStringRef;
+    private addKnownString;
+    private addStringPrimitive;
+    private addStringElement;
+    private stringPrefix;
+    private stringSuffix;
+    private stringToRef;
+    private cvtSetToRef;
+    private dedupeSetRefs;
+    private createUniqueKeys;
+    private cvtMapToRef;
+    private dedupeMapRefs;
+    private cvtRegExpToRef;
+    private cvtDateToRef;
+    private cvtBigintToRef;
+    private cvtObjToRef;
+    private dedupeObject;
+    /**
+     *
+     * @param value - The array converted to an ArrayRefElement.
+     * @param element - the element to dedupe.
+     * @param cacheValue - Whether to cache the value. It is false when it is a dynamic array, like object keys,
+     *      in that case, we want to dedupe the keys and values.
+     * @returns the element to use.
+     */
+    private dedupeArray;
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    private arrToRef;
+    private valueToRef;
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    private softReset;
+    toJSON(): Flatpacked;
+    static fromJSON(data: Flatpacked): FlatpackStore;
+    static parse(content: string): FlatpackStore;
+    stringify(): string;
+    toValue(): Unpacked;
+}
+export declare function toJSON<V extends Serializable>(json: V, options?: FlatpackOptions): Flatpacked;
+export declare function stringify(data: Unpacked, pretty?: boolean): string;
+//# sourceMappingURL=Flatpack.d.mts.map

package/dist/Flatpack.mjs

@@ -0,0 +1,560 @@
+import { FlatpackedWrapper } from './flatpackUtil.mjs';
+import { ArrayRefElement, BigIntRefElement, DateRefElement, isStringRefElements, MapRefElement, ObjectRefElement, ObjectWrapperRefElement, PrimitiveRefElement, RegExpRefElement, SetRefElement, StringConcatRefElement, StringPrimitiveRefElement, SubStringRefElement, } from './RefElements.mjs';
+import { stringifyFlatpacked } from './stringify.mjs';
+import { Trie } from './Trie.mjs';
+import { blockSplitRegex, dataHeader } from './types.mjs';
+import { fromJSON } from './unpack.mjs';
+const collator = new Intl.Collator('en', {
+    usage: 'sort',
+    numeric: true,
+    sensitivity: 'variant',
+    caseFirst: 'upper',
+    ignorePunctuation: false,
+});
+const compare = collator.compare;
+const forceStringPrimitives = false;
+const minSubStringLen = 8;
+const minSubStringSuffixLen = 16;
+const useSuffix = true;
+const maxCachedStringLen = 256;
+export class FlatpackStore {
+    options;
+    knownElements = new Set();
+    assignedElements = new Map();
+    elements = [undefined];
+    root = undefined;
+    dedupe = true;
+    sortKeys = true;
+    emptyObjIdx = undefined;
+    ids = 0;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    cache = new Map();
+    /**
+     * Set of elements that have been referenced by other indexes.
+     */
+    referenced = new Set();
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    cachedArrays = new Map();
+    cachedObjects = new Map();
+    cachedSets = new Map();
+    cachedMaps = new Map();
+    /**
+     * Cache of strings that have been deduped and stored in the data array.
+     */
+    knownStrings = new Trie();
+    /**
+     * Cache of reversed strings that have been deduped and stored in the data array.
+     * This is used to find matching suffixes.
+     */
+    knownStringsRev = new Trie();
+    refUndefined;
+    constructor(value, options) {
+        this.options = options;
+        this.dedupe = options?.dedupe ?? true;
+        this.sortKeys = options?.sortKeys || this.dedupe;
+        this.refUndefined = this.addValueAndElement(undefined, new PrimitiveRefElement(undefined));
+        if (value instanceof FlatpackedWrapper) {
+            this.#fromWrapper(value);
+        }
+        else {
+            this.#setValue(value);
+        }
+    }
+    #fromWrapper(wrapper) {
+        this.elements = wrapper.toRefElements();
+        this.root = this.elements[1];
+        for (let i = 1; i < this.elements.length; i++) {
+            const element = this.elements[i];
+            if (!element)
+                continue;
+            this.knownElements.add(element);
+            if (element instanceof PrimitiveRefElement) {
+                this.addValueAndElement(element.value, element);
+            }
+            if (isStringRefElements(element)) {
+                this.addStringElement(element.value, element);
+            }
+        }
+        this.ids = this.elements.length;
+        this.#resolveRefs();
+    }
+    setValue(value) {
+        this.#setValue(value);
+    }
+    #setValue(value) {
+        this.softReset();
+        this.root = this.valueToRef(value);
+        this.#resolveRefs();
+        return this.root;
+    }
+    nextId() {
+        return this.ids++;
+    }
+    addElement(element) {
+        if (this.knownElements.has(element))
+            return element;
+        element.setId(this.nextId());
+        this.knownElements.add(element);
+        return element;
+    }
+    addValueAndElement(value, element, cache = true) {
+        this.addElement(element);
+        if (cache) {
+            this.cache.set(value, element);
+        }
+        return element;
+    }
+    primitiveToRef(value) {
+        if (typeof value === 'string')
+            return this.stringToRef(value);
+        if (typeof value === 'bigint')
+            return this.cvtBigintToRef(value);
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        return this.addValueAndElement(value, new PrimitiveRefElement(value));
+    }
+    createSubStringRef(baseString, value, offset) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        return this.addStringElement(value, new SubStringRefElement(baseString, value.length, offset));
+    }
+    addKnownString(ref, value) {
+        if (value.length >= minSubStringLen) {
+            this.knownStrings.add(prefix(value, maxCachedStringLen), ref);
+            const rev = reverse(suffix(value, maxCachedStringLen));
+            this.knownStringsRev.add(rev, ref);
+        }
+    }
+    addStringPrimitive(value) {
+        return this.addStringElement(value, new StringPrimitiveRefElement(value));
+    }
+    addStringElement(value, element) {
+        this.addKnownString(element, value);
+        return this.addValueAndElement(value, element);
+    }
+    stringPrefix(value) {
+        // if (value.length < maxCachedStringLen * 2) return undefined;
+        const trieFound = this.knownStrings.find(value);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+            return undefined;
+        }
+        const { data: tData, found: subStr } = trieFound;
+        // assert(subStr === value.slice(0, subStr.length));
+        return this.createSubStringRef(tData, subStr);
+    }
+    stringSuffix(value) {
+        if (!useSuffix)
+            return undefined;
+        const rev = reverse(value);
+        const trieFound = this.knownStringsRev.find(rev);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringSuffixLen) {
+            return undefined;
+        }
+        const { data: tData, found: subStr } = trieFound;
+        // assert(subStr === value.slice(0, subStr.length));
+        return this.createSubStringRef(tData, value.slice(-subStr.length), tData.length - subStr.length);
+    }
+    stringToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        if (forceStringPrimitives || value.length < minSubStringLen || blockSplitRegex.test(value)) {
+            return this.addStringPrimitive(value);
+        }
+        const partialsPfx = [];
+        const partialsSfx = [];
+        let subStr = value;
+        while (subStr.length) {
+            const prefix = this.stringPrefix(subStr);
+            const suffix = this.stringSuffix(subStr);
+            if (!prefix && !suffix)
+                break;
+            if (prefix && prefix.length >= (suffix?.length || 0)) {
+                partialsPfx.push(prefix);
+                subStr = subStr.slice(prefix.length);
+            }
+            else {
+                const sfx = suffix;
+                partialsSfx.push(sfx);
+                subStr = subStr.slice(0, -sfx.length);
+            }
+        }
+        partialsSfx.reverse();
+        if (!partialsPfx.length && !partialsSfx.length) {
+            return this.addStringPrimitive(value);
+        }
+        if (subStr.length) {
+            partialsPfx.push(this.stringToRef(subStr));
+        }
+        const partials = [...partialsPfx, ...partialsSfx];
+        return this.addStringElement(value, partials.length === 1 ? partials[0] : new StringConcatRefElement(partials));
+    }
+    cvtSetToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const element = this.addValueAndElement(value, new SetRefElement());
+        element.setValues(this.createUniqueKeys([...value]));
+        return this.dedupeSetRefs(value, element);
+    }
+    dedupeSetRefs(value, element) {
+        if (!this.dedupe)
+            return element;
+        const values = element.values();
+        const found = this.cachedSets.get(values);
+        if (!found) {
+            this.cachedSets.set(values, element);
+            return element;
+        }
+        if (this.referenced.has(element))
+            return element;
+        this.knownElements.delete(element);
+        this.cache.set(value, found);
+        return found;
+    }
+    createUniqueKeys(keys) {
+        const cacheValue = false;
+        let k = this.arrToRef(keys, cacheValue);
+        const uniqueKeys = new Set(k.valueRefs());
+        if (uniqueKeys.size !== keys.length) {
+            // one or more of the keys got deduped. We need to duplicate it.
+            uniqueKeys.clear();
+            const values = k.valueRefs().map((ref) => {
+                if (uniqueKeys.has(ref)) {
+                    return this.addElement(ref.clone());
+                }
+                uniqueKeys.add(ref);
+                return ref;
+            });
+            k = this.addValueAndElement(keys, new ArrayRefElement(values), cacheValue);
+        }
+        return k;
+    }
+    cvtMapToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const element = this.addValueAndElement(value, new MapRefElement());
+        element.setKeysAndValues(this.createUniqueKeys([...value.keys()]), this.arrToRef([...value.values()], false));
+        return this.dedupeMapRefs(value, element);
+    }
+    dedupeMapRefs(value, element) {
+        if (!this.dedupe)
+            return element;
+        const keys = element.keys();
+        const values = element.values();
+        let found = this.cachedMaps.get(keys);
+        if (!found) {
+            found = new Map();
+            found.set(values, element);
+            this.cachedMaps.set(keys, found);
+            return element;
+        }
+        const foundValue = found.get(values);
+        if (foundValue) {
+            if (this.referenced.has(element))
+                return element;
+            this.knownElements.delete(element);
+            this.cache.set(value, foundValue);
+            return foundValue;
+        }
+        found.set(values, element);
+        return element;
+    }
+    cvtRegExpToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        return this.addValueAndElement(value, new RegExpRefElement(this.stringToRef(value.source), this.stringToRef(value.flags)));
+    }
+    cvtDateToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        return this.addValueAndElement(value, new DateRefElement(value.getTime()));
+    }
+    cvtBigintToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const valElement = this.primitiveToRef(value <= Number.MAX_SAFE_INTEGER && value >= -Number.MAX_SAFE_INTEGER ? Number(value) : value.toString());
+        return this.addValueAndElement(value, new BigIntRefElement(valElement));
+    }
+    cvtObjToRef(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        if (isObjectWrapper(value)) {
+            const element = this.addValueAndElement(value, new ObjectWrapperRefElement());
+            element.setValue(this.valueToRef(value.valueOf()));
+            return element;
+        }
+        const entries = Object.entries(value);
+        if (!entries.length) {
+            if (this.emptyObjIdx) {
+                return this.emptyObjIdx;
+            }
+            this.emptyObjIdx = this.addValueAndElement(value, new ObjectRefElement());
+            return this.emptyObjIdx;
+        }
+        if (this.sortKeys) {
+            entries.sort((a, b) => compare(a[0], b[0]));
+        }
+        const element = this.addValueAndElement(value, new ObjectRefElement());
+        const k = this.arrToRef(entries.map(([key]) => key), false);
+        const v = this.arrToRef(entries.map(([, value]) => value), false);
+        element.setKeysAndValues(k, v);
+        return this.dedupeObject(value, element);
+    }
+    dedupeObject(value, element) {
+        if (!this.dedupe)
+            return element;
+        const keys = element.keys();
+        const values = element.values();
+        let found = this.cachedObjects.get(keys);
+        if (!found) {
+            found = new Map();
+            found.set(values, element);
+            this.cachedObjects.set(keys, found);
+            return element;
+        }
+        const foundValue = found.get(values);
+        if (foundValue) {
+            if (this.referenced.has(element))
+                return element;
+            this.knownElements.delete(element);
+            this.cache.set(value, foundValue);
+            return foundValue;
+        }
+        found.set(values, element);
+        return element;
+    }
+    /**
+     *
+     * @param value - The array converted to an ArrayRefElement.
+     * @param element - the element to dedupe.
+     * @param cacheValue - Whether to cache the value. It is false when it is a dynamic array, like object keys,
+     *      in that case, we want to dedupe the keys and values.
+     * @returns the element to use.
+     */
+    dedupeArray(value, element, cacheValue) {
+        if (cacheValue && !this.dedupe)
+            return element;
+        const indexHash = element.hash;
+        let cached = this.cachedArrays.get(indexHash);
+        if (!cached) {
+            cached = [];
+            this.cachedArrays.set(indexHash, cached);
+        }
+        const found = cached.find((entry) => element.isEqual(entry));
+        if (found) {
+            if (this.referenced.has(element))
+                return element;
+            this.knownElements.delete(element);
+            if (cacheValue || this.cache.has(value)) {
+                this.cache.set(value, found);
+            }
+            return found;
+        }
+        cached.push(element);
+        return element;
+    }
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    arrToRef(value, cacheValue = true) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const element = this.addValueAndElement(value, new ArrayRefElement(), cacheValue);
+        element.setValues(value.map((v) => this.valueToRef(v)));
+        return this.dedupeArray(value, element, cacheValue);
+    }
+    valueToRef(value) {
+        if (value === null) {
+            return this.primitiveToRef(value);
+        }
+        if (typeof value === 'object') {
+            if (value instanceof Set) {
+                return this.cvtSetToRef(value);
+            }
+            if (value instanceof Map) {
+                return this.cvtMapToRef(value);
+            }
+            if (value instanceof RegExp) {
+                return this.cvtRegExpToRef(value);
+            }
+            if (Array.isArray(value)) {
+                return this.arrToRef(value);
+            }
+            if (value instanceof Date) {
+                return this.cvtDateToRef(value);
+            }
+            return this.cvtObjToRef(value);
+        }
+        return this.primitiveToRef(value);
+    }
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    softReset() {
+        this.referenced.clear();
+        if (this.root) {
+            const idx = this.assignedElements.get(this.root);
+            if (idx) {
+                this.elements[idx] = undefined;
+                this.assignedElements.delete(this.root);
+            }
+        }
+        this.root = undefined;
+    }
+    #resolveRefs() {
+        if (!this.root)
+            return;
+        const elements = this.elements;
+        const assigned = this.assignedElements;
+        const referenced = this.referenced;
+        const availableIndexes = [];
+        function addElement(ref) {
+            if (assigned.has(ref))
+                return false;
+            const emptyCell = availableIndexes.pop();
+            if (emptyCell) {
+                assigned.set(ref, emptyCell);
+                elements[emptyCell] = ref;
+                return true;
+            }
+            const i = elements.push(ref) - 1;
+            assigned.set(ref, i);
+            return true;
+        }
+        function walk(ref, action) {
+            function _walk(ref) {
+                if (!action(ref))
+                    return;
+                const deps = ref.getDependencies();
+                if (!deps)
+                    return;
+                for (const dep of deps) {
+                    _walk(dep);
+                }
+            }
+            _walk(ref);
+        }
+        function calcReferences(root) {
+            referenced.clear();
+            walk(root, (ref) => {
+                if (referenced.has(ref))
+                    return false;
+                referenced.add(ref);
+                return true;
+            });
+        }
+        function calcAvailableIndexes() {
+            availableIndexes.length = 0;
+            for (let i = 1; i < elements.length; i++) {
+                const ref = elements[i];
+                if (!ref || !referenced.has(ref)) {
+                    availableIndexes.push(i);
+                    if (ref) {
+                        assigned.delete(ref);
+                    }
+                    elements[i] = undefined;
+                }
+                else {
+                    assigned.set(ref, i);
+                }
+            }
+            availableIndexes.reverse();
+        }
+        function addElements(root) {
+            walk(root, addElement);
+            let i = elements.length - 1;
+            for (; i > 0 && elements[i] === undefined; i--) {
+                // empty
+            }
+            elements.length = i + 1;
+        }
+        calcReferences(this.root);
+        elements[0] = this.refUndefined;
+        assigned.set(this.refUndefined, 0);
+        calcAvailableIndexes();
+        addElements(this.root);
+    }
+    toJSON() {
+        const data = [dataHeader];
+        const idxLookup = this.assignedElements;
+        const lookup = (ref) => (ref && idxLookup.get(ref)) || 0;
+        const elements = this.elements;
+        for (let i = 1; i < elements.length; i++) {
+            const element = elements[i];
+            if (!element) {
+                data.push(0);
+                continue;
+            }
+            const value = element.toElement(lookup);
+            if (value === undefined)
+                continue;
+            data.push(value);
+        }
+        return data;
+    }
+    static fromJSON(data) {
+        return new FlatpackStore(new FlatpackedWrapper(data));
+    }
+    static parse(content) {
+        return new FlatpackStore(FlatpackedWrapper.parse(content));
+    }
+    stringify() {
+        return stringifyFlatpacked(this.toJSON());
+    }
+    toValue() {
+        return fromJSON(this.toJSON());
+    }
+}
+function isObjectWrapper(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.valueOf === 'function' &&
+        value.valueOf() !== value);
+}
+function prefix(value, len) {
+    return value.length > len ? [...value].slice(0, len) : value;
+}
+function suffix(value, len) {
+    return value.length > len ? [...value].slice(-len) : value;
+}
+function reverse(value) {
+    return [...value].reverse();
+}
+export function toJSON(json, options) {
+    return new FlatpackStore(json, options).toJSON();
+}
+export function stringify(data, pretty = true) {
+    return pretty ? stringifyFlatpacked(toJSON(data)) : JSON.stringify(toJSON(data));
+}
+//# sourceMappingURL=Flatpack.mjs.map