flatpack-json 9.7.0 → 9.8.0

package/dist/storage.d.mts

@@ -1,67 +1,6 @@
+import type { StringifyOptions } from './stringify.mjs';
 import type { Flatpacked, FlatpackOptions, Serializable, Unpacked } from './types.mjs';
-export declare class CompactStorage {
-    readonly options?: FlatpackOptions | undefined;
-    private data;
-    private dedupe;
-    private sortKeys;
-    private emptyObjIdx;
-    /**
-     * Cache of primitives and objects that have been added to the data.
-     */
-    private cache;
-    /**
-     * Set of indexes that have been referenced by other indexes.
-     */
-    private referenced;
-    /**
-     * Cache of arrays that have been deduped.
-     * The key is a hash of the array elements as a function of the index of the element.
-     */
-    private cachedArrays;
-    /**
-     * Cache of strings used for prefix matching.
-     */
-    private knownStrings;
-    /**
-     * Cache of reversed strings used for suffix matching.
-     */
-    private knownStringsRev;
-    private cachedElements;
-    constructor(options?: FlatpackOptions | undefined);
-    private primitiveToIdx;
-    private addSubStringRef;
-    private estimateSubStringCost;
-    private addKnownString;
-    private addStringPrimitive;
-    private duplicateIndex;
-    private stringToIdx;
-    private findPrefix;
-    private findSuffix;
-    private objSetToIdx;
-    private createUniqueKeys;
-    private objMapToIdx;
-    private objRegExpToIdx;
-    private objDateToIdx;
-    private bigintToIdx;
-    private objToIdx;
-    private storeElement;
-    private cacheElement;
-    private stashArray;
-    private createArrayElementFromIndexValues;
-    /**
-     * Convert an array to an index.
-     * @param value - The array to convert to an index.
-     * @param cacheValue - Whether to cache the value.
-     * @returns the index of the array.
-     */
-    private arrToIdx;
-    private valueToIdx;
-    /**
-     * Reset things in a way that allows for reuse.
-     */
-    private softReset;
-    toJSON<V extends Serializable>(json: V): Flatpacked;
-}
 export declare function toJSON<V extends Serializable>(json: V, options?: FlatpackOptions): Flatpacked;
-export declare function stringify(data: Unpacked, pretty?: boolean, options?: FlatpackOptions): string;
+export declare function stringify(data: Unpacked, pretty?: boolean, options?: FlatpackOptions & StringifyOptions): string;
+export declare function normalizeOptions(options?: FlatpackOptions): Required<FlatpackOptions>;
 //# sourceMappingURL=storage.d.mts.map

package/dist/storage.mjs

@@ -1,445 +1,37 @@
-import assert from 'node:assert';
-import { optimizeFlatpacked } from './optimizeFlatpacked.mjs';
+import { CompactStorageV1 } from './storageV1.mjs';
+import { CompactStorageV2 } from './storageV2.mjs';
 import { stringifyFlatpacked } from './stringify.mjs';
-import { Trie } from './Trie.mjs';
-import { blockSplitRegex, dataHeader, ElementType } from './types.mjs';
-const collator = new Intl.Collator('en', {
-    usage: 'sort',
-    numeric: true,
-    sensitivity: 'variant',
-    caseFirst: 'upper',
-    ignorePunctuation: false,
-});
-const compare = collator.compare;
-const forceStringPrimitives = false;
-const minSubStringLen = 4;
-export class CompactStorage {
-    options;
-    data = [dataHeader];
-    dedupe = true;
-    sortKeys = true;
-    emptyObjIdx = 0;
-    /**
-     * Cache of primitives and objects that have been added to the data.
-     */
-    cache = new Map([[undefined, 0]]);
-    /**
-     * Set of indexes that have been referenced by other indexes.
-     */
-    referenced = new Set();
-    /**
-     * Cache of arrays that have been deduped.
-     * The key is a hash of the array elements as a function of the index of the element.
-     */
-    cachedArrays = new Map();
-    /**
-     * Cache of strings used for prefix matching.
-     */
-    knownStrings = new Trie();
-    /**
-     * Cache of reversed strings used for suffix matching.
-     */
-    knownStringsRev = new Trie();
-    cachedElements = new Map();
-    constructor(options) {
-        this.options = options;
-        this.dedupe = options?.dedupe ?? true;
-        this.sortKeys = options?.sortKeys || this.dedupe;
-    }
-    primitiveToIdx(value) {
-        if (typeof value === 'string')
-            return this.stringToIdx(value);
-        if (typeof value === 'bigint')
-            return this.bigintToIdx(value);
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        const idx = this.data.push(value) - 1;
-        this.cache.set(value, idx);
-        return idx;
-    }
-    addSubStringRef(idxString, value, offset) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        const sub = offset
-            ? [ElementType.SubString, idxString, value.length, offset]
-            : [ElementType.SubString, idxString, value.length];
-        const idx = this.data.push(sub) - 1;
-        this.cache.set(value, idx);
-        return idx;
-    }
-    estimateSubStringCost(idxString, value, offset) {
-        if (this.cache.has(value)) {
-            return 0;
-        }
-        let cost = 5;
-        cost += Math.ceil(Math.log10(idxString));
-        cost += Math.ceil(Math.log10(value.length));
-        if (offset) {
-            cost += Math.ceil(Math.log10(offset)) + 1;
-        }
-        return cost;
-    }
-    addKnownString(idx, value) {
-        if (value.length >= minSubStringLen) {
-            const data = { idx, value };
-            this.knownStrings.add(value.length > 256 ? [...value].slice(0, 256) : value, data);
-            this.knownStringsRev.add(revStr(value, 256), data);
-        }
-    }
-    addStringPrimitive(value) {
-        const idx = this.data.push(value) - 1;
-        this.addKnownString(idx, value);
-        this.cache.set(value, idx);
-        return idx;
-    }
-    duplicateIndex(idx) {
-        const element = this.data[idx];
-        const duplicate = this.data.push(element) - 1;
-        return duplicate;
-    }
-    stringToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        if (forceStringPrimitives || value.length < minSubStringLen || blockSplitRegex.test(value)) {
-            return this.addStringPrimitive(value);
-        }
-        const foundPrefix = this.findPrefix(value);
-        const foundSuffix = this.findSuffix(value);
-        if (!foundPrefix && !foundSuffix) {
-            return this.addStringPrimitive(value);
-        }
-        if (foundPrefix &&
-            (!foundSuffix ||
-                foundPrefix.subStr.length - foundPrefix.cost >= foundSuffix.subStr.length - foundSuffix.cost)) {
-            const { idx: pfIdx, subStr, offset } = foundPrefix;
-            const sIdx = this.addSubStringRef(pfIdx, subStr, offset);
-            if (subStr === value)
-                return sIdx;
-            const v = [sIdx, this.stringToIdx(value.slice(subStr.length))];
-            const idx = this.data.push([ElementType.String, ...v]) - 1;
-            this.cache.set(value, idx);
-            this.addKnownString(idx, value);
-            return idx;
-        }
-        if (!foundSuffix) {
-            return this.addStringPrimitive(value);
-        }
-        const { idx: pfIdx, subStr, offset } = foundSuffix;
-        const sIdx = this.addSubStringRef(pfIdx, subStr, offset);
-        const v = [this.stringToIdx(value.slice(0, -subStr.length)), sIdx];
-        const idx = this.data.push([ElementType.String, ...v]) - 1;
-        this.cache.set(value, idx);
-        this.addKnownString(idx, value);
-        return idx;
-    }
-    findPrefix(value) {
-        const trieFound = this.knownStrings.find(value);
-        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
-            return undefined;
-        }
-        const { data: tData, found: subStr } = trieFound;
-        const cost = this.estimateSubStringCost(tData.idx, subStr, undefined);
-        if (cost > subStr.length) {
-            return undefined;
-        }
-        return { idx: tData.idx, subStr, offset: undefined, cost };
-    }
-    findSuffix(value) {
-        const rev = revStr(value, 256);
-        const trieFound = this.knownStringsRev.find(rev);
-        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
-            return undefined;
-        }
-        const { data: tData, found: subStrRev } = trieFound;
-        const offset = tData.value.length - subStrRev.length;
-        const cost = this.estimateSubStringCost(tData.idx, subStrRev, offset);
-        if (cost > subStrRev.length) {
-            return undefined;
-        }
-        const srcStr = tData.value;
-        const subStr = srcStr.slice(-subStrRev.length);
-        return { idx: tData.idx, subStr, offset, cost };
-    }
-    objSetToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        const keys = [...value];
-        const k = this.createUniqueKeys(keys, false);
-        const element = [ElementType.Set, k];
-        return this.storeElement(value, idx, element);
-    }
-    createUniqueKeys(keys, cacheValue = true) {
-        let k = this.arrToIdx(keys, cacheValue);
-        const elementKeys = this.data[k];
-        const uniqueKeys = new Set(elementKeys.slice(1));
-        if (uniqueKeys.size !== keys.length) {
-            // one or more of the keys got deduped. We need to duplicate it.
-            uniqueKeys.clear();
-            const indexes = elementKeys.slice(1).map((idx) => {
-                if (uniqueKeys.has(idx)) {
-                    return this.duplicateIndex(idx);
-                }
-                uniqueKeys.add(idx);
-                return idx;
-            });
-            k = this.createArrayElementFromIndexValues(this.data.length, indexes);
-        }
-        return k;
-    }
-    objMapToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        const entries = [...value.entries()];
-        const k = this.createUniqueKeys(entries.map(([key]) => key), false);
-        const v = this.arrToIdx(entries.map(([, value]) => value), false);
-        const element = [ElementType.Map, k, v];
-        return this.storeElement(value, idx, element);
-    }
-    objRegExpToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        const element = [
-            ElementType.RegExp,
-            this.stringToIdx(value.source),
-            this.stringToIdx(value.flags),
-        ];
-        return this.storeElement(value, idx, element);
-    }
-    objDateToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        const element = [ElementType.Date, value.getTime()];
-        return this.storeElement(value, idx, element);
-    }
-    bigintToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            return found;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        const element = [
-            ElementType.BigInt,
-            this.primitiveToIdx(value <= Number.MAX_SAFE_INTEGER && value >= -Number.MAX_SAFE_INTEGER
-                ? Number(value)
-                : value.toString()),
-        ];
-        return this.storeElement(value, idx, element);
-    }
-    objToIdx(value) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        if (isObjectWrapper(value)) {
-            const idx = this.data.push({}) - 1;
-            this.cache.set(value, idx);
-            const element = [ElementType.Object, 0, this.valueToIdx(value.valueOf())];
-            return this.storeElement(value, idx, element);
-        }
-        const entries = Object.entries(value);
-        if (!entries.length) {
-            if (this.emptyObjIdx) {
-                return this.emptyObjIdx;
-            }
-            const idx = this.data.push({}) - 1;
-            this.emptyObjIdx = idx;
-            return idx;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        if (this.sortKeys) {
-            entries.sort(([a], [b]) => compare(a, b));
-        }
-        const k = this.arrToIdx(entries.map(([key]) => key));
-        const v = this.arrToIdx(entries.map(([, value]) => value));
-        const element = [ElementType.Object, k, v];
-        return this.storeElement(value, idx, element);
-    }
-    storeElement(value, idx, element) {
-        const useIdx = this.dedupe ? this.cacheElement(idx, element) : idx;
-        if (useIdx !== idx && idx === this.data.length - 1) {
-            this.data.length = idx;
-            this.cache.set(value, useIdx);
-            return useIdx;
-        }
-        this.data[idx] = element;
-        return idx;
-    }
-    cacheElement(elemIdx, element) {
-        let map = this.cachedElements;
-        for (let i = 0; i < element.length - 1; i++) {
-            const idx = element[i];
-            let found = map.get(idx);
-            if (!found) {
-                found = new Map();
-                map.set(idx, found);
-            }
-            assert(found instanceof Map);
-            map = found;
-        }
-        const idx = element[element.length - 1];
-        const foundIdx = map.get(idx);
-        if (typeof foundIdx === 'number') {
-            return this.referenced.has(elemIdx) ? elemIdx : foundIdx;
-        }
-        map.set(idx, elemIdx);
-        return elemIdx;
-    }
-    stashArray(idx, element) {
-        const indexHash = simpleHash(element);
-        let found = this.cachedArrays.get(indexHash);
-        if (!found) {
-            found = [];
-            this.cachedArrays.set(indexHash, found);
-        }
-        const foundIdx = found.find((entry) => isEqual(entry.v, element));
-        if (foundIdx) {
-            return this.referenced.has(idx) ? idx : foundIdx.idx;
-        }
-        found.push({ idx, v: element });
-        return idx;
-    }
-    createArrayElementFromIndexValues(idx, indexValues) {
-        const element = [ElementType.Array, ...indexValues];
-        const useIdx = this.dedupe ? this.stashArray(idx, element) : idx;
-        if (useIdx !== idx) {
-            assert(this.data.length === idx + 1, `Expected ${idx + 1} but got ${this.data.length}`);
-            this.data.length = idx;
-            return useIdx;
-        }
-        this.data[idx] = element;
-        return idx;
-    }
-    /**
-     * Convert an array to an index.
-     * @param value - The array to convert to an index.
-     * @param cacheValue - Whether to cache the value.
-     * @returns the index of the array.
-     */
-    arrToIdx(value, cacheValue = true) {
-        const found = this.cache.get(value);
-        if (found !== undefined) {
-            this.referenced.add(found);
-            return found;
-        }
-        const idx = this.data.push(0) - 1;
-        this.cache.set(value, idx);
-        const useIdx = this.createArrayElementFromIndexValues(idx, value.map((idx) => this.valueToIdx(idx)));
-        if (cacheValue) {
-            this.cache.set(value, useIdx);
-        }
-        return useIdx;
-    }
-    valueToIdx(value) {
-        if (value === null) {
-            return this.primitiveToIdx(value);
-        }
-        if (typeof value === 'object') {
-            if (value instanceof Set) {
-                return this.objSetToIdx(value);
-            }
-            if (value instanceof Map) {
-                return this.objMapToIdx(value);
-            }
-            if (value instanceof RegExp) {
-                return this.objRegExpToIdx(value);
-            }
-            if (Array.isArray(value)) {
-                return this.arrToIdx(value);
-            }
-            if (value instanceof Date) {
-                return this.objDateToIdx(value);
-            }
-            return this.objToIdx(value);
-        }
-        return this.primitiveToIdx(value);
-    }
-    /**
-     * Reset things in a way that allows for reuse.
-     */
-    softReset() {
-        this.cache.clear();
-        this.cache.set(undefined, 0);
-    }
-    toJSON(json) {
-        this.softReset();
-        this.valueToIdx(json);
-        return this.options?.optimize ? optimizeFlatpacked(this.data) : this.data;
-    }
-}
-function* revStr(str, limit) {
-    let i = str.length - 1;
-    let n = 0;
-    while (i >= 0 && n < limit) {
-        // eslint-disable-next-line unicorn/prefer-code-point
-        const code = str.charCodeAt(i);
-        if (code & 0xfc00) {
-            // surrogate pair
-            i -= 1;
-            yield str.slice(i, i + 2);
-        }
-        else {
-            yield str[i];
-        }
-        i -= 1;
-        n += 1;
-    }
-}
-function isEqual(a, b) {
-    if (a === b)
-        return true;
-    if (a.length !== b.length)
-        return false;
-    for (let i = 0; i < a.length; i++) {
-        if (a[i] !== b[i])
-            return false;
-    }
-    return true;
-}
-function simpleHash(values) {
-    let hash = Math.sqrt(values.length);
-    for (const value of values) {
-        hash += value * value;
-    }
-    return hash;
-}
-function isObjectWrapper(value) {
-    return (typeof value === 'object' &&
-        value !== null &&
-        typeof value.valueOf === 'function' &&
-        value.valueOf() !== value);
-}
+import { dataHeader, dataHeaderV1_0, dataHeaderV2_0 } from './types.mjs';
 export function toJSON(json, options) {
-    return new CompactStorage(options).toJSON(json);
+    options = normalizeOptions(options);
+    return options.format === 'V1'
+        ? new CompactStorageV1(options).toJSON(json)
+        : new CompactStorageV2(options).toJSON(json);
 }
 export function stringify(data, pretty = true, options) {
-    const json = toJSON(data, options);
-    return pretty ? stringifyFlatpacked(json) : JSON.stringify(json);
+    const normalizedOptions = normalizeOptions(options);
+    const json = toJSON(data, normalizedOptions);
+    const stringifyOptions = normalizedOptions.format === 'V1' ? undefined : options || {};
+    return pretty ? stringifyFlatpacked(json, stringifyOptions) : JSON.stringify(json);
+}
+export function normalizeOptions(options) {
+    let header = dataHeader;
+    if (options?.format === 'V1') {
+        header = dataHeaderV1_0;
+    }
+    if (options?.format === 'V2') {
+        header = dataHeaderV2_0;
+    }
+    const dedupe = options?.dedupe ?? true;
+    const sortKeys = options?.sortKeys || dedupe;
+    const result = {
+        ...options,
+        format: header === dataHeaderV1_0 ? 'V1' : 'V2',
+        dedupe,
+        sortKeys,
+        optimize: options?.optimize ?? false,
+        meta: options?.meta,
+    };
+    return result;
 }
 //# sourceMappingURL=storage.mjs.map

package/dist/storageV1.d.mts

@@ -0,0 +1,67 @@
+import { CompactStorage } from './CompactStorage.mjs';
+import type { Flatpacked, FlatpackOptions, Serializable, Unpacked } from './types.mjs';
+export declare class CompactStorageV1 extends CompactStorage {
+    private data;
+    private dedupe;
+    private sortKeys;
+    private emptyObjIdx;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    private cache;
+    /**
+     * Set of indexes that have been referenced by other indexes.
+     */
+    private referenced;
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    private cachedArrays;
+    /**
+     * Cache of strings used for prefix matching.
+     */
+    private knownStrings;
+    /**
+     * Cache of reversed strings used for suffix matching.
+     */
+    private knownStringsRev;
+    private cachedElements;
+    constructor(options?: FlatpackOptions | undefined);
+    private primitiveToIdx;
+    private addSubStringRef;
+    private estimateSubStringCost;
+    private addKnownString;
+    private addStringPrimitive;
+    private duplicateIndex;
+    private stringToIdx;
+    private findPrefix;
+    private findSuffix;
+    private objSetToIdx;
+    private createUniqueKeys;
+    private objMapToIdx;
+    private objRegExpToIdx;
+    private objDateToIdx;
+    private bigintToIdx;
+    private objToIdx;
+    private storeElement;
+    private cacheElement;
+    private stashArray;
+    private createArrayElementFromIndexValues;
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    private arrToIdx;
+    private valueToIdx;
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    private softReset;
+    toJSON<V extends Serializable>(json: V): Flatpacked;
+}
+export declare function toJSON<V extends Serializable>(json: V, options?: FlatpackOptions): Flatpacked;
+export declare function stringify(data: Unpacked, pretty?: boolean, options?: FlatpackOptions): string;
+//# sourceMappingURL=storageV1.d.mts.map