flatpack-json 8.19.3 → 9.0.0

package/dist/flatpackUtil.mjs

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { ArrayRefElement, BigIntRefElement, DateRefElement, MapRefElement, NumberRefElement, ObjectRefElement, ObjectWrapperRefElement, PrimitiveRefElement, RegExpRefElement, SetRefElement, StringConcatRefElement, StringPrimitiveRefElement, SubStringRefElement, } from './RefElements.mjs';
-import { dataHeader, ElementType, } from './types.mjs';
+import { ElementType, supportedHeaders, } from './types.mjs';
 export function fromElement(elem, resolve) {
     function handleArrayElement(element) {
         switch (element[0]) {
@@ -90,6 +90,6 @@ export class FlatpackedWrapper {
     }
 }
 export function isFlatpacked(value) {
-    return Array.isArray(value) && value[0] === dataHeader;
+    return Array.isArray(value) && typeof value[0] === 'string' && supportedHeaders.has(value[0]);
 }
 //# sourceMappingURL=flatpackUtil.mjs.map

package/dist/optimizeFlatpacked.d.mts

@@ -0,0 +1,3 @@
+import { Flatpacked } from './types.mjs';
+export declare function optimizeFlatpacked(data: Flatpacked): Flatpacked;
+//# sourceMappingURL=optimizeFlatpacked.d.mts.map

package/dist/optimizeFlatpacked.mjs

@@ -0,0 +1,147 @@
+import assert from 'node:assert';
+import { ElementType, supportedHeaders } from './types.mjs';
+export function optimizeFlatpacked(data) {
+    const [header] = data;
+    if (!supportedHeaders.has(header)) {
+        throw new Error('Invalid header');
+    }
+    const elementRefs = data.map((element, index) => ({ origIndex: index, refCount: 0, index: 0, element }));
+    const indexToRefElement = new Map(elementRefs.entries());
+    for (const refElement of elementRefs) {
+        if (refElement.origIndex === 0) {
+            continue;
+        }
+        const indexes = getRefIndexes(refElement.element);
+        for (const index of indexes) {
+            const ref = indexToRefElement.get(index);
+            assert(ref, `Invalid reference index: ${index}`);
+            ref.refCount++;
+        }
+    }
+    const sortedRefElements = elementRefs.slice(2).sort((a, b) => b.refCount - a.refCount || a.origIndex - b.origIndex);
+    sortedRefElements.forEach((refElement, index) => {
+        refElement.index = index + 2;
+    });
+    const indexMap = new Map([
+        [0, 0],
+        [1, 1],
+    ]);
+    sortedRefElements.forEach((refElement) => {
+        indexMap.set(refElement.origIndex, refElement.index);
+    });
+    const optimizedElements = [
+        data[1],
+        ...sortedRefElements.map((refElement) => refElement.element),
+    ].map((element) => patchIndexes(element, indexMap));
+    return [header, ...optimizedElements];
+}
+function patchIndexes(elem, indexMap) {
+    function mapIndex(index) {
+        const v = indexMap.get(index);
+        assert(v !== undefined, `Invalid index: ${index}`);
+        return v;
+    }
+    function mapIndexes(indexes) {
+        return indexes.map(mapIndex);
+    }
+    function handleArrayElement(element) {
+        switch (element[0]) {
+            case ElementType.Array: {
+                return [element[0], ...mapIndexes(element.slice(1))];
+            }
+            case ElementType.Object: {
+                return [element[0], mapIndex(element[1]), mapIndex(element[2])];
+            }
+            case ElementType.String: {
+                return [element[0], ...mapIndexes(element.slice(1))];
+            }
+            case ElementType.SubString: {
+                return element[3] !== undefined
+                    ? [element[0], mapIndex(element[1]), element[2], element[3]]
+                    : [element[0], mapIndex(element[1]), element[2]];
+            }
+            case ElementType.Set: {
+                return [element[0], mapIndex(element[1])];
+            }
+            case ElementType.Map: {
+                return [element[0], mapIndex(element[1]), mapIndex(element[2])];
+            }
+            case ElementType.RegExp: {
+                return [element[0], mapIndex(element[1]), mapIndex(element[2])];
+            }
+            case ElementType.Date: {
+                return element;
+            }
+            case ElementType.BigInt: {
+                return [element[0], mapIndex(element[1])];
+            }
+        }
+        assert(false, 'Invalid element type');
+    }
+    if (Array.isArray(elem)) {
+        return handleArrayElement(elem);
+    }
+    if (typeof elem === 'string') {
+        return elem;
+    }
+    if (typeof elem === 'number') {
+        return elem;
+    }
+    if (typeof elem === 'object') {
+        return elem;
+    }
+    assert(typeof elem === 'boolean');
+    return elem;
+}
+function getRefIndexes(elem) {
+    function handleArrayElement(element) {
+        switch (element[0]) {
+            case ElementType.Array: {
+                return element.slice(1);
+            }
+            case ElementType.Object: {
+                return element.slice(1).filter((v) => !!v);
+            }
+            case ElementType.String: {
+                return element.slice(1);
+            }
+            case ElementType.SubString: {
+                return [element[1]];
+            }
+            case ElementType.Set: {
+                return element.slice(1);
+            }
+            case ElementType.Map: {
+                return element.slice(1);
+            }
+            case ElementType.RegExp: {
+                return element.slice(1);
+            }
+            case ElementType.Date: {
+                return [];
+            }
+            case ElementType.BigInt: {
+                return [element[1]];
+            }
+        }
+        assert(false, 'Invalid element type');
+    }
+    if (Array.isArray(elem)) {
+        return handleArrayElement(elem);
+    }
+    if (typeof elem === 'string') {
+        return [];
+    }
+    if (typeof elem === 'number') {
+        return [];
+    }
+    if (typeof elem === 'object') {
+        if (elem === null) {
+            return [];
+        }
+        return [];
+    }
+    assert(typeof elem === 'boolean');
+    return [];
+}
+//# sourceMappingURL=optimizeFlatpacked.mjs.map

package/dist/storage.d.mts

@@ -19,17 +19,24 @@ export declare class CompactStorage {
      */
     private cachedArrays;
     /**
-     * Cache of strings that have been deduped and stored in the data array.
+     * Cache of strings used for prefix matching.
      */
     private knownStrings;
+    /**
+     * Cache of reversed strings used for suffix matching.
+     */
+    private knownStringsRev;
     private cachedElements;
     constructor(options?: FlatpackOptions | undefined);
     private primitiveToIdx;
     private addSubStringRef;
+    private estimateSubStringCost;
     private addKnownString;
     private addStringPrimitive;
     private duplicateIndex;
     private stringToIdx;
+    private findPrefix;
+    private findSuffix;
     private objSetToIdx;
     private createUniqueKeys;
     private objMapToIdx;
@@ -56,5 +63,5 @@ export declare class CompactStorage {
     toJSON<V extends Serializable>(json: V): Flatpacked;
 }
 export declare function toJSON<V extends Serializable>(json: V, options?: FlatpackOptions): Flatpacked;
-export declare function stringify(data: Unpacked, pretty?: boolean): string;
+export declare function stringify(data: Unpacked, pretty?: boolean, options?: FlatpackOptions): string;
 //# sourceMappingURL=storage.d.mts.map

package/dist/storage.mjs

@@ -1,4 +1,5 @@
 import assert from 'node:assert';
+import { optimizeFlatpacked } from './optimizeFlatpacked.mjs';
 import { stringifyFlatpacked } from './stringify.mjs';
 import { Trie } from './Trie.mjs';
 import { blockSplitRegex, dataHeader, ElementType } from './types.mjs';
@@ -32,9 +33,13 @@ export class CompactStorage {
      */
     cachedArrays = new Map();
     /**
-     * Cache of strings that have been deduped and stored in the data array.
+     * Cache of strings used for prefix matching.
      */
     knownStrings = new Trie();
+    /**
+     * Cache of reversed strings used for suffix matching.
+     */
+    knownStringsRev = new Trie();
     cachedElements = new Map();
     constructor(options) {
         this.options = options;
@@ -66,9 +71,23 @@ export class CompactStorage {
         this.cache.set(value, idx);
         return idx;
     }
+    estimateSubStringCost(idxString, value, offset) {
+        if (this.cache.has(value)) {
+            return 0;
+        }
+        let cost = 5;
+        cost += Math.ceil(Math.log10(idxString));
+        cost += Math.ceil(Math.log10(value.length));
+        if (offset) {
+            cost += Math.ceil(Math.log10(offset)) + 1;
+        }
+        return cost;
+    }
     addKnownString(idx, value) {
         if (value.length >= minSubStringLen) {
-            this.knownStrings.add(value.length > 256 ? value.slice(0, 256) : value, { idx });
+            const data = { idx, value };
+            this.knownStrings.add(value.length > 256 ? [...value].slice(0, 256) : value, data);
+            this.knownStringsRev.add(revStr(value, 256), data);
         }
     }
     addStringPrimitive(value) {
@@ -90,20 +109,63 @@ export class CompactStorage {
         if (forceStringPrimitives || value.length < minSubStringLen || blockSplitRegex.test(value)) {
             return this.addStringPrimitive(value);
         }
-        const trieFound = this.knownStrings.find(value);
-        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+        const foundPrefix = this.findPrefix(value);
+        const foundSuffix = this.findSuffix(value);
+        if (!foundPrefix && !foundSuffix) {
             return this.addStringPrimitive(value);
         }
-        const { data: tData, found: subStr } = trieFound;
-        const sIdx = this.addSubStringRef(tData.idx, subStr, tData.offset);
-        if (subStr === value)
-            return sIdx;
-        const v = [sIdx, this.stringToIdx(value.slice(subStr.length))];
+        if (foundPrefix &&
+            (!foundSuffix ||
+                foundPrefix.subStr.length - foundPrefix.cost >= foundSuffix.subStr.length - foundSuffix.cost)) {
+            const { idx: pfIdx, subStr, offset } = foundPrefix;
+            const sIdx = this.addSubStringRef(pfIdx, subStr, offset);
+            if (subStr === value)
+                return sIdx;
+            const v = [sIdx, this.stringToIdx(value.slice(subStr.length))];
+            const idx = this.data.push([ElementType.String, ...v]) - 1;
+            this.cache.set(value, idx);
+            this.addKnownString(idx, value);
+            return idx;
+        }
+        if (!foundSuffix) {
+            return this.addStringPrimitive(value);
+        }
+        const { idx: pfIdx, subStr, offset } = foundSuffix;
+        const sIdx = this.addSubStringRef(pfIdx, subStr, offset);
+        const v = [this.stringToIdx(value.slice(0, -subStr.length)), sIdx];
         const idx = this.data.push([ElementType.String, ...v]) - 1;
         this.cache.set(value, idx);
         this.addKnownString(idx, value);
         return idx;
     }
+    findPrefix(value) {
+        const trieFound = this.knownStrings.find(value);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+            return undefined;
+        }
+        const { data: tData, found: subStr } = trieFound;
+        const cost = this.estimateSubStringCost(tData.idx, subStr, undefined);
+        if (cost > subStr.length) {
+            return undefined;
+        }
+        return { idx: tData.idx, subStr, offset: undefined, cost };
+    }
+    findSuffix(value) {
+        const rev = revStr(value, 256);
+        const trieFound = this.knownStringsRev.find(rev);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+            return undefined;
+        }
+        const { data: tData, found: subStrRev } = trieFound;
+        const offset = tData.value.length - subStrRev.length;
+        const cost = this.estimateSubStringCost(tData.idx, subStrRev, offset);
+        if (cost > subStrRev.length) {
+            return undefined;
+        }
+        const srcStr = tData.value;
+        const subStr = srcStr.slice(-subStrRev.length);
+        return { idx: tData.idx, subStr, offset, cost };
+    }
     objSetToIdx(value) {
         const found = this.cache.get(value);
         if (found !== undefined) {
@@ -328,7 +390,25 @@ export class CompactStorage {
     toJSON(json) {
         this.softReset();
         this.valueToIdx(json);
-        return this.data;
+        return this.options?.optimize ? optimizeFlatpacked(this.data) : this.data;
+    }
+}
+function* revStr(str, limit) {
+    let i = str.length - 1;
+    let n = 0;
+    while (i >= 0 && n < limit) {
+        // eslint-disable-next-line unicorn/prefer-code-point
+        const code = str.charCodeAt(i);
+        if (code & 0xfc00) {
+            // surrogate pair
+            i -= 1;
+            yield str.slice(i, i + 2);
+        }
+        else {
+            yield str[i];
+        }
+        i -= 1;
+        n += 1;
     }
 }
 function isEqual(a, b) {
@@ -358,7 +438,8 @@ function isObjectWrapper(value) {
 export function toJSON(json, options) {
     return new CompactStorage(options).toJSON(json);
 }
-export function stringify(data, pretty = true) {
-    return pretty ? stringifyFlatpacked(toJSON(data)) : JSON.stringify(toJSON(data));
+export function stringify(data, pretty = true, options) {
+    const json = toJSON(data, options);
+    return pretty ? stringifyFlatpacked(json) : JSON.stringify(json);
 }
 //# sourceMappingURL=storage.mjs.map

package/dist/types.d.mts

@@ -93,7 +93,22 @@ export interface FlatpackOptions {
      * @default true
      */
     dedupe?: boolean;
+    /**
+     * Try to optimize the size of the output.
+     */
+    optimize?: boolean;
 }
-export declare const dataHeader: "Dehydrated JSON v1";
+/**
+ * Legacy header for Flatpack JSON.
+ */
+export declare const dataHeaderV0_1: "Dehydrated JSON v1";
+/**
+ * The current header for Flatpack JSON.
+ */
+export declare const dataHeader: "Flatpack JSON v1";
+/**
+ * The set of supported headers for Flatpack JSON.
+ */
+export declare const supportedHeaders: Set<string>;
 export {};
 //# sourceMappingURL=types.d.mts.map

package/dist/types.mjs

@@ -11,5 +11,16 @@ export var ElementType;
     ElementType[ElementType["BigInt"] = 8] = "BigInt";
 })(ElementType || (ElementType = {}));
 export const blockSplitRegex = /^sha\d/;
-export const dataHeader = 'Dehydrated JSON v1';
+/**
+ * Legacy header for Flatpack JSON.
+ */
+export const dataHeaderV0_1 = 'Dehydrated JSON v1';
+/**
+ * The current header for Flatpack JSON.
+ */
+export const dataHeader = 'Flatpack JSON v1';
+/**
+ * The set of supported headers for Flatpack JSON.
+ */
+export const supportedHeaders = new Set([dataHeaderV0_1, dataHeader]);
 //# sourceMappingURL=types.mjs.map

package/dist/unpack.mjs

@@ -1,8 +1,8 @@
 import assert from 'node:assert';
-import { dataHeader, ElementType, } from './types.mjs';
+import { ElementType, supportedHeaders, } from './types.mjs';
 export function fromJSON(data) {
     const [header] = data;
-    if (header !== dataHeader) {
+    if (!supportedHeaders.has(header)) {
         throw new Error('Invalid header');
     }
     const cache = new Map([[0, undefined]]);

package/package.json

@@ -4,8 +4,8 @@
     "access": "public",
     "provenance": true
   },
-  "version": "8.19.3",
-  "description": "A library to normalize JSON objects to reduce the size.",
+  "version": "9.0.0",
+  "description": "A library to normalize / flatten JSON objects to reduce the size.",
   "keywords": [
     "cspell",
     "json",
@@ -50,11 +50,12 @@
     "url": "https://github.com/streetsidesoftware/cspell/labels/filetype"
   },
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "devDependencies": {
-    "@cspell/filetypes": "8.19.3",
-    "diff": "^7.0.0"
+    "@cspell/filetypes": "9.0.0",
+    "diff": "^7.0.0",
+    "flatted": "^3.3.3"
   },
-  "gitHead": "cb6946b709da590d0c6175f9962799d18aa43e93"
+  "gitHead": "f7c4be398734894d817d9b60214731a516cff7d2"
 }