flatpack-json 8.14.3

package/dist/storage.d.mts

@@ -0,0 +1,60 @@
+import type { Flatpacked, FlatpackOptions, Serializable, Unpacked } from './types.mjs';
+export declare class CompactStorage {
+    readonly options?: FlatpackOptions | undefined;
+    private data;
+    private dedupe;
+    private sortKeys;
+    private emptyObjIdx;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    private cache;
+    /**
+     * Set of indexes that have been referenced by other indexes.
+     */
+    private referenced;
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    private cachedArrays;
+    /**
+     * Cache of strings that have been deduped and stored in the data array.
+     */
+    private knownStrings;
+    private cachedElements;
+    constructor(options?: FlatpackOptions | undefined);
+    private primitiveToIdx;
+    private addSubStringRef;
+    private addKnownString;
+    private addStringPrimitive;
+    private duplicateIndex;
+    private stringToIdx;
+    private objSetToIdx;
+    private createUniqueKeys;
+    private objMapToIdx;
+    private objRegExpToIdx;
+    private objDateToIdx;
+    private bigintToIdx;
+    private objToIdx;
+    private storeElement;
+    private cacheElement;
+    private stashArray;
+    private createArrayElementFromIndexValues;
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    private arrToIdx;
+    private valueToIdx;
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    private softReset;
+    toJSON<V extends Serializable>(json: V): Flatpacked;
+}
+export declare function toJSON<V extends Serializable>(json: V, options?: FlatpackOptions): Flatpacked;
+export declare function stringify(data: Unpacked, pretty?: boolean): string;
+//# sourceMappingURL=storage.d.mts.map

package/dist/storage.mjs

@@ -0,0 +1,364 @@
+import assert from 'node:assert';
+import { stringifyFlatpacked } from './stringify.mjs';
+import { Trie } from './Trie.mjs';
+import { blockSplitRegex, dataHeader, ElementType } from './types.mjs';
+const collator = new Intl.Collator('en', {
+    usage: 'sort',
+    numeric: true,
+    sensitivity: 'variant',
+    caseFirst: 'upper',
+    ignorePunctuation: false,
+});
+const compare = collator.compare;
+const forceStringPrimitives = false;
+const minSubStringLen = 4;
+export class CompactStorage {
+    options;
+    data = [dataHeader];
+    dedupe = true;
+    sortKeys = true;
+    emptyObjIdx = 0;
+    /**
+     * Cache of primitives and objects that have been added to the data.
+     */
+    cache = new Map([[undefined, 0]]);
+    /**
+     * Set of indexes that have been referenced by other indexes.
+     */
+    referenced = new Set();
+    /**
+     * Cache of arrays that have been deduped.
+     * The key is a hash of the array elements as a function of the index of the element.
+     */
+    cachedArrays = new Map();
+    /**
+     * Cache of strings that have been deduped and stored in the data array.
+     */
+    knownStrings = new Trie();
+    cachedElements = new Map();
+    constructor(options) {
+        this.options = options;
+        this.dedupe = options?.dedupe ?? true;
+        this.sortKeys = options?.sortKeys || this.dedupe;
+    }
+    primitiveToIdx(value) {
+        if (typeof value === 'string')
+            return this.stringToIdx(value);
+        if (typeof value === 'bigint')
+            return this.bigintToIdx(value);
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(value) - 1;
+        this.cache.set(value, idx);
+        return idx;
+    }
+    addSubStringRef(idxString, value, offset) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const sub = offset
+            ? [ElementType.SubString, idxString, value.length, offset]
+            : [ElementType.SubString, idxString, value.length];
+        const idx = this.data.push(sub) - 1;
+        this.cache.set(value, idx);
+        return idx;
+    }
+    addKnownString(idx, value) {
+        if (value.length >= minSubStringLen) {
+            this.knownStrings.add(value.length > 256 ? value.slice(0, 256) : value, { idx });
+        }
+    }
+    addStringPrimitive(value) {
+        const idx = this.data.push(value) - 1;
+        this.addKnownString(idx, value);
+        this.cache.set(value, idx);
+        return idx;
+    }
+    duplicateIndex(idx) {
+        const element = this.data[idx];
+        const duplicate = this.data.push(element) - 1;
+        return duplicate;
+    }
+    stringToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        if (forceStringPrimitives || value.length < minSubStringLen || blockSplitRegex.test(value)) {
+            return this.addStringPrimitive(value);
+        }
+        const trieFound = this.knownStrings.find(value);
+        if (!trieFound || !trieFound.data || trieFound.found.length < minSubStringLen) {
+            return this.addStringPrimitive(value);
+        }
+        const { data: tData, found: subStr } = trieFound;
+        const sIdx = this.addSubStringRef(tData.idx, subStr, tData.offset);
+        if (subStr === value)
+            return sIdx;
+        const v = [sIdx, this.stringToIdx(value.slice(subStr.length))];
+        const idx = this.data.push([ElementType.String, ...v]) - 1;
+        this.cache.set(value, idx);
+        this.addKnownString(idx, value);
+        return idx;
+    }
+    objSetToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const keys = [...value];
+        const k = this.createUniqueKeys(keys, false);
+        const element = [ElementType.Set, k];
+        return this.storeElement(value, idx, element);
+    }
+    createUniqueKeys(keys, cacheValue = true) {
+        let k = this.arrToIdx(keys, cacheValue);
+        const elementKeys = this.data[k];
+        const uniqueKeys = new Set(elementKeys.slice(1));
+        if (uniqueKeys.size !== keys.length) {
+            // one or more of the keys got deduped. We need to duplicate it.
+            uniqueKeys.clear();
+            const indexes = elementKeys.slice(1).map((idx) => {
+                if (uniqueKeys.has(idx)) {
+                    return this.duplicateIndex(idx);
+                }
+                uniqueKeys.add(idx);
+                return idx;
+            });
+            k = this.createArrayElementFromIndexValues(this.data.length, indexes);
+        }
+        return k;
+    }
+    objMapToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const entries = [...value.entries()];
+        const k = this.createUniqueKeys(entries.map(([key]) => key), false);
+        const v = this.arrToIdx(entries.map(([, value]) => value), false);
+        const element = [ElementType.Map, k, v];
+        return this.storeElement(value, idx, element);
+    }
+    objRegExpToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const element = [
+            ElementType.RegExp,
+            this.stringToIdx(value.source),
+            this.stringToIdx(value.flags),
+        ];
+        return this.storeElement(value, idx, element);
+    }
+    objDateToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const element = [ElementType.Date, value.getTime()];
+        return this.storeElement(value, idx, element);
+    }
+    bigintToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const element = [
+            ElementType.BigInt,
+            this.primitiveToIdx(value <= Number.MAX_SAFE_INTEGER && value >= -Number.MAX_SAFE_INTEGER
+                ? Number(value)
+                : value.toString()),
+        ];
+        return this.storeElement(value, idx, element);
+    }
+    objToIdx(value) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        if (isObjectWrapper(value)) {
+            const idx = this.data.push({}) - 1;
+            this.cache.set(value, idx);
+            const element = [ElementType.Object, 0, this.valueToIdx(value.valueOf())];
+            return this.storeElement(value, idx, element);
+        }
+        const entries = Object.entries(value);
+        if (!entries.length) {
+            if (this.emptyObjIdx) {
+                return this.emptyObjIdx;
+            }
+            const idx = this.data.push({}) - 1;
+            this.emptyObjIdx = idx;
+            return idx;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        if (this.sortKeys) {
+            entries.sort(([a], [b]) => compare(a, b));
+        }
+        const k = this.arrToIdx(entries.map(([key]) => key));
+        const v = this.arrToIdx(entries.map(([, value]) => value));
+        const element = [ElementType.Object, k, v];
+        return this.storeElement(value, idx, element);
+    }
+    storeElement(value, idx, element) {
+        const useIdx = this.dedupe ? this.cacheElement(idx, element) : idx;
+        if (useIdx !== idx && idx === this.data.length - 1) {
+            this.data.length = idx;
+            this.cache.set(value, useIdx);
+            return useIdx;
+        }
+        this.data[idx] = element;
+        return idx;
+    }
+    cacheElement(elemIdx, element) {
+        let map = this.cachedElements;
+        for (let i = 0; i < element.length - 1; i++) {
+            const idx = element[i];
+            let found = map.get(idx);
+            if (!found) {
+                found = new Map();
+                map.set(idx, found);
+            }
+            assert(found instanceof Map);
+            map = found;
+        }
+        const idx = element[element.length - 1];
+        const foundIdx = map.get(idx);
+        if (typeof foundIdx === 'number') {
+            return this.referenced.has(elemIdx) ? elemIdx : foundIdx;
+        }
+        map.set(idx, elemIdx);
+        return elemIdx;
+    }
+    stashArray(idx, element) {
+        const indexHash = simpleHash(element);
+        let found = this.cachedArrays.get(indexHash);
+        if (!found) {
+            found = [];
+            this.cachedArrays.set(indexHash, found);
+        }
+        const foundIdx = found.find((entry) => isEqual(entry.v, element));
+        if (foundIdx) {
+            return this.referenced.has(idx) ? idx : foundIdx.idx;
+        }
+        found.push({ idx, v: element });
+        return idx;
+    }
+    createArrayElementFromIndexValues(idx, indexValues) {
+        const element = [ElementType.Array, ...indexValues];
+        const useIdx = this.dedupe ? this.stashArray(idx, element) : idx;
+        if (useIdx !== idx) {
+            assert(this.data.length == idx + 1, `Expected ${idx + 1} but got ${this.data.length}`);
+            this.data.length = idx;
+            return useIdx;
+        }
+        this.data[idx] = element;
+        return idx;
+    }
+    /**
+     * Convert an array to an index.
+     * @param value - The array to convert to an index.
+     * @param cacheValue - Whether to cache the value.
+     * @returns the index of the array.
+     */
+    arrToIdx(value, cacheValue = true) {
+        const found = this.cache.get(value);
+        if (found !== undefined) {
+            this.referenced.add(found);
+            return found;
+        }
+        const idx = this.data.push(0) - 1;
+        this.cache.set(value, idx);
+        const useIdx = this.createArrayElementFromIndexValues(idx, value.map((idx) => this.valueToIdx(idx)));
+        if (cacheValue) {
+            this.cache.set(value, useIdx);
+        }
+        return useIdx;
+    }
+    valueToIdx(value) {
+        if (value === null) {
+            return this.primitiveToIdx(value);
+        }
+        if (typeof value === 'object') {
+            if (value instanceof Set) {
+                return this.objSetToIdx(value);
+            }
+            if (value instanceof Map) {
+                return this.objMapToIdx(value);
+            }
+            if (value instanceof RegExp) {
+                return this.objRegExpToIdx(value);
+            }
+            if (Array.isArray(value)) {
+                return this.arrToIdx(value);
+            }
+            if (value instanceof Date) {
+                return this.objDateToIdx(value);
+            }
+            return this.objToIdx(value);
+        }
+        return this.primitiveToIdx(value);
+    }
+    /**
+     * Reset things in a way that allows for reuse.
+     */
+    softReset() {
+        this.cache.clear();
+        this.cache.set(undefined, 0);
+    }
+    toJSON(json) {
+        this.softReset();
+        this.valueToIdx(json);
+        return this.data;
+    }
+}
+function isEqual(a, b) {
+    if (a === b)
+        return true;
+    if (a.length !== b.length)
+        return false;
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] !== b[i])
+            return false;
+    }
+    return true;
+}
+function simpleHash(values) {
+    let hash = Math.sqrt(values.length);
+    for (const value of values) {
+        hash += value * value;
+    }
+    return hash;
+}
+function isObjectWrapper(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.valueOf === 'function' &&
+        value.valueOf() !== value);
+}
+export function toJSON(json, options) {
+    return new CompactStorage(options).toJSON(json);
+}
+export function stringify(data, pretty = true) {
+    return pretty ? stringifyFlatpacked(toJSON(data)) : JSON.stringify(toJSON(data));
+}
+//# sourceMappingURL=storage.mjs.map

package/dist/stringify.d.mts

@@ -0,0 +1,3 @@
+import type { Flatpacked } from './types.mjs';
+export declare function stringifyFlatpacked(input: Flatpacked): string;
+//# sourceMappingURL=stringify.d.mts.map

package/dist/stringify.mjs

@@ -0,0 +1,13 @@
+export function stringifyFlatpacked(input) {
+    let result = '[\n' + JSON.stringify(input[0]);
+    let prev = '';
+    for (let i = 1; i < input.length; i++) {
+        const next = JSON.stringify(input[i]);
+        result += prev == next ? ',' : ',\n';
+        result += next;
+        prev = next;
+    }
+    result += '\n]\n';
+    return result;
+}
+//# sourceMappingURL=stringify.mjs.map

package/dist/types.d.mts

@@ -0,0 +1,99 @@
+export type SimplePrimitive = string | number | boolean | null | undefined;
+export type Primitive = SimplePrimitive | RegExp | Date | bigint;
+export type PrimitiveSet = Set<Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap>;
+export type PrimitiveMap = Map<Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap, Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap>;
+export interface PrimitiveObject {
+    readonly [key: string]: Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap;
+}
+export interface ObjectWrapper {
+    valueOf(): PrimitiveObject;
+}
+export type PrimitiveArray = readonly (Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap)[];
+type PrimitiveElement = Primitive;
+export type Serializable = Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap;
+export declare enum ElementType {
+    Array = 0,
+    Object = 1,
+    String = 2,
+    SubString = 3,
+    Set = 4,
+    Map = 5,
+    RegExp = 6,
+    Date = 7,
+    BigInt = 8
+}
+interface EmptyObject {
+    readonly t?: ElementType.Object;
+}
+type ObjectBasedElements = EmptyObject;
+/**
+ * Non-primitive elements.
+ * An array is used to represent the element.
+ * The first element is the type of the element.
+ */
+export type ArrayBasedElements = ArrayElement | BigIntElement | DateElement | MapElement | ObjectElement | ObjectWrapperElement | RegExpElement | SetElement | StringElement | SubStringElement;
+export type Index = number;
+/**
+ * A Compound string element. Each index is a reference to a string element that is concatenated
+ * to form the final string.
+ */
+export type StringElement = readonly [type: ElementType.String, ...Index[]];
+/**
+ * A substring element. The first index is a reference to a string element.
+ * The second index is the length of the substring.
+ * The third index is the offset of the substring, defaults to 0.
+ */
+export type SubStringElement = readonly [type: ElementType.SubString, idx: Index, len: number, offset?: number];
+/**
+ * An object element. The first index is a reference to an array of keys.
+ * The second index is a reference to an array of values.
+ */
+export type ObjectElement = readonly [type: ElementType.Object, keys: Index, values: Index];
+/**
+ * A Object wrapper element.
+ */
+export type ObjectWrapperElement = readonly [type: ElementType.Object, keys: 0, values: Index];
+/**
+ * A set element. The first index is a reference to an array of keys.
+ */
+export type SetElement = readonly [type: ElementType.Set, keys: Index];
+/**
+ * A map element. The first index is a reference to an array of keys.
+ * The second index is a reference to an array of values.
+ */
+export type MapElement = readonly [type: ElementType.Map, keys: Index, values: Index];
+/**
+ * A regular expression element. The first index is a reference to a string element that represents the pattern.
+ * The second index is a reference to a string element that represents the flags.
+ */
+export type RegExpElement = readonly [type: ElementType.RegExp, pattern: Index, flags: Index];
+/**
+ * A date element. The first index is the number of milliseconds since the epoch.
+ */
+export type DateElement = readonly [type: ElementType.Date, value: number];
+export type BigIntElement = readonly [type: ElementType.BigInt, value: Index];
+/**
+ * An array element. Each index is a reference to an element.
+ */
+export type ArrayElement = readonly [type: ElementType.Array, ...Index[]];
+export type FlattenedElement = Readonly<PrimitiveElement | ObjectBasedElements | ArrayBasedElements>;
+type Header = string;
+export type Flatpacked = [Header, ...FlattenedElement[]];
+export type Unpacked = Readonly<Serializable>;
+export declare const blockSplitRegex: RegExp;
+export interface FlatpackOptions {
+    /**
+     * Sort keys in objects.
+     * Does not affect arrays, sets, or maps.
+     */
+    sortKeys?: boolean;
+    /**
+     * Dedupe objects and arrays.
+     * Implies {@linkcode sortKeys}.
+     * @default true
+     */
+    dedupe?: boolean;
+}
+export declare const dataHeader: "Dehydrated JSON v1";
+export {};
+//# sourceMappingURL=types.d.mts.map

package/dist/types.mjs

@@ -0,0 +1,15 @@
+export var ElementType;
+(function (ElementType) {
+    ElementType[ElementType["Array"] = 0] = "Array";
+    ElementType[ElementType["Object"] = 1] = "Object";
+    ElementType[ElementType["String"] = 2] = "String";
+    ElementType[ElementType["SubString"] = 3] = "SubString";
+    ElementType[ElementType["Set"] = 4] = "Set";
+    ElementType[ElementType["Map"] = 5] = "Map";
+    ElementType[ElementType["RegExp"] = 6] = "RegExp";
+    ElementType[ElementType["Date"] = 7] = "Date";
+    ElementType[ElementType["BigInt"] = 8] = "BigInt";
+})(ElementType || (ElementType = {}));
+export const blockSplitRegex = /^sha\d/;
+export const dataHeader = 'Dehydrated JSON v1';
+//# sourceMappingURL=types.mjs.map

package/dist/unpack.d.mts

@@ -0,0 +1,4 @@
+import { Flatpacked, Unpacked } from './types.mjs';
+export declare function fromJSON(data: Flatpacked): Unpacked;
+export declare function parse(data: string): Unpacked;
+//# sourceMappingURL=unpack.d.mts.map