flatpack-json 9.7.0 → 10.0.0

package/dist/stringify.d.mts

@@ -1,3 +1,8 @@
 import type { Flatpacked } from './types.mjs';
-export declare function stringifyFlatpacked(input: Flatpacked): string;
+export interface StringifyOptions {
+    maxLineLength?: number;
+    maxBatchSize?: number;
+}
+export declare const DEFAULT_STRINGIFY_OPTIONS: Required<StringifyOptions>;
+export declare function stringifyFlatpacked(input: Flatpacked, options?: StringifyOptions): string;
 //# sourceMappingURL=stringify.d.mts.map

package/dist/stringify.mjs

@@ -1,8 +1,14 @@
-export function stringifyFlatpacked(input) {
+const maxLineLength = 512;
+const maxBatchSize = 64;
+export const DEFAULT_STRINGIFY_OPTIONS = {
+    maxLineLength,
+    maxBatchSize,
+};
+export function stringifyFlatpacked(input, options) {
     let result = '[\n' + JSON.stringify(input[0]);
     let prev = '';
     for (let i = 1; i < input.length; i++) {
-        const next = JSON.stringify(input[i]);
+        const next = formatLine(input[i], options);
         result += prev === next ? ',' : ',\n';
         result += next;
         prev = next;
@@ -10,4 +16,27 @@ export function stringifyFlatpacked(input) {
     result += '\n]\n';
     return result;
 }
+function formatLine(elem, options) {
+    if (!options || !Array.isArray(elem)) {
+        return JSON.stringify(elem);
+    }
+    const { maxLineLength, maxBatchSize } = { ...DEFAULT_STRINGIFY_OPTIONS, ...options };
+    const input = elem;
+    const result = [];
+    let line = '';
+    for (let i = 0; i < input.length; i++) {
+        if (line)
+            line += ',';
+        const next = JSON.stringify(input[i]);
+        if (line.length + next.length > maxLineLength || (i && i % maxBatchSize === 0)) {
+            result.push(line);
+            line = '';
+        }
+        line += next;
+    }
+    if (line) {
+        result.push(line);
+    }
+    return '[' + result.join('\n') + ']';
+}
 //# sourceMappingURL=stringify.mjs.map

package/dist/types.d.mts

@@ -1,3 +1,4 @@
+import type { RefCounter } from './RefCounter.mjs';
 export type SimplePrimitive = string | number | boolean | null | undefined;
 export type Primitive = SimplePrimitive | RegExp | Date | bigint;
 export type PrimitiveSet = Set<Primitive | PrimitiveObject | PrimitiveArray | PrimitiveSet | PrimitiveMap>;
@@ -20,7 +21,8 @@ export declare enum ElementType {
     Map = 5,
     RegExp = 6,
     Date = 7,
-    BigInt = 8
+    BigInt = 8,
+    StringTable = 128
 }
 interface EmptyObject {
     readonly t?: ElementType.Object;
@@ -31,55 +33,63 @@ type ObjectBasedElements = EmptyObject;
  * An array is used to represent the element.
  * The first element is the type of the element.
  */
-export type ArrayBasedElements = ArrayElement | BigIntElement | DateElement | MapElement | ObjectElement | ObjectWrapperElement | RegExpElement | SetElement | StringElement | SubStringElement;
-export type Index = number;
+export type ArrayBasedElements = ArrayElement | BigIntElement | DateElement | MapElement | ObjectElement | ObjectWrapperElement | RegExpElement | SetElement | StringElement | SubStringElement | StringTableElement;
+/**
+ * The absolute index of an element in the Flatpacked array. The first element is the header,
+ * so the first element is at index 1.
+ */
+export type FlatpackIndex = number;
 /**
  * A Compound string element. Each index is a reference to a string element that is concatenated
  * to form the final string.
  */
-export type StringElement = readonly [type: ElementType.String, ...Index[]];
+export type StringElement = readonly [type: ElementType.String, ...FlatpackIndex[]];
 /**
  * A substring element. The first index is a reference to a string element.
  * The second index is the length of the substring.
  * The third index is the offset of the substring, defaults to 0.
  */
-export type SubStringElement = readonly [type: ElementType.SubString, idx: Index, len: number, offset?: number];
+export type SubStringElement = readonly [type: ElementType.SubString, idx: FlatpackIndex, len: number, offset?: number];
 /**
  * An object element. The first index is a reference to an array of keys.
  * The second index is a reference to an array of values.
  */
-export type ObjectElement = readonly [type: ElementType.Object, keys: Index, values: Index];
+export type ObjectElement = readonly [type: ElementType.Object, keys: FlatpackIndex, values: FlatpackIndex];
 /**
  * A Object wrapper element.
  */
-export type ObjectWrapperElement = readonly [type: ElementType.Object, keys: 0, values: Index];
+export type ObjectWrapperElement = readonly [type: ElementType.Object, keys: 0, values: FlatpackIndex];
 /**
  * A set element. The first index is a reference to an array of keys.
  */
-export type SetElement = readonly [type: ElementType.Set, keys: Index];
+export type SetElement = readonly [type: ElementType.Set, keys: FlatpackIndex];
 /**
  * A map element. The first index is a reference to an array of keys.
  * The second index is a reference to an array of values.
  */
-export type MapElement = readonly [type: ElementType.Map, keys: Index, values: Index];
+export type MapElement = readonly [type: ElementType.Map, keys: FlatpackIndex, values: FlatpackIndex];
 /**
  * A regular expression element. The first index is a reference to a string element that represents the pattern.
  * The second index is a reference to a string element that represents the flags.
  */
-export type RegExpElement = readonly [type: ElementType.RegExp, pattern: Index, flags: Index];
+export type RegExpElement = readonly [type: ElementType.RegExp, pattern: FlatpackIndex, flags: FlatpackIndex];
 /**
  * A date element. The first index is the number of milliseconds since the epoch.
  */
 export type DateElement = readonly [type: ElementType.Date, value: number];
-export type BigIntElement = readonly [type: ElementType.BigInt, value: Index];
+export type BigIntElement = readonly [type: ElementType.BigInt, value: FlatpackIndex];
 /**
  * An array element. Each index is a reference to an element.
  */
-export type ArrayElement = readonly [type: ElementType.Array, ...Index[]];
-export type FlattenedElement = Readonly<PrimitiveElement | ObjectBasedElements | ArrayBasedElements>;
+export type ArrayElement = readonly [type: ElementType.Array, ...FlatpackIndex[]];
+export type EmptyElement = readonly [];
+export type StringTableEntry = string | number[];
+export type StringTableElement = readonly [type: ElementType.StringTable, ...StringTableEntry[]];
+export type FlattenedElement = Readonly<PrimitiveElement | ObjectBasedElements | ArrayBasedElements | StringTableElement | EmptyElement>;
 type Header = string;
 export type Flatpacked = [Header, ...FlattenedElement[]];
-export type Unpacked = Readonly<Serializable>;
+export type RawUnpacked = Serializable;
+export type Unpacked = AnnotateUnpacked<RawUnpacked>;
 export declare const blockSplitRegex: RegExp;
 export interface FlatpackOptions {
     /**
@@ -97,18 +107,56 @@ export interface FlatpackOptions {
      * Try to optimize the size of the output.
      */
     optimize?: boolean;
+    /**
+     * The format of the output. If not specified, the latest format will be used.
+     */
+    format?: 'V1' | 'V2';
+    /**
+     * Meta data to use when packing a value. This is used mainly to minimize the difference between the output of two similar values.
+     */
+    meta?: UnpackMetaData | undefined;
 }
 /**
  * Legacy header for Flatpack JSON.
  */
 export declare const dataHeaderV0_1: "Dehydrated JSON v1";
+export declare const dataHeaderV1_0: "Flatpack JSON v1";
+export declare const dataHeaderV2_0: "Flatpack JSON v2";
+export type Headers = typeof dataHeaderV0_1 | typeof dataHeaderV1_0 | typeof dataHeaderV2_0;
 /**
  * The current header for Flatpack JSON.
  */
-export declare const dataHeader: "Flatpack JSON v1";
+export declare const dataHeader: string;
 /**
  * The set of supported headers for Flatpack JSON.
  */
 export declare const supportedHeaders: Set<string>;
+export interface FlatpackApi {
+    setValue(value: Serializable): void;
+    toJSON(): Flatpacked;
+    stringify(): string;
+    toValue(): Unpacked;
+}
+export declare const symbolFlatpackAnnotation: unique symbol;
+export interface UnpackMetaData {
+    /** The source of the unpacked data */
+    flatpack: Flatpacked;
+    /** The reference count of elements.  */
+    referenced: RefCounter<number>;
+    /**
+     * The index of the root element in the flatpack.
+     * In most cases, this will be 2 or 1 (if there is NOT a string table), but can be higher if there are more meta elements.
+     * The string table is a meta element, and should be included before the root element if it exists.
+     */
+    rootIndex: FlatpackIndex;
+}
+export interface UnpackedAnnotation {
+    meta: UnpackMetaData;
+    index: number;
+}
+export interface UnpackedAnnotated {
+    [symbolFlatpackAnnotation]?: UnpackedAnnotation;
+}
+export type AnnotateUnpacked<T> = T extends null ? T : T extends object ? T & UnpackedAnnotated : T;
 export {};
 //# sourceMappingURL=types.d.mts.map

package/dist/types.mjs

@@ -9,18 +9,22 @@ export var ElementType;
     ElementType[ElementType["RegExp"] = 6] = "RegExp";
     ElementType[ElementType["Date"] = 7] = "Date";
     ElementType[ElementType["BigInt"] = 8] = "BigInt";
+    ElementType[ElementType["StringTable"] = 128] = "StringTable";
 })(ElementType || (ElementType = {}));
 export const blockSplitRegex = /^sha\d/;
 /**
  * Legacy header for Flatpack JSON.
  */
 export const dataHeaderV0_1 = 'Dehydrated JSON v1';
+export const dataHeaderV1_0 = 'Flatpack JSON v1';
+export const dataHeaderV2_0 = 'Flatpack JSON v2';
 /**
  * The current header for Flatpack JSON.
  */
-export const dataHeader = 'Flatpack JSON v1';
+export const dataHeader = dataHeaderV2_0;
 /**
  * The set of supported headers for Flatpack JSON.
  */
-export const supportedHeaders = new Set([dataHeaderV0_1, dataHeader]);
+export const supportedHeaders = new Set([dataHeaderV0_1, dataHeaderV1_0, dataHeaderV2_0]);
+export const symbolFlatpackAnnotation = Symbol.for('flatpackAnnotation');
 //# sourceMappingURL=types.mjs.map

package/dist/unpack.d.mts

@@ -1,4 +1,4 @@
 import type { Flatpacked, Unpacked } from './types.mjs';
-export declare function fromJSON(data: Flatpacked): Unpacked;
+export declare function fromJSON<T = Unpacked>(data: Flatpacked): T;
 export declare function parse(data: string): Unpacked;
 //# sourceMappingURL=unpack.d.mts.map

package/dist/unpack.mjs

@@ -1,28 +1,48 @@
 import assert from 'node:assert';
-import { ElementType, supportedHeaders } from './types.mjs';
+import { getFlatpackedRootIdx } from './flatpacked.mjs';
+import { RefCounter } from './RefCounter.mjs';
+import { StringTable } from './stringTable.mjs';
+import { ElementType, supportedHeaders, symbolFlatpackAnnotation } from './types.mjs';
 export function fromJSON(data) {
     const [header] = data;
+    let stringTable;
     if (!supportedHeaders.has(header)) {
         throw new Error('Invalid header');
     }
     const cache = new Map([[0, undefined]]);
     /**
-     * indexes that have been referenced by other objects.
+     * indexes that have been referenced by multiple objects.
+     * A count of 1 means that there is only 1 reference.
      */
-    const referenced = new Set();
+    const referenced = new RefCounter();
+    const meta = {
+        flatpack: data,
+        referenced,
+        rootIndex: getFlatpackedRootIdx(data),
+    };
+    return idxToValue(1);
+    function cacheValue(idx, value) {
+        assert(!cache.has(idx), `Index ${idx} already exists in cache`);
+        cache.set(idx, value);
+        return value;
+    }
+    function getCachedValue(idx) {
+        referenced.add(idx);
+        return cache.get(idx);
+    }
     function mergeKeysValues(keys, values) {
         return keys.map((key, i) => [key, values[i]]);
     }
     function toSet(idx, elem) {
         const [_, k] = elem;
         const s = k ? new Set(idxToArr(k)) : new Set();
-        cache.set(idx, s);
+        cacheValue(idx, s);
         return s;
     }
     function toMap(idx, elem) {
         const [_, k, v] = elem;
         const m = !k || !v ? new Map() : new Map(mergeKeysValues(idxToArr(k), idxToArr(v)));
-        cache.set(idx, m);
+        cacheValue(idx, m);
         return m;
     }
     function toRegExp(idx, elem) {
@@ -30,24 +50,24 @@ export function fromJSON(data) {
         const p = idxToValue(pattern);
         const f = idxToValue(flags);
         const r = new RegExp(p, f);
-        cache.set(idx, r);
+        cacheValue(idx, r);
         return r;
     }
     function toBigInt(idx, elem) {
         const [_, vIdx] = elem;
         const r = BigInt(idxToValue(vIdx));
-        cache.set(idx, r);
+        cacheValue(idx, r);
         return r;
     }
     function toDate(idx, elem) {
         const [_, value] = elem;
         const r = new Date(value);
-        cache.set(idx, r);
+        cacheValue(idx, r);
         return r;
     }
     function toString(idx, elem) {
-        const s = typeof elem === 'string' ? elem : idxToValue(elem.slice(1));
-        cache.set(idx, s);
+        const s = typeof elem === 'string' ? elem : idxToString(elem.slice(1));
+        cacheValue(idx, s);
         return s;
     }
     function toObj(idx, elem) {
@@ -55,11 +75,11 @@ export function fromJSON(data) {
         // Object Wrapper
         if (!k && v) {
             const obj = Object(idxToValue(v));
-            cache.set(idx, obj);
+            cacheValue(idx, obj);
             return obj;
         }
         const obj = {};
-        cache.set(idx, obj);
+        cacheValue(idx, obj);
         if (!k || !v)
             return obj;
         const keys = idxToArr(k);
@@ -68,6 +88,10 @@ export function fromJSON(data) {
         return obj;
     }
     function idxToArr(idx) {
+        const found = getCachedValue(idx);
+        if (found !== undefined) {
+            return found;
+        }
         const element = data[idx];
         assert(isArrayElement(element));
         return toArr(idx, element);
@@ -75,12 +99,12 @@ export function fromJSON(data) {
     function toArr(idx, element) {
         const placeHolder = [];
         const refs = element.slice(1);
-        cache.set(idx, placeHolder);
+        cacheValue(idx, placeHolder);
         const arr = refs.map(idxToValue);
         // check if the array has been referenced by another object.
-        if (!referenced.has(idx)) {
+        if (!referenced.isReferenced(idx)) {
             // It has not, just replace the placeholder with the array.
-            cache.set(idx, arr);
+            cacheValue(idx, arr);
             return arr;
         }
         placeHolder.push(...arr);
@@ -89,13 +113,13 @@ export function fromJSON(data) {
     function handleSubStringElement(idx, refs) {
         const [_t, sIdx, len, offset = 0] = refs;
         const s = `${idxToValue(sIdx)}`.slice(offset, offset + len);
-        cache.set(idx, s);
+        cacheValue(idx, s);
         return s;
     }
     function handleArrayElement(idx, element) {
         switch (element[0]) {
             case ElementType.Array: {
-                break;
+                return toArr(idx, element);
             }
             case ElementType.Object: {
                 return toObj(idx, element);
@@ -121,22 +145,26 @@ export function fromJSON(data) {
             case ElementType.BigInt: {
                 return toBigInt(idx, element);
             }
+            case ElementType.StringTable: {
+                stringTable = new StringTable(element);
+                return idxToValue(idx + 1);
+            }
         }
-        return toArr(idx, element);
+        return undefined;
+    }
+    function idxToString(idx) {
+        return joinToString(idx.map((i) => idxToValue(i)));
     }
     function idxToValue(idx) {
         if (!idx)
             return undefined;
-        const found = cache.get(idx);
-        if (found !== undefined) {
-            if (typeof idx === 'number')
-                referenced.add(idx);
-            return found;
+        if (idx < 0) {
+            referenced.add(idx);
+            return stringTable?.get(-idx);
         }
-        if (Array.isArray(idx)) {
-            // it is a nested string;
-            const parts = idx.map((i) => idxToValue(i));
-            return joinToString(parts);
+        const found = getCachedValue(idx);
+        if (found !== undefined) {
+            return annotateUnpacked(found, { meta, index: idx });
         }
         const element = data[idx];
         if (typeof element === 'object') {
@@ -144,15 +172,17 @@ export function fromJSON(data) {
             if (element === null)
                 return null;
             if (Array.isArray(element))
-                return handleArrayElement(idx, element);
-            return {};
+                return annotateUnpacked(handleArrayElement(idx, element), {
+                    meta,
+                    index: idx,
+                });
+            return annotateUnpacked({}, { meta, index: idx });
         }
         return element;
     }
-    return idxToValue(1);
 }
 function joinToString(parts) {
-    return parts.map((a) => (Array.isArray(a) ? joinToString(a) : a)).join('');
+    return parts.flat().join('');
 }
 function isArrayElement(value) {
     return Array.isArray(value) && value[0] === ElementType.Array;
@@ -160,4 +190,13 @@ function isArrayElement(value) {
 export function parse(data) {
     return fromJSON(JSON.parse(data));
 }
+function annotateUnpacked(value, meta) {
+    if (value && typeof value === 'object') {
+        if (Object.hasOwn(value, symbolFlatpackAnnotation)) {
+            return value;
+        }
+        return Object.defineProperty(value, symbolFlatpackAnnotation, { value: meta });
+    }
+    return value;
+}
 //# sourceMappingURL=unpack.mjs.map

package/dist/unpackedAnnotation.d.mts

@@ -0,0 +1,23 @@
+import type { Serializable, UnpackedAnnotated, UnpackedAnnotation, UnpackMetaData } from './types.mjs';
+/**
+ * Get the annotation from an unpacked value. This is only available from values that have been unpacked with the
+ * `UnpackedAnnotated` type. If the value is not annotated, this will return undefined.
+ * @param data - The unpacked value to extract the meta data from.
+ * @returns The meta data or undefined if the value is not annotated.
+ */
+export declare function extractUnpackedAnnotation(data: Serializable): UnpackedAnnotation | undefined;
+/**
+ * Get the meta data from an unpacked value. This is only available from values that have been unpacked with the
+ * `UnpackedAnnotated` type. If the value is not annotated, this will return undefined.
+ * @param data - The unpacked value to extract the meta data from.
+ * @returns The meta data or undefined if the value is not annotated.
+ */
+export declare function extractUnpackedMetaData(data: Serializable): UnpackMetaData | undefined;
+/**
+ * Check if a value has an unpacked annotation. This is only available from values that have been unpacked with the
+ * `UnpackedAnnotated` type.
+ * @param value - any value to test
+ * @returns `value` has UnpackedAnnotation.
+ */
+export declare function isUnpackedAnnotated<T>(value: T): value is T & UnpackedAnnotated;
+//# sourceMappingURL=unpackedAnnotation.d.mts.map

package/dist/unpackedAnnotation.mjs

@@ -0,0 +1,32 @@
+import { symbolFlatpackAnnotation } from './types.mjs';
+/**
+ * Get the annotation from an unpacked value. This is only available from values that have been unpacked with the
+ * `UnpackedAnnotated` type. If the value is not annotated, this will return undefined.
+ * @param data - The unpacked value to extract the meta data from.
+ * @returns The meta data or undefined if the value is not annotated.
+ */
+export function extractUnpackedAnnotation(data) {
+    if (isUnpackedAnnotated(data)) {
+        return data[symbolFlatpackAnnotation];
+    }
+    return undefined;
+}
+/**
+ * Get the meta data from an unpacked value. This is only available from values that have been unpacked with the
+ * `UnpackedAnnotated` type. If the value is not annotated, this will return undefined.
+ * @param data - The unpacked value to extract the meta data from.
+ * @returns The meta data or undefined if the value is not annotated.
+ */
+export function extractUnpackedMetaData(data) {
+    return extractUnpackedAnnotation(data)?.meta;
+}
+/**
+ * Check if a value has an unpacked annotation. This is only available from values that have been unpacked with the
+ * `UnpackedAnnotated` type.
+ * @param value - any value to test
+ * @returns `value` has UnpackedAnnotation.
+ */
+export function isUnpackedAnnotated(value) {
+    return typeof value === 'object' && value !== null && Object.hasOwn(value, symbolFlatpackAnnotation);
+}
+//# sourceMappingURL=unpackedAnnotation.mjs.map

package/package.json

@@ -4,7 +4,7 @@
     "access": "public",
     "provenance": true
   },
-  "version": "9.7.0",
+  "version": "10.0.0",
   "description": "A library to normalize / flatten JSON objects to reduce the size.",
   "keywords": [
     "cspell",
@@ -50,12 +50,12 @@
     "url": "https://github.com/streetsidesoftware/cspell/labels/filetype"
   },
   "engines": {
-    "node": ">=20"
+    "node": ">=22.18.0"
   },
   "devDependencies": {
-    "@cspell/filetypes": "9.7.0",
-    "diff": "^8.0.3",
-    "flatted": "^3.3.3"
+    "@cspell/filetypes": "10.0.0",
+    "diff": "^8.0.4",
+    "flatted": "^3.4.2"
   },
-  "gitHead": "48f64e0bd95b39011af6dc80cd8ae4d519511f73"
+  "gitHead": "6ddfd576b6bb554a7cb8dba1ab053a0b6ff8021f"
 }