jsonschema-key-compression 1.6.1 → 2.0.0

package/src/create-compression-table.ts

@@ -7,6 +7,7 @@ import {
     numberToLetter,
     alphabeticCompare
 } from './util';
+import { getEnumCompressionTable } from './enum-compression';
 
 /**
  * Compressed property-names begin with the compression-flag
@@ -16,10 +17,17 @@ import {
  */
 export const DEFAULT_COMPRESSION_FLAG = '|';
 
+/**
+ * @param compressEnums If true, the values of string-enums are compressed
+ * into the index of the value inside of the enum.
+ * This changes the stored format, so it is opt-in
+ * to keep already compressed data readable.
+ */
 export function createCompressionTable(
     schema: JsonSchema,
     compressionFlag: string = DEFAULT_COMPRESSION_FLAG,
-    ignoreProperties: string[] = []
+    ignoreProperties: string[] = [],
+    compressEnums: boolean = false
 ): CompressionTable {
     const table = compressedToUncompressedTable(
         schema,
@@ -35,6 +43,20 @@ export function createCompressionTable(
         compressionFlag
     };
 
+    if (compressEnums) {
+        const enumCompression = getEnumCompressionTable(
+            schema,
+            ignoreProperties
+        );
+        /**
+         * The field stays undefined when there is nothing to compress,
+         * so that the compression does not have to look it up on each key.
+         */
+        if (enumCompression.size > 0) {
+            compressionTable.enumCompression = enumCompression;
+        }
+    }
+
     return compressionTable;
 }
 
@@ -44,35 +66,37 @@ export function createCompressionTable(
  */
 export function getPropertiesOfSchema(schema: JsonSchema): Set<string> {
     const ret: Set<string> = new Set();
+    addPropertiesOfSchema(schema, ret);
+    return ret;
+}
 
-    function addSchema(innerSchema: JsonSchema) {
-        const keys = getPropertiesOfSchema(innerSchema);
-        Array.from(keys).forEach(k => ret.add(k));
-    }
-
-    if (schema.properties) {
+/**
+ * Adds all property names of the schema to the given set.
+ * Using a single accumulator set avoids creating
+ * a new set and array copy on each nesting level.
+ */
+function addPropertiesOfSchema(schema: JsonSchema, ret: Set<string>) {
+    const properties = schema.properties;
+    if (properties) {
         // do not use Object.entries, it is transpiled shitty
-        Object.keys(schema.properties).forEach(property => {
-            const deepSchema = (schema as any).properties[property];
+        const propertyNames = Object.keys(properties);
+        for (let i = 0; i < propertyNames.length; i++) {
+            const property = propertyNames[i] as string;
             ret.add(property);
-            if (!schema.properties) {
-                return;
-            }
-            addSchema(deepSchema);
-        });
+            addPropertiesOfSchema(properties[property] as JsonSchema, ret);
+        }
     }
 
-    if (schema.items) {
-        if (Array.isArray(schema.items)) {
-            schema.items.forEach(subSchema => {
-                addSchema(subSchema);
-            });
+    const items = schema.items;
+    if (items) {
+        if (Array.isArray(items)) {
+            for (let i = 0; i < items.length; i++) {
+                addPropertiesOfSchema(items[i] as JsonSchema, ret);
+            }
         } else {
-            addSchema(schema.items);
+            addPropertiesOfSchema(items, ret);
         }
     }
-
-    return ret;
 }
 
 export function compressedToUncompressedTable(
@@ -83,13 +107,13 @@ export function compressedToUncompressedTable(
     const schemaKeysSorted: string[] = Array.from(attributes).sort(alphabeticCompare);
     const table: TableType = new Map();
     let lastKeyNumber: number = 0;
-    schemaKeysSorted
-        .filter(k => k.length > 3 && !ignoreProperties.includes(k))
-        .forEach(k => {
-            const compressKey = numberToLetter(lastKeyNumber);
+    for (let i = 0; i < schemaKeysSorted.length; i++) {
+        const k = schemaKeysSorted[i] as string;
+        if (k.length > 3 && !ignoreProperties.includes(k)) {
+            table.set(k, numberToLetter(lastKeyNumber));
             lastKeyNumber++;
-            table.set(k, compressKey);
-        });
+        }
+    }
     return table;
 }
 
@@ -99,9 +123,13 @@ export function uncompressedToCompressedTable(
     ignoreProperties: string[]
 ): TableType {
     const reverseTable: TableType = new Map();
-    Array.from(table.keys()).forEach(key => {
-        const value = table.get(key) as string;
-        if (!ignoreProperties.includes(value)) {
+    table.forEach((value, key) => {
+        /**
+         * The ignored properties are property names,
+         * so they must be compared to the key and not to the
+         * compressed value which could randomly be equal to an ignored name.
+         */
+        if (!ignoreProperties.includes(key)) {
             reverseTable.set(compressionFlag + value, key);
         }
     });

package/src/decompress.ts

@@ -1,32 +1,44 @@
 import type {
     PlainJsonObject,
+    PlainJsonObjectNotArray,
     CompressionTable
 } from './types';
+import { decompressEnumValue } from './enum-compression';
 
 export function decompressObject(
     table: CompressionTable,
     obj: PlainJsonObject
 ): PlainJsonObject {
     if (typeof obj !== 'object' || obj === null) return obj;
-    else if (Array.isArray(obj)) {
+    if (Array.isArray(obj)) {
         // array
-        return obj.map(item => decompressObject(table, item));
-    } else {
-        // object
-        const ret: PlainJsonObject = {};
-        Object.keys(obj).forEach(key => {
-            const decompressed = decompressedKey(
-                table,
-                key
-            );
-            const value = decompressObject(
-                table,
-                obj[key]
-            );
-            ret[decompressed] = value;
-        });
-        return ret;
+        const retArray: PlainJsonObjectNotArray[] = new Array(obj.length);
+        for (let index = 0; index < obj.length; index++) {
+            const item = obj[index];
+            // primitives do not need a recursive call
+            retArray[index] = (typeof item === 'object' && item !== null) ? decompressObject(table, item) as any : item;
+        }
+        return retArray;
     }
+    // object
+    const ret: PlainJsonObjectNotArray = {};
+    const keys = Object.keys(obj);
+    const enumCompression = table.enumCompression;
+    for (let index = 0; index < keys.length; index++) {
+        const key = keys[index] as string;
+        const value = (obj as PlainJsonObjectNotArray)[key];
+        const decompressed = decompressedKey(table, key);
+        const enumValues = enumCompression && enumCompression.get(decompressed);
+        if (enumValues && (typeof value === 'number' || Array.isArray(value))) {
+            ret[decompressed] = decompressEnumValue(enumValues, value);
+        } else {
+            // primitives do not need a recursive call
+            ret[decompressed] = (typeof value === 'object' && value !== null) ?
+                decompressObject(table, value) :
+                value;
+        }
+    }
+    return ret;
 }
 
 /**
@@ -57,17 +69,20 @@ export function decompressedKey(
 ): string {
 
     /**
-        * keys could be array-accessors like myArray[4]
-        * we have to split and readd the squared brackets value
-        */
-    const splitSquaredBrackets = key.split('[');
-    key = splitSquaredBrackets.shift() as string;
-
-    const decompressed = table.uncompressedToCompressed.get(key);
+     * keys could be array-accessors like myArray[4]
+     * so the part before the squared bracket is looked up
+     * and the bracket part is re-added.
+     * Most keys have no bracket, so the plain lookup is the fast path.
+     */
+    const bracketIndex = key.indexOf('[');
+    if (bracketIndex === -1) {
+        const directDecompressed = table.uncompressedToCompressed.get(key);
+        return directDecompressed ? directDecompressed : key;
+    }
+    const plainKey = key.slice(0, bracketIndex);
+    const decompressed = table.uncompressedToCompressed.get(plainKey);
     if (!decompressed) {
         return key;
-    } else {
-        const readdSquared = splitSquaredBrackets.length ? '[' + splitSquaredBrackets.join('[') : '';
-        return decompressed + readdSquared;
     }
+    return decompressed + key.slice(bracketIndex);
 }

package/src/enum-compression.ts

@@ -0,0 +1,240 @@
+import type {
+    EnumCompressionTable,
+    JsonSchema
+} from './types';
+import { alphabeticCompare } from './util';
+
+/**
+ * Enum-compression replaces the value of a property
+ * by the index of that value inside of the enum of the property.
+ * For example with the enum ['admin', 'editor', 'viewer']
+ * the value 'editor' is stored as the number 1.
+ *
+ * Only enums where all values are strings can be compressed:
+ * - The compressed value is a number, so a number inside of the enum
+ *   could not be told apart from an index.
+ * - Strings are also where the saved space actually is.
+ *
+ * The values are sorted before the indexes are assigned, so that the order
+ * of the indexes is equal to the alphabetic order of the values.
+ * This keeps sorting and range-queries like $gt working on compressed data.
+ */
+export function compressibleEnumValues(values: any[]): string[] | null {
+    if (values.length === 0) {
+        return null;
+    }
+    const unique: string[] = [];
+    for (let i = 0; i < values.length; i++) {
+        const value = values[i];
+        if (typeof value !== 'string') {
+            return null;
+        }
+        if (!unique.includes(value)) {
+            unique.push(value);
+        }
+    }
+    return unique.sort(alphabeticCompare);
+}
+
+/**
+ * Returns the enum that applies to the values which are stored
+ * at a property, or null if the property has no compressible enum.
+ * The enum can either be defined at the property itself,
+ * or at the items of an array-property.
+ */
+function valueEnumOfSchema(schema: JsonSchema): string[] | null {
+    if (Array.isArray(schema.enum)) {
+        return compressibleEnumValues(schema.enum);
+    }
+    const items = schema.items;
+    if (items && !Array.isArray(items) && Array.isArray(items.enum)) {
+        return compressibleEnumValues(items.enum);
+    }
+    return null;
+}
+
+function equalEnumValues(a: string[], b: string[]): boolean {
+    if (a.length !== b.length) {
+        return false;
+    }
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] !== b[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+/**
+ * Collects the enum of each property name for every place
+ * where the name occurs in the schema.
+ * All sub-schema keywords are walked, not only the ones that
+ * the key-compression uses, so that a property name which is used
+ * with different enums is detected as ambiguous.
+ * @recursive
+ */
+function addEnumCandidates(
+    schema: JsonSchema,
+    candidates: Map<string, (string[] | null)[]>
+) {
+    const properties = schema.properties;
+    if (properties) {
+        // do not use Object.entries, it is transpiled shitty
+        const propertyNames = Object.keys(properties);
+        for (let i = 0; i < propertyNames.length; i++) {
+            const property = propertyNames[i] as string;
+            const subSchema = properties[property] as JsonSchema;
+            let found = candidates.get(property);
+            if (!found) {
+                found = [];
+                candidates.set(property, found);
+            }
+            found.push(valueEnumOfSchema(subSchema));
+            addEnumCandidates(subSchema, candidates);
+        }
+    }
+
+    const items = schema.items;
+    if (items) {
+        if (Array.isArray(items)) {
+            for (let i = 0; i < items.length; i++) {
+                addEnumCandidates(items[i] as JsonSchema, candidates);
+            }
+        } else {
+            addEnumCandidates(items, candidates);
+        }
+    }
+    if (schema.additionalItems && typeof schema.additionalItems === 'object') {
+        addEnumCandidates(schema.additionalItems, candidates);
+    }
+
+    const subSchemaLists: (JsonSchema[] | undefined)[] = [schema.allOf, schema.anyOf, schema.oneOf];
+    for (let i = 0; i < subSchemaLists.length; i++) {
+        const subSchemas = subSchemaLists[i];
+        if (subSchemas) {
+            for (let j = 0; j < subSchemas.length; j++) {
+                addEnumCandidates(subSchemas[j] as JsonSchema, candidates);
+            }
+        }
+    }
+    if (schema.not) {
+        addEnumCandidates(schema.not, candidates);
+    }
+
+    const subSchemaMaps: ({ [k: string]: JsonSchema } | undefined)[] = [schema.patternProperties, schema.definitions];
+    for (let i = 0; i < subSchemaMaps.length; i++) {
+        const schemaMap = subSchemaMaps[i];
+        if (schemaMap) {
+            Object.keys(schemaMap).forEach(key => addEnumCandidates(schemaMap[key] as JsonSchema, candidates));
+        }
+    }
+
+    const dependencies = schema.dependencies;
+    if (dependencies) {
+        Object.keys(dependencies).forEach(key => {
+            const dependency = dependencies[key] as JsonSchema | string[];
+            if (!Array.isArray(dependency)) {
+                addEnumCandidates(dependency, candidates);
+            }
+        });
+    }
+}
+
+/**
+ * Creates the table that maps a property name to the enum of its values.
+ * A property name is only compressed when it has the same compressible enum
+ * at every place where it occurs in the schema. Otherwise the same value
+ * could mean different things depending on where it is stored.
+ */
+export function getEnumCompressionTable(
+    schema: JsonSchema,
+    ignoreProperties: string[]
+): EnumCompressionTable {
+    const candidates: Map<string, (string[] | null)[]> = new Map();
+    addEnumCandidates(schema, candidates);
+
+    const table: EnumCompressionTable = new Map();
+    const propertyNames = Array.from(candidates.keys()).sort(alphabeticCompare);
+    for (let i = 0; i < propertyNames.length; i++) {
+        const property = propertyNames[i] as string;
+        if (ignoreProperties.includes(property)) {
+            continue;
+        }
+        const occurrences = candidates.get(property) as (string[] | null)[];
+        const first = occurrences[0];
+        if (!first) {
+            continue;
+        }
+        let isAmbiguous = false;
+        for (let j = 1; j < occurrences.length; j++) {
+            const other = occurrences[j];
+            if (!other || !equalEnumValues(first, other)) {
+                isAmbiguous = true;
+                break;
+            }
+        }
+        if (!isAmbiguous) {
+            table.set(property, first);
+        }
+    }
+    return table;
+}
+
+/**
+ * Returns true if the given number can be an index of the enum.
+ * Number.isInteger() is not used to stay usable on old runtimes.
+ */
+function isEnumIndex(value: number, enumValues: string[]): boolean {
+    return value >= 0 && value < enumValues.length && value % 1 === 0;
+}
+
+/**
+ * Replaces the enum-value by its index.
+ * Values that are not part of the enum are kept as they are,
+ * so that data which does not match the schema is not destroyed.
+ */
+export function compressEnumValue(
+    enumValues: string[],
+    value: any
+): any {
+    if (typeof value === 'string') {
+        const index = enumValues.indexOf(value);
+        return index === -1 ? value : index;
+    }
+    if (Array.isArray(value)) {
+        const ret: any[] = new Array(value.length);
+        for (let i = 0; i < value.length; i++) {
+            const item = value[i];
+            if (typeof item === 'string') {
+                const index = enumValues.indexOf(item);
+                ret[i] = index === -1 ? item : index;
+            } else {
+                ret[i] = item;
+            }
+        }
+        return ret;
+    }
+    return value;
+}
+
+/**
+ * Replaces the index by its enum-value.
+ * The exact counterpart of compressEnumValue().
+ */
+export function decompressEnumValue(
+    enumValues: string[],
+    value: any
+): any {
+    if (typeof value === 'number') {
+        return isEnumIndex(value, enumValues) ? enumValues[value] : value;
+    }
+    if (Array.isArray(value)) {
+        const ret: any[] = new Array(value.length);
+        for (let i = 0; i < value.length; i++) {
+            const item = value[i];
+            ret[i] = (typeof item === 'number' && isEnumIndex(item, enumValues)) ? enumValues[item] : item;
+        }
+        return ret;
+    }
+    return value;
+}

package/src/index.ts

@@ -1,6 +1,7 @@
 export type {
     TableType,
     CompressionTable,
+    EnumCompressionTable,
     PlainJsonObjectNotArray,
     PlainJsonObject,
     SortDirection,

package/src/types.d.ts

@@ -1,9 +1,20 @@
 export type TableType = Map<string, string>;
 
+/**
+ * Maps a property name to the sorted enum-values of that property.
+ * A value is compressed by replacing it with its index in this list.
+ */
+export type EnumCompressionTable = Map<string, string[]>;
+
 export type CompressionTable = {
     compressedToUncompressed: TableType;
     uncompressedToCompressed: TableType;
     compressionFlag: string;
+    /**
+     * Only set when enum-compression is enabled
+     * and the schema has at least one compressible enum.
+     */
+    enumCompression?: EnumCompressionTable;
 };