jsonschema-key-compression 1.7.0 → 2.0.0

package/src/compress.ts

@@ -1,8 +1,10 @@
 import type {
     PlainJsonObject,
+    PlainJsonObjectNotArray,
     CompressionTable,
     MangoQuery
 } from './types';
+import { compressEnumValue } from './enum-compression';
 
 /**
  * compress the keys of an object via the compression-table
@@ -13,28 +15,34 @@ export function compressObject(
     obj: PlainJsonObject
 ): PlainJsonObject {
     if (typeof obj !== 'object' || obj === null) return obj;
-    else if (Array.isArray(obj)) {
+    if (Array.isArray(obj)) {
         // array
-        return obj
-            .map(item => compressObject(table, item));
-    } else {
-        // object
-        const ret: PlainJsonObject = {};
-        const keys = Object.keys(obj);
-        for (let index = 0; index < keys.length; index++) {
-            const key = keys[index];
-            const compressedKey = compressedAndFlaggedKey(
-                table,
-                key as any
-            );
-            const value = compressObject(
-                table,
-                obj[key as any]
-            );
-            ret[compressedKey] = value;
+        const retArray: PlainJsonObjectNotArray[] = new Array(obj.length);
+        for (let index = 0; index < obj.length; index++) {
+            const item = obj[index];
+            // primitives do not need a recursive call
+            retArray[index] = (typeof item === 'object' && item !== null) ? compressObject(table, item) as any : item;
         }
-        return ret;
+        return retArray;
     }
+    // object
+    const ret: PlainJsonObjectNotArray = {};
+    const keys = Object.keys(obj);
+    const enumCompression = table.enumCompression;
+    for (let index = 0; index < keys.length; index++) {
+        const key = keys[index] as string;
+        const value = (obj as PlainJsonObjectNotArray)[key];
+        const enumValues = enumCompression && enumCompression.get(key);
+        if (enumValues && (typeof value === 'string' || Array.isArray(value))) {
+            ret[compressedAndFlaggedKey(table, key)] = compressEnumValue(enumValues, value);
+        } else {
+            // primitives do not need a recursive call
+            ret[compressedAndFlaggedKey(table, key)] = (typeof value === 'object' && value !== null) ?
+                compressObject(table, value) :
+                value;
+        }
+    }
+    return ret;
 }
 
 /**
@@ -71,6 +79,26 @@ export function throwErrorIfCompressionFlagUsed(
     }
 }
 
+/**
+ * Cache of the flagged compressed keys per table,
+ * so that the flag does not have to be concatenated on each use.
+ * The cache is keyed by the table object, so a table must not be
+ * mutated after it has been used for compression.
+ */
+const flaggedKeysCache: WeakMap<CompressionTable, Map<string, string>> = new WeakMap();
+function getFlaggedKeys(table: CompressionTable): Map<string, string> {
+    let flaggedKeys = flaggedKeysCache.get(table);
+    if (!flaggedKeys) {
+        const newFlaggedKeys: Map<string, string> = new Map();
+        table.compressedToUncompressed.forEach((compressedKey, key) => {
+            newFlaggedKeys.set(key, table.compressionFlag + compressedKey);
+        });
+        flaggedKeysCache.set(table, newFlaggedKeys);
+        flaggedKeys = newFlaggedKeys;
+    }
+    return flaggedKeys;
+}
+
 export function compressedAndFlaggedKey(
     table: CompressionTable,
     key: string
@@ -81,17 +109,22 @@ export function compressedAndFlaggedKey(
     );
     /**
      * keys could be array-accessors like myArray[4]
-     * we have to split and read the squared brackets value
+     * so the part before the squared bracket is looked up
+     * and the bracket part is re-added.
+     * Most keys have no bracket, so the plain lookup is the fast path.
      */
-    const splitSquaredBrackets = key.split('[');
-    const plainKey = splitSquaredBrackets.shift() as string;
-    const compressedKey = table.compressedToUncompressed.get(plainKey);
-    if (!compressedKey) {
+    const flaggedKeys = getFlaggedKeys(table);
+    const bracketIndex = key.indexOf('[');
+    if (bracketIndex === -1) {
+        const directFlaggedKey = flaggedKeys.get(key);
+        return directFlaggedKey ? directFlaggedKey : key;
+    }
+    const plainKey = key.slice(0, bracketIndex);
+    const flaggedKey = flaggedKeys.get(plainKey);
+    if (!flaggedKey) {
         return key;
-    } else {
-        const readdSquared = splitSquaredBrackets.length ? '[' + splitSquaredBrackets.join('[') : '';
-        return table.compressionFlag + compressedKey + readdSquared;
     }
+    return flaggedKey + key.slice(bracketIndex);
 }
 
 
@@ -161,38 +194,155 @@ export function compressQuery(
     return ret;
 }
 
+/**
+ * Operators whose value is a document-value and therefore
+ * has to be enum-compressed like the documents themselves.
+ * Any other operator, like $type or $mod, gets its value unchanged.
+ */
+const ENUM_VALUE_OPERATORS: string[] = [
+    '$eq',
+    '$ne',
+    '$gt',
+    '$gte',
+    '$lt',
+    '$lte',
+    '$in',
+    '$nin',
+    '$all',
+    '$not',
+    '$elemMatch'
+];
+
+/**
+ * Returns the enum of the property that the path points to,
+ * or undefined when that property is not enum-compressed.
+ */
+function enumValuesOfPath(
+    table: CompressionTable,
+    path: string
+): string[] | undefined {
+    const enumCompression = table.enumCompression;
+    if (!enumCompression) {
+        return undefined;
+    }
+    const splitted = path.split('.');
+    let lastKey = splitted[splitted.length - 1] as string;
+    const bracketIndex = lastKey.indexOf('[');
+    if (bracketIndex !== -1) {
+        lastKey = lastKey.slice(0, bracketIndex);
+    }
+    return enumCompression.get(lastKey);
+}
+
+/**
+ * A regular expression cannot run on the compressed number of an enum-value,
+ * so it is resolved up front into the indexes of the matching enum-values.
+ */
+function enumIndexesOfRegex(
+    enumValues: string[],
+    pattern: any,
+    options: any
+): number[] {
+    let source: string;
+    let flags: string;
+    if (pattern instanceof RegExp) {
+        source = pattern.source;
+        flags = (pattern.ignoreCase ? 'i' : '') + (pattern.multiline ? 'm' : '');
+    } else {
+        source = String(pattern);
+        flags = '';
+    }
+    if (typeof options === 'string') {
+        flags = options;
+    }
+    /**
+     * The global flags make .test() stateful
+     * and are meaningless for a full check of a single value.
+     */
+    flags = flags.replace(/[gy]/g, '');
+    const regex = new RegExp(source, flags);
+    const indexes: number[] = [];
+    for (let i = 0; i < enumValues.length; i++) {
+        if (regex.test(enumValues[i] as string)) {
+            indexes.push(i);
+        }
+    }
+    return indexes;
+}
+
 /**
  * @recursive
+ * @param enumValues The enum of the property that the selector belongs to,
+ * so that the compared values can be enum-compressed.
  */
 export function compressQuerySelector(
     table: CompressionTable,
-    selector: any
+    selector: any,
+    enumValues?: string[]
 ): any {
     if (Array.isArray(selector)) {
-        return selector.map(item => compressQuerySelector(table, item));
+        return selector.map(item => compressQuerySelector(table, item, enumValues));
     } else if (selector instanceof RegExp) {
         return selector;
     } else if (typeof selector === 'object' && selector !== null) {
         const ret: any = {};
+        /**
+         * $regex and its $options are replaced by a single $in
+         * with the indexes of the matching enum-values.
+         */
+        let regexIndexes: number[] | undefined;
+        if (enumValues && typeof selector['$regex'] !== 'undefined') {
+            regexIndexes = enumIndexesOfRegex(
+                enumValues,
+                selector['$regex'],
+                selector['$options']
+            );
+        }
         Object.keys(selector).forEach(key => {
+            if (regexIndexes && (key === '$regex' || key === '$options')) {
+                return;
+            }
             let useKey;
+            let useEnumValues;
             if (key.startsWith('$')) {
                 // operator
                 useKey = key;
+                useEnumValues = ENUM_VALUE_OPERATORS.indexOf(key) === -1 ? undefined : enumValues;
             } else {
                 // property path
                 useKey = compressedPath(
                     table,
                     key
                 );
+                useEnumValues = enumValuesOfPath(
+                    table,
+                    key
+                );
+            }
+            const value = selector[key];
+            if (useEnumValues && value instanceof RegExp) {
+                ret[useKey] = {
+                    $in: enumIndexesOfRegex(useEnumValues, value, undefined)
+                };
+            } else {
+                ret[useKey] = compressQuerySelector(
+                    table,
+                    value,
+                    useEnumValues
+                );
             }
-            ret[useKey] = compressQuerySelector(
-                table,
-                selector[key]
-            );
         });
+        if (regexIndexes) {
+            const alreadyIn = ret['$in'];
+            ret['$in'] = Array.isArray(alreadyIn) ?
+                regexIndexes.filter(index => alreadyIn.includes(index)) :
+                regexIndexes;
+        }
         return ret;
     } else {
+        if (enumValues) {
+            return compressEnumValue(enumValues, selector);
+        }
         return selector;
     }
 }

package/src/create-compressed-json-schema.ts

@@ -1,53 +1,176 @@
 import type {
     JsonSchema,
+    JsonSchemaTypes,
     CompressionTable
 } from './types';
 import { flatClone } from './util';
-import { compressedPath } from './compress';
+import {
+    compressedPath,
+    compressObject
+} from './compress';
 
+/**
+ * A value that is enum-compressed is stored as the index of the value,
+ * so the type of the property changes from string to number.
+ */
+function toNumberType(
+    type: JsonSchemaTypes | JsonSchemaTypes[] | undefined
+): JsonSchemaTypes | JsonSchemaTypes[] | undefined {
+    if (!type) {
+        return type;
+    }
+    if (Array.isArray(type)) {
+        return type.map(single => single === 'string' ? 'number' : single);
+    }
+    return type === 'string' ? 'number' : type;
+}
+
+/**
+ * Replaces the enum-values of an enum-compressed property
+ * by the indexes that the compressed documents contain.
+ * The enum can sit at the property itself or at the items of an array-property.
+ * @recursive
+ */
+function compressEnumOfSchema(
+    schema: JsonSchema,
+    enumValues: string[]
+): JsonSchema {
+    let ret = schema;
+    if (Array.isArray(schema.enum)) {
+        ret = flatClone(schema);
+        ret.enum = enumValues.map((_value, index) => index);
+        ret.type = toNumberType(schema.type);
+    }
+    const items = ret.items;
+    if (items && !Array.isArray(items) && Array.isArray(items.enum)) {
+        if (ret === schema) {
+            ret = flatClone(schema);
+        }
+        ret.items = compressEnumOfSchema(items, enumValues);
+    }
+    return ret;
+}
+
+/**
+ * Transforms the schema so that it describes the compressed objects.
+ * It recurses into all keywords that can contain sub-schemas,
+ * independent of the 'type' field. This mirrors how compressObject() works,
+ * which compresses every key that is known to the compression-table,
+ * wherever the key occurs in the document. For example a nullable nested object
+ * has type: ['object', 'null'] and its properties must still be compressed,
+ * and properties inside of oneOf/anyOf/allOf must be compressed as well.
+ */
 export function createCompressedJsonSchema(
     compressionTable: CompressionTable,
     schema: JsonSchema
 ): JsonSchema {
-    if (schema.type === 'array' && schema.items) {
-        const cloned = flatClone(schema);
+    const hasNested = schema.items ||
+        schema.additionalItems ||
+        schema.properties ||
+        schema.patternProperties ||
+        schema.required ||
+        schema.allOf ||
+        schema.anyOf ||
+        schema.oneOf ||
+        schema.not ||
+        schema.dependencies ||
+        schema.definitions ||
+        schema.enum;
+    if (!hasNested) {
+        // no deeper fields in the schema
+        return schema;
+    }
+
+    const cloned = flatClone(schema);
+    const enumCompression = compressionTable.enumCompression;
+    const compressSchema = (subSchema: JsonSchema) => createCompressedJsonSchema(compressionTable, subSchema);
+    const compressKey = (key: string) => compressedPath(compressionTable, key);
+    const compressSchemaMap = (
+        schemaMap: { [k: string]: JsonSchema },
+        compressKeys: boolean
+    ): { [k: string]: JsonSchema } => {
+        const ret: { [k: string]: JsonSchema } = {};
+        // do not use Object.entries, it is transpiled shitty
+        Object.keys(schemaMap).forEach(key => {
+            const useKey = compressKeys ? compressKey(key) : key;
+            const subSchema = compressSchema(schemaMap[key] as JsonSchema);
+            /**
+             * Only real property names can be enum-compressed,
+             * the keys of patternProperties and definitions are no property names.
+             */
+            const enumValues = (compressKeys && enumCompression) ? enumCompression.get(key) : undefined;
+            ret[useKey] = enumValues ? compressEnumOfSchema(subSchema, enumValues) : subSchema;
+        });
+        return ret;
+    };
+
+    if (schema.items) {
         if (Array.isArray(schema.items)) {
-            const newItems = schema.items.map(item => createCompressedJsonSchema(
-                compressionTable,
-                item
-            ));
-            cloned.items = newItems;
+            cloned.items = schema.items.map(compressSchema);
         } else {
-            const newItems = createCompressedJsonSchema(
-                compressionTable,
-                schema.items
-            );
-            cloned.items = newItems;
+            cloned.items = compressSchema(schema.items);
         }
-        return cloned;
-    } else if (schema.type === 'object' && schema.properties) {
-        const cloned = flatClone(schema);
-
-        // compress all property names
-        const newProperties: {
-            [k: string]: JsonSchema
-        } = {};
-        // do not use Object.entries, it is transpiled shitty
-        Object.keys(schema.properties).forEach(key => {
-            const property = (schema as any).properties[key];
-            const compressedKey = compressedPath(compressionTable, key);
-            newProperties[compressedKey] = createCompressedJsonSchema(compressionTable, property);
+    }
+    if (schema.additionalItems && typeof schema.additionalItems === 'object') {
+        cloned.additionalItems = compressSchema(schema.additionalItems);
+    }
+
+    // compress all property names
+    if (schema.properties) {
+        cloned.properties = compressSchemaMap(schema.properties, true);
+    }
+    // the patterns are not property names, only the sub-schemas are compressed
+    if (schema.patternProperties) {
+        cloned.patternProperties = compressSchemaMap(schema.patternProperties, false);
+    }
+    // also compress the required array
+    if (schema.required) {
+        cloned.required = schema.required.map(compressKey);
+    }
+
+    if (schema.allOf) {
+        cloned.allOf = schema.allOf.map(compressSchema);
+    }
+    if (schema.anyOf) {
+        cloned.anyOf = schema.anyOf.map(compressSchema);
+    }
+    if (schema.oneOf) {
+        cloned.oneOf = schema.oneOf.map(compressSchema);
+    }
+    if (schema.not) {
+        cloned.not = compressSchema(schema.not);
+    }
+
+    /**
+     * dependencies are keyed by property name
+     * and contain either a list of property names
+     * or a sub-schema
+     */
+    if (schema.dependencies) {
+        const dependencies = schema.dependencies;
+        const newDependencies: { [k: string]: JsonSchema | string[] } = {};
+        Object.keys(dependencies).forEach(key => {
+            const dependency = dependencies[key] as JsonSchema | string[];
+            newDependencies[compressKey(key)] = Array.isArray(dependency) ?
+                dependency.map(compressKey) :
+                compressSchema(dependency);
         });
-        cloned.properties = newProperties;
+        cloned.dependencies = newDependencies;
+    }
 
-        // also compress the required array
-        if (cloned.required) {
-            cloned.required = cloned.required.map(key => compressedPath(compressionTable, key));
-        }
+    // definition names are not property names, only the sub-schemas are compressed
+    if (schema.definitions) {
+        cloned.definitions = compressSchemaMap(schema.definitions, false);
+    }
 
-        return cloned;
-    } else {
-        // no deeper fields in the schema
-        return schema;
+    /**
+     * enum values are document fragments,
+     * so object values contain property names that must be compressed.
+     * Primitive values are returned unchanged by compressObject().
+     */
+    if (schema.enum) {
+        cloned.enum = schema.enum.map(value => compressObject(compressionTable, value));
     }
+
+    return cloned;
 }

package/src/create-compression-table.ts

@@ -7,6 +7,7 @@ import {
     numberToLetter,
     alphabeticCompare
 } from './util';
+import { getEnumCompressionTable } from './enum-compression';
 
 /**
  * Compressed property-names begin with the compression-flag
@@ -16,10 +17,17 @@ import {
  */
 export const DEFAULT_COMPRESSION_FLAG = '|';
 
+/**
+ * @param compressEnums If true, the values of string-enums are compressed
+ * into the index of the value inside of the enum.
+ * This changes the stored format, so it is opt-in
+ * to keep already compressed data readable.
+ */
 export function createCompressionTable(
     schema: JsonSchema,
     compressionFlag: string = DEFAULT_COMPRESSION_FLAG,
-    ignoreProperties: string[] = []
+    ignoreProperties: string[] = [],
+    compressEnums: boolean = false
 ): CompressionTable {
     const table = compressedToUncompressedTable(
         schema,
@@ -35,6 +43,20 @@ export function createCompressionTable(
         compressionFlag
     };
 
+    if (compressEnums) {
+        const enumCompression = getEnumCompressionTable(
+            schema,
+            ignoreProperties
+        );
+        /**
+         * The field stays undefined when there is nothing to compress,
+         * so that the compression does not have to look it up on each key.
+         */
+        if (enumCompression.size > 0) {
+            compressionTable.enumCompression = enumCompression;
+        }
+    }
+
     return compressionTable;
 }
 
@@ -44,35 +66,37 @@ export function createCompressionTable(
  */
 export function getPropertiesOfSchema(schema: JsonSchema): Set<string> {
     const ret: Set<string> = new Set();
+    addPropertiesOfSchema(schema, ret);
+    return ret;
+}
 
-    function addSchema(innerSchema: JsonSchema) {
-        const keys = getPropertiesOfSchema(innerSchema);
-        Array.from(keys).forEach(k => ret.add(k));
-    }
-
-    if (schema.properties) {
+/**
+ * Adds all property names of the schema to the given set.
+ * Using a single accumulator set avoids creating
+ * a new set and array copy on each nesting level.
+ */
+function addPropertiesOfSchema(schema: JsonSchema, ret: Set<string>) {
+    const properties = schema.properties;
+    if (properties) {
         // do not use Object.entries, it is transpiled shitty
-        Object.keys(schema.properties).forEach(property => {
-            const deepSchema = (schema as any).properties[property];
+        const propertyNames = Object.keys(properties);
+        for (let i = 0; i < propertyNames.length; i++) {
+            const property = propertyNames[i] as string;
             ret.add(property);
-            if (!schema.properties) {
-                return;
-            }
-            addSchema(deepSchema);
-        });
+            addPropertiesOfSchema(properties[property] as JsonSchema, ret);
+        }
     }
 
-    if (schema.items) {
-        if (Array.isArray(schema.items)) {
-            schema.items.forEach(subSchema => {
-                addSchema(subSchema);
-            });
+    const items = schema.items;
+    if (items) {
+        if (Array.isArray(items)) {
+            for (let i = 0; i < items.length; i++) {
+                addPropertiesOfSchema(items[i] as JsonSchema, ret);
+            }
         } else {
-            addSchema(schema.items);
+            addPropertiesOfSchema(items, ret);
         }
     }
-
-    return ret;
 }
 
 export function compressedToUncompressedTable(
@@ -83,13 +107,13 @@ export function compressedToUncompressedTable(
     const schemaKeysSorted: string[] = Array.from(attributes).sort(alphabeticCompare);
     const table: TableType = new Map();
     let lastKeyNumber: number = 0;
-    schemaKeysSorted
-        .filter(k => k.length > 3 && !ignoreProperties.includes(k))
-        .forEach(k => {
-            const compressKey = numberToLetter(lastKeyNumber);
+    for (let i = 0; i < schemaKeysSorted.length; i++) {
+        const k = schemaKeysSorted[i] as string;
+        if (k.length > 3 && !ignoreProperties.includes(k)) {
+            table.set(k, numberToLetter(lastKeyNumber));
             lastKeyNumber++;
-            table.set(k, compressKey);
-        });
+        }
+    }
     return table;
 }
 
@@ -99,9 +123,13 @@ export function uncompressedToCompressedTable(
     ignoreProperties: string[]
 ): TableType {
     const reverseTable: TableType = new Map();
-    Array.from(table.keys()).forEach(key => {
-        const value = table.get(key) as string;
-        if (!ignoreProperties.includes(value)) {
+    table.forEach((value, key) => {
+        /**
+         * The ignored properties are property names,
+         * so they must be compared to the key and not to the
+         * compressed value which could randomly be equal to an ignored name.
+         */
+        if (!ignoreProperties.includes(key)) {
             reverseTable.set(compressionFlag + value, key);
         }
     });