jsonschema-key-compression 1.6.1 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jsonschema-key-compression",
-  "version": "1.6.1",
+  "version": "2.0.0",
   "description": "Compress json-data based on its json-schema",
   "author": "pubkey",
   "keywords": [
@@ -29,26 +29,28 @@
     "lint:fix": "tslint --project . --fix",
     "transpile": "tsc -p ./ && copyfiles -u 1 \"src/**/*.d.ts\" dist/lib && echo '# transpile es5 (require) sucess!'",
     "transpile:es": "tsc -p ./ --target ES5 --module ES6 --outDir ./dist/es && copyfiles -u 1 \"src/**/*.d.ts\" dist/es && echo '# transpile es6 (modules) sucess!'",
-    "build": "rimraf -r ./dist && npm run transpile && npm run transpile:es",
-    "test": "mocha -r ts-node/register test/unit/index.test.ts",
+    "build": "rimraf -fr ./dist && npm run transpile && npm run transpile:es",
+    "test": "mocha -r ts-node/register test/unit/index.test.ts --bail",
     "test:performance": "mocha -r ts-node/register --bail --exit ./test/performance.test.ts",
     "test:efficiency": "mocha -r ts-node/register --bail --exit ./test/efficiency.test.ts",
     "dev": "mocha -r ts-node/register test/unit/index.test.ts  --watch --watch-extensions ts"
   },
   "devDependencies": {
     "@types/faker": "5.5.9",
-    "@types/mocha": "9.0.0",
-    "@types/node": "16.11.6",
-    "@types/node-gzip": "1.1.0",
-    "assert": "2.0.0",
-    "async-test-util": "2.0.0",
+    "@types/mocha": "10.0.10",
+    "@types/node": "26.6.6",
+    "@types/node-gzip": "1.1.3",
+    "ajv": "8.20.0",
+    "assert": "2.1.0",
+    "async-test-util": "2.5.0",
     "copyfiles": "2.4.1",
     "faker": "5.5.3",
-    "mocha": "9.1.3",
+    "mingo": "7.2.7",
+    "mocha": "12.0.3",
     "node-gzip": "1.1.2",
-    "rimraf": "3.0.2",
-    "ts-node": "10.4.0",
+    "rimraf": "6.1.4",
+    "ts-node": "10.9.2",
     "tslint": "6.1.3",
-    "typescript": "4.4.4"
+    "typescript": "5.9.3"
   }
 }

package/src/compress.ts

@@ -1,8 +1,10 @@
 import type {
     PlainJsonObject,
+    PlainJsonObjectNotArray,
     CompressionTable,
     MangoQuery
 } from './types';
+import { compressEnumValue } from './enum-compression';
 
 /**
  * compress the keys of an object via the compression-table
@@ -13,26 +15,34 @@ export function compressObject(
     obj: PlainJsonObject
 ): PlainJsonObject {
     if (typeof obj !== 'object' || obj === null) return obj;
-    else if (Array.isArray(obj)) {
+    if (Array.isArray(obj)) {
         // array
-        return obj
-            .map(item => compressObject(table, item));
-    } else {
-        // object
-        const ret: PlainJsonObject = {};
-        Object.keys(obj).forEach(key => {
-            const compressedKey = compressedAndFlaggedKey(
-                table,
-                key
-            );
-            const value = compressObject(
-                table,
-                obj[key]
-            );
-            ret[compressedKey] = value;
-        });
-        return ret;
+        const retArray: PlainJsonObjectNotArray[] = new Array(obj.length);
+        for (let index = 0; index < obj.length; index++) {
+            const item = obj[index];
+            // primitives do not need a recursive call
+            retArray[index] = (typeof item === 'object' && item !== null) ? compressObject(table, item) as any : item;
+        }
+        return retArray;
+    }
+    // object
+    const ret: PlainJsonObjectNotArray = {};
+    const keys = Object.keys(obj);
+    const enumCompression = table.enumCompression;
+    for (let index = 0; index < keys.length; index++) {
+        const key = keys[index] as string;
+        const value = (obj as PlainJsonObjectNotArray)[key];
+        const enumValues = enumCompression && enumCompression.get(key);
+        if (enumValues && (typeof value === 'string' || Array.isArray(value))) {
+            ret[compressedAndFlaggedKey(table, key)] = compressEnumValue(enumValues, value);
+        } else {
+            // primitives do not need a recursive call
+            ret[compressedAndFlaggedKey(table, key)] = (typeof value === 'object' && value !== null) ?
+                compressObject(table, value) :
+                value;
+        }
     }
+    return ret;
 }
 
 /**
@@ -69,6 +79,26 @@ export function throwErrorIfCompressionFlagUsed(
     }
 }
 
+/**
+ * Cache of the flagged compressed keys per table,
+ * so that the flag does not have to be concatenated on each use.
+ * The cache is keyed by the table object, so a table must not be
+ * mutated after it has been used for compression.
+ */
+const flaggedKeysCache: WeakMap<CompressionTable, Map<string, string>> = new WeakMap();
+function getFlaggedKeys(table: CompressionTable): Map<string, string> {
+    let flaggedKeys = flaggedKeysCache.get(table);
+    if (!flaggedKeys) {
+        const newFlaggedKeys: Map<string, string> = new Map();
+        table.compressedToUncompressed.forEach((compressedKey, key) => {
+            newFlaggedKeys.set(key, table.compressionFlag + compressedKey);
+        });
+        flaggedKeysCache.set(table, newFlaggedKeys);
+        flaggedKeys = newFlaggedKeys;
+    }
+    return flaggedKeys;
+}
+
 export function compressedAndFlaggedKey(
     table: CompressionTable,
     key: string
@@ -77,21 +107,24 @@ export function compressedAndFlaggedKey(
         table,
         key
     );
-
     /**
      * keys could be array-accessors like myArray[4]
-     * we have to split and readd the squared brackets value
+     * so the part before the squared bracket is looked up
+     * and the bracket part is re-added.
+     * Most keys have no bracket, so the plain lookup is the fast path.
      */
-    const splitSquaredBrackets = key.split('[');
-    key = splitSquaredBrackets.shift() as string;
-
-    const compressedKey = table.compressedToUncompressed.get(key);
-    if (!compressedKey) {
+    const flaggedKeys = getFlaggedKeys(table);
+    const bracketIndex = key.indexOf('[');
+    if (bracketIndex === -1) {
+        const directFlaggedKey = flaggedKeys.get(key);
+        return directFlaggedKey ? directFlaggedKey : key;
+    }
+    const plainKey = key.slice(0, bracketIndex);
+    const flaggedKey = flaggedKeys.get(plainKey);
+    if (!flaggedKey) {
         return key;
-    } else {
-        const readdSquared = splitSquaredBrackets.length ? '[' + splitSquaredBrackets.join('[') : '';
-        return table.compressionFlag + compressedKey + readdSquared;
     }
+    return flaggedKey + key.slice(bracketIndex);
 }
 
 
@@ -161,38 +194,155 @@ export function compressQuery(
     return ret;
 }
 
+/**
+ * Operators whose value is a document-value and therefore
+ * has to be enum-compressed like the documents themselves.
+ * Any other operator, like $type or $mod, gets its value unchanged.
+ */
+const ENUM_VALUE_OPERATORS: string[] = [
+    '$eq',
+    '$ne',
+    '$gt',
+    '$gte',
+    '$lt',
+    '$lte',
+    '$in',
+    '$nin',
+    '$all',
+    '$not',
+    '$elemMatch'
+];
+
+/**
+ * Returns the enum of the property that the path points to,
+ * or undefined when that property is not enum-compressed.
+ */
+function enumValuesOfPath(
+    table: CompressionTable,
+    path: string
+): string[] | undefined {
+    const enumCompression = table.enumCompression;
+    if (!enumCompression) {
+        return undefined;
+    }
+    const splitted = path.split('.');
+    let lastKey = splitted[splitted.length - 1] as string;
+    const bracketIndex = lastKey.indexOf('[');
+    if (bracketIndex !== -1) {
+        lastKey = lastKey.slice(0, bracketIndex);
+    }
+    return enumCompression.get(lastKey);
+}
+
+/**
+ * A regular expression cannot run on the compressed number of an enum-value,
+ * so it is resolved up front into the indexes of the matching enum-values.
+ */
+function enumIndexesOfRegex(
+    enumValues: string[],
+    pattern: any,
+    options: any
+): number[] {
+    let source: string;
+    let flags: string;
+    if (pattern instanceof RegExp) {
+        source = pattern.source;
+        flags = (pattern.ignoreCase ? 'i' : '') + (pattern.multiline ? 'm' : '');
+    } else {
+        source = String(pattern);
+        flags = '';
+    }
+    if (typeof options === 'string') {
+        flags = options;
+    }
+    /**
+     * The global flags make .test() stateful
+     * and are meaningless for a full check of a single value.
+     */
+    flags = flags.replace(/[gy]/g, '');
+    const regex = new RegExp(source, flags);
+    const indexes: number[] = [];
+    for (let i = 0; i < enumValues.length; i++) {
+        if (regex.test(enumValues[i] as string)) {
+            indexes.push(i);
+        }
+    }
+    return indexes;
+}
+
 /**
  * @recursive
+ * @param enumValues The enum of the property that the selector belongs to,
+ * so that the compared values can be enum-compressed.
  */
 export function compressQuerySelector(
     table: CompressionTable,
-    selector: any
+    selector: any,
+    enumValues?: string[]
 ): any {
     if (Array.isArray(selector)) {
-        return selector.map(item => compressQuerySelector(table, item));
+        return selector.map(item => compressQuerySelector(table, item, enumValues));
     } else if (selector instanceof RegExp) {
         return selector;
     } else if (typeof selector === 'object' && selector !== null) {
         const ret: any = {};
+        /**
+         * $regex and its $options are replaced by a single $in
+         * with the indexes of the matching enum-values.
+         */
+        let regexIndexes: number[] | undefined;
+        if (enumValues && typeof selector['$regex'] !== 'undefined') {
+            regexIndexes = enumIndexesOfRegex(
+                enumValues,
+                selector['$regex'],
+                selector['$options']
+            );
+        }
         Object.keys(selector).forEach(key => {
+            if (regexIndexes && (key === '$regex' || key === '$options')) {
+                return;
+            }
             let useKey;
+            let useEnumValues;
             if (key.startsWith('$')) {
                 // operator
                 useKey = key;
+                useEnumValues = ENUM_VALUE_OPERATORS.indexOf(key) === -1 ? undefined : enumValues;
             } else {
                 // property path
                 useKey = compressedPath(
                     table,
                     key
                 );
+                useEnumValues = enumValuesOfPath(
+                    table,
+                    key
+                );
+            }
+            const value = selector[key];
+            if (useEnumValues && value instanceof RegExp) {
+                ret[useKey] = {
+                    $in: enumIndexesOfRegex(useEnumValues, value, undefined)
+                };
+            } else {
+                ret[useKey] = compressQuerySelector(
+                    table,
+                    value,
+                    useEnumValues
+                );
             }
-            ret[useKey] = compressQuerySelector(
-                table,
-                selector[key]
-            );
         });
+        if (regexIndexes) {
+            const alreadyIn = ret['$in'];
+            ret['$in'] = Array.isArray(alreadyIn) ?
+                regexIndexes.filter(index => alreadyIn.includes(index)) :
+                regexIndexes;
+        }
         return ret;
     } else {
+        if (enumValues) {
+            return compressEnumValue(enumValues, selector);
+        }
         return selector;
     }
 }

package/src/create-compressed-json-schema.ts

@@ -1,53 +1,176 @@
 import type {
     JsonSchema,
+    JsonSchemaTypes,
     CompressionTable
 } from './types';
 import { flatClone } from './util';
-import { compressedPath } from './compress';
+import {
+    compressedPath,
+    compressObject
+} from './compress';
 
+/**
+ * A value that is enum-compressed is stored as the index of the value,
+ * so the type of the property changes from string to number.
+ */
+function toNumberType(
+    type: JsonSchemaTypes | JsonSchemaTypes[] | undefined
+): JsonSchemaTypes | JsonSchemaTypes[] | undefined {
+    if (!type) {
+        return type;
+    }
+    if (Array.isArray(type)) {
+        return type.map(single => single === 'string' ? 'number' : single);
+    }
+    return type === 'string' ? 'number' : type;
+}
+
+/**
+ * Replaces the enum-values of an enum-compressed property
+ * by the indexes that the compressed documents contain.
+ * The enum can sit at the property itself or at the items of an array-property.
+ * @recursive
+ */
+function compressEnumOfSchema(
+    schema: JsonSchema,
+    enumValues: string[]
+): JsonSchema {
+    let ret = schema;
+    if (Array.isArray(schema.enum)) {
+        ret = flatClone(schema);
+        ret.enum = enumValues.map((_value, index) => index);
+        ret.type = toNumberType(schema.type);
+    }
+    const items = ret.items;
+    if (items && !Array.isArray(items) && Array.isArray(items.enum)) {
+        if (ret === schema) {
+            ret = flatClone(schema);
+        }
+        ret.items = compressEnumOfSchema(items, enumValues);
+    }
+    return ret;
+}
+
+/**
+ * Transforms the schema so that it describes the compressed objects.
+ * It recurses into all keywords that can contain sub-schemas,
+ * independent of the 'type' field. This mirrors how compressObject() works,
+ * which compresses every key that is known to the compression-table,
+ * wherever the key occurs in the document. For example a nullable nested object
+ * has type: ['object', 'null'] and its properties must still be compressed,
+ * and properties inside of oneOf/anyOf/allOf must be compressed as well.
+ */
 export function createCompressedJsonSchema(
     compressionTable: CompressionTable,
     schema: JsonSchema
 ): JsonSchema {
-    if (schema.type === 'array' && schema.items) {
-        const cloned = flatClone(schema);
+    const hasNested = schema.items ||
+        schema.additionalItems ||
+        schema.properties ||
+        schema.patternProperties ||
+        schema.required ||
+        schema.allOf ||
+        schema.anyOf ||
+        schema.oneOf ||
+        schema.not ||
+        schema.dependencies ||
+        schema.definitions ||
+        schema.enum;
+    if (!hasNested) {
+        // no deeper fields in the schema
+        return schema;
+    }
+
+    const cloned = flatClone(schema);
+    const enumCompression = compressionTable.enumCompression;
+    const compressSchema = (subSchema: JsonSchema) => createCompressedJsonSchema(compressionTable, subSchema);
+    const compressKey = (key: string) => compressedPath(compressionTable, key);
+    const compressSchemaMap = (
+        schemaMap: { [k: string]: JsonSchema },
+        compressKeys: boolean
+    ): { [k: string]: JsonSchema } => {
+        const ret: { [k: string]: JsonSchema } = {};
+        // do not use Object.entries, it is transpiled shitty
+        Object.keys(schemaMap).forEach(key => {
+            const useKey = compressKeys ? compressKey(key) : key;
+            const subSchema = compressSchema(schemaMap[key] as JsonSchema);
+            /**
+             * Only real property names can be enum-compressed,
+             * the keys of patternProperties and definitions are no property names.
+             */
+            const enumValues = (compressKeys && enumCompression) ? enumCompression.get(key) : undefined;
+            ret[useKey] = enumValues ? compressEnumOfSchema(subSchema, enumValues) : subSchema;
+        });
+        return ret;
+    };
+
+    if (schema.items) {
         if (Array.isArray(schema.items)) {
-            const newItems = schema.items.map(item => createCompressedJsonSchema(
-                compressionTable,
-                item
-            ));
-            cloned.items = newItems;
+            cloned.items = schema.items.map(compressSchema);
         } else {
-            const newItems = createCompressedJsonSchema(
-                compressionTable,
-                schema.items
-            );
-            cloned.items = newItems;
+            cloned.items = compressSchema(schema.items);
         }
-        return cloned;
-    } else if (schema.type === 'object' && schema.properties) {
-        const cloned = flatClone(schema);
-
-        // compress all property names
-        const newProperties: {
-            [k: string]: JsonSchema
-        } = {};
-        // do not use Object.entries, it is transpiled shitty
-        Object.keys(schema.properties).forEach(key => {
-            const property = (schema as any).properties[key];
-            const compressedKey = compressedPath(compressionTable, key);
-            newProperties[compressedKey] = createCompressedJsonSchema(compressionTable, property);
+    }
+    if (schema.additionalItems && typeof schema.additionalItems === 'object') {
+        cloned.additionalItems = compressSchema(schema.additionalItems);
+    }
+
+    // compress all property names
+    if (schema.properties) {
+        cloned.properties = compressSchemaMap(schema.properties, true);
+    }
+    // the patterns are not property names, only the sub-schemas are compressed
+    if (schema.patternProperties) {
+        cloned.patternProperties = compressSchemaMap(schema.patternProperties, false);
+    }
+    // also compress the required array
+    if (schema.required) {
+        cloned.required = schema.required.map(compressKey);
+    }
+
+    if (schema.allOf) {
+        cloned.allOf = schema.allOf.map(compressSchema);
+    }
+    if (schema.anyOf) {
+        cloned.anyOf = schema.anyOf.map(compressSchema);
+    }
+    if (schema.oneOf) {
+        cloned.oneOf = schema.oneOf.map(compressSchema);
+    }
+    if (schema.not) {
+        cloned.not = compressSchema(schema.not);
+    }
+
+    /**
+     * dependencies are keyed by property name
+     * and contain either a list of property names
+     * or a sub-schema
+     */
+    if (schema.dependencies) {
+        const dependencies = schema.dependencies;
+        const newDependencies: { [k: string]: JsonSchema | string[] } = {};
+        Object.keys(dependencies).forEach(key => {
+            const dependency = dependencies[key] as JsonSchema | string[];
+            newDependencies[compressKey(key)] = Array.isArray(dependency) ?
+                dependency.map(compressKey) :
+                compressSchema(dependency);
         });
-        cloned.properties = newProperties;
+        cloned.dependencies = newDependencies;
+    }
 
-        // also compress the required array
-        if (cloned.required) {
-            cloned.required = cloned.required.map(key => compressedPath(compressionTable, key));
-        }
+    // definition names are not property names, only the sub-schemas are compressed
+    if (schema.definitions) {
+        cloned.definitions = compressSchemaMap(schema.definitions, false);
+    }
 
-        return cloned;
-    } else {
-        // no deeper fields in the schema
-        return schema;
+    /**
+     * enum values are document fragments,
+     * so object values contain property names that must be compressed.
+     * Primitive values are returned unchanged by compressObject().
+     */
+    if (schema.enum) {
+        cloned.enum = schema.enum.map(value => compressObject(compressionTable, value));
     }
+
+    return cloned;
 }