@contrail/transform-data 1.0.5 → 1.0.7

package/lib/index.d.ts

@@ -1,3 +1,7 @@
 export * from './conditional';
 export * from './map-file';
+export * from './morph';
+export * from './processor';
 export * from './rekey';
+export * from './remove';
+export * from './value-function';

package/lib/index.js

@@ -16,4 +16,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./conditional"), exports);
 __exportStar(require("./map-file"), exports);
+__exportStar(require("./morph"), exports);
+__exportStar(require("./processor"), exports);
 __exportStar(require("./rekey"), exports);
+__exportStar(require("./remove"), exports);
+__exportStar(require("./value-function"), exports);

package/lib/map-file/map-file-util.d.ts

@@ -1,8 +1,12 @@
+import { TransformTask } from "../processor";
 export declare class MapFileUtil {
     private entities;
     private cache;
+    static FILE_NOT_FOUND: string;
+    static ERROR_RETRIEVING: string;
     constructor(entities: any);
     getMapFile(fileId: string): Promise<any>;
-    getMappingSectionFromMap(mapFile: any, objectClass: string, direction: string): any;
-    getMappingSection(fileId: string, objectClass: string, direction: string): Promise<any>;
+    getMappingSectionFromMap(mapFile: any, objectClass: string, direction: string): {};
+    getMappingSection(fileId: string, objectClass: string, direction: string): Promise<{}>;
+    static getTransformTasks(mapSection: object): TransformTask[];
 }

package/lib/map-file/map-file-util.js

@@ -7,6 +7,9 @@ class MapFileUtil {
         this.cache = {};
     }
     async getMapFile(fileId) {
+        if (!fileId) {
+            return {};
+        }
         if (this.cache[fileId]) {
             return this.cache[fileId];
         }
@@ -15,14 +18,35 @@ class MapFileUtil {
             id: fileId,
         };
         const file = await this.entities.get(options);
-        const downloadUrl = file['downloadUrl'];
-        const response = await fetch(downloadUrl);
-        const mappingFile = await response.json();
+        if (!file) {
+            throw new Error(MapFileUtil.FILE_NOT_FOUND + fileId);
+        }
+        let mappingFile = {};
+        try {
+            const downloadUrl = file['downloadUrl'];
+            const contentType = '' + file['contentType'];
+            if (contentType && contentType.toLowerCase().indexOf('json') > -1) {
+                console.log('json file');
+                const response = await fetch(downloadUrl);
+                mappingFile = await response.json();
+                console.log(JSON.stringify(mappingFile));
+            }
+            else {
+                console.log('javascript file');
+                const requireFromUrl = require('require-from-url/sync');
+                const maps = requireFromUrl(downloadUrl);
+                mappingFile = maps['mapping'];
+                console.log(JSON.stringify(mappingFile));
+            }
+        }
+        catch (e) {
+            throw new Error(MapFileUtil.ERROR_RETRIEVING + e);
+        }
         this.cache[fileId] = mappingFile;
         return mappingFile;
     }
     getMappingSectionFromMap(mapFile, objectClass, direction) {
-        let mapping = undefined;
+        let mapping = {};
         if (mapFile[objectClass]) {
             const classMapping = mapFile[objectClass];
             if (classMapping[direction]) {
@@ -36,5 +60,27 @@ class MapFileUtil {
         const mapSection = this.getMappingSectionFromMap(map, objectClass, direction);
         return mapSection;
     }
+    static getTransformTasks(mapSection) {
+        const tasks = [];
+        const order = mapSection['transformOrder'];
+        if (order) {
+            for (const step of order) {
+                const task = {
+                    processor: step['processor'],
+                    rekeyDelete: step['rekeyDelete']
+                };
+                for (const mapping of ['conditionalTransformDefinitions', 'functionTransformers', 'rekeyTransformers', 'removeKeys']) {
+                    const mapKey = step[mapping + 'Key'];
+                    if (mapKey) {
+                        task[mapping] = mapSection[mapKey];
+                    }
+                }
+                tasks.push(task);
+            }
+        }
+        return tasks;
+    }
 }
 exports.MapFileUtil = MapFileUtil;
+MapFileUtil.FILE_NOT_FOUND = 'Mapping File not found for id: ';
+MapFileUtil.ERROR_RETRIEVING = 'Error getting mapping file: ';

package/lib/morph/index.d.ts

@@ -0,0 +1 @@
+export * from './morph-transformer';

package/lib/morph/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./morph-transformer"), exports);

package/lib/morph/morph-transformer.d.ts

@@ -0,0 +1,4 @@
+export declare class MorphTransformer {
+    static transformData(rows: any[], transformFunctions: Record<string, Function>, dependencies: Record<string, any>): void;
+    static transformObject(row: any, transformFunctions: Record<string, Function>, dependencies: Record<string, any>): void;
+}

package/lib/morph/morph-transformer.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MorphTransformer = void 0;
+class MorphTransformer {
+    static transformData(rows, transformFunctions, dependencies) {
+        for (const row of rows) {
+            MorphTransformer.transformObject(row, transformFunctions, dependencies);
+        }
+    }
+    static transformObject(row, transformFunctions, dependencies) {
+        for (const [key, fun] of Object.entries(transformFunctions)) {
+            fun(row, dependencies);
+        }
+    }
+}
+exports.MorphTransformer = MorphTransformer;

package/lib/processor/index.d.ts

@@ -0,0 +1,2 @@
+export * from './process-interfaces';
+export * from './transform-processor';

package/lib/processor/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./process-interfaces"), exports);
+__exportStar(require("./transform-processor"), exports);

package/lib/processor/process-interfaces.d.ts

@@ -0,0 +1,17 @@
+import { TransformDefinition } from "../conditional";
+export declare enum ProcessorKeys {
+    CONDITIONAL = "CONDITIONAL",
+    MORPH = "MORPH",
+    REKEY = "REKEY",
+    REMOVE = "REMOVE",
+    VALUE_TRANSFORM = "VALUE_TRANSFORM"
+}
+export interface TransformTask {
+    processor: ProcessorKeys;
+    conditionalTransformDefinitions?: TransformDefinition[];
+    functionTransformers?: Record<string, Function>;
+    rekeyTransformers?: Record<string, string>;
+    rekeyDelete?: boolean;
+    removeKeys?: string[];
+    dependencies?: object;
+}

package/lib/processor/process-interfaces.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProcessorKeys = void 0;
+var ProcessorKeys;
+(function (ProcessorKeys) {
+    ProcessorKeys["CONDITIONAL"] = "CONDITIONAL";
+    ProcessorKeys["MORPH"] = "MORPH";
+    ProcessorKeys["REKEY"] = "REKEY";
+    ProcessorKeys["REMOVE"] = "REMOVE";
+    ProcessorKeys["VALUE_TRANSFORM"] = "VALUE_TRANSFORM";
+})(ProcessorKeys = exports.ProcessorKeys || (exports.ProcessorKeys = {}));

package/lib/processor/transform-processor.d.ts

@@ -0,0 +1,4 @@
+import { TransformTask } from "./process-interfaces";
+export declare class TransformProcessor {
+    static transformData(rows: any[], transformTasks: TransformTask[]): any[];
+}

package/lib/processor/transform-processor.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformProcessor = void 0;
+const conditional_1 = require("../conditional");
+const morph_transformer_1 = require("../morph/morph-transformer");
+const rekey_1 = require("../rekey");
+const remove_1 = require("../remove");
+const value_function_1 = require("../value-function");
+class TransformProcessor {
+    static transformData(rows, transformTasks) {
+        for (const transformTask of transformTasks) {
+            switch (transformTask.processor) {
+                case 'CONDITIONAL':
+                    conditional_1.ConditionalTransformer.evaluateTransformDefinitionsOnBatch(transformTask.conditionalTransformDefinitions, rows);
+                    break;
+                case 'MORPH':
+                    morph_transformer_1.MorphTransformer.transformData(rows, transformTask.functionTransformers, transformTask.dependencies);
+                    break;
+                case 'REKEY':
+                    rekey_1.RekeyTransformer.transformData(rows, transformTask.rekeyTransformers, transformTask.rekeyDelete);
+                    break;
+                case 'REMOVE':
+                    remove_1.RemoveTransformer.transformData(rows, transformTask.removeKeys);
+                    break;
+                case 'VALUE_TRANSFORM':
+                    value_function_1.ValueFunctionTransformer.transformData(rows, transformTask.functionTransformers, transformTask.dependencies);
+                    break;
+            }
+        }
+        return rows;
+    }
+}
+exports.TransformProcessor = TransformProcessor;

package/lib/rekey/rekey-transformer.d.ts

@@ -1,4 +1,4 @@
 export declare class RekeyTransformer {
-    static transformData(csvJSON: any[], federatedMappings: Record<string, string>, deleteOldKey?: boolean): any[];
-    static transformObject(row: any, federatedMappings: Record<string, string>, deleteOldKey?: boolean): any;
+    static transformData(csvJSON: any[], federatedMappings: Record<string, string>, deleteOldKey?: boolean): void;
+    static transformObject(row: any, federatedMappings: Record<string, string>, deleteOldKey?: boolean): void;
 }

package/lib/rekey/rekey-transformer.js

@@ -6,7 +6,6 @@ class RekeyTransformer {
         for (const row of csvJSON) {
             RekeyTransformer.transformObject(row, federatedMappings, deleteOldKey);
         }
-        return csvJSON;
     }
     static transformObject(row, federatedMappings, deleteOldKey = false) {
         for (const [key, value] of Object.entries(federatedMappings)) {
@@ -19,7 +18,6 @@ class RekeyTransformer {
                 delete row[value];
             }
         }
-        return row;
     }
 }
 exports.RekeyTransformer = RekeyTransformer;

package/lib/remove/index.d.ts

@@ -0,0 +1 @@
+export * from './remove-transformer';

package/lib/remove/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./remove-transformer"), exports);

package/lib/remove/remove-transformer.d.ts

@@ -0,0 +1,4 @@
+export declare class RemoveTransformer {
+    static transformData(rows: any[], keys: string[]): void;
+    static transformObject(row: any, keys: string[]): void;
+}

package/lib/remove/remove-transformer.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoveTransformer = void 0;
+class RemoveTransformer {
+    static transformData(rows, keys) {
+        for (const row of rows) {
+            RemoveTransformer.transformObject(row, keys);
+        }
+    }
+    static transformObject(row, keys) {
+        for (const key of keys) {
+            delete row[key];
+        }
+    }
+}
+exports.RemoveTransformer = RemoveTransformer;

package/lib/value-function/index.d.ts

@@ -0,0 +1 @@
+export * from './value-function-transformer';

package/lib/value-function/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./value-function-transformer"), exports);

package/lib/value-function/value-function-transformer.d.ts

@@ -0,0 +1,4 @@
+export declare class ValueFunctionTransformer {
+    static transformData(rows: any[], transformFunctions: Record<string, Function>, dependencies: Record<string, any>): void;
+    static transformObject(row: any, transformFunctions: Record<string, Function>, dependencies: Record<string, any>): void;
+}

package/lib/value-function/value-function-transformer.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValueFunctionTransformer = void 0;
+class ValueFunctionTransformer {
+    static transformData(rows, transformFunctions, dependencies) {
+        for (const row of rows) {
+            ValueFunctionTransformer.transformObject(row, transformFunctions, dependencies);
+        }
+    }
+    static transformObject(row, transformFunctions, dependencies) {
+        for (const [key, fun] of Object.entries(transformFunctions)) {
+            row[key] = fun(row, dependencies);
+        }
+    }
+}
+exports.ValueFunctionTransformer = ValueFunctionTransformer;

package/package.json

@@ -1,27 +1,34 @@
 {
   "name": "@contrail/transform-data",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "Libraries for transforming data",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
     "build": "tsc",
     "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"",
-    "lint": "tslint -p tsconfig.json",
-    "test": "jest"
+    "lint": "tslint --fix -p tsconfig.json",
+    "test": "jest",
+    "test-watch": "jest --watch"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@types/jest": "^23.3.14",
-    "@types/node": "^14.14.6",
-    "jest": "^23.6.0",
+    "@types/jest": "^29.4.0",
+    "@types/node": "^18.14.6",
+    "@typescript-eslint/eslint-plugin": "^5.48.0",
+    "@typescript-eslint/parser": "^5.48.0",
+    "body-parser": "^1.20.2",
+    "eslint": "^8.31.0",
+    "eslint-plugin-unused-imports": "^2.0.0",
+    "jest": "^29.2.2",
     "prettier": "^1.19.1",
-    "ts-jest": "^23.10.5",
     "tslint": "^5.11.0",
     "tslint-config-prettier": "^1.18.0",
-    "typescript": "^4.0.0"
+    "ts-jest": "^29.0.3",
+    "ts-node": "^7.0.1",
+    "typescript": "^4.7.4"
   },
   "jest": {
     "moduleFileExtensions": [
@@ -38,9 +45,9 @@
     "testEnvironment": "node"
   },
   "dependencies": {
-    "@contrail/core": "1.5.56",
     "@contrail/sdk": "^1.2.1",
     "@contrail/util": "^1.0.26",
-    "node-fetch": "^2.6.0"
+    "node-fetch": "^3.2.10",
+    "require-from-url": "^3.1.3"
   }
 }