@restorecommerce/gql-bot 0.1.18 → 0.2.2

package/CHANGELOG.md

@@ -3,6 +3,39 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.2.2](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.2.1...@restorecommerce/gql-bot@0.2.2) (2022-06-20)
+
+
+### Bug Fixes
+
+* **gql-bot:** resume if errors are skipped ([7f01a84](https://github.com/restorecommerce/libs/commit/7f01a84f19a81360b94d8dc9421cb572d0fb3cd5))
+
+
+
+
+
+## [0.2.1](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.2.0...@restorecommerce/gql-bot@0.2.1) (2022-06-20)
+
+
+### Bug Fixes
+
+* **gql-bot:** add option to ignore ssl errors ([b82d302](https://github.com/restorecommerce/libs/commit/b82d3020318ec5495ad1c6143cd5c80cb1657f80))
+
+
+
+
+
+# [0.2.0](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.18...@restorecommerce/gql-bot@0.2.0) (2022-06-20)
+
+
+### Features
+
+* **gql-bot:** correctly handle errors, more readable output ([b773d2b](https://github.com/restorecommerce/libs/commit/b773d2b94d41233f9660a96de5cac4e85f305e66))
+
+
+
+
+
 ## [0.1.18](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.17...@restorecommerce/gql-bot@0.1.18) (2022-06-10)
 
 

package/lib/client.d.ts

@@ -5,5 +5,5 @@ export declare class Client {
     constructor(opts: any);
     _buildURLs(): any;
     _normalizeUrl(source?: any): string;
-    post(source: any, job?: any, accessControl?: any, formOptions?: any): Promise<any>;
+    post(source: any, job?: any, verbose?: boolean, ignoreSelfSigned?: boolean): Promise<any>;
 }

package/lib/client.js

@@ -18,7 +18,8 @@ const apollo_client_1 = require("apollo-client");
 const apollo_cache_inmemory_1 = require("apollo-cache-inmemory");
 const node_fetch_1 = require("node-fetch"); // required for apollo-link-http
 const apollo_link_http_1 = require("apollo-link-http");
-function _checkVariableMutation(mutation) {
+const https = require("https");
+const _checkVariableMutation = (mutation) => {
     const mutationName = mutation.slice(mutation.indexOf(' '), mutation.indexOf('($'));
     if (mutationName.indexOf('$') > 0) {
         return false;
@@ -26,12 +27,12 @@ function _checkVariableMutation(mutation) {
     else {
         return new RegExp('\\b' + mutationName + '\\(', 'i').test(mutation);
     }
-}
-function _replaceInlineVars(mutation, args) {
+};
+const _replaceInlineVars = (mutation, args) => {
     if (mutation)
         return mutation.replace(/\${(\w+)}/g, (_, v) => args[v]);
-}
-function _createQueryVariables(inputVarName, queryVarKey, varValue) {
+};
+const _createQueryVariables = (inputVarName, queryVarKey, varValue) => {
     if (queryVarKey) {
         return {
             [inputVarName]: {
@@ -42,7 +43,44 @@ function _createQueryVariables(inputVarName, queryVarKey, varValue) {
     return {
         [inputVarName]: JSON.parse(varValue)
     };
-}
+};
+const checkError = (data) => {
+    if (typeof data === 'object') {
+        if (Array.isArray(data)) {
+            const result = data.map(value => {
+                const inner = checkError(value);
+                if (inner) {
+                    return inner;
+                }
+            }).filter(value => !!value);
+            if (result.length > 0) {
+                return result;
+            }
+        }
+        else {
+            if ('__typename' in data) {
+                switch (data['__typename']) {
+                    case 'IoRestorecommerceStatusOperationStatus':
+                    case 'IoRestorecommerceStatusStatus':
+                        if ('code' in data) {
+                            const code = data['code'];
+                            if (code != '' && code != '200' && code != 0 && code != 200) {
+                                return data;
+                            }
+                        }
+                        break;
+                }
+            }
+            for (const value of Object.values(data)) {
+                const inner = checkError(value);
+                if (inner) {
+                    return inner;
+                }
+            }
+        }
+    }
+    return undefined;
+};
 class Client {
     constructor(opts) {
         if (_.isNil(opts)) {
@@ -95,7 +133,7 @@ class Client {
         }
         return url.resolve(this.entryBaseUrl, extendURL);
     }
-    post(source, job, accessControl, formOptions) {
+    post(source, job, verbose = false, ignoreSelfSigned = false) {
         return __awaiter(this, void 0, void 0, function* () {
             const normalUrl = this._normalizeUrl();
             let mutation;
@@ -103,7 +141,7 @@ class Client {
                 mutation = JSON.stringify(job.mutation);
             }
             else {
-                throw new Error('mutation not present in job config');
+                throw new Error(`mutation not present in job config (${job.name})`);
             }
             const apiKey = JSON.stringify(this.opts.apiKey);
             let resource_list = JSON.stringify(source);
@@ -134,16 +172,33 @@ class Client {
             if (this.opts.headers) {
                 apolloLinkOpts['headers'] = this.opts.headers;
             }
+            if (ignoreSelfSigned) {
+                apolloLinkOpts.fetchOptions = {
+                    agent: new https.Agent({ rejectUnauthorized: false }),
+                };
+            }
             let apolloLink = apollo_link_http_1.createHttpLink(apolloLinkOpts);
             const apolloCache = new apollo_cache_inmemory_1.InMemoryCache();
             const apolloClient = new apollo_client_1.ApolloClient({
                 cache: apolloCache,
                 link: apolloLink
             });
-            return apolloClient.mutate({
+            const response = yield apolloClient.mutate({
                 mutation: graphql_tag_1.default `${mutation}`,
                 variables
             });
+            const error = checkError(response);
+            if (error) {
+                if (verbose) {
+                    console.error(JSON.stringify({
+                        request: mutation,
+                        variables,
+                        response
+                    }));
+                }
+                throw new Error(JSON.stringify(error));
+            }
+            return response;
         });
     }
 }

package/lib/job_processor.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+import * as ps from 'promise-streams';
 import { Readable } from 'stream';
 import { EventEmitter } from 'events';
 export declare class ReadArrayStream extends Readable {
@@ -8,14 +9,16 @@ export declare class ReadArrayStream extends Readable {
 }
 export declare class Job extends EventEmitter {
     opts: any;
+    done: boolean;
+    error: any;
     constructor(opts?: any);
+    wait(): Promise<unknown>;
 }
 export declare class JobProcessor {
     jobInfo: any;
-    processed: number;
     processedTasks: number;
-    taskStream: any;
+    taskStream: ps.PromiseStream<any>;
     constructor(jobInfo: any);
-    start(tasks?: any, job?: Job): Promise<any>;
+    start(tasks?: any, job?: Job, verbose?: boolean, ignoreErrors?: boolean, ignoreSelfSigned?: boolean): Promise<any>;
     sync(task: any, job: Job): Promise<any>;
 }

package/lib/job_processor.js

@@ -17,6 +17,34 @@ const through2 = require("through2");
 const readdirp = require("readdirp");
 const path = require("path");
 const events_1 = require("events");
+const utils_1 = require("./utils");
+const unwrap = (data) => {
+    let result = data;
+    while (typeof result === 'object' && Object.keys(result).length == 1) {
+        result = result[Object.keys(result)[0]];
+    }
+    return result;
+};
+const removeType = (data) => {
+    if (typeof data === 'object') {
+        if (Array.isArray(data)) {
+            data = data.map(removeType);
+        }
+        else {
+            delete data['__typename'];
+            Object.keys(data).forEach(k => data[k] = removeType(data[k]));
+        }
+    }
+    return data;
+};
+const processResponse = (body) => {
+    const result = [];
+    for (const response of Array.isArray(body) ? body : [body]) {
+        const clean = unwrap(removeType(response));
+        result.push(clean);
+    }
+    return result;
+};
 class ReadArrayStream extends stream_1.Readable {
     constructor(opts, array) {
         super(opts);
@@ -40,29 +68,52 @@ exports.ReadArrayStream = ReadArrayStream;
 class Job extends events_1.EventEmitter {
     constructor(opts) {
         super();
+        this.done = false;
+        this.error = undefined;
         this.setMaxListeners(100);
         this.opts = opts || {};
+        this.once('done', () => this.done = true);
+        this.once('error', (err) => this.error = err);
+    }
+    wait() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.error) {
+                throw this.error;
+            }
+            if (this.done) {
+                return;
+            }
+            return new Promise((resolve, reject) => {
+                this.once('done', resolve);
+                this.once('error', reject);
+            });
+        });
     }
 }
 exports.Job = Job;
 class JobProcessor {
     constructor(jobInfo) {
         this.jobInfo = jobInfo;
-        this.processed = 0;
         _.defaults(this.jobInfo, {
             concurrency: 3,
             processor: null
         });
     }
-    start(tasks, job) {
+    start(tasks, job, verbose = false, ignoreErrors = false, ignoreSelfSigned = false) {
         return __awaiter(this, void 0, void 0, function* () {
             job = job || new Job();
             tasks = tasks || this.jobInfo.tasks;
             const concurrency = this.jobInfo.options.concurrency;
             this.taskStream = ps.map({ concurrent: concurrency }, (task) => {
-                return this.jobInfo.options.processor.process(task).then((body) => {
-                    console.log('Processed task [' + this.processed + '] and status is:' + JSON.stringify(body));
-                    this.processed++;
+                return this.jobInfo.options.processor.process(task, verbose, ignoreErrors, ignoreSelfSigned).then((body) => {
+                    const logColor = utils_1.stringToChalk(task.name);
+                    if (verbose) {
+                        const processed = processResponse(body);
+                        console.log(`[${logColor(task.name)}] Completed successfully`, JSON.stringify(processed));
+                    }
+                    else {
+                        console.log(`[${logColor(task.name)}] Completed successfully`);
+                    }
                     task.inputTask.processing--;
                     task.progress.value = 100; // task complete
                     job.emit('progress', task);
@@ -78,9 +129,8 @@ class JobProcessor {
                 cb();
             })));
             yield ps.wait(inputTaskStream);
-            this.taskStream.on('error', (err) => { throw err; });
-            this.taskStream.on('end', () => {
-                // console.log('Stream ended');
+            this.taskStream.on('error', (err) => {
+                job.emit('error', err);
             });
             const tasksStreamEnded = ps.wait(this.taskStream);
             // Wait until the task stream emitted 'end'
@@ -93,7 +143,9 @@ class JobProcessor {
     sync(task, job) {
         return __awaiter(this, void 0, void 0, function* () {
             const pathOptions = {
-                fileFilter: (entry) => { return true; },
+                fileFilter: (entry) => {
+                    return true;
+                },
                 depth: 1,
                 lstat: true
             };
@@ -141,8 +193,9 @@ class JobProcessor {
                     // Check processedTasks tasks
                     // Manually emit `end` event for all tasks finished
                     if (this.processedTasks === this.jobInfo.tasks.length) {
-                        this.taskStream._flush(() => { });
-                        this.taskStream.emit('end');
+                        this.taskStream._flush(() => {
+                        });
+                        this.taskStream.end();
                     }
                 })
                     .pipe(this.taskStream, { end: false });

package/lib/job_processor_gql.d.ts

@@ -6,5 +6,5 @@ export declare class GraphQLProcessor {
     opts: any;
     client: Client;
     constructor(opts: any);
-    process(job: any): Promise<any>;
+    process(task: any, verbose?: boolean, ignoreErrors?: boolean, ignoreSelfSigned?: boolean): Promise<any>;
 }

package/lib/job_processor_gql.js

@@ -13,7 +13,8 @@ exports.GraphQLProcessor = void 0;
 const _ = require("lodash");
 const fs = require("fs");
 const index_1 = require("./index");
-const { YamlStreamReadTransformer } = require('yaml-document-stream');
+const yaml_document_stream_1 = require("yaml-document-stream");
+const utils_1 = require("./utils");
 /**
  * GraphQL-specific job processor.
  */
@@ -29,74 +30,118 @@ class GraphQLProcessor {
         this.opts = opts;
         this.client = new index_1.Client(opts);
     }
-    process(job) {
+    process(task, verbose = false, ignoreErrors = false, ignoreSelfSigned = false) {
         return __awaiter(this, void 0, void 0, function* () {
-            let yamlStream = new YamlStreamReadTransformer();
-            let jobPath = job.path;
+            let yamlStream = new yaml_document_stream_1.YamlStreamReadTransformer();
+            let jobPath = task.path;
             let data = false;
-            switch (job.operation) {
+            const logColor = utils_1.stringToChalk(task.name);
+            switch (task.operation) {
                 case 'sync': { //  synchronous operation
-                    const fileStream = fs.createReadStream(job.fullPath);
-                    fileStream.pipe(yamlStream);
-                    let batchsize;
-                    if (job.batchSize) {
-                        batchsize = job.batchSize;
-                        console.log('Batch size:', batchsize);
-                    }
-                    let counter = 0;
-                    let batchCounter = 0;
-                    let docArr = [];
-                    let resultArr = [];
-                    // Here we read from the readable stream each yaml document parsed
-                    // as an object, and if we have batching enabled we first batch this
-                    // documents into smaller sets. When a dataset is ready to be imported,
-                    // we pause the readable stream and emit a 'pause' event and
-                    // right after that we reset the dataset object 'docArr'.
-                    yamlStream.on('data', (doc) => {
-                        data = true;
-                        if (batchsize && batchsize != undefined) {
-                            docArr.push(doc);
-                            counter++;
-                            if (counter === batchsize) {
-                                counter = 0;
-                                batchCounter++;
-                                console.log('Processing batch:', batchCounter);
-                                yamlStream.pause();
-                                docArr = [];
-                            }
-                        }
-                        else {
-                            docArr.push(doc);
-                        }
-                    });
-                    // On 'pause' event we create a post request using the GQL Client
-                    // using the accumulated resources inside the 'docArr' dataset,
-                    // we wait for the response, store the response inside an array and
-                    // only then we emit a 'resume' event, resuming reading from the
-                    // 'yamlStream' readable stream.
-                    yamlStream.on('pause', () => __awaiter(this, void 0, void 0, function* () {
-                        let result = yield this.client.post(docArr, job);
-                        resultArr.push(result);
-                        yamlStream.resume();
-                    }));
-                    // On 'end' if we still have data accumulated inside the 'docArr'
-                    // dataset, we create a final post request to import this data as-well,
-                    // store the response inside the array and finally resolve this as a
-                    // Promise to return all the responses back to the initial caller.
                     return new Promise((resolve, reject) => {
-                        yamlStream.on('end', () => __awaiter(this, void 0, void 0, function* () {
-                            if (data === false) {
-                                throw new Error(`Could not import resources from ${jobPath}. Readable stream is empty. Please provide a file with YAML multi-document format.`);
+                        try {
+                            yamlStream.on('error', (err) => {
+                                !ignoreErrors && reject(err);
+                            });
+                            const fileStream = fs.createReadStream(task.fullPath);
+                            fileStream.pipe(yamlStream);
+                            let batchsize;
+                            if (task.batchSize) {
+                                batchsize = task.batchSize;
+                                console.log(`[${logColor(task.name)}] Batch size:`, batchsize);
                             }
-                            if (docArr && !_.isEmpty(docArr)) {
-                                batchCounter++;
-                                console.log('Processing batch:', batchCounter);
-                                let result = yield this.client.post(docArr, job);
-                                resultArr.push(result);
-                                docArr = [];
+                            let counter = 0;
+                            let batchCounter = 0;
+                            let docArr = [];
+                            let resultArr = [];
+                            // Here we read from the readable stream each yaml document parsed
+                            // as an object, and if we have batching enabled we first batch this
+                            // documents into smaller sets. When a dataset is ready to be imported,
+                            // we pause the readable stream and emit a 'pause' event and
+                            // right after that we reset the dataset object 'docArr'.
+                            yamlStream.on('data', (doc) => {
+                                data = true;
+                                if (batchsize) {
+                                    docArr.push(doc);
+                                    counter++;
+                                    if (counter === batchsize) {
+                                        counter = 0;
+                                        let batchText = '';
+                                        if (batchsize > 0) {
+                                            const from = batchCounter * batchsize + 1;
+                                            const to = from + (docArr.length - 1);
+                                            batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
+                                        }
+                                        batchCounter++;
+                                        console.log(`[${logColor(task.name)}] Processing batch: ${batchCounter}${batchText}`);
+                                        yamlStream.pause();
+                                        docArr = [];
+                                    }
+                                }
+                                else {
+                                    docArr.push(doc);
+                                }
+                            });
+                            // On 'pause' event we create a post request using the GQL Client
+                            // using the accumulated resources inside the 'docArr' dataset,
+                            // we wait for the response, store the response inside an array and
+                            // only then we emit a 'resume' event, resuming reading from the
+                            // 'yamlStream' readable stream.
+                            yamlStream.on('pause', () => __awaiter(this, void 0, void 0, function* () {
+                                try {
+                                    resultArr.push(yield this.client.post(docArr, task, verbose, ignoreSelfSigned));
+                                    yamlStream.resume();
+                                }
+                                catch (e) {
+                                    !ignoreErrors && reject(e);
+                                    ignoreErrors && yamlStream.resume();
+                                }
+                            }));
+                            let runOnResume;
+                            yamlStream.on('resume', () => __awaiter(this, void 0, void 0, function* () {
+                                if (runOnResume) {
+                                    runOnResume();
+                                }
+                            }));
+                            // On 'end' if we still have data accumulated inside the 'docArr'
+                            // dataset, we create a final post request to import this data as-well,
+                            // store the response inside the array and finally resolve this as a
+                            // Promise to return all the responses back to the initial caller.
+                            const endFunc = () => __awaiter(this, void 0, void 0, function* () {
+                                if (data === false) {
+                                    throw new Error(`Could not import resources from ${jobPath}. Readable stream is empty. Please provide a file with YAML multi-document format.`);
+                                }
+                                if (docArr && !_.isEmpty(docArr)) {
+                                    let batchText = '';
+                                    if (batchsize > 0) {
+                                        const from = batchCounter * batchsize + 1;
+                                        const to = from + (docArr.length - 1);
+                                        batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
+                                    }
+                                    batchCounter++;
+                                    console.log(`[${logColor(task.name)}] Processing batch: ${batchCounter}${batchText}`);
+                                    try {
+                                        resultArr.push(yield this.client.post(docArr, task, verbose, ignoreSelfSigned));
+                                    }
+                                    catch (e) {
+                                        !ignoreErrors && reject(e);
+                                    }
+                                    docArr = [];
+                                }
                                 resolve(resultArr);
-                            }
-                        }));
+                            });
+                            yamlStream.on('end', () => {
+                                if (yamlStream.isPaused()) {
+                                    runOnResume = endFunc;
+                                }
+                                else {
+                                    endFunc();
+                                }
+                            });
+                        }
+                        catch (e) {
+                            !ignoreErrors && reject(e);
+                        }
                     });
                 }
                 default: {

package/lib/utils.d.ts

@@ -0,0 +1,2 @@
+import * as chalk from 'chalk';
+export declare const stringToChalk: (str: any) => chalk.Chalk;

package/lib/utils.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stringToChalk = void 0;
+const color_hash_1 = require("color-hash");
+const chalk = require("chalk");
+const colorHash = new color_hash_1.default({
+    lightness: [0.45, 0.6, 0.75]
+});
+const stringToChalk = (str) => {
+    return chalk.hex(colorHash.hex(str));
+};
+exports.stringToChalk = stringToChalk;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@restorecommerce/gql-bot",
   "description": "GraphQL Client Automated Task Processor",
   "main": "lib/index",
-  "version": "0.1.18",
+  "version": "0.2.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/restorecommerce/libs.git"
@@ -19,6 +19,8 @@
     "apollo-cache-inmemory": "^1.6.6",
     "apollo-client": "^2.6.10",
     "apollo-link-http": "^1.5.17",
+    "chalk": "^4.1.2",
+    "color-hash": "^2.0.1",
     "graphql": "^15.5.0",
     "graphql-tag": "^2.11.0",
     "js-yaml": "^4.1.0",
@@ -30,28 +32,29 @@
     "yaml-document-stream": "^1.1.0"
   },
   "devDependencies": {
+    "@types/color-hash": "^1.0.2",
     "@types/mocha": "^8.2.2",
     "@types/node": "^14.14.41",
     "@typescript-eslint/eslint-plugin": "^4.22.0",
     "@typescript-eslint/parser": "^4.22.0",
     "coveralls": "^3.1.0",
     "eslint": "^7.24.0",
+    "eslint-plugin-prefer-arrow-functions": "^3.1.4",
     "mocha": "^8.3.2",
     "nock": "^13.0.11",
     "npm-run-all": "^4.1.5",
     "nyc": "^15.1.0",
     "should": "^13.2.3",
+    "ts-node": "^10.5.0",
     "typescript": "^4.2.4"
   },
   "scripts": {
-    "tsctests": "tsc test/*.ts --module commonjs -target ES6",
-    "pretest": "npm run tsctests && npm run lint",
+    "pretest": "npm run lint",
     "test": "nyc mocha",
     "lint": "eslint './src/**/*.ts'",
     "build:tsc": "tsc -d",
     "build:clean": "rimraf lib",
     "build": "npm-run-all lint build:clean build:tsc",
-    "mocha": "mocha --timeout 5000 ./test/*.js --trace-warnings",
     "mocha-debug": "mocha -R spec test/*.js --full-trace --inspect-brk",
     "lcov-report": "nyc report --reporter=lcov",
     "coveralls": "nyc report --reporter=text-lcov | coveralls"
@@ -59,5 +62,14 @@
   "engines": {
     "node": ">= 12.0.0"
   },
-  "gitHead": "2d453885b1c82df3e6aabbfbfe3b5a9c0b240277"
+  "nx": {
+    "targets": {
+      "build": {
+        "outputs": [
+          "./lib"
+        ]
+      }
+    }
+  },
+  "gitHead": "a40423a5fca103646c2adbc2530cf096a190139f"
 }

package/src/client.ts

@@ -5,9 +5,10 @@ import gql from 'graphql-tag';
 import { ApolloClient } from 'apollo-client';
 import { InMemoryCache } from 'apollo-cache-inmemory';
 import fetch from 'node-fetch'; // required for apollo-link-http
-import { createHttpLink } from 'apollo-link-http';
+import { createHttpLink, HttpLink } from 'apollo-link-http';
+import * as https from 'https';
 
-function _checkVariableMutation(mutation: string): Boolean {
+const _checkVariableMutation = (mutation: string): Boolean => {
   const mutationName = mutation.slice(mutation.indexOf(' '),
     mutation.indexOf('($'));
   if (mutationName.indexOf('$') > 0) {
@@ -15,14 +16,14 @@ function _checkVariableMutation(mutation: string): Boolean {
   } else {
     return new RegExp('\\b' + mutationName + '\\(', 'i').test(mutation);
   }
-}
+};
 
-function _replaceInlineVars(mutation: string, args: any): string {
+const _replaceInlineVars = (mutation: string, args: any): string => {
   if (mutation)
     return mutation.replace(/\${(\w+)}/g, (_, v) => args[v]);
-}
+};
 
-function _createQueryVariables(inputVarName: string, queryVarKey: string, varValue: any): Object {
+const _createQueryVariables = (inputVarName: string, queryVarKey: string, varValue: any): Object => {
   if (queryVarKey) {
     return {
       [inputVarName]: {
@@ -34,7 +35,45 @@ function _createQueryVariables(inputVarName: string, queryVarKey: string, varVal
   return {
     [inputVarName]: JSON.parse(varValue)
   };
-}
+};
+
+const checkError = (data: any): any => {
+  if (typeof data === 'object') {
+    if (Array.isArray(data)) {
+      const result = data.map(value => {
+        const inner = checkError(value);
+        if (inner) {
+          return inner;
+        }
+      }).filter(value => !!value);
+      if (result.length > 0) {
+        return result;
+      }
+    } else {
+      if ('__typename' in data) {
+        switch (data['__typename']) {
+          case 'IoRestorecommerceStatusOperationStatus':
+          case 'IoRestorecommerceStatusStatus':
+            if ('code' in data) {
+              const code = data['code'];
+              if (code != '' && code != '200' && code != 0 && code != 200) {
+                return data;
+              }
+            }
+            break;
+        }
+      }
+
+      for (const value of Object.values(data)) {
+        const inner = checkError(value);
+        if (inner) {
+          return inner;
+        }
+      }
+    }
+  }
+  return undefined;
+};
 
 export class Client {
   opts: any;
@@ -100,16 +139,14 @@ export class Client {
     return url.resolve(this.entryBaseUrl, extendURL);
   }
 
-  async post(source: any, job?: any, accessControl?: any,
-    formOptions?: any): Promise<any> {
-
+  async post(source: any, job?: any, verbose = false, ignoreSelfSigned = false): Promise<any> {
     const normalUrl = this._normalizeUrl();
 
     let mutation;
     if (job && job.mutation) {
       mutation = JSON.stringify(job.mutation);
     } else {
-      throw new Error('mutation not present in job config');
+      throw new Error(`mutation not present in job config (${job.name})`);
     }
 
     const apiKey = JSON.stringify(this.opts.apiKey);
@@ -136,7 +173,7 @@ export class Client {
       mutation = _replaceInlineVars(mutation, { resource_list, apiKey });
     }
 
-    const apolloLinkOpts = {
+    const apolloLinkOpts: HttpLink.Options = {
       uri: normalUrl,
       fetch
     };
@@ -145,6 +182,12 @@ export class Client {
       apolloLinkOpts['headers'] = this.opts.headers;
     }
 
+    if (ignoreSelfSigned) {
+      apolloLinkOpts.fetchOptions = {
+        agent: new https.Agent({rejectUnauthorized: false}),
+      };
+    }
+
     let apolloLink = createHttpLink(apolloLinkOpts);
 
     const apolloCache = new InMemoryCache();
@@ -153,10 +196,23 @@ export class Client {
       link: apolloLink
     });
 
-    return apolloClient.mutate({
+    const response = await apolloClient.mutate({
       mutation: gql`${mutation}`,
       variables
     });
 
+    const error = checkError(response);
+    if (error) {
+      if (verbose) {
+        console.error(JSON.stringify({
+          request: mutation,
+          variables,
+          response
+        }));
+      }
+      throw new Error(JSON.stringify(error));
+    }
+
+    return response;
   }
 }

package/src/job_processor.ts

@@ -5,9 +5,40 @@ import * as through2 from 'through2';
 import * as readdirp from 'readdirp';
 import * as path from 'path';
 import { EventEmitter } from 'events';
+import { stringToChalk } from './utils';
+
+const unwrap = (data: any): any => {
+  let result = data;
+  while (typeof result === 'object' && Object.keys(result).length == 1) {
+    result = result[Object.keys(result)[0]];
+  }
+  return result;
+};
+
+const removeType = (data: any): any => {
+  if (typeof data === 'object') {
+    if (Array.isArray(data)) {
+      data = data.map(removeType);
+    } else {
+      delete data['__typename'];
+      Object.keys(data).forEach(k => data[k] = removeType(data[k]));
+    }
+  }
+  return data;
+};
+
+const processResponse = (body: any | any[]): any => {
+  const result = [];
+  for (const response of Array.isArray(body) ? body : [body]) {
+    const clean = unwrap(removeType(response));
+    result.push(clean);
+  }
+  return result;
+};
 
 export class ReadArrayStream extends Readable {
   array: any[];
+
   constructor(opts: any, array: any[]) {
     super(opts);
     this.array = _.clone(array);
@@ -29,21 +60,40 @@ export class ReadArrayStream extends Readable {
  */
 export class Job extends EventEmitter {
   opts: any;
+  done = false;
+  error = undefined;
+
   constructor(opts?: any) {
     super();
     this.setMaxListeners(100);
     this.opts = opts || {};
+    this.once('done', () => this.done = true);
+    this.once('error', (err) => this.error = err);
+  }
+
+  async wait() {
+    if (this.error) {
+      throw this.error;
+    }
+
+    if (this.done) {
+      return;
+    }
+
+    return new Promise((resolve, reject) => {
+      this.once('done', resolve);
+      this.once('error', reject);
+    });
   }
 }
 
 export class JobProcessor {
   jobInfo: any;
-  processed: number;
   processedTasks: number;
-  taskStream: any;
+  taskStream: ps.PromiseStream<any>;
+
   constructor(jobInfo: any) {
     this.jobInfo = jobInfo;
-    this.processed = 0;
 
     _.defaults(this.jobInfo, {
       concurrency: 3,
@@ -51,15 +101,22 @@ export class JobProcessor {
     });
   }
 
-  async start(tasks?: any, job?: Job): Promise<any> {
+  async start(tasks?: any, job?: Job, verbose = false, ignoreErrors = false, ignoreSelfSigned = false): Promise<any> {
     job = job || new Job();
     tasks = tasks || this.jobInfo.tasks;
 
     const concurrency = this.jobInfo.options.concurrency;
-    this.taskStream = ps.map({ concurrent: concurrency }, (task: any) => {
-      return this.jobInfo.options.processor.process(task).then((body) => {
-        console.log('Processed task [' + this.processed + '] and status is:' + JSON.stringify(body));
-        this.processed++;
+    this.taskStream = ps.map({concurrent: concurrency}, (task: any) => {
+      return this.jobInfo.options.processor.process(task, verbose, ignoreErrors, ignoreSelfSigned).then((body) => {
+        const logColor = stringToChalk(task.name);
+
+        if (verbose) {
+          const processed = processResponse(body);
+          console.log(`[${logColor(task.name)}] Completed successfully`, JSON.stringify(processed));
+        } else {
+          console.log(`[${logColor(task.name)}] Completed successfully`);
+        }
+
         task.inputTask.processing--;
         task.progress.value = 100; // task complete
         job.emit('progress', task);
@@ -79,9 +136,8 @@ export class JobProcessor {
 
     await ps.wait(inputTaskStream);
 
-    this.taskStream.on('error', (err) => { throw err; });
-    this.taskStream.on('end', () => {
-      // console.log('Stream ended');
+    this.taskStream.on('error', (err) => {
+      job.emit('error', err);
     });
 
     const tasksStreamEnded = ps.wait(this.taskStream);
@@ -96,7 +152,9 @@ export class JobProcessor {
 
   async sync(task: any, job: Job): Promise<any> {
     const pathOptions = {
-      fileFilter: (entry) => { return true; },
+      fileFilter: (entry) => {
+        return true;
+      },
       depth: 1,
       lstat: true
     };
@@ -147,11 +205,12 @@ export class JobProcessor {
           // Check processedTasks tasks
           // Manually emit `end` event for all tasks finished
           if (this.processedTasks === this.jobInfo.tasks.length) {
-            this.taskStream._flush(() => { });
-            this.taskStream.emit('end');
+            this.taskStream._flush(() => {
+            });
+            this.taskStream.end();
           }
         })
-        .pipe(this.taskStream, { end: false });
+        .pipe(this.taskStream, {end: false});
     });
   }
 }

package/src/job_processor_gql.ts

@@ -1,7 +1,8 @@
 import * as _ from 'lodash';
 import * as fs from 'fs';
 import { Client } from './index';
-const {YamlStreamReadTransformer} = require('yaml-document-stream');
+import { YamlStreamReadTransformer } from 'yaml-document-stream';
+import { stringToChalk } from './utils';
 
 /**
  * GraphQL-specific job processor.
@@ -9,6 +10,7 @@ const {YamlStreamReadTransformer} = require('yaml-document-stream');
 export class GraphQLProcessor {
   opts: any;
   client: Client;
+
   constructor(opts: any) {
     if (_.isNil(opts)) {
       throw new Error('Missing options parameter');
@@ -21,81 +23,123 @@ export class GraphQLProcessor {
     this.client = new Client(opts);
   }
 
-  async process(job: any): Promise<any> {
+  async process(task: any, verbose = false, ignoreErrors = false, ignoreSelfSigned = false): Promise<any> {
     let yamlStream = new YamlStreamReadTransformer();
-    let jobPath = job.path;
+    let jobPath = task.path;
     let data = false;
-    switch (job.operation) {
+    const logColor = stringToChalk(task.name);
+    switch (task.operation) {
       case 'sync': {  //  synchronous operation
-        const fileStream = fs.createReadStream(job.fullPath);
-        fileStream.pipe(yamlStream);
-        let batchsize;
-        if (job.batchSize) {
-          batchsize = job.batchSize;
-          console.log('Batch size:', batchsize);
-        }
+        return new Promise<any>((resolve, reject) => {
+          try {
+            yamlStream.on('error', (err) => {
+              !ignoreErrors && reject(err);
+            });
 
-        let counter = 0;
-        let batchCounter = 0;
-        let docArr: any[] = [];
-        let resultArr: any[] = [];
+            const fileStream = fs.createReadStream(task.fullPath);
+            fileStream.pipe(yamlStream);
+            let batchsize;
+            if (task.batchSize) {
+              batchsize = task.batchSize;
+              console.log(`[${logColor(task.name)}] Batch size:`, batchsize);
+            }
 
-        // Here we read from the readable stream each yaml document parsed
-        // as an object, and if we have batching enabled we first batch this
-        // documents into smaller sets. When a dataset is ready to be imported,
-        // we pause the readable stream and emit a 'pause' event and
-        // right after that we reset the dataset object 'docArr'.
-        yamlStream.on('data', (doc) => {
-          data = true;
-          if (batchsize && batchsize != undefined) {
-            docArr.push(doc);
-            counter++;
+            let counter = 0;
+            let batchCounter = 0;
+            let docArr: any[] = [];
+            let resultArr: any[] = [];
 
-            if (counter === batchsize) {
-              counter = 0;
-              batchCounter++;
-              console.log('Processing batch:', batchCounter);
-              yamlStream.pause();
-              docArr = [];
-            }
-          } else {
-            docArr.push(doc);
-          }
-        });
+            // Here we read from the readable stream each yaml document parsed
+            // as an object, and if we have batching enabled we first batch this
+            // documents into smaller sets. When a dataset is ready to be imported,
+            // we pause the readable stream and emit a 'pause' event and
+            // right after that we reset the dataset object 'docArr'.
+            yamlStream.on('data', (doc) => {
+              data = true;
+              if (batchsize) {
+                docArr.push(doc);
+                counter++;
 
-        // On 'pause' event we create a post request using the GQL Client
-        // using the accumulated resources inside the 'docArr' dataset,
-        // we wait for the response, store the response inside an array and
-        // only then we emit a 'resume' event, resuming reading from the
-        // 'yamlStream' readable stream.
-        yamlStream.on('pause', async () => {
-          let result = await this.client.post(docArr, job);
-          resultArr.push(result);
-          yamlStream.resume();
-        });
+                if (counter === batchsize) {
+                  counter = 0;
+                  let batchText = '';
+                  if (batchsize > 0) {
+                    const from = batchCounter * batchsize + 1;
+                    const to = from + (docArr.length - 1);
+                    batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
+                  }
+                  batchCounter++;
+                  console.log(`[${logColor(task.name)}] Processing batch: ${batchCounter}${batchText}`);
+                  yamlStream.pause();
+                  docArr = [];
+                }
+              } else {
+                docArr.push(doc);
+              }
+            });
 
-        // On 'end' if we still have data accumulated inside the 'docArr'
-        // dataset, we create a final post request to import this data as-well,
-        // store the response inside the array and finally resolve this as a
-        // Promise to return all the responses back to the initial caller.
-        return new Promise<any>((resolve, reject) => {
-          yamlStream.on('end', async () => {
-            if (data === false) {
-              throw new Error(`Could not import resources from ${jobPath}. Readable stream is empty. Please provide a file with YAML multi-document format.`);
-            }
+            // On 'pause' event we create a post request using the GQL Client
+            // using the accumulated resources inside the 'docArr' dataset,
+            // we wait for the response, store the response inside an array and
+            // only then we emit a 'resume' event, resuming reading from the
+            // 'yamlStream' readable stream.
+            yamlStream.on('pause', async () => {
+              try {
+                resultArr.push(await this.client.post(docArr, task, verbose, ignoreSelfSigned));
+                yamlStream.resume();
+              } catch (e) {
+                !ignoreErrors && reject(e);
+                ignoreErrors && yamlStream.resume();
+              }
+            });
+
+            let runOnResume: undefined | (() => void);
+            yamlStream.on('resume', async () => {
+              if (runOnResume) {
+                runOnResume();
+              }
+            });
 
-            if (docArr && !_.isEmpty(docArr)) {
-              batchCounter++;
-              console.log('Processing batch:', batchCounter);
-              let result = await this.client.post(docArr, job);
-              resultArr.push(result);
-              docArr = [];
+            // On 'end' if we still have data accumulated inside the 'docArr'
+            // dataset, we create a final post request to import this data as-well,
+            // store the response inside the array and finally resolve this as a
+            // Promise to return all the responses back to the initial caller.
+            const endFunc = async () => {
+              if (data === false) {
+                throw new Error(`Could not import resources from ${jobPath}. Readable stream is empty. Please provide a file with YAML multi-document format.`);
+              }
+
+              if (docArr && !_.isEmpty(docArr)) {
+                let batchText = '';
+                if (batchsize > 0) {
+                  const from = batchCounter * batchsize + 1;
+                  const to = from + (docArr.length - 1);
+                  batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
+                }
+                batchCounter++;
+                console.log(`[${logColor(task.name)}] Processing batch: ${batchCounter}${batchText}`);
+                try {
+                  resultArr.push(await this.client.post(docArr, task, verbose, ignoreSelfSigned));
+                } catch (e) {
+                  !ignoreErrors && reject(e);
+                }
+                docArr = [];
+              }
 
               resolve(resultArr as any);
-            }
-          });
-        });
+            };
 
+            yamlStream.on('end', () => {
+              if (yamlStream.isPaused()) {
+                runOnResume = endFunc;
+              } else {
+                endFunc();
+              }
+            });
+          } catch (e) {
+            !ignoreErrors && reject(e);
+          }
+        });
       }
       default: {
         throw new Error('Unsupported job operation');

package/src/utils.ts

@@ -0,0 +1,10 @@
+import ColorHash from 'color-hash';
+import * as chalk from 'chalk';
+
+const colorHash = new ColorHash({
+  lightness: [0.45, 0.6, 0.75]
+});
+
+export const stringToChalk = (str) => {
+  return chalk.hex(colorHash.hex(str));
+};