@restorecommerce/gql-bot 0.2.0 → 0.2.3

package/CHANGELOG.md

@@ -3,6 +3,39 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.2.3](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.2.2...@restorecommerce/gql-bot@0.2.3) (2022-06-20)
+
+
+### Bug Fixes
+
+* **gql-bot:** correctly handle null objects ([e7e44ef](https://github.com/restorecommerce/libs/commit/e7e44ef95585d423ddee7d41b6cc196ad1668284))
+
+
+
+
+
+## [0.2.2](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.2.1...@restorecommerce/gql-bot@0.2.2) (2022-06-20)
+
+
+### Bug Fixes
+
+* **gql-bot:** resume if errors are skipped ([7f01a84](https://github.com/restorecommerce/libs/commit/7f01a84f19a81360b94d8dc9421cb572d0fb3cd5))
+
+
+
+
+
+## [0.2.1](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.2.0...@restorecommerce/gql-bot@0.2.1) (2022-06-20)
+
+
+### Bug Fixes
+
+* **gql-bot:** add option to ignore ssl errors ([b82d302](https://github.com/restorecommerce/libs/commit/b82d3020318ec5495ad1c6143cd5c80cb1657f80))
+
+
+
+
+
 # [0.2.0](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.18...@restorecommerce/gql-bot@0.2.0) (2022-06-20)
 
 

package/lib/client.d.ts

@@ -5,5 +5,5 @@ export declare class Client {
     constructor(opts: any);
     _buildURLs(): any;
     _normalizeUrl(source?: any): string;
-    post(source: any, job?: any, verbose?: boolean): Promise<any>;
+    post(source: any, job?: any, verbose?: boolean, ignoreSelfSigned?: boolean): Promise<any>;
 }

package/lib/client.js

@@ -18,6 +18,7 @@ const apollo_client_1 = require("apollo-client");
 const apollo_cache_inmemory_1 = require("apollo-cache-inmemory");
 const node_fetch_1 = require("node-fetch"); // required for apollo-link-http
 const apollo_link_http_1 = require("apollo-link-http");
+const https = require("https");
 const _checkVariableMutation = (mutation) => {
     const mutationName = mutation.slice(mutation.indexOf(' '), mutation.indexOf('($'));
     if (mutationName.indexOf('$') > 0) {
@@ -132,7 +133,7 @@ class Client {
         }
         return url.resolve(this.entryBaseUrl, extendURL);
     }
-    post(source, job, verbose = false) {
+    post(source, job, verbose = false, ignoreSelfSigned = false) {
         return __awaiter(this, void 0, void 0, function* () {
             const normalUrl = this._normalizeUrl();
             let mutation;
@@ -171,6 +172,11 @@ class Client {
             if (this.opts.headers) {
                 apolloLinkOpts['headers'] = this.opts.headers;
             }
+            if (ignoreSelfSigned) {
+                apolloLinkOpts.fetchOptions = {
+                    agent: new https.Agent({ rejectUnauthorized: false }),
+                };
+            }
             let apolloLink = apollo_link_http_1.createHttpLink(apolloLinkOpts);
             const apolloCache = new apollo_cache_inmemory_1.InMemoryCache();
             const apolloClient = new apollo_client_1.ApolloClient({

package/lib/job_processor.d.ts

@@ -19,6 +19,6 @@ export declare class JobProcessor {
     processedTasks: number;
     taskStream: ps.PromiseStream<any>;
     constructor(jobInfo: any);
-    start(tasks?: any, job?: Job, verbose?: boolean, ignoreErrors?: boolean): Promise<any>;
+    start(tasks?: any, job?: Job, verbose?: boolean, ignoreErrors?: boolean, ignoreSelfSigned?: boolean): Promise<any>;
     sync(task: any, job: Job): Promise<any>;
 }

package/lib/job_processor.js

@@ -99,13 +99,13 @@ class JobProcessor {
             processor: null
         });
     }
-    start(tasks, job, verbose = false, ignoreErrors = false) {
+    start(tasks, job, verbose = false, ignoreErrors = false, ignoreSelfSigned = false) {
         return __awaiter(this, void 0, void 0, function* () {
             job = job || new Job();
             tasks = tasks || this.jobInfo.tasks;
             const concurrency = this.jobInfo.options.concurrency;
             this.taskStream = ps.map({ concurrent: concurrency }, (task) => {
-                return this.jobInfo.options.processor.process(task, verbose, ignoreErrors).then((body) => {
+                return this.jobInfo.options.processor.process(task, verbose, ignoreErrors, ignoreSelfSigned).then((body) => {
                     const logColor = utils_1.stringToChalk(task.name);
                     if (verbose) {
                         const processed = processResponse(body);

package/lib/job_processor_gql.d.ts

@@ -6,5 +6,5 @@ export declare class GraphQLProcessor {
     opts: any;
     client: Client;
     constructor(opts: any);
-    process(task: any, verbose?: boolean, ignoreErrors?: boolean): Promise<any>;
+    process(task: any, verbose?: boolean, ignoreErrors?: boolean, ignoreSelfSigned?: boolean): Promise<any>;
 }

package/lib/job_processor_gql.js

@@ -30,7 +30,7 @@ class GraphQLProcessor {
         this.opts = opts;
         this.client = new index_1.Client(opts);
     }
-    process(task, verbose = false, ignoreErrors = false) {
+    process(task, verbose = false, ignoreErrors = false, ignoreSelfSigned = false) {
         return __awaiter(this, void 0, void 0, function* () {
             let yamlStream = new yaml_document_stream_1.YamlStreamReadTransformer();
             let jobPath = task.path;
@@ -68,7 +68,7 @@ class GraphQLProcessor {
                                         counter = 0;
                                         let batchText = '';
                                         if (batchsize > 0) {
-                                            const from = batchCounter * batchsize;
+                                            const from = batchCounter * batchsize + 1;
                                             const to = from + (docArr.length - 1);
                                             batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
                                         }
@@ -89,11 +89,12 @@ class GraphQLProcessor {
                             // 'yamlStream' readable stream.
                             yamlStream.on('pause', () => __awaiter(this, void 0, void 0, function* () {
                                 try {
-                                    resultArr.push(yield this.client.post(docArr, task, verbose));
+                                    resultArr.push(yield this.client.post(docArr, task, verbose, ignoreSelfSigned));
                                     yamlStream.resume();
                                 }
                                 catch (e) {
                                     !ignoreErrors && reject(e);
+                                    ignoreErrors && yamlStream.resume();
                                 }
                             }));
                             let runOnResume;
@@ -113,14 +114,14 @@ class GraphQLProcessor {
                                 if (docArr && !_.isEmpty(docArr)) {
                                     let batchText = '';
                                     if (batchsize > 0) {
-                                        const from = batchCounter * batchsize;
+                                        const from = batchCounter * batchsize + 1;
                                         const to = from + (docArr.length - 1);
                                         batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
                                     }
                                     batchCounter++;
                                     console.log(`[${logColor(task.name)}] Processing batch: ${batchCounter}${batchText}`);
                                     try {
-                                        resultArr.push(yield this.client.post(docArr, task, verbose));
+                                        resultArr.push(yield this.client.post(docArr, task, verbose, ignoreSelfSigned));
                                     }
                                     catch (e) {
                                         !ignoreErrors && reject(e);

package/package.json

@@ -2,7 +2,7 @@
   "name": "@restorecommerce/gql-bot",
   "description": "GraphQL Client Automated Task Processor",
   "main": "lib/index",
-  "version": "0.2.0",
+  "version": "0.2.3",
   "repository": {
     "type": "git",
     "url": "https://github.com/restorecommerce/libs.git"
@@ -71,5 +71,5 @@
       }
     }
   },
-  "gitHead": "65fa379f6d4a215fb6c760431f9718c25b6f51ad"
+  "gitHead": "6a7808de256668f2681bf1ef62c2090a2a0d1d22"
 }

package/src/client.ts

@@ -5,7 +5,9 @@ import gql from 'graphql-tag';
 import { ApolloClient } from 'apollo-client';
 import { InMemoryCache } from 'apollo-cache-inmemory';
 import fetch from 'node-fetch'; // required for apollo-link-http
-import { createHttpLink } from 'apollo-link-http';
+import { createHttpLink, HttpLink } from 'apollo-link-http';
+import * as https from 'https';
+import { processResponse } from './utils';
 
 const _checkVariableMutation = (mutation: string): Boolean => {
   const mutationName = mutation.slice(mutation.indexOf(' '),
@@ -37,7 +39,7 @@ const _createQueryVariables = (inputVarName: string, queryVarKey: string, varVal
 };
 
 const checkError = (data: any): any => {
-  if (typeof data === 'object') {
+  if (typeof data === 'object' && data !== null) {
     if (Array.isArray(data)) {
       const result = data.map(value => {
         const inner = checkError(value);
@@ -138,7 +140,7 @@ export class Client {
     return url.resolve(this.entryBaseUrl, extendURL);
   }
 
-  async post(source: any, job?: any, verbose = false): Promise<any> {
+  async post(source: any, job?: any, verbose = false, ignoreErrors = false, ignoreSelfSigned = false): Promise<any> {
     const normalUrl = this._normalizeUrl();
 
     let mutation;
@@ -172,7 +174,7 @@ export class Client {
       mutation = _replaceInlineVars(mutation, { resource_list, apiKey });
     }
 
-    const apolloLinkOpts = {
+    const apolloLinkOpts: HttpLink.Options = {
       uri: normalUrl,
       fetch
     };
@@ -181,6 +183,12 @@ export class Client {
       apolloLinkOpts['headers'] = this.opts.headers;
     }
 
+    if (ignoreSelfSigned) {
+      apolloLinkOpts.fetchOptions = {
+        agent: new https.Agent({rejectUnauthorized: false}),
+      };
+    }
+
     let apolloLink = createHttpLink(apolloLinkOpts);
 
     const apolloCache = new InMemoryCache();
@@ -202,6 +210,9 @@ export class Client {
           variables,
           response
         }));
+      } else if (ignoreErrors) {
+        const processed = processResponse(response);
+        console.error(JSON.stringify(processed));
       }
       throw new Error(JSON.stringify(error));
     }

package/src/job_processor.ts

@@ -5,36 +5,7 @@ import * as through2 from 'through2';
 import * as readdirp from 'readdirp';
 import * as path from 'path';
 import { EventEmitter } from 'events';
-import { stringToChalk } from './utils';
-
-const unwrap = (data: any): any => {
-  let result = data;
-  while (typeof result === 'object' && Object.keys(result).length == 1) {
-    result = result[Object.keys(result)[0]];
-  }
-  return result;
-};
-
-const removeType = (data: any): any => {
-  if (typeof data === 'object') {
-    if (Array.isArray(data)) {
-      data = data.map(removeType);
-    } else {
-      delete data['__typename'];
-      Object.keys(data).forEach(k => data[k] = removeType(data[k]));
-    }
-  }
-  return data;
-};
-
-const processResponse = (body: any | any[]): any => {
-  const result = [];
-  for (const response of Array.isArray(body) ? body : [body]) {
-    const clean = unwrap(removeType(response));
-    result.push(clean);
-  }
-  return result;
-};
+import { stringToChalk, processResponse } from './utils';
 
 export class ReadArrayStream extends Readable {
   array: any[];
@@ -101,20 +72,20 @@ export class JobProcessor {
     });
   }
 
-  async start(tasks?: any, job?: Job, verbose = false, ignoreErrors = false): Promise<any> {
+  async start(tasks?: any, job?: Job, verbose = false, ignoreErrors = false, ignoreSelfSigned = false): Promise<any> {
     job = job || new Job();
     tasks = tasks || this.jobInfo.tasks;
 
     const concurrency = this.jobInfo.options.concurrency;
     this.taskStream = ps.map({concurrent: concurrency}, (task: any) => {
-      return this.jobInfo.options.processor.process(task, verbose, ignoreErrors).then((body) => {
+      return this.jobInfo.options.processor.process(task, verbose, ignoreErrors, ignoreSelfSigned).then((body) => {
         const logColor = stringToChalk(task.name);
 
         if (verbose) {
           const processed = processResponse(body);
-          console.log(`[${logColor(task.name)}] Completed successfully`, JSON.stringify(processed));
+          console.log(`[${logColor(task.name)}] Completed`, JSON.stringify(processed));
         } else {
-          console.log(`[${logColor(task.name)}] Completed successfully`);
+          console.log(`[${logColor(task.name)}] Completed`);
         }
 
         task.inputTask.processing--;

package/src/job_processor_gql.ts

@@ -23,7 +23,7 @@ export class GraphQLProcessor {
     this.client = new Client(opts);
   }
 
-  async process(task: any, verbose = false, ignoreErrors = false): Promise<any> {
+  async process(task: any, verbose = false, ignoreErrors = false, ignoreSelfSigned = false): Promise<any> {
     let yamlStream = new YamlStreamReadTransformer();
     let jobPath = task.path;
     let data = false;
@@ -64,7 +64,7 @@ export class GraphQLProcessor {
                   counter = 0;
                   let batchText = '';
                   if (batchsize > 0) {
-                    const from = batchCounter * batchsize;
+                    const from = batchCounter * batchsize + 1;
                     const to = from + (docArr.length - 1);
                     batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
                   }
@@ -85,10 +85,11 @@ export class GraphQLProcessor {
             // 'yamlStream' readable stream.
             yamlStream.on('pause', async () => {
               try {
-                resultArr.push(await this.client.post(docArr, task, verbose));
+                resultArr.push(await this.client.post(docArr, task, verbose, ignoreErrors, ignoreSelfSigned));
                 yamlStream.resume();
               } catch (e) {
                 !ignoreErrors && reject(e);
+                ignoreErrors && yamlStream.resume();
               }
             });
 
@@ -111,14 +112,14 @@ export class GraphQLProcessor {
               if (docArr && !_.isEmpty(docArr)) {
                 let batchText = '';
                 if (batchsize > 0) {
-                  const from = batchCounter * batchsize;
+                  const from = batchCounter * batchsize + 1;
                   const to = from + (docArr.length - 1);
                   batchText = from == to ? ` (${from})` : ` (${from} - ${to})`;
                 }
                 batchCounter++;
                 console.log(`[${logColor(task.name)}] Processing batch: ${batchCounter}${batchText}`);
                 try {
-                  resultArr.push(await this.client.post(docArr, task, verbose));
+                  resultArr.push(await this.client.post(docArr, task, verbose, ignoreErrors, ignoreSelfSigned));
                 } catch (e) {
                   !ignoreErrors && reject(e);
                 }

package/src/utils.ts

@@ -8,3 +8,32 @@ const colorHash = new ColorHash({
 export const stringToChalk = (str) => {
   return chalk.hex(colorHash.hex(str));
 };
+
+const unwrap = (data: any): any => {
+  let result = data;
+  while (typeof result === 'object' && Object.keys(result).length == 1) {
+    result = result[Object.keys(result)[0]];
+  }
+  return result;
+};
+
+const removeType = (data: any): any => {
+  if (typeof data === 'object' && data !== null) {
+    if (Array.isArray(data)) {
+      data = data.map(removeType);
+    } else {
+      delete data['__typename'];
+      Object.keys(data).forEach(k => data[k] = removeType(data[k]));
+    }
+  }
+  return data;
+};
+
+export const processResponse = (body: any | any[]): any => {
+  const result = [];
+  for (const response of Array.isArray(body) ? body : [body]) {
+    const clean = unwrap(removeType(response));
+    result.push(clean);
+  }
+  return result;
+};