@restorecommerce/gql-bot 0.1.11 → 0.1.16

package/CHANGELOG.md

@@ -3,6 +3,59 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.1.16](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.15...@restorecommerce/gql-bot@0.1.16) (2022-02-14)
+
+**Note:** Version bump only for package @restorecommerce/gql-bot
+
+
+
+
+
+## [0.1.15](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.14...@restorecommerce/gql-bot@0.1.15) (2021-08-23)
+
+
+### Bug Fixes
+
+* **version:** up version to be in sync in package-lock ([b8f22c1](https://github.com/restorecommerce/libs/commit/b8f22c1268ee2af4beff7d88bda30f197896e3d2))
+
+
+
+
+
+## [0.1.14](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.13...@restorecommerce/gql-bot@0.1.14) (2021-08-10)
+
+
+### Bug Fixes
+
+* **acs-client, gql-bot, kafka-client, koa-health-check:** eslintrc added root to uniquely identify eslint plugin package to avoid error building facade-srv ([c2e446b](https://github.com/restorecommerce/libs/commit/c2e446bf0f09d7fa4f000da3bb09fd612cb9526c))
+
+
+
+
+
+## [0.1.13](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.12...@restorecommerce/gql-bot@0.1.13) (2021-08-03)
+
+
+### Bug Fixes
+
+* up pkg locks ([8ed92d6](https://github.com/restorecommerce/libs/commit/8ed92d613b9a095e4b5066056ac566e5dbcf1472))
+* updated githead ([2904d30](https://github.com/restorecommerce/libs/commit/2904d30e5773dc8a87c01a08ff6481f99d692354))
+
+
+
+
+
+## [0.1.12](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.11...@restorecommerce/gql-bot@0.1.12) (2021-08-03)
+
+
+### Bug Fixes
+
+* **koa-health-check:** added missing .eslintrc.js ([45af632](https://github.com/restorecommerce/libs/commit/45af632955d2dd448e7a27f4e8c4b971412cd004))
+
+
+
+
+
 ## [0.1.11](https://github.com/restorecommerce/libs/compare/@restorecommerce/gql-bot@0.1.10...@restorecommerce/gql-bot@0.1.11) (2021-06-26)
 
 **Note:** Version bump only for package @restorecommerce/gql-bot

package/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) Invend GmbH and other contributors.
+Copyright (c) n-fuse GmbH and other contributors.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -16,4 +16,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

package/lib/client.d.ts

@@ -0,0 +1,9 @@
+export declare class Client {
+    opts: any;
+    metaQs: any;
+    entryBaseUrl: string;
+    constructor(opts: any);
+    _buildURLs(): any;
+    _normalizeUrl(source?: any): string;
+    post(source: any, job?: any, accessControl?: any, formOptions?: any): Promise<any>;
+}

package/lib/client.js

@@ -0,0 +1,150 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Client = void 0;
+const _ = require("lodash");
+const url = require("url");
+const fs = require("fs");
+const graphql_tag_1 = require("graphql-tag");
+const apollo_client_1 = require("apollo-client");
+const apollo_cache_inmemory_1 = require("apollo-cache-inmemory");
+const node_fetch_1 = require("node-fetch"); // required for apollo-link-http
+const apollo_link_http_1 = require("apollo-link-http");
+function _checkVariableMutation(mutation) {
+    const mutationName = mutation.slice(mutation.indexOf(' '), mutation.indexOf('($'));
+    if (mutationName.indexOf('$') > 0) {
+        return false;
+    }
+    else {
+        return new RegExp('\\b' + mutationName + '\\(', 'i').test(mutation);
+    }
+}
+function _replaceInlineVars(mutation, args) {
+    if (mutation)
+        return mutation.replace(/\${(\w+)}/g, (_, v) => args[v]);
+}
+function _createQueryVariables(inputVarName, queryVarKey, varValue) {
+    if (queryVarKey) {
+        return {
+            [inputVarName]: {
+                [queryVarKey]: JSON.parse(varValue)
+            }
+        };
+    }
+    return {
+        [inputVarName]: JSON.parse(varValue)
+    };
+}
+class Client {
+    constructor(opts) {
+        if (_.isNil(opts)) {
+            throw new Error('Missing options parameter');
+        }
+        this.metaQs = { meta: true };
+        _.defaults(opts, {
+            entry: null,
+            protocol: 'http',
+            apiKey: null,
+            headers: {}
+        });
+        const required = ['entry'];
+        required.forEach((key) => {
+            if (!opts[key]) {
+                throw new Error('Missing option: \'' + key + '\'');
+            }
+        });
+        this.opts = opts;
+        this._buildURLs();
+        _.assign(this.opts.headers, {
+            Accept: 'application/json',
+            Origin: this.entryBaseUrl
+        });
+    }
+    _buildURLs() {
+        const entry = this.opts.entry;
+        if (!String(entry).startsWith('http')) {
+            this.opts.entry = this.opts.protocol + '://' + entry;
+        }
+        const parsedEntry = url.parse(entry);
+        if (parsedEntry.protocol) {
+            this.opts.protocol = parsedEntry.protocol;
+        }
+        const protocol = this.opts.protocol + '//';
+        this.entryBaseUrl = protocol + parsedEntry.host + parsedEntry.path;
+    }
+    _normalizeUrl(source) {
+        let extendURL = '';
+        if (source) {
+            // this is source.path /restore/oss-client/test/folder/file.1
+            if (!source.path && source != true) {
+                extendURL = source;
+            }
+            if (source.path) {
+                // TODO: read this 'the Node way'
+                extendURL = fs.readFileSync(source.path).toString();
+                extendURL = '?query=' + extendURL;
+            }
+        }
+        return url.resolve(this.entryBaseUrl, extendURL);
+    }
+    post(source, job, accessControl, formOptions) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const normalUrl = this._normalizeUrl();
+            let mutation;
+            if (job && job.mutation) {
+                mutation = JSON.stringify(job.mutation);
+            }
+            else {
+                throw new Error('mutation not present in job config');
+            }
+            const apiKey = JSON.stringify(this.opts.apiKey);
+            let resource_list = JSON.stringify(source);
+            if (mutation) {
+                // don't replace quoted strings inside outer quotes
+                // (i.e. if the quote is preceded by a backslash)
+                // make sure to also match line/expression start
+                // and keep the symbol preceding the quote
+                mutation = mutation.replace(/(^|[^\\])\"/g, '$1');
+                // afterwards, replace escaped quotes with regular ones
+                mutation = mutation.replace(/\\\"/g, '\"');
+            }
+            let variables;
+            if (_checkVariableMutation(mutation)) {
+                const queryVarKey = job.queryVariables;
+                const inputVarName = mutation.slice(mutation.indexOf('$') + 1, mutation.indexOf(':'));
+                variables = _createQueryVariables(inputVarName, queryVarKey, resource_list);
+            }
+            else {
+                // To remove double quotes from the keys in JSON data
+                resource_list = resource_list.replace(/\"([^(\")"]+)\":/g, '$1:');
+                mutation = _replaceInlineVars(mutation, { resource_list, apiKey });
+            }
+            const apolloLinkOpts = {
+                uri: normalUrl,
+                fetch: node_fetch_1.default
+            };
+            if (this.opts.headers) {
+                apolloLinkOpts['headers'] = this.opts.headers;
+            }
+            let apolloLink = apollo_link_http_1.createHttpLink(apolloLinkOpts);
+            const apolloCache = new apollo_cache_inmemory_1.InMemoryCache();
+            const apolloClient = new apollo_client_1.ApolloClient({
+                cache: apolloCache,
+                link: apolloLink
+            });
+            return apolloClient.mutate({
+                mutation: graphql_tag_1.default `${mutation}`,
+                variables
+            });
+        });
+    }
+}
+exports.Client = Client;

package/lib/index.d.ts

@@ -0,0 +1,6 @@
+import { Client } from './client';
+export { Client };
+import { GraphQLProcessor } from './job_processor_gql';
+export { GraphQLProcessor };
+import { JobProcessor, Job } from './job_processor';
+export { JobProcessor, Job };

package/lib/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Job = exports.JobProcessor = exports.GraphQLProcessor = exports.Client = void 0;
+const client_1 = require("./client");
+Object.defineProperty(exports, "Client", { enumerable: true, get: function () { return client_1.Client; } });
+const job_processor_gql_1 = require("./job_processor_gql");
+Object.defineProperty(exports, "GraphQLProcessor", { enumerable: true, get: function () { return job_processor_gql_1.GraphQLProcessor; } });
+const job_processor_1 = require("./job_processor");
+Object.defineProperty(exports, "JobProcessor", { enumerable: true, get: function () { return job_processor_1.JobProcessor; } });
+Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return job_processor_1.Job; } });

package/lib/job_processor.d.ts

@@ -0,0 +1,21 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { EventEmitter } from 'events';
+export declare class ReadArrayStream extends Readable {
+    array: any[];
+    constructor(opts: any, array: any[]);
+    _read(): void;
+}
+export declare class Job extends EventEmitter {
+    opts: any;
+    constructor(opts?: any);
+}
+export declare class JobProcessor {
+    jobInfo: any;
+    processed: number;
+    processedTasks: number;
+    taskStream: any;
+    constructor(jobInfo: any);
+    start(tasks?: any, job?: Job): Promise<any>;
+    sync(task: any, job: Job): Promise<any>;
+}

package/lib/job_processor.js

@@ -0,0 +1,153 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JobProcessor = exports.Job = exports.ReadArrayStream = void 0;
+const _ = require("lodash");
+const ps = require("promise-streams");
+const stream_1 = require("stream");
+const through2 = require("through2");
+const readdirp = require("readdirp");
+const path = require("path");
+const events_1 = require("events");
+class ReadArrayStream extends stream_1.Readable {
+    constructor(opts, array) {
+        super(opts);
+        this.array = _.clone(array);
+    }
+    _read() {
+        if (this.array && this.array.length > 0) {
+            const data = this.array.shift();
+            this.push(data);
+        }
+        else {
+            this.push(null);
+        }
+    }
+}
+exports.ReadArrayStream = ReadArrayStream;
+/*
+ * A class to represent a job and its end result as promise.
+ * It also reports the progress as event emitter.
+ */
+class Job extends events_1.EventEmitter {
+    constructor(opts) {
+        super();
+        this.setMaxListeners(100);
+        this.opts = opts || {};
+    }
+}
+exports.Job = Job;
+class JobProcessor {
+    constructor(jobInfo) {
+        this.jobInfo = jobInfo;
+        this.processed = 0;
+        _.defaults(this.jobInfo, {
+            concurrency: 3,
+            processor: null
+        });
+    }
+    start(tasks, job) {
+        return __awaiter(this, void 0, void 0, function* () {
+            job = job || new Job();
+            tasks = tasks || this.jobInfo.tasks;
+            const concurrency = this.jobInfo.options.concurrency;
+            this.taskStream = ps.map({ concurrent: concurrency }, (task) => {
+                return this.jobInfo.options.processor.process(task).then((body) => {
+                    console.log('Processed task [' + this.processed + '] and status is:' + JSON.stringify(body));
+                    this.processed++;
+                    task.inputTask.processing--;
+                    task.progress.value = 100; // task complete
+                    job.emit('progress', task);
+                });
+            });
+            this.taskStream.setMaxListeners(100);
+            const inputTaskStream = new ReadArrayStream({
+                objectMode: true
+            }, tasks);
+            inputTaskStream.pipe(through2.obj((task, enc, cb) => __awaiter(this, void 0, void 0, function* () {
+                const operation = task.operation;
+                yield this[operation].apply(this, [task, job]);
+                cb();
+            })));
+            yield ps.wait(inputTaskStream);
+            this.taskStream.on('error', (err) => { throw err; });
+            this.taskStream.on('end', () => {
+                // console.log('Stream ended');
+            });
+            const tasksStreamEnded = ps.wait(this.taskStream);
+            // Wait until the task stream emitted 'end'
+            tasksStreamEnded.then(() => {
+                job.emit('done');
+            });
+            return job;
+        });
+    }
+    sync(task, job) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const pathOptions = {
+                fileFilter: (entry) => { return true; },
+                depth: 1,
+                lstat: true
+            };
+            const pathSegments = [process.cwd()];
+            if (!_.isUndefined(this.jobInfo.base)) {
+                pathSegments.push(this.jobInfo.base);
+            }
+            if (!_.isUndefined(task.src)) {
+                pathSegments.push(task.src);
+            }
+            pathOptions.fileFilter = task.filter || pathOptions.fileFilter;
+            if (task.depth) {
+                pathOptions.depth = task.depth;
+            }
+            else {
+                delete pathOptions.depth;
+            }
+            const fileItemStream = readdirp(path.join.apply(this, pathSegments), pathOptions);
+            yield new Promise((resolve, reject) => {
+                fileItemStream
+                    .on('warn', (warn) => {
+                    job.emit('warn', warn);
+                })
+                    .on('error', (err) => {
+                    job.emit('error', err);
+                })
+                    .on('data', (fileItem) => {
+                    fileItem.progress = {
+                        value: 0
+                    };
+                    _.merge(fileItem, task);
+                    if (!task.processing) {
+                        task.processing = 0;
+                    }
+                    task.processing++;
+                    fileItem.inputTask = task;
+                    job.emit('progress', fileItem);
+                    resolve(job);
+                })
+                    .on('end', () => {
+                    if (!this.processedTasks) {
+                        this.processedTasks = 0;
+                    }
+                    this.processedTasks++;
+                    // Check processedTasks tasks
+                    // Manually emit `end` event for all tasks finished
+                    if (this.processedTasks === this.jobInfo.tasks.length) {
+                        this.taskStream._flush(() => { });
+                        this.taskStream.emit('end');
+                    }
+                })
+                    .pipe(this.taskStream, { end: false });
+            });
+        });
+    }
+}
+exports.JobProcessor = JobProcessor;

package/lib/job_processor_gql.d.ts

@@ -0,0 +1,10 @@
+import { Client } from './index';
+/**
+ * GraphQL-specific job processor.
+ */
+export declare class GraphQLProcessor {
+    opts: any;
+    client: Client;
+    constructor(opts: any);
+    process(job: any): Promise<any>;
+}

package/lib/job_processor_gql.js

@@ -0,0 +1,109 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GraphQLProcessor = void 0;
+const _ = require("lodash");
+const fs = require("fs");
+const index_1 = require("./index");
+const { YamlStreamReadTransformer } = require('yaml-document-stream');
+/**
+ * GraphQL-specific job processor.
+ */
+class GraphQLProcessor {
+    constructor(opts) {
+        if (_.isNil(opts)) {
+            throw new Error('Missing options parameter');
+        }
+        const defaults = {
+            diffBase: 'md5'
+        };
+        _.defaults(opts, defaults);
+        this.opts = opts;
+        this.client = new index_1.Client(opts);
+    }
+    process(job) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let yamlStream = new YamlStreamReadTransformer();
+            let jobPath = job.path;
+            let data = false;
+            switch (job.operation) {
+                case 'sync': { //  synchronous operation
+                    const fileStream = fs.createReadStream(job.fullPath);
+                    fileStream.pipe(yamlStream);
+                    let batchsize;
+                    if (job.batchSize) {
+                        batchsize = job.batchSize;
+                        console.log('Batch size:', batchsize);
+                    }
+                    let counter = 0;
+                    let batchCounter = 0;
+                    let docArr = [];
+                    let resultArr = [];
+                    // Here we read from the readable stream each yaml document parsed
+                    // as an object, and if we have batching enabled we first batch this
+                    // documents into smaller sets. When a dataset is ready to be imported,
+                    // we pause the readable stream and emit a 'pause' event and
+                    // right after that we reset the dataset object 'docArr'.
+                    yamlStream.on('data', (doc) => {
+                        data = true;
+                        if (batchsize && batchsize != undefined) {
+                            docArr.push(doc);
+                            counter++;
+                            if (counter === batchsize) {
+                                counter = 0;
+                                batchCounter++;
+                                console.log('Processing batch:', batchCounter);
+                                yamlStream.pause();
+                                docArr = [];
+                            }
+                        }
+                        else {
+                            docArr.push(doc);
+                        }
+                    });
+                    // On 'pause' event we create a post request using the GQL Client
+                    // using the accumulated resources inside the 'docArr' dataset,
+                    // we wait for the response, store the response inside an array and
+                    // only then we emit a 'resume' event, resuming reading from the
+                    // 'yamlStream' readable stream.
+                    yamlStream.on('pause', () => __awaiter(this, void 0, void 0, function* () {
+                        let result = yield this.client.post(docArr, job);
+                        resultArr.push(result);
+                        yamlStream.resume();
+                    }));
+                    // On 'end' if we still have data accumulated inside the 'docArr'
+                    // dataset, we create a final post request to import this data as-well,
+                    // store the response inside the array and finally resolve this as a
+                    // Promise to return all the responses back to the initial caller.
+                    return new Promise((resolve, reject) => {
+                        yamlStream.on('end', () => __awaiter(this, void 0, void 0, function* () {
+                            if (data === false) {
+                                throw new Error(`Could not import resources from ${jobPath}. Readable stream is empty. Please provide a file with YAML multi-document format.`);
+                            }
+                            if (docArr && !_.isEmpty(docArr)) {
+                                batchCounter++;
+                                console.log('Processing batch:', batchCounter);
+                                let result = yield this.client.post(docArr, job);
+                                resultArr.push(result);
+                                docArr = [];
+                                resolve(resultArr);
+                            }
+                        }));
+                    });
+                }
+                default: {
+                    throw new Error('Unsupported job operation');
+                }
+            }
+        });
+    }
+}
+exports.GraphQLProcessor = GraphQLProcessor;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@restorecommerce/gql-bot",
   "description": "GraphQL Client Automated Task Processor",
   "main": "lib/index",
-  "version": "0.1.11",
+  "version": "0.1.16",
   "repository": {
     "type": "git",
     "url": "https://github.com/restorecommerce/libs.git"
@@ -59,5 +59,5 @@
   "engines": {
     "node": ">= 12.0.0"
   },
-  "gitHead": "6f9b0654e7ac016896e05c176d3e2b312f57f9cd"
+  "gitHead": "e97bbfe2fe8166dfe1cd47ae60bce54347a4f1c9"
 }