@stemy/backend 3.0.1 → 3.1.2

package/fesm2015/stemy-backend.js

@@ -25,7 +25,7 @@ import { GridFSBucket } from 'mongodb';
 import dotenv from 'dotenv';
 import { validate, schedule } from 'node-cron';
 import { socket } from 'zeromq';
-import socket_io_client from 'socket.io-client';
+import { filter as filter$1, map } from 'rxjs/operators';
 import { createServer } from 'http';
 import express_, { static as static$1 } from 'express';
 import socket_io from 'socket.io';
@@ -118,6 +118,22 @@ function firstItem(value) {
 function lastItem(value) {
     return value[value.length - 1];
 }
+function regroup(value, comparator) {
+    const result = [];
+    if (!isArray(value) || value.length == 0)
+        return result;
+    value = Array.from(value);
+    result.push([value.shift()]);
+    value.forEach(item => {
+        const group = result.find(g => g.some(a => comparator(a, item)));
+        if (group) {
+            group.push(item);
+            return;
+        }
+        result.push([item]);
+    });
+    return result;
+}
 function getValue(obj, key, defaultValue, treeFallback = false) {
     key = key || "";
     const keys = key.split(".");
@@ -680,6 +696,9 @@ const defaultColors = {
     falseColor: ConsoleColor.FgRed,
     nullColor: ConsoleColor.BgMagenta
 };
+function colorize(input, color) {
+    return `${color}${input}${ConsoleColor.Reset}`;
+}
 function jsonHighlight(input, colorOptions) {
     const colors = Object.assign({}, defaultColors, colorOptions);
     const json = (isString(input) ? input : JSON.stringify(input, null, 2)).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
@@ -1459,11 +1478,10 @@ var __awaiter$p = (this && this.__awaiter) || function (thisArg, _arguments, P,
     });
 };
 class LazyAsset extends BaseEntity {
-    constructor(id, data, collection, assets, progresses, jobMan) {
+    constructor(id, data, collection, assets, progresses) {
         super(id, data, collection);
         this.assets = assets;
         this.progresses = progresses;
-        this.jobMan = jobMan;
     }
     get jobName() {
         return this.data.jobName;
@@ -1531,7 +1549,7 @@ class LazyAsset extends BaseEntity {
             this.data.progressId = (yield this.progresses.create()).id;
             this.data.assetId = null;
             yield this.save();
-            yield this.jobMan.enqueueWithName(this.data.jobName, Object.assign(Object.assign({}, this.data.jobParams), { lazyId: this.id, fromLoad }));
+            yield this.progresses.jobMan.enqueueWithName(this.data.jobName, Object.assign(Object.assign({}, this.data.jobParams), { lazyId: this.id, fromLoad }));
         });
     }
 }
@@ -1565,10 +1583,23 @@ let JobManager = class JobManager {
         this.jobs = this.jobTypes.reduce((res, jobType) => {
             res[getConstructorName(jobType)] = (jobParams) => {
                 const job = this.resolveJobInstance(jobType, jobParams);
-                return job.process();
+                return job.process(this.messageBridge);
             };
             return res;
         }, {});
+        this.messages = new Subject();
+        this.messageBridge = {
+            sendMessage: (message, params) => {
+                this.workerPush.send([message, JSON.stringify(params)]);
+            }
+        };
+        this.processing = false;
+        this.maxTimeout = this.config.resolve("jobTimeout");
+    }
+    on(message, cb) {
+        return this.messages
+            .pipe(filter$1(t => t.message === message))
+            .pipe(map(t => t.params)).subscribe(cb);
     }
     process(jobType, params = {}) {
         return __awaiter$o(this, void 0, void 0, function* () {
@@ -1585,20 +1616,12 @@ let JobManager = class JobManager {
     }
     enqueueWithName(name, params = {}) {
         return __awaiter$o(this, void 0, void 0, function* () {
-            const jobName = yield this.tryResolveFromName(name, params);
-            return this.sendToWorkers(jobName, params);
+            return this.sendToWorkers(this.tryResolveFromName(name, params), params);
         });
     }
     enqueue(jobType, params = {}) {
         return __awaiter$o(this, void 0, void 0, function* () {
-            const jobName = yield this.tryResolveAndConnect(jobType, params);
-            return this.sendToWorkers(jobName, params);
-        });
-    }
-    sendToWorkers(jobName, params) {
-        return __awaiter$o(this, void 0, void 0, function* () {
-            const publisher = yield this.scheduler;
-            yield publisher.send([jobName, JSON.stringify(params), new ObjectId().toHexString()]);
+            return this.sendToWorkers(this.tryResolveAndInit(jobType, params), params);
         });
     }
     schedule(minute, hour, dayOfMonth, month, dayOfWeek, jobType, params = {}) {
@@ -1624,32 +1647,45 @@ let JobManager = class JobManager {
         });
     }
     startProcessing() {
-        const host = this.config.resolve("zmqRemoteHost");
-        this.worker = socket("pull");
-        this.worker.connect(host);
-        this.worker.on("message", (name, args, uniqueId) => __awaiter$o(this, void 0, void 0, function* () {
-            try {
-                const jobName = name.toString("utf8");
-                const jobParams = JSON.parse(args.toString("utf8"));
-                const timerId = uniqueId === null || uniqueId === void 0 ? void 0 : uniqueId.toString("utf8");
-                const jobNameLog = `\x1b[36m"${jobName}"\x1b[0m`;
-                const jobArgsLog = `\n${jsonHighlight(jobParams)}`;
-                console.time(timerId);
-                console.timeLog(timerId, `Started working on background job: ${jobNameLog} with args: ${jobArgsLog}\n\n`);
+        return __awaiter$o(this, void 0, void 0, function* () {
+            if (this.processing)
+                return null;
+            this.processing = true;
+            if (!this.config.resolve("isWorker")) {
+                console.log(colorize(`Processing can not be started because this is NOT a worker process!`, ConsoleColor.FgRed));
+                return null;
+            }
+            const host = this.config.resolve("zmqRemoteHost");
+            const pushHost = `${host}:${this.config.resolve("zmqBackPort")}`;
+            this.workerPush = socket("push");
+            yield this.workerPush.connect(pushHost);
+            console.log(`Worker producer connected to: ${pushHost}`);
+            const pullHost = `${host}:${this.config.resolve("zmqPort")}`;
+            this.workerPull = socket("pull");
+            yield this.workerPull.connect(pullHost);
+            console.log(`Worker consumer connected to: ${pullHost}`);
+            this.workerPull.on("message", (name, args, uniqueId) => __awaiter$o(this, void 0, void 0, function* () {
                 try {
-                    yield Promise.race([this.jobs[jobName](jobParams), promiseTimeout(15000, true)]);
-                    console.timeLog(timerId, `Finished working on background job: ${jobNameLog}\n\n`);
+                    const jobName = name.toString("utf8");
+                    const jobParams = JSON.parse(args.toString("utf8"));
+                    const timerId = uniqueId === null || uniqueId === void 0 ? void 0 : uniqueId.toString("utf8");
+                    console.time(timerId);
+                    console.timeLog(timerId, `Started working on background job: ${colorize(jobName, ConsoleColor.FgCyan)} with args: \n${jsonHighlight(jobParams)}\n\n`);
+                    this.messageBridge.sendMessage(`job-started`, { name: jobName });
+                    try {
+                        yield Promise.race([this.jobs[jobName](jobParams), promiseTimeout(this.maxTimeout, true)]);
+                        console.timeLog(timerId, `Finished working on background job: ${colorize(jobName, ConsoleColor.FgCyan)}\n\n`);
+                    }
+                    catch (e) {
+                        console.timeLog(timerId, `Background job failed: ${colorize(jobName, ConsoleColor.FgRed)}\n${e}\n\n`);
+                    }
+                    console.timeEnd(timerId);
                 }
                 catch (e) {
-                    console.timeLog(timerId, `Background job failed: ${jobNameLog}\n${e.message}\n\n`);
+                    console.log(`Failed to start job: ${e.message}`);
                 }
-                console.timeEnd(timerId);
-            }
-            catch (e) {
-                console.log(`Failed to start job: ${e.message}`);
-            }
-        }));
-        console.log(`Waiting for jobs at: ${host}`);
+            }));
+        });
     }
     tryResolve(jobType, params) {
         const jobName = getConstructorName(jobType);
@@ -1671,19 +1707,28 @@ let JobManager = class JobManager {
         if (!jobType) {
             throw `Can't find job type with name: ${jobName} so it can't be enqueued!`;
         }
-        return this.tryResolveAndConnect(jobType, params);
-    }
-    tryResolveAndConnect(jobType, params) {
-        return __awaiter$o(this, void 0, void 0, function* () {
-            this.scheduler = this.scheduler || new Promise((resolve) => __awaiter$o(this, void 0, void 0, function* () {
-                const port = this.config.resolve("zmqPort");
-                const publisher = socket("push");
-                yield publisher.bind(`tcp://0.0.0.0:${port}`);
-                console.log(`Publisher bound to port: ${port}`);
-                resolve(publisher);
-            }));
-            return this.tryResolve(jobType, params);
-        });
+        return this.tryResolveAndInit(jobType, params);
+    }
+    tryResolveAndInit(jobType, params) {
+        if (!this.apiPush) {
+            const port = this.config.resolve("zmqPort");
+            this.apiPush = socket("push");
+            this.apiPush.bind(`tcp://0.0.0.0:${port}`);
+            console.log(`API producer bound to port: ${port}`);
+        }
+        if (!this.apiPull) {
+            const backPort = this.config.resolve("zmqBackPort");
+            this.apiPull = socket("pull");
+            this.apiPull.bind(`tcp://0.0.0.0:${backPort}`);
+            this.apiPull.on("message", (name, args) => {
+                const message = name.toString("utf8");
+                const params = JSON.parse((args === null || args === void 0 ? void 0 : args.toString("utf8")) || "{}");
+                console.log(`Received a message from worker: "${colorize(message, ConsoleColor.FgCyan)}" with args: ${jsonHighlight(params)}\n\n`);
+                this.messages.next({ message, params });
+            });
+            console.log(`API consumer bound to port: ${backPort}`);
+        }
+        return this.tryResolve(jobType, params);
     }
     resolveJobInstance(jobType, params) {
         const container = this.container.createChildContainer();
@@ -1693,6 +1738,12 @@ let JobManager = class JobManager {
         container.register(jobType, jobType);
         return container.resolve(jobType);
     }
+    sendToWorkers(jobName, params) {
+        return __awaiter$o(this, void 0, void 0, function* () {
+            const publisher = yield this.apiPush;
+            yield publisher.send([jobName, JSON.stringify(params), new ObjectId().toHexString()]);
+        });
+    }
 };
 JobManager = __decorate$s([
     injectable(),
@@ -1711,9 +1762,8 @@ var __awaiter$n = (this && this.__awaiter) || function (thisArg, _arguments, P,
     });
 };
 class Progress extends BaseEntity {
-    constructor(id, data, collection, client) {
+    constructor(id, data, collection) {
         super(id, data, collection);
-        this.client = client;
     }
     get current() {
         return this.data.current;
@@ -1736,6 +1786,10 @@ class Progress extends BaseEntity {
     get remaining() {
         return this.max > 0 ? this.max - this.current : 0;
     }
+    setMessageBridge(messageBridge) {
+        this.messageBridge = messageBridge || this.messageBridge;
+        return this;
+    }
     createSubProgress(progressValue, max, message) {
         return __awaiter$n(this, void 0, void 0, function* () {
             if (max <= 0 && progressValue > 0) {
@@ -1779,9 +1833,6 @@ class Progress extends BaseEntity {
                 return null;
             this.data.current = Math.min(this.max, this.current + value);
             yield this.save();
-            if (!this.client)
-                return;
-            this.client.emit("background-progress", this.id);
         });
     }
     cancel() {
@@ -1790,6 +1841,12 @@ class Progress extends BaseEntity {
             yield this.save();
         });
     }
+    save() {
+        if (this.messageBridge) {
+            this.messageBridge.sendMessage(`progress-changed`, this.toJSON());
+        }
+        return super.save();
+    }
 }
 class SubProgress {
     constructor(parent, progressFrom, progressValue, mMax = 100) {
@@ -1829,6 +1886,12 @@ class SubProgress {
     get canceled() {
         return !this.parent || this.parent.canceled;
     }
+    setMessageBridge(messageBridge) {
+        if (!this.parent)
+            return this;
+        this.parent.setMessageBridge(messageBridge);
+        return this;
+    }
     createSubProgress(progressValue, max, message) {
         return __awaiter$n(this, void 0, void 0, function* () {
             if (max <= 0 && progressValue > 0) {
@@ -1917,33 +1980,34 @@ var __awaiter$m = (this && this.__awaiter) || function (thisArg, _arguments, P,
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const socketIOClient = socket_io_client;
 let Progresses = class Progresses {
-    constructor(connector, config) {
+    constructor(connector, jobMan) {
         this.connector = connector;
-        this.config = config;
-        const mainEndpoint = this.config.resolve("mainEndpoint");
-        this.client = !mainEndpoint ? null : socketIOClient(mainEndpoint, { path: "/socket" });
+        this.jobMan = jobMan;
         this.collection = connector.database.collection("progresses");
+        this.progresses = {};
+        this.jobMan.on("progress-changed", progress => {
+            const id = progress.id;
+            this.progresses[id] = new Progress(new ObjectId(id), progress, this.collection);
+        });
     }
     waitToFinish(id) {
         return __awaiter$m(this, void 0, void 0, function* () {
-            let isFinished = false;
-            let progress = null;
-            while (!isFinished) {
-                progress = yield this.get(id);
-                if (!progress) {
-                    throw `Progress does not exists with id: ${id}`;
-                }
-                if (progress.error) {
-                    throw progress.error;
-                }
-                isFinished = progress.percent == 100;
-                if (!isFinished) {
-                    yield promiseTimeout(50);
-                }
-            }
-            return progress;
+            return Promise.race([
+                this.waitForProgress(id, () => __awaiter$m(this, void 0, void 0, function* () {
+                    let progress = this.progresses[id];
+                    if (!progress || progress.percent < 100) {
+                        progress = yield this.get(id);
+                    }
+                    if (!progress) {
+                        throw new Error(`Progress does not exists with id: ${id}`);
+                    }
+                    return progress;
+                }), 500),
+                this.waitForProgress(id, () => __awaiter$m(this, void 0, void 0, function* () {
+                    return this.progresses[id] || null;
+                }), 25)
+            ]);
         });
     }
     get(id) {
@@ -1954,13 +2018,13 @@ let Progresses = class Progresses {
     find(where) {
         return __awaiter$m(this, void 0, void 0, function* () {
             const data = yield this.collection.findOne(where);
-            return !data ? null : new Progress(data._id, data, this.collection, this.client);
+            return !data ? null : new Progress(data._id, data, this.collection);
         });
     }
     create(max = 100) {
         return __awaiter$m(this, void 0, void 0, function* () {
             if (isNaN(max) || max <= 0) {
-                throw "Max progress value must be bigger than zero";
+                throw new Error(`Max progress value must be bigger than zero`);
             }
             const data = {
                 current: 0,
@@ -1970,7 +2034,7 @@ let Progresses = class Progresses {
                 canceled: false
             };
             const res = yield this.collection.insertOne(data);
-            return new Progress(res.insertedId, data, this.collection, this.client);
+            return new Progress(res.insertedId, data, this.collection);
         });
     }
     remove(id) {
@@ -1979,11 +2043,35 @@ let Progresses = class Progresses {
             return id;
         });
     }
+    waitForProgress(id, cb, delay) {
+        return __awaiter$m(this, void 0, void 0, function* () {
+            let isFinished = false;
+            let progress = null;
+            let waitTime = 0;
+            while (!isFinished) {
+                progress = yield cb();
+                waitTime += delay;
+                if (progress) {
+                    if (progress.error) {
+                        throw new Error(progress.error);
+                    }
+                    isFinished = progress.percent >= 100;
+                }
+                if (!isFinished) {
+                    if (waitTime >= this.jobMan.maxTimeout) {
+                        throw new Error(`Progress with id: ${id} probably never will be finished!`);
+                    }
+                    yield promiseTimeout(delay);
+                }
+            }
+            return progress;
+        });
+    }
 };
 Progresses = __decorate$r([
     injectable(),
     singleton(),
-    __metadata$l("design:paramtypes", [MongoConnector, Configuration])
+    __metadata$l("design:paramtypes", [MongoConnector, JobManager])
 ], Progresses);
 
 var __decorate$q = (this && this.__decorate) || function (decorators, target, key, desc) {
@@ -2024,7 +2112,7 @@ let LazyAssets = class LazyAssets {
             if (existingAsset)
                 return existingAsset;
             const res = yield this.collection.insertOne(data);
-            return new LazyAsset(res.insertedId, data, this.collection, this.assets, this.progresses, this.jobMan);
+            return new LazyAsset(res.insertedId, data, this.collection, this.assets, this.progresses);
         });
     }
     read(id) {
@@ -2037,7 +2125,7 @@ let LazyAssets = class LazyAssets {
             const data = yield this.collection.findOne(where);
             return !data
                 ? null
-                : new LazyAsset(data._id, data, this.collection, this.assets, this.progresses, this.jobMan);
+                : new LazyAsset(data._id, data, this.collection, this.assets, this.progresses);
         });
     }
     unlink(id) {
@@ -2052,8 +2140,10 @@ let LazyAssets = class LazyAssets {
 LazyAssets = __decorate$q([
     injectable(),
     scoped(Lifecycle.ContainerScoped),
-    __metadata$k("design:paramtypes", [MongoConnector, Assets,
-        Progresses, JobManager])
+    __metadata$k("design:paramtypes", [MongoConnector,
+        Assets,
+        Progresses,
+        JobManager])
 ], LazyAssets);
 
 var __decorate$p = (this && this.__decorate) || function (decorators, target, key, desc) {
@@ -3754,14 +3844,15 @@ class LazyAssetGenerator {
     get lazyAssets() {
         return this.assetResolver.lazyAssets;
     }
-    process() {
+    process(messaging) {
         return __awaiter$1(this, void 0, void 0, function* () {
             const lazyAsset = yield this.lazyAssets.read(this.lazyId);
             let progress = yield this.progresses.get(lazyAsset.progressId);
             if (!progress || progress.canceled)
                 return null;
+            progress.setMessageBridge(messaging);
             try {
-                const asset = yield this.generate(progress);
+                const asset = yield this.generate(progress, messaging);
                 progress = yield progress.load();
                 if (!progress || progress.canceled)
                     return null;
@@ -3825,10 +3916,11 @@ function createServices() {
         new Parameter("mongoDb", "node-backend"),
         new Parameter("mongoUser", null),
         new Parameter("mongoPassword", null),
-        new Parameter("nodeEnv", "development"),
+        new Parameter("nodeEnv", "production"),
         new Parameter("appPort", 80),
         new Parameter("zmqPort", 3000),
-        new Parameter("zmqRemoteHost", "tcp://127.0.0.1:3000"),
+        new Parameter("zmqBackPort", 3100),
+        new Parameter("zmqRemoteHost", "tcp://127.0.0.1"),
         new Parameter("isWorker", false),
         new Parameter("mainEndpoint", ""),
         new Parameter("idChars", "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"),
@@ -3836,6 +3928,7 @@ function createServices() {
         new Parameter("idPrefix", "ID-"),
         new Parameter("idParts", [4, 4]),
         new Parameter("jsonLimit", "250mb"),
+        new Parameter("jobTimeout", 5 * 60 * 1000),
         new Parameter("cacheCollection", "cache"),
     ];
     // Convert parameters to providers
@@ -4034,5 +4127,5 @@ function setupBackend(config, providers, parent) {
  * Generated bundle index. Do not edit.
  */
 
-export { AssetProcessor, AssetResolver, Assets, AuthController, BackendProvider, Cache, CacheProcessor, Configuration, ConsoleColor, DI_CONTAINER, EXPRESS, EndpointProvider, ErrorHandlerMiddleware, FIXTURE, Fixtures, Gallery, GalleryCache, GalleryController, HTTP_SERVER, IdGenerator, IsFile, IsObjectId, JOB, JobManager, LanguageMiddleware, LazyAssetGenerator, LazyAssets, MailSender, MemoryCache, MongoConnector, PARAMETER, Parameter, Progresses, ResolveEntity, SOCKET_SERVER, TemplateRenderer, TranslationProvider, Translator, Type, UserManager, assign, broadcast, bufferToStream, convertValue, copy, copyStream, createServices, createTransformer, deleteFile, deleteFromBucket, filter, firstItem, getConstructorName, getExtension, getFileName, getFunctionParams, getType, getValue, groupBy, hydratePopulated, idToString, injectServices, isArray, isBoolean, isConstructor, isDate, isDefined, isFunction, isInterface, isNullOrUndefined, isObject, isPrimitive, isString, isType, jsonHighlight, lastItem, lcFirst, lookupPipelines, md5, mkdirRecursive, multiSubscription, observableFromFunction, padLeft, padRight, paginate, paginateAggregations, promiseTimeout, proxyFunction, proxyFunctions, rand, random, readAndDeleteFile, readFile, runCommand, setupBackend, streamToBuffer, ucFirst, valueToPromise, writeFile };
+export { AssetProcessor, AssetResolver, Assets, AuthController, BackendProvider, Cache, CacheProcessor, Configuration, ConsoleColor, DI_CONTAINER, EXPRESS, EndpointProvider, ErrorHandlerMiddleware, FIXTURE, Fixtures, Gallery, GalleryCache, GalleryController, HTTP_SERVER, IdGenerator, IsFile, IsObjectId, JOB, JobManager, LanguageMiddleware, LazyAssetGenerator, LazyAssets, MailSender, MemoryCache, MongoConnector, PARAMETER, Parameter, Progresses, ResolveEntity, SOCKET_SERVER, TemplateRenderer, TranslationProvider, Translator, Type, UserManager, assign, broadcast, bufferToStream, colorize, convertValue, copy, copyStream, createServices, createTransformer, deleteFile, deleteFromBucket, filter, firstItem, getConstructorName, getExtension, getFileName, getFunctionParams, getType, getValue, groupBy, hydratePopulated, idToString, injectServices, isArray, isBoolean, isConstructor, isDate, isDefined, isFunction, isInterface, isNullOrUndefined, isObject, isPrimitive, isString, isType, jsonHighlight, lastItem, lcFirst, lookupPipelines, md5, mkdirRecursive, multiSubscription, observableFromFunction, padLeft, padRight, paginate, paginateAggregations, promiseTimeout, proxyFunction, proxyFunctions, rand, random, readAndDeleteFile, readFile, regroup, runCommand, setupBackend, streamToBuffer, ucFirst, valueToPromise, writeFile };
 //# sourceMappingURL=stemy-backend.js.map