@stemy/backend 3.0.0 → 3.1.1

package/fesm2015/stemy-backend.js

@@ -25,7 +25,7 @@ import { GridFSBucket } from 'mongodb';
 import dotenv from 'dotenv';
 import { validate, schedule } from 'node-cron';
 import { socket } from 'zeromq';
-import socket_io_client from 'socket.io-client';
+import { filter as filter$1, map } from 'rxjs/operators';
 import { createServer } from 'http';
 import express_, { static as static$1 } from 'express';
 import socket_io from 'socket.io';
@@ -661,6 +661,7 @@ var ConsoleColor;
     ConsoleColor["FgMagenta"] = "\u001B[35m";
     ConsoleColor["FgCyan"] = "\u001B[36m";
     ConsoleColor["FgWhite"] = "\u001B[37m";
+    ConsoleColor["FgDefault"] = "\u001B[38m";
     ConsoleColor["BgBlack"] = "\u001B[40m";
     ConsoleColor["BgRed"] = "\u001B[41m";
     ConsoleColor["BgGreen"] = "\u001B[42m";
@@ -669,15 +670,20 @@ var ConsoleColor;
     ConsoleColor["BgMagenta"] = "\u001B[45m";
     ConsoleColor["BgCyan"] = "\u001B[46m";
     ConsoleColor["BgWhite"] = "\u001B[47m";
+    ConsoleColor["BgDefault"] = "\u001B[48m";
 })(ConsoleColor || (ConsoleColor = {}));
 const defaultColors = {
-    keyColor: ConsoleColor.Dim,
+    keyColor: ConsoleColor.FgWhite,
     numberColor: ConsoleColor.FgBlue,
-    stringColor: ConsoleColor.FgCyan,
+    stringColor: ConsoleColor.FgYellow,
     trueColor: ConsoleColor.FgGreen,
     falseColor: ConsoleColor.FgRed,
     nullColor: ConsoleColor.BgMagenta
 };
+const MAX_TIMEOUT = 120000;
+function colorize(input, color) {
+    return `${color}${input}${ConsoleColor.Reset}`;
+}
 function jsonHighlight(input, colorOptions) {
     const colors = Object.assign({}, defaultColors, colorOptions);
     const json = (isString(input) ? input : JSON.stringify(input, null, 2)).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
@@ -1457,11 +1463,10 @@ var __awaiter$p = (this && this.__awaiter) || function (thisArg, _arguments, P,
     });
 };
 class LazyAsset extends BaseEntity {
-    constructor(id, data, collection, assets, progresses, jobMan) {
+    constructor(id, data, collection, assets, progresses) {
         super(id, data, collection);
         this.assets = assets;
         this.progresses = progresses;
-        this.jobMan = jobMan;
     }
     get jobName() {
         return this.data.jobName;
@@ -1494,7 +1499,7 @@ class LazyAsset extends BaseEntity {
             this.progresses.get(this.progressId).then(p => {
                 p === null || p === void 0 ? void 0 : p.cancel();
             });
-            this.startWorkingOnAsset().then(() => {
+            this.startWorkingOnAsset(false).then(() => {
                 console.log(`Started working on lazy asset: ${this.id}`);
             }).catch(reason => {
                 console.log(`Can't start working on lazy asset: ${this.id}\nReason: ${reason}`);
@@ -1513,7 +1518,7 @@ class LazyAsset extends BaseEntity {
                 yield this.progresses.waitToFinish(this.progressId);
                 return this.loadAsset();
             }
-            yield this.startWorkingOnAsset();
+            yield this.startWorkingOnAsset(true);
             return this.loadAsset();
         });
     }
@@ -1524,12 +1529,12 @@ class LazyAsset extends BaseEntity {
             return asset;
         });
     }
-    startWorkingOnAsset() {
+    startWorkingOnAsset(fromLoad) {
         return __awaiter$p(this, void 0, void 0, function* () {
             this.data.progressId = (yield this.progresses.create()).id;
             this.data.assetId = null;
             yield this.save();
-            yield this.jobMan.enqueueWithName(this.data.jobName, Object.assign(Object.assign({}, this.data.jobParams), { lazyId: this.id }));
+            yield this.progresses.jobMan.enqueueWithName(this.data.jobName, Object.assign(Object.assign({}, this.data.jobParams), { lazyId: this.id, fromLoad }));
         });
     }
 }
@@ -1563,10 +1568,22 @@ let JobManager = class JobManager {
         this.jobs = this.jobTypes.reduce((res, jobType) => {
             res[getConstructorName(jobType)] = (jobParams) => {
                 const job = this.resolveJobInstance(jobType, jobParams);
-                return job.process();
+                return job.process(this.messageBridge);
             };
             return res;
         }, {});
+        this.messages = new Subject();
+        this.messageBridge = {
+            sendMessage: (message, params) => {
+                this.workerPush.send([message, JSON.stringify(params)]);
+            }
+        };
+        this.processing = false;
+    }
+    on(message, cb) {
+        return this.messages
+            .pipe(filter$1(t => t.message === message))
+            .pipe(map(t => t.params)).subscribe(cb);
     }
     process(jobType, params = {}) {
         return __awaiter$o(this, void 0, void 0, function* () {
@@ -1583,20 +1600,12 @@ let JobManager = class JobManager {
     }
     enqueueWithName(name, params = {}) {
         return __awaiter$o(this, void 0, void 0, function* () {
-            const jobName = yield this.tryResolveFromName(name, params);
-            return this.sendToWorkers(jobName, params);
+            return this.sendToWorkers(this.tryResolveFromName(name, params), params);
         });
     }
     enqueue(jobType, params = {}) {
         return __awaiter$o(this, void 0, void 0, function* () {
-            const jobName = yield this.tryResolveAndConnect(jobType, params);
-            return this.sendToWorkers(jobName, params);
-        });
-    }
-    sendToWorkers(jobName, params) {
-        return __awaiter$o(this, void 0, void 0, function* () {
-            const publisher = yield this.scheduler;
-            yield publisher.send([jobName, JSON.stringify(params), new ObjectId().toHexString()]);
+            return this.sendToWorkers(this.tryResolveAndInit(jobType, params), params);
         });
     }
     schedule(minute, hour, dayOfMonth, month, dayOfWeek, jobType, params = {}) {
@@ -1622,32 +1631,45 @@ let JobManager = class JobManager {
         });
     }
     startProcessing() {
-        const host = this.config.resolve("zmqRemoteHost");
-        this.worker = socket("pull");
-        this.worker.connect(host);
-        this.worker.on("message", (name, args, uniqueId) => __awaiter$o(this, void 0, void 0, function* () {
-            try {
-                const jobName = name.toString("utf8");
-                const jobParams = JSON.parse(args.toString("utf8"));
-                const timerId = uniqueId === null || uniqueId === void 0 ? void 0 : uniqueId.toString("utf8");
-                const jobNameLog = `\x1b[36m"${jobName}"\x1b[0m`;
-                const jobArgsLog = `\n${jsonHighlight(jobParams)}\n`;
-                console.time(timerId);
-                console.timeLog(timerId, `Started working on background job: ${jobNameLog} with args: ${jobArgsLog}`);
+        return __awaiter$o(this, void 0, void 0, function* () {
+            if (this.processing)
+                return null;
+            this.processing = true;
+            if (!this.config.resolve("isWorker")) {
+                console.log(colorize(`Processing can not be started because this is NOT a worker process!`, ConsoleColor.FgRed));
+                return null;
+            }
+            const host = this.config.resolve("zmqRemoteHost");
+            const pushHost = `${host}:${this.config.resolve("zmqBackPort")}`;
+            this.workerPush = socket("push");
+            yield this.workerPush.connect(pushHost);
+            console.log(`Worker producer connected to: ${pushHost}`);
+            const pullHost = `${host}:${this.config.resolve("zmqPort")}`;
+            this.workerPull = socket("pull");
+            yield this.workerPull.connect(pullHost);
+            console.log(`Worker consumer connected to: ${pullHost}`);
+            this.workerPull.on("message", (name, args, uniqueId) => __awaiter$o(this, void 0, void 0, function* () {
                 try {
-                    yield Promise.race([this.jobs[jobName](jobParams), promiseTimeout(15000, true)]);
-                    console.timeLog(timerId, `Finished working on background job: ${jobNameLog} with args: ${jobArgsLog}`);
+                    const jobName = name.toString("utf8");
+                    const jobParams = JSON.parse(args.toString("utf8"));
+                    const timerId = uniqueId === null || uniqueId === void 0 ? void 0 : uniqueId.toString("utf8");
+                    console.time(timerId);
+                    console.timeLog(timerId, `Started working on background job: ${colorize(jobName, ConsoleColor.FgCyan)} with args: \n${jsonHighlight(jobParams)}\n\n`);
+                    this.messageBridge.sendMessage(`job-started`, { name: jobName });
+                    try {
+                        yield Promise.race([this.jobs[jobName](jobParams), promiseTimeout(MAX_TIMEOUT, true)]);
+                        console.timeLog(timerId, `Finished working on background job: ${colorize(jobName, ConsoleColor.FgCyan)}\n\n`);
+                    }
+                    catch (e) {
+                        console.timeLog(timerId, `Background job failed: ${colorize(jobName, ConsoleColor.FgRed)}\n${e}\n\n`);
+                    }
+                    console.timeEnd(timerId);
                 }
                 catch (e) {
-                    console.timeLog(timerId, `Background job failed: ${jobNameLog} with args: ${jobArgsLog}${e.message}\n\n`);
+                    console.log(`Failed to start job: ${e.message}`);
                 }
-                console.timeEnd(timerId);
-            }
-            catch (e) {
-                console.log(`Failed to start job: ${e.message}`);
-            }
-        }));
-        console.log(`Waiting for jobs at: ${host}`);
+            }));
+        });
     }
     tryResolve(jobType, params) {
         const jobName = getConstructorName(jobType);
@@ -1669,19 +1691,28 @@ let JobManager = class JobManager {
         if (!jobType) {
             throw `Can't find job type with name: ${jobName} so it can't be enqueued!`;
         }
-        return this.tryResolveAndConnect(jobType, params);
-    }
-    tryResolveAndConnect(jobType, params) {
-        return __awaiter$o(this, void 0, void 0, function* () {
-            this.scheduler = this.scheduler || new Promise((resolve) => __awaiter$o(this, void 0, void 0, function* () {
-                const port = this.config.resolve("zmqPort");
-                const publisher = socket("push");
-                yield publisher.bind(`tcp://0.0.0.0:${port}`);
-                console.log(`Publisher bound to port: ${port}`);
-                resolve(publisher);
-            }));
-            return this.tryResolve(jobType, params);
-        });
+        return this.tryResolveAndInit(jobType, params);
+    }
+    tryResolveAndInit(jobType, params) {
+        if (!this.apiPush) {
+            const port = this.config.resolve("zmqPort");
+            this.apiPush = socket("push");
+            this.apiPush.bind(`tcp://0.0.0.0:${port}`);
+            console.log(`API producer bound to port: ${port}`);
+        }
+        if (!this.apiPull) {
+            const backPort = this.config.resolve("zmqBackPort");
+            this.apiPull = socket("pull");
+            this.apiPull.bind(`tcp://0.0.0.0:${backPort}`);
+            this.apiPull.on("message", (name, args) => {
+                const message = name.toString("utf8");
+                const params = JSON.parse((args === null || args === void 0 ? void 0 : args.toString("utf8")) || "{}");
+                console.log(`Received a message from worker: "${colorize(message, ConsoleColor.FgCyan)}" with args: ${jsonHighlight(params)}\n\n`);
+                this.messages.next({ message, params });
+            });
+            console.log(`API consumer bound to port: ${backPort}`);
+        }
+        return this.tryResolve(jobType, params);
     }
     resolveJobInstance(jobType, params) {
         const container = this.container.createChildContainer();
@@ -1691,6 +1722,12 @@ let JobManager = class JobManager {
         container.register(jobType, jobType);
         return container.resolve(jobType);
     }
+    sendToWorkers(jobName, params) {
+        return __awaiter$o(this, void 0, void 0, function* () {
+            const publisher = yield this.apiPush;
+            yield publisher.send([jobName, JSON.stringify(params), new ObjectId().toHexString()]);
+        });
+    }
 };
 JobManager = __decorate$s([
     injectable(),
@@ -1709,9 +1746,8 @@ var __awaiter$n = (this && this.__awaiter) || function (thisArg, _arguments, P,
     });
 };
 class Progress extends BaseEntity {
-    constructor(id, data, collection, client) {
+    constructor(id, data, collection) {
         super(id, data, collection);
-        this.client = client;
     }
     get current() {
         return this.data.current;
@@ -1734,6 +1770,10 @@ class Progress extends BaseEntity {
     get remaining() {
         return this.max > 0 ? this.max - this.current : 0;
     }
+    setMessageBridge(messageBridge) {
+        this.messageBridge = messageBridge || this.messageBridge;
+        return this;
+    }
     createSubProgress(progressValue, max, message) {
         return __awaiter$n(this, void 0, void 0, function* () {
             if (max <= 0 && progressValue > 0) {
@@ -1777,9 +1817,6 @@ class Progress extends BaseEntity {
                 return null;
             this.data.current = Math.min(this.max, this.current + value);
             yield this.save();
-            if (!this.client)
-                return;
-            this.client.emit("background-progress", this.id);
         });
     }
     cancel() {
@@ -1788,6 +1825,12 @@ class Progress extends BaseEntity {
             yield this.save();
         });
     }
+    save() {
+        if (this.messageBridge) {
+            this.messageBridge.sendMessage(`progress-changed`, this.toJSON());
+        }
+        return super.save();
+    }
 }
 class SubProgress {
     constructor(parent, progressFrom, progressValue, mMax = 100) {
@@ -1827,6 +1870,12 @@ class SubProgress {
     get canceled() {
         return !this.parent || this.parent.canceled;
     }
+    setMessageBridge(messageBridge) {
+        if (!this.parent)
+            return this;
+        this.parent.setMessageBridge(messageBridge);
+        return this;
+    }
     createSubProgress(progressValue, max, message) {
         return __awaiter$n(this, void 0, void 0, function* () {
             if (max <= 0 && progressValue > 0) {
@@ -1915,33 +1964,34 @@ var __awaiter$m = (this && this.__awaiter) || function (thisArg, _arguments, P,
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const socketIOClient = socket_io_client;
 let Progresses = class Progresses {
-    constructor(connector, config) {
+    constructor(connector, jobMan) {
         this.connector = connector;
-        this.config = config;
-        const mainEndpoint = this.config.resolve("mainEndpoint");
-        this.client = !mainEndpoint ? null : socketIOClient(mainEndpoint, { path: "/socket" });
+        this.jobMan = jobMan;
         this.collection = connector.database.collection("progresses");
+        this.progresses = {};
+        this.jobMan.on("progress-changed", progress => {
+            const id = progress.id;
+            this.progresses[id] = new Progress(new ObjectId(id), progress, this.collection);
+        });
     }
     waitToFinish(id) {
         return __awaiter$m(this, void 0, void 0, function* () {
-            let isFinished = false;
-            let progress = null;
-            while (!isFinished) {
-                progress = yield this.get(id);
-                if (!progress) {
-                    throw `Progress does not exists with id: ${id}`;
-                }
-                if (progress.error) {
-                    throw progress.error;
-                }
-                isFinished = progress.percent == 100;
-                if (!isFinished) {
-                    yield promiseTimeout(50);
-                }
-            }
-            return progress;
+            return Promise.race([
+                this.waitForProgress(id, () => __awaiter$m(this, void 0, void 0, function* () {
+                    let progress = this.progresses[id];
+                    if (!progress || progress.percent < 100) {
+                        progress = yield this.get(id);
+                    }
+                    if (!progress) {
+                        throw new Error(`Progress does not exists with id: ${id}`);
+                    }
+                    return progress;
+                }), 500),
+                this.waitForProgress(id, () => __awaiter$m(this, void 0, void 0, function* () {
+                    return this.progresses[id] || null;
+                }), 25)
+            ]);
         });
     }
     get(id) {
@@ -1952,13 +2002,13 @@ let Progresses = class Progresses {
     find(where) {
         return __awaiter$m(this, void 0, void 0, function* () {
             const data = yield this.collection.findOne(where);
-            return !data ? null : new Progress(data._id, data, this.collection, this.client);
+            return !data ? null : new Progress(data._id, data, this.collection);
         });
     }
     create(max = 100) {
         return __awaiter$m(this, void 0, void 0, function* () {
             if (isNaN(max) || max <= 0) {
-                throw "Max progress value must be bigger than zero";
+                throw new Error(`Max progress value must be bigger than zero`);
             }
             const data = {
                 current: 0,
@@ -1968,7 +2018,7 @@ let Progresses = class Progresses {
                 canceled: false
             };
             const res = yield this.collection.insertOne(data);
-            return new Progress(res.insertedId, data, this.collection, this.client);
+            return new Progress(res.insertedId, data, this.collection);
         });
     }
     remove(id) {
@@ -1977,11 +2027,35 @@ let Progresses = class Progresses {
             return id;
         });
     }
+    waitForProgress(id, cb, delay) {
+        return __awaiter$m(this, void 0, void 0, function* () {
+            let isFinished = false;
+            let progress = null;
+            let waitTime = 0;
+            while (!isFinished) {
+                progress = yield cb();
+                waitTime += delay;
+                if (progress) {
+                    if (progress.error) {
+                        throw new Error(progress.error);
+                    }
+                    isFinished = progress.percent >= 100;
+                }
+                if (!isFinished) {
+                    if (waitTime >= MAX_TIMEOUT) {
+                        throw new Error(`Progress with id: ${id} probably never will be finished!`);
+                    }
+                    yield promiseTimeout(delay);
+                }
+            }
+            return progress;
+        });
+    }
 };
 Progresses = __decorate$r([
     injectable(),
     singleton(),
-    __metadata$l("design:paramtypes", [MongoConnector, Configuration])
+    __metadata$l("design:paramtypes", [MongoConnector, JobManager])
 ], Progresses);
 
 var __decorate$q = (this && this.__decorate) || function (decorators, target, key, desc) {
@@ -2022,7 +2096,7 @@ let LazyAssets = class LazyAssets {
             if (existingAsset)
                 return existingAsset;
             const res = yield this.collection.insertOne(data);
-            return new LazyAsset(res.insertedId, data, this.collection, this.assets, this.progresses, this.jobMan);
+            return new LazyAsset(res.insertedId, data, this.collection, this.assets, this.progresses);
         });
     }
     read(id) {
@@ -2035,7 +2109,7 @@ let LazyAssets = class LazyAssets {
             const data = yield this.collection.findOne(where);
             return !data
                 ? null
-                : new LazyAsset(data._id, data, this.collection, this.assets, this.progresses, this.jobMan);
+                : new LazyAsset(data._id, data, this.collection, this.assets, this.progresses);
         });
     }
     unlink(id) {
@@ -2050,8 +2124,10 @@ let LazyAssets = class LazyAssets {
 LazyAssets = __decorate$q([
     injectable(),
     scoped(Lifecycle.ContainerScoped),
-    __metadata$k("design:paramtypes", [MongoConnector, Assets,
-        Progresses, JobManager])
+    __metadata$k("design:paramtypes", [MongoConnector,
+        Assets,
+        Progresses,
+        JobManager])
 ], LazyAssets);
 
 var __decorate$p = (this && this.__decorate) || function (decorators, target, key, desc) {
@@ -3752,14 +3828,15 @@ class LazyAssetGenerator {
     get lazyAssets() {
         return this.assetResolver.lazyAssets;
     }
-    process() {
+    process(messaging) {
         return __awaiter$1(this, void 0, void 0, function* () {
             const lazyAsset = yield this.lazyAssets.read(this.lazyId);
             let progress = yield this.progresses.get(lazyAsset.progressId);
             if (!progress || progress.canceled)
                 return null;
+            progress.setMessageBridge(messaging);
             try {
-                const asset = yield this.generate(progress);
+                const asset = yield this.generate(progress, messaging);
                 progress = yield progress.load();
                 if (!progress || progress.canceled)
                     return null;
@@ -3826,7 +3903,8 @@ function createServices() {
         new Parameter("nodeEnv", "development"),
         new Parameter("appPort", 80),
         new Parameter("zmqPort", 3000),
-        new Parameter("zmqRemoteHost", "tcp://127.0.0.1:3000"),
+        new Parameter("zmqBackPort", 3100),
+        new Parameter("zmqRemoteHost", "tcp://127.0.0.1"),
         new Parameter("isWorker", false),
         new Parameter("mainEndpoint", ""),
         new Parameter("idChars", "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"),
@@ -4032,5 +4110,5 @@ function setupBackend(config, providers, parent) {
  * Generated bundle index. Do not edit.
  */
 
-export { AssetProcessor, AssetResolver, Assets, AuthController, BackendProvider, Cache, CacheProcessor, Configuration, ConsoleColor, DI_CONTAINER, EXPRESS, EndpointProvider, ErrorHandlerMiddleware, FIXTURE, Fixtures, Gallery, GalleryCache, GalleryController, HTTP_SERVER, IdGenerator, IsFile, IsObjectId, JOB, JobManager, LanguageMiddleware, LazyAssetGenerator, LazyAssets, MailSender, MemoryCache, MongoConnector, PARAMETER, Parameter, Progresses, ResolveEntity, SOCKET_SERVER, TemplateRenderer, TranslationProvider, Translator, Type, UserManager, assign, broadcast, bufferToStream, convertValue, copy, copyStream, createServices, createTransformer, deleteFile, deleteFromBucket, filter, firstItem, getConstructorName, getExtension, getFileName, getFunctionParams, getType, getValue, groupBy, hydratePopulated, idToString, injectServices, isArray, isBoolean, isConstructor, isDate, isDefined, isFunction, isInterface, isNullOrUndefined, isObject, isPrimitive, isString, isType, jsonHighlight, lastItem, lcFirst, lookupPipelines, md5, mkdirRecursive, multiSubscription, observableFromFunction, padLeft, padRight, paginate, paginateAggregations, promiseTimeout, proxyFunction, proxyFunctions, rand, random, readAndDeleteFile, readFile, runCommand, setupBackend, streamToBuffer, ucFirst, valueToPromise, writeFile };
+export { AssetProcessor, AssetResolver, Assets, AuthController, BackendProvider, Cache, CacheProcessor, Configuration, ConsoleColor, DI_CONTAINER, EXPRESS, EndpointProvider, ErrorHandlerMiddleware, FIXTURE, Fixtures, Gallery, GalleryCache, GalleryController, HTTP_SERVER, IdGenerator, IsFile, IsObjectId, JOB, JobManager, LanguageMiddleware, LazyAssetGenerator, LazyAssets, MailSender, MemoryCache, MongoConnector, PARAMETER, Parameter, Progresses, ResolveEntity, SOCKET_SERVER, TemplateRenderer, TranslationProvider, Translator, Type, UserManager, assign, broadcast, bufferToStream, colorize, convertValue, copy, copyStream, createServices, createTransformer, deleteFile, deleteFromBucket, filter, firstItem, getConstructorName, getExtension, getFileName, getFunctionParams, getType, getValue, groupBy, hydratePopulated, idToString, injectServices, isArray, isBoolean, isConstructor, isDate, isDefined, isFunction, isInterface, isNullOrUndefined, isObject, isPrimitive, isString, isType, jsonHighlight, lastItem, lcFirst, lookupPipelines, md5, mkdirRecursive, multiSubscription, observableFromFunction, padLeft, padRight, paginate, paginateAggregations, promiseTimeout, proxyFunction, proxyFunctions, rand, random, readAndDeleteFile, readFile, runCommand, setupBackend, streamToBuffer, ucFirst, valueToPromise, writeFile };
 //# sourceMappingURL=stemy-backend.js.map