@stemy/backend 6.0.2 → 6.0.4

package/fesm2022/stemy-backend.mjs

@@ -39,6 +39,7 @@ import { compare } from 'bcrypt';
 import moment from 'moment';
 import { GridFSBucket } from 'mongodb/lib/gridfs';
 import { writeFile as writeFile$2, rm } from 'fs/promises';
+import got from 'got';
 import { getModelForClass } from '@typegoose/typegoose';
 import { getValue as getValue$1, setValue } from 'mongoose/lib/utils';
 
@@ -57,6 +58,8 @@ const DI_CONTAINER = Symbol.for("di-container-token");
 const OPENAPI_VALIDATION = Symbol.for("openapi-validation-token");
 const LOCAL_DIR = Symbol.for('asset-local-dir');
 const ASSET_DRIVER = Symbol.for('assets-driver');
+const ASSET_MAIN_DRIVER = Symbol.for('assets-main-driver');
+const ASSET_FALLBACK_DRIVER = Symbol.for('assets-fallback-driver');
 class Parameter {
     name;
     defaultValue;
@@ -1009,17 +1012,11 @@ class Asset extends BaseEntity {
     async unlink() {
         try {
             await this.driver.delete(this.oid);
-            await this.collection.deleteOne({ _id: this.oid });
         }
         catch (error) {
-            let err = error;
-            if (error) {
-                err = error.message || error || "";
-                if (!isString(err) || !err.startsWith("FileNotFound")) {
-                    throw err;
-                }
-            }
+            console.log("Failed to unlink", error?.message);
         }
+        await this.collection.deleteOne({ _id: this.oid });
         return this.id;
     }
     async setMeta(metadata) {
@@ -1355,7 +1352,6 @@ let JobManager = class JobManager {
     jobTypes;
     jobs;
     messages;
-    processing;
     apiPush;
     apiPull;
     workerPush;
@@ -1373,7 +1369,9 @@ let JobManager = class JobManager {
                 const messageBridge = {
                     sendMessage: (message, params) => {
                         params.uniqueId = uniqueId;
-                        this.workerPush.send([message, JSON.stringify(params)]);
+                        this.workerPush.then(sock => {
+                            sock.send([message, JSON.stringify(params)]);
+                        });
                     }
                 };
                 messageBridge.sendMessage(`job-started`, { name: jobName });
@@ -1382,7 +1380,6 @@ let JobManager = class JobManager {
             return res;
         }, {});
         this.messages = new Subject();
-        this.processing = null;
         this.maxTimeout = this.config.resolve("jobTimeout");
     }
     on(message, cb) {
@@ -1429,40 +1426,46 @@ let JobManager = class JobManager {
             });
         });
     }
-    async initProcessing() {
+    async startProcessing() {
         const host = this.config.resolve("zmqRemoteHost");
         const pushHost = `${host}:${this.config.resolve("zmqBackPort")}`;
-        this.workerPush = socket("push");
-        this.workerPush.connect(pushHost);
-        this.logger.log("job-manager", `Worker producer connected to: ${pushHost}`);
         const pullHost = `${host}:${this.config.resolve("zmqPort")}`;
-        this.workerPull = socket("pull");
-        this.workerPull.connect(pullHost);
-        this.logger.log("job-manager", `Worker consumer connected to: ${pullHost}`);
-        this.workerPull.on("message", async (name, args, uniqId) => {
-            try {
-                const jobName = name.toString("utf8");
-                const jobParams = JSON.parse(args.toString("utf8"));
-                const uniqueId = uniqId?.toString("utf8");
-                console.time(uniqueId);
-                console.timeLog(uniqueId, `Started working on background job: ${colorize(jobName, ConsoleColor.FgCyan)} with args: \n${jsonHighlight(jobParams)}\n\n`);
+        this.workerPush = this.workerPush || new Promise(resolve => {
+            const sock = socket("push");
+            sock.connect(pushHost);
+            this.logger.log("job-manager", `Worker producer connected to: ${pushHost}`);
+            resolve(sock);
+        });
+        this.workerPull = this.workerPull || new Promise(resolve => {
+            const sock = socket("pull");
+            sock.connect(pullHost);
+            sock.on("message", async (name, args, uniqId) => {
                 try {
-                    await Promise.race([this.jobs[jobName](jobParams, uniqueId), promiseTimeout(this.maxTimeout, true)]);
-                    console.timeLog(uniqueId, `Finished working on background job: ${colorize(jobName, ConsoleColor.FgCyan)}\n\n`);
+                    const jobName = name.toString("utf8");
+                    const jobParams = JSON.parse(args.toString("utf8"));
+                    const uniqueId = uniqId?.toString("utf8");
+                    console.time(uniqueId);
+                    console.timeLog(uniqueId, `Started working on background job: ${colorize(jobName, ConsoleColor.FgCyan)} with args: \n${jsonHighlight(jobParams)}\n\n`);
+                    try {
+                        await Promise.race([this.jobs[jobName](jobParams, uniqueId), promiseTimeout(this.maxTimeout, true)]);
+                        console.timeLog(uniqueId, `Finished working on background job: ${colorize(jobName, ConsoleColor.FgCyan)}\n\n`);
+                    }
+                    catch (e) {
+                        console.timeLog(uniqueId, `Background job failed: ${colorize(jobName, ConsoleColor.FgRed)}\n${e}\n\n`);
+                    }
+                    console.timeEnd(uniqueId);
                 }
                 catch (e) {
-                    console.timeLog(uniqueId, `Background job failed: ${colorize(jobName, ConsoleColor.FgRed)}\n${e}\n\n`);
+                    this.logger.log("job-manager", `Failed to start job: ${e.message}`);
                 }
-                console.timeEnd(uniqueId);
-            }
-            catch (e) {
-                this.logger.log("job-manager", `Failed to start job: ${e.message}`);
-            }
+            });
+            this.logger.log("job-manager", `Worker consumer connected to: ${pullHost}`);
+            resolve(sock);
         });
-    }
-    startProcessing() {
-        this.processing = this.processing || this.initProcessing();
-        return this.processing;
+        await Promise.allSettled([
+            this.workerPush,
+            this.workerPull,
+        ]);
     }
     tryResolve(jobType, params) {
         const jobName = getConstructorName(jobType);
@@ -1487,28 +1490,32 @@ let JobManager = class JobManager {
         return this.tryResolveAndInit(jobType, params);
     }
     tryResolveAndInit(jobType, params) {
-        if (!this.apiPush) {
+        this.apiPush = this.apiPush || new Promise(resolve => {
             const port = this.config.resolve("zmqPort");
-            this.apiPush = socket("push");
-            this.apiPush.bind(`tcp://0.0.0.0:${port}`);
-            this.logger.log("job-manager", `API producer bound to port: ${port}`);
-        }
-        if (!this.apiPull) {
+            const sock = socket("push");
+            sock.bind(`tcp://0.0.0.0:${port}`, () => {
+                this.logger.log("job-manager", `API producer bound to port: ${port}`);
+                resolve(sock);
+            });
+        });
+        this.apiPull = this.apiPush || new Promise(resolve => {
             const backPort = this.config.resolve("zmqBackPort");
-            this.apiPull = socket("pull");
-            this.apiPull.bind(`tcp://0.0.0.0:${backPort}`);
-            this.apiPull.on("message", (name, args) => {
-                const message = name.toString("utf8");
-                const params = JSON.parse(args?.toString("utf8") || "{}");
-                const paramTypes = Object.keys(params).reduce((res, key) => {
-                    res[key] = getType(params[key]);
-                    return res;
-                }, {});
-                this.logger.log("job-manager", `Received a message from worker: "${colorize(message, ConsoleColor.FgCyan)}" with args: ${jsonHighlight(paramTypes)}\n\n`);
-                this.messages.next({ message, params });
+            const sock = socket("pull");
+            sock.bind(`tcp://0.0.0.0:${backPort}`, () => {
+                sock.on("message", (name, args) => {
+                    const message = name.toString("utf8");
+                    const params = JSON.parse(args?.toString("utf8") || "{}");
+                    const paramTypes = Object.keys(params).reduce((res, key) => {
+                        res[key] = getType(params[key]);
+                        return res;
+                    }, {});
+                    this.logger.log("job-manager", `Received a message from worker: "${colorize(message, ConsoleColor.FgCyan)}" with args: ${jsonHighlight(paramTypes)}\n\n`);
+                    this.messages.next({ message, params });
+                });
+                this.logger.log("job-manager", `API consumer bound to port: ${backPort}`);
+                resolve(sock);
             });
-            this.logger.log("job-manager", `API consumer bound to port: ${backPort}`);
-        }
+        });
         return this.tryResolve(jobType, params);
     }
     resolveJobInstance(jobType, params, uniqueId = "") {
@@ -1522,7 +1529,8 @@ let JobManager = class JobManager {
     }
     async sendToWorkers(jobName, params) {
         const uniqueId = new ObjectId$1().toHexString();
-        this.apiPush.send([jobName, JSON.stringify(params), uniqueId]);
+        const sock = await this.apiPush;
+        sock.send([jobName, JSON.stringify(params), uniqueId]);
         return uniqueId;
     }
 };
@@ -4007,11 +4015,9 @@ const fixtures = [
 ];
 
 let AssetGridDriver = class AssetGridDriver {
-    metaCollection;
     bucket;
     constructor(connector) {
         this.bucket = new GridFSBucket(connector.database, { bucketName: 'assets' });
-        this.metaCollection = "assets.files";
     }
     openUploadStream(filename, opts) {
         return this.bucket.openUploadStream(filename, opts);
@@ -4030,10 +4036,8 @@ AssetGridDriver = __decorate([
 
 let AssetLocalDriver = class AssetLocalDriver {
     dir;
-    metaCollection;
     constructor(dir) {
         this.dir = dir;
-        this.metaCollection = "assets.local";
     }
     openUploadStream(filename, opts) {
         const id = new Types.ObjectId();
@@ -4062,6 +4066,144 @@ AssetLocalDriver = __decorate([
     __metadata("design:paramtypes", [String])
 ], AssetLocalDriver);
 
+function createFallbackReadable(primary, fallbackFactory) {
+    const proxy = new PassThrough();
+    let hasSwitched = false;
+    const switchToFallback = () => {
+        if (hasSwitched)
+            return;
+        hasSwitched = true;
+        // 1. Cleanup the failing primary
+        primary.unpipe(proxy);
+        primary.destroy();
+        // 2. Initialize and pipe the fallback
+        const fallback = fallbackFactory();
+        // On the fallback, we allow it to end the proxy naturally
+        fallback.pipe(proxy);
+        fallback.on('error', (err) => {
+            proxy.emit('error', err); // If fallback fails too, it's a hard error
+        });
+    };
+    // Pipe primary but prevent it from closing the proxy automatically
+    primary.pipe(proxy, { end: false });
+    primary.on('error', () => {
+        switchToFallback();
+    });
+    // Handle the "Early End" case (e.g., stream closed before data finished)
+    primary.on('end', () => {
+        if (!hasSwitched) {
+            // If we finished successfully without erroring, close the proxy
+            proxy.end();
+        }
+    });
+    return proxy;
+}
+/**
+ * Creates a Writable proxy that switches to a fallback destination on error.
+ * @param primary The initial Writable destination.
+ * @param fallbackFactory A function that returns a new Writable destination.
+ */
+function createFallbackWritable(primary, fallbackFactory) {
+    // 1. The Proxy acts as the stable "entry point" for your data
+    const proxy = new PassThrough();
+    let currentDestination = primary;
+    let hasSwitched = false;
+    // Function to handle the swap
+    const handleSwitch = (err) => {
+        if (hasSwitched)
+            return; // Prevent infinite loops if fallback also fails
+        hasSwitched = true;
+        // 2. Unpipe from the failed stream and destroy it
+        proxy.unpipe(currentDestination);
+        currentDestination.destroy();
+        // 3. Create the new destination and pipe the remaining data to it
+        const fallback = fallbackFactory();
+        currentDestination = fallback;
+        // Pipe the proxy into the fallback
+        proxy.pipe(fallback);
+        // Ensure errors on the fallback are bubbled up
+        fallback.on('error', (fallbackErr) => {
+            proxy.emit('error', fallbackErr);
+        });
+    };
+    // Initial setup
+    primary.on('error', handleSwitch);
+    proxy.on('finish', () => {
+        const final = proxy;
+        final.id = currentDestination.id;
+        final.done = true;
+    });
+    proxy.pipe(primary);
+    return proxy;
+}
+
+let AssetFallbackDriver = class AssetFallbackDriver {
+    main;
+    fallback;
+    constructor(main, fallback) {
+        this.main = main;
+        this.fallback = fallback;
+    }
+    openDownloadStream(id) {
+        return createFallbackReadable(this.main.openDownloadStream(id), () => this.fallback.openDownloadStream(id));
+    }
+    openUploadStream(filename, opts) {
+        return createFallbackWritable(this.main.openUploadStream(filename, opts), () => this.fallback.openUploadStream(filename, opts));
+    }
+    async delete(id) {
+        try {
+            await this.main.delete(id);
+        }
+        catch (e) {
+            console.log("Failed to delete from primary: ", e.message);
+            await this.fallback.delete(id);
+        }
+    }
+};
+AssetFallbackDriver = __decorate([
+    injectable(),
+    __param(0, inject(ASSET_MAIN_DRIVER)),
+    __param(1, inject(ASSET_FALLBACK_DRIVER)),
+    __metadata("design:paramtypes", [Object, Object])
+], AssetFallbackDriver);
+
+let AssetStorageProxyDriver = class AssetStorageProxyDriver {
+    config;
+    baseUrl;
+    url;
+    constructor(config) {
+        this.config = config;
+        this.baseUrl = this.config.resolve("storageProxyUri");
+        this.url = this.baseUrl + this.config.resolve("storageProxyBucket");
+    }
+    openDownloadStream(id) {
+        return got.stream.get(this.getUrl(id));
+    }
+    openUploadStream(_, opts) {
+        const id = new ObjectId$1();
+        const stream = got.stream.put(this.getUrl(id), { headers: {
+                'Content-Type': opts?.contentType || 'application/octet-stream'
+            } });
+        stream.done = false;
+        stream.on('finish', () => {
+            stream.id = id;
+            stream.done = true;
+        });
+        return stream;
+    }
+    async delete(id) {
+        console.log("DELETE", id.toHexString());
+        await got.delete(this.getUrl(id));
+    }
+    getUrl(id) {
+        return `${this.url}/${id.toHexString()}`;
+    }
+};
+AssetStorageProxyDriver = __decorate([
+    injectable(),
+    __metadata("design:paramtypes", [Configuration])
+], AssetStorageProxyDriver);
+
 class BaseDoc {
     _id;
     /**
@@ -4439,6 +4581,8 @@ function createServices() {
         new Parameter("mongoDb", "node-backend"),
         new Parameter("mongoUser", null),
         new Parameter("mongoPassword", null),
+        new Parameter("storageProxyUri", "http://localhost:4500/", prepareUrl),
+        new Parameter("storageProxyBucket", "something"),
         new Parameter("nodeEnv", "production"),
         new Parameter("appPort", 80),
         new Parameter("zmqPort", 3000),
@@ -4612,7 +4756,13 @@ async function setupBackend(config, providers, parent) {
         useValue: config.assetLocalDir || "assets_files"
     });
     diContainer.register(ASSET_DRIVER, {
-        useClass: config.assetDriver || AssetGridDriver
+        useClass: config.assetDriver || AssetFallbackDriver
+    });
+    diContainer.register(ASSET_MAIN_DRIVER, {
+        useClass: config.assetMainDriver || AssetGridDriver
+    });
+    diContainer.register(ASSET_FALLBACK_DRIVER, {
+        useClass: config.assetFallbackDriver || AssetLocalDriver
     });
     diContainers.appContainer = diContainers.appContainer || diContainer;
     // Authentication
@@ -4683,5 +4833,5 @@ async function setupBackend(config, providers, parent) {
  * Generated bundle index. Do not edit.
  */
 
-export { AssetGridDriver, AssetImageParams, AssetLocalDriver, AssetProcessor, AssetResolver, Assets, AuthController, BackendProvider, BaseDoc, Cache, CacheProcessor, Configuration, ConsoleColor, DI_CONTAINER, DocumentArray, EXPRESS, EndpointProvider, ErrorHandlerMiddleware, FIXTURE, Fixtures, Gallery, GalleryCache, GalleryController, HTTP_SERVER, IdGenerator, IsDocumented, IsFile, IsObjectId, JOB, JobManager, JsonResponse, LanguageMiddleware, LazyAssetGenerator, LazyAssets, Logger, MailSender, MemoryCache, MongoConnector, OPENAPI_VALIDATION, OpenApi, PARAMETER, Parameter, PrimitiveArray, Progresses, ResolveEntity, ResponseType, SOCKET_CONTROLLERS, SOCKET_SERVER, TERMINAL_COMMAND, TemplateRenderer, TerminalManager, TokenGenerator, TranslationProvider, Translator, Type, UserManager, assign, broadcast, bufferToStream, camelCaseToDash, colorize, convertValue, copy, copyStream, createIdString, createServices, createTransformer, deleteFile, deleteFromBucket, fileTypeFromBuffer, filter, firstItem, flatten, getConstructorName, getDirName, getExtension, getFileName, getFunctionParams, getType, getValue, groupBy, gunzipPromised, gzipPromised, hydratePopulated, idToString, injectServices, isArray, isBoolean, isBuffer, isConstructor, isDate, isDefined, isFunction, isInterface, isNullOrUndefined, isObject, isObjectId, isPrimitive, isString, isType, jsonHighlight, lastItem, lcFirst, letsLookupStage, lookupStages, matchField, matchFieldStages, matchStage, md5, mkdirRecursive, multiSubscription, observableFromFunction, padLeft, padRight, paginate, paginateAggregations, prepareUrl, prepareUrlEmpty, prepareUrlSlash, projectStage, promiseTimeout, rand, random, readAndDeleteFile, readFile, regexEscape, regroup, replaceSpecialChars, resolveUser, runCommand, service, setupBackend, streamToBuffer, toImage, ucFirst, uniqueItems, unwindStage, valueToPromise, wrapError, writeFile };
+export { AssetGridDriver, AssetImageParams, AssetLocalDriver, AssetProcessor, AssetResolver, AssetStorageProxyDriver, Assets, AuthController, BackendProvider, BaseDoc, Cache, CacheProcessor, Configuration, ConsoleColor, DI_CONTAINER, DocumentArray, EXPRESS, EndpointProvider, ErrorHandlerMiddleware, FIXTURE, Fixtures, Gallery, GalleryCache, GalleryController, HTTP_SERVER, IdGenerator, IsDocumented, IsFile, IsObjectId, JOB, JobManager, JsonResponse, LanguageMiddleware, LazyAssetGenerator, LazyAssets, Logger, MailSender, MemoryCache, MongoConnector, OPENAPI_VALIDATION, OpenApi, PARAMETER, Parameter, PrimitiveArray, Progresses, ResolveEntity, ResponseType, SOCKET_CONTROLLERS, SOCKET_SERVER, TERMINAL_COMMAND, TemplateRenderer, TerminalManager, TokenGenerator, TranslationProvider, Translator, Type, UserManager, assign, broadcast, bufferToStream, camelCaseToDash, colorize, convertValue, copy, copyStream, createIdString, createServices, createTransformer, deleteFile, deleteFromBucket, fileTypeFromBuffer, filter, firstItem, flatten, getConstructorName, getDirName, getExtension, getFileName, getFunctionParams, getType, getValue, groupBy, gunzipPromised, gzipPromised, hydratePopulated, idToString, injectServices, isArray, isBoolean, isBuffer, isConstructor, isDate, isDefined, isFunction, isInterface, isNullOrUndefined, isObject, isObjectId, isPrimitive, isString, isType, jsonHighlight, lastItem, lcFirst, letsLookupStage, lookupStages, matchField, matchFieldStages, matchStage, md5, mkdirRecursive, multiSubscription, observableFromFunction, padLeft, padRight, paginate, paginateAggregations, prepareUrl, prepareUrlEmpty, prepareUrlSlash, projectStage, promiseTimeout, rand, random, readAndDeleteFile, readFile, regexEscape, regroup, replaceSpecialChars, resolveUser, runCommand, service, setupBackend, streamToBuffer, toImage, ucFirst, uniqueItems, unwindStage, valueToPromise, wrapError, writeFile };
 //# sourceMappingURL=stemy-backend.mjs.map