@social-mail/social-mail-web-server 1.7.6 → 1.7.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.7.6",
+  "version": "1.7.7",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/common/spawnPromise.ts

@@ -31,10 +31,10 @@ export const spawnPromise = (path, args?: string[], options?: SpawnOptionsWithou
     });
     cd.on("close", (status) => {
         if (status>0) {
+            if (logError) {
+                console.error(all);
+            }
             if (throwOnFail) {
-                if (logError) {
-                    console.error(all);
-                }
                 reject(new Error(all.join("\n")));
                 return;
             }

package/src/isolated/FileInfo.ts

@@ -0,0 +1,13 @@
+import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
+
+export default class FileInfo {
+
+
+    public static from(file: LocalFile) {
+        return new FileInfo(file.path, file.contentType, file.fileName);
+    }
+
+    constructor(public readonly path: string, public readonly contentType: string, public readonly fileName: string) {
+
+    }
+}

package/src/isolated/IsolatedProcess.ts

@@ -0,0 +1,33 @@
+import { fileURLToPath } from "url";
+import { spawnPromise } from "../common/spawnPromise.js";
+import FileInfo from "./FileInfo.js";
+import IImageConverter from "./image-converter/IImageConverter.js";
+import IExtract from "./extract/IExtract.js";
+
+export default class IsolatedProcess {
+
+    static async convertImage(p: IImageConverter) {
+        return spawnPromise(process.execPath, [
+            "--enable-source-maps",
+            fileURLToPath(import.meta.resolve("./image-converter/index.js")),
+            JSON.stringify(p),
+        ], {
+            logCommand: false,
+            logError: true
+        });
+    }
+
+    static async extractText(p: IExtract) {
+        const result = await spawnPromise(process.execPath, [
+            "--enable-source-maps",
+            fileURLToPath(import.meta.resolve("./extract/index.js")),
+            JSON.stringify(p),
+        ], {
+            logCommand: false,
+            logError: true
+        });
+
+        return result.all;
+    }
+
+}

package/src/isolated/extract/IExtract.ts

@@ -0,0 +1,4 @@
+export default interface IExtract {
+    input: { contentType, fileName, path },
+    useOcr: boolean;
+}

package/src/isolated/extract/index.ts

@@ -0,0 +1,59 @@
+/* eslint-disable no-console */
+import { readFile } from "fs/promises";
+import tesseract from "tesseract.js";
+import FileInfo from "../FileInfo.js";
+import IExtract from "./IExtract.js";
+const { recognize } = tesseract;
+
+export class ExtractWorker {
+    static async text({ input: file, useOcr }: IExtract) {
+        try {
+            if (/^image\//.test(file.contentType)) {
+                if (useOcr) {
+                    return this.ocr(file);
+                }
+            }
+
+            return await readFile(file.path, "utf8");
+        } catch (error) {
+            console.error(error);
+        }
+        return "";
+
+    }
+
+    static async ocr(file: FileInfo) {
+        try {
+            const buffer = await readFile(file.path);
+            const result = await recognize(buffer);
+            return result.data.text;
+        } catch (error) {
+            console.error(error);
+        }
+        return "";
+    }
+};
+
+
+const [execPath, self, p] = process.argv;
+
+
+ExtractWorker.text(JSON.parse(p))
+    .then((r) => console.log(r))
+    .catch(console.error);
+
+
+// parentPort.on("message", async ({ file, useOcr }) => {
+//     try {
+//         const result = await ExtractWorker.text(file, useOcr);
+//         parentPort.postMessage({ result });
+//         process.exit(0);
+//     } catch (error) {
+//         try {
+//             parentPort.postMessage({ error: error.stack ?? error.toString() });
+//         } catch (er1) {
+//             // we will ignore this..  we need to close process successfully.
+//         }
+//         process.exit(0);
+//     }
+// });

package/src/isolated/image-converter/IImageConverter.ts

@@ -0,0 +1,6 @@
+export default interface IImageConverter {
+    type: string;
+    input: { path: string; },
+    output: { path: string;},
+    outputFileName: string;
+}

package/src/{server/services → isolated}/image-converter/formats/size.ts

@@ -1,8 +1,7 @@
 /* eslint-disable no-bitwise */
-import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import sharp from "sharp";
 
-export default async function (file: LocalFile, ... args: any[] ): Promise<sharp.Sharp> {
+export default async function (file: { path: string }, ... args: any[] ): Promise<sharp.Sharp> {
 
     const size = {
         failOnError: false,

package/src/isolated/image-converter/index.ts

@@ -0,0 +1,73 @@
+import sharp from "sharp";
+import { parse } from "path";
+import IImageConverter from "./IImageConverter.js";
+
+async function transform({
+        type,
+        input,
+        output,
+        outputFileName = "file.jpg"
+    }: IImageConverter
+) {
+
+    let { ext } = parse(outputFileName);
+    let fileName = type;
+    const matches = type.match(/(\w+)\(([^\)]+)\)/);
+    let args = [];
+    if (matches) {
+        fileName = matches[1];
+        args = matches[2].split(',').map( (x) => parseFloat(x) );
+        switch(fileName) {
+            case "jpg":
+                ext = ".jpg";
+                fileName = "size";
+                break;
+            case "png":
+                ext = ".png";
+                fileName = "size";
+                break;
+            case "gif":
+                ext = ".gif";
+                fileName = "size";
+                break;
+            case "webp":
+                ext = ".webp";
+                fileName = "size";
+                break;
+            case "ico":
+                ext = ".ico";
+                fileName = "size";
+                break;
+        }
+    }
+
+    const { default: fx } = await import("./formats/" + fileName + ".js");
+    let r: sharp.Sharp = await fx(input, ... args);
+
+    switch(ext) {
+        case ".jpg":
+            r = r.jpeg({ quality: 95});
+            break;
+        case ".png":
+            r = r.png({
+                compressionLevel: 9
+            });
+            break;
+        case ".webp":
+            r = r.webp({
+                nearLossless: true,
+                quality: 95
+            });
+            break;
+        case ".gif":
+            r = r.gif();
+            break;
+    }
+
+    await r.toFile(output.path);
+}
+
+const [execPath, self, p] = process.argv;
+
+transform(JSON.parse(p))
+    .catch(console.error);

package/src/server/services/convert/FileConversionService.ts

@@ -2,7 +2,6 @@ import Inject, { RegisterScoped, ServiceProvider } from "@entity-access/entity-a
 import TempFileService from "../../storage/TempFileService.js";
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import { Convert } from "./Convert.js";
-import PdfDoc from "../extract/pdf/PdfDoc.js";
 import FFCommand from "./FFCommand.js";
 import AppFile from "../../model/entities/AppFile.js";
 import TimeSpan from "@entity-access/entity-access/dist/types/TimeSpan.js";
@@ -13,6 +12,7 @@ import Page from "@entity-access/server-pages/dist/Page.js";
 import StringFormat from "../../../common/StringFormat.js";
 import RateLimiterService from "../rate-limiter/RateLimiterService.js";
 import { globalEnv } from "../../../common/globalEnv.js";
+import PdfDoc from "../extract/pdf/PdfDoc.js";
 
 interface IDownloadConvertedFile {
     appFile: AppFile;

package/src/server/services/extract/Extract.ts

@@ -6,10 +6,11 @@ import FileInfo from "./FileInfo.js";
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import TempFolder from "@entity-access/server-pages/dist/core/TempFolder.js";
 import { FileType } from "../../../common/FileType.js";
-import PdfDoc from "./pdf/PdfDoc.js";
 import { readFile } from "fs/promises";
 import { Convert } from "../convert/Convert.js";
 import LockFile from "../../storage/LockFile.js";
+import PdfDoc from "./pdf/PdfDoc.js";
+import IsolatedProcess from "../../../isolated/IsolatedProcess.js";
 
 export class Extract {
     static async text(src: LocalFile, useOcr = false) {
@@ -44,52 +45,6 @@ export class Extract {
 
         using lock = await LockFile.lock("extract-worker-lock");
 
-        return await new Promise<string>((resolve, reject) => {
-            let resolved = false;
-            console.log(`Extracting text from ${file.path}`);
-            const execArgv = ["--enable-source-maps"];
-            if (process.argv.includes("--trace-deprecation")) {
-                execArgv.push("--trace-deprecation");
-            }
-            const w = new Worker( dirname(fileURLToPath(import.meta.url)) +  "/ExtractWorker.js",{
-                execArgv,
-                resourceLimits: {
-                    maxOldGenerationSizeMb: 1024
-                }
-            });
-            w.on("message", ({ result, error }) => {
-                if (resolved) {
-                    return;
-                }
-                if (result) {
-                    resolved = true;
-                    resolve(result);
-                    return;
-                }
-                if (error) {
-                    reject(error);
-                }
-            });
-            w.on("error", (error) => {
-                if(resolved) {
-                    return;
-                }
-                console.error(error);
-                resolved = true;
-                // reject(error);
-                console.error(`Failed to extract text for ${src.fileName} ${error.stack ?? error}`);
-                resolve("");
-            });
-            w.on("exit", (n) => {
-                if (n>0) {
-                    if (resolved) {
-                        return;
-                    }
-                    resolved = true;
-                    reject(new Error(`Worker exited with ${n}`));
-                }
-            });
-            w.postMessage({ file: { ... FileInfo.from(file)}, useOcr });
-        });
+        return await IsolatedProcess.extractText({ input: { fileName: file.fileName, contentType: file.contentType, path: file.path }, useOcr });
     }
 };

package/src/server/services/image-converter/ImageConverterService.ts

@@ -1,6 +1,5 @@
 import Inject, { RegisterScoped } from "@entity-access/entity-access/dist/di/di.js";
 import TempFileService from "../../storage/TempFileService.js";
-import sharp from "sharp";
 import { join, parse } from "path";
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import { Convert } from "../convert/Convert.js";
@@ -9,6 +8,7 @@ import { spawnPromise } from "../../../common/spawnPromise.js";
 import { copyFile, readdir } from "fs/promises";
 import FFCommand from "../convert/FFCommand.js";
 import { ImagesFolder } from "../../../common/ImagesFolder.js";
+import IsolatedProcess from "../../../isolated/IsolatedProcess.js";
 
 @RegisterScoped
 export default class ImageConverterService {
@@ -92,6 +92,7 @@ export default class ImageConverterService {
     async transform(type, file: LocalFile, outputFileName = "file.jpg") {
 
         let { ext } = parse(outputFileName);
+
         let fileName = type;
         const matches = type.match(/(\w+)\(([^\)]+)\)/);
         let args = [];
@@ -127,31 +128,15 @@ export default class ImageConverterService {
             file = ImagesFolder.localFile("png", "unavailable.png");
         }
 
-        const { default: fx } = await import("./formats/" + fileName + ".js");
-        let r: sharp.Sharp = await fx(file, ... args);
-
-        switch(ext) {
-            case ".jpg":
-                r = r.jpeg({ quality: 95});
-                break;
-            case ".png":
-                r = r.png({
-                    compressionLevel: 9
-                });
-                break;
-            case ".webp":
-                r = r.webp({
-                    nearLossless: true,
-                    quality: 95
-                });
-                break;
-            case ".gif":
-                r = r.gif();
-                break;
-        }
-
         const tf = await this.tempFileService.createTempFile(ext, outputFileName);
-        await r.toFile(tf.path);
+
+        await IsolatedProcess.convertImage({
+            type,
+            input: { path: file.path },
+            output: { path: tf.path },
+            outputFileName
+        });
+
         return tf;
     }
 

package/dist/server/services/image-converter/formats/size.d.ts

@@ -1,4 +0,0 @@
-import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
-import sharp from "sharp";
-export default function (file: LocalFile, ...args: any[]): Promise<sharp.Sharp>;
-//# sourceMappingURL=size.d.ts.map

package/dist/server/services/image-converter/formats/size.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"size.d.ts","sourceRoot":"","sources":["../../../../../src/server/services/image-converter/formats/size.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;AAC/E,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,yBAA+B,IAAI,EAAE,SAAS,EAAE,GAAI,IAAI,EAAE,GAAG,EAAE,GAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAmCtF"}

package/dist/server/services/image-converter/formats/size.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"size.js","sourceRoot":"","sources":["../../../../../src/server/services/image-converter/formats/size.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,CAAC,OAAO,CAAC,KAAK,WAAW,IAAe,EAAE,GAAI,IAAW;IAE3D,MAAM,IAAI,GAAG;QACT,WAAW,EAAE,KAAK;KACd,CAAC;IAET,IAAI,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;QAExB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,uCAAuC;YACvC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;YAC7B,uCAAuC;YACvC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;YACrB,uCAAuC;YACvC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;YACvB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC;QACvB,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,MAAO,KAAK,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,EAAC,CAAS,CAAC;QACrE,CAAC;QACD,OAAO,MAAO,KAAK,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,EAAC,CAAS;aAC/D,MAAM,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC"}