@social-mail/social-mail-web-server 1.8.39 → 1.8.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.8.39",
+  "version": "1.8.41",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/common/BufferHelper.ts

@@ -0,0 +1,18 @@
+export default class BufferHelper {
+
+
+    static replace00(b: Buffer, replace: Buffer) {
+        let start = 0;
+        for(;;) {
+            const i = b.indexOf(0x0, start);
+            if (i === -1) {
+                break;
+            }
+            b.set(replace, i);
+            start = i + 1;
+        }
+        return b;
+    }
+
+
+}

package/src/isolated/extract/IExtract.ts

@@ -1,4 +1,7 @@
 export default interface IExtract {
     input: { contentType, fileName, path },
+    output: {
+        path: string,
+    },
     useOcr: boolean;
 }

package/src/isolated/extract/index.ts

@@ -1,36 +1,30 @@
 /* eslint-disable no-console */
-import { readFile } from "fs/promises";
+import { copyFile, readFile, writeFile } from "fs/promises";
 import tesseract from "tesseract.js";
 import FileInfo from "../FileInfo.js";
 import IExtract from "./IExtract.js";
 const { recognize } = tesseract;
 
 export class ExtractWorker {
-    static async text({ input: file, useOcr }: IExtract) {
+    static async text({ input: file, useOcr, output }: IExtract) {
         try {
             if (/^image\//.test(file.contentType)) {
                 if (useOcr) {
-                    return this.ocr(file);
+                    return this.ocr(file, output);
                 }
             }
 
-            return await readFile(file.path, "utf-8");
+            await copyFile(file.path, output.path);
+            return;
         } catch (error) {
             console.error(error);
         }
-        return "";
-
     }
 
-    static async ocr(file: FileInfo) {
-        try {
-            const buffer = await readFile(file.path);
-            const result = await recognize(buffer);
-            return result.data.text;
-        } catch (error) {
-            console.error(error);
-        }
-        return "";
+    static async ocr(file: FileInfo, output: { path: string}) {
+        const buffer = await readFile(file.path);
+        const result = await recognize(buffer);
+        await writeFile(output.path, result.data.text, "utf-8");
     }
 };
 

package/src/server/seed/ui/seed-ui.ts

@@ -5,7 +5,7 @@ export default async function seedUI(config: DBConfig) {
     await config.saveVersion(UIPackageConfig, {
         package: "@social-mail/social-mail-client",
         view: "dist/web/AppIndex",
-        version: "1.8.215"
+        version: "1.8.216"
     });
 
     await config.saveVersion(WebComponentsPackageConfig, {

package/src/server/services/convert/Convert.ts

@@ -40,10 +40,16 @@ export const Convert = {
             if (args[1] === "txt") {
                 // txt hangs, so lets use to html and convert
                 args[1] = "html";
+
                 await this.execute(file, args, /\.html$/, output);
-                const htmlText = await readFile(output, "utf-8");
-                const { window } = new JSDOM(htmlText);
-                const text = window.document.documentElement.innerText ?? window.document.body.innerText ?? htmlText;
+
+                let text = "";
+                if (file.contentSize < 10*1024*1024) {
+
+                    const htmlText = await readFile(output, "utf-8");
+                    const { window } = new JSDOM(htmlText);
+                    text = window.document.documentElement.innerText ?? window.document.body.innerText ?? htmlText;
+                }
                 await writeFile(output, text);
                 return output;
             }

package/src/server/services/extract/Extract.ts

@@ -1,25 +1,20 @@
 /* eslint-disable no-console */
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import { FileType } from "../../../common/FileType.js";
-import { readFile } from "fs/promises";
+import { copyFile, readFile } from "fs/promises";
 import { Convert } from "../convert/Convert.js";
 import LockFile from "../../storage/LockFile.js";
 import PdfDoc from "./pdf/PdfDoc.js";
 import IsolatedProcess from "../../../isolated/IsolatedProcess.js";
 import { tempDiskCache } from "../../storage/tempDiskCache.js";
-import { AppStringHelper } from "../../../common/AppStringHelper.js";
 
 export class Extract {
 
-    static async text(src: LocalFile, useOcr = false) {
-        const text = await this.internalText(src, useOcr);
-        if (text) {
-            return AppStringHelper.remove00(text);
-        }
-        return text;
+    static async text(src: LocalFile, outputFile: LocalFile, useOcr = false) {
+        await this.internalText(src, outputFile, useOcr);
     }
 
-    private static async internalText(src: LocalFile, useOcr = false) {
+    private static async internalText(src: LocalFile, outputFile: LocalFile, useOcr = false) {
 
         if (FileType.isMedia(src)) {
             // currently we are not going to support
@@ -28,7 +23,8 @@ export class Extract {
             // in future might provide some sort of
             // caption extraction or title information
             // extraction
-            return "";
+            await outputFile.writeAllText("");
+            return;
         }
 
         using folder = tempDiskCache.newFolder("to-text-"+ Date.now());
@@ -36,11 +32,13 @@ export class Extract {
         await src.copyTo(file);
 
         if (FileType.isPdf(file)) {
-            return await PdfDoc.extract(file);
+            await PdfDoc.extract(file, outputFile);
+            return;
         }
 
         if (FileType.isCode(file)) {
-            return await readFile(file.path, "utf-8");
+            await copyFile(file.path, outputFile.path);
+            return;
         }
 
         if (!FileType.isImage(file)) {
@@ -49,18 +47,22 @@ export class Extract {
                 return "";
             }
 
-
-            const output = folder.get(file.fileName + ".txt", "text/plain");
-            await Convert.convert(file, "text", output.path);
-            const { contentSize } = output;
-            if (contentSize > 10*1024*1024) {
-                throw new Error(`File size too big ${contentSize}`);
-            }
-            return await output.readAsText();
+            await Convert.convert(file, "text", outputFile.path);
+            return;
         }
 
-        using lock = await LockFile.lock("extract-worker-lock");
+        using _lock = await LockFile.lock("extract-worker-lock");
 
-        return await IsolatedProcess.extractText({ input: { fileName: file.fileName, contentType: file.contentType, path: file.path }, useOcr });
+        return await IsolatedProcess.extractText({
+            input: {
+                fileName: file.fileName,
+                contentType: file.contentType,
+                path: file.path
+            },
+            output: {
+                path: outputFile.path
+            },
+            useOcr
+        });
     }
 };

package/src/server/services/extract/pdf/PdfDoc.ts

@@ -7,15 +7,15 @@ import { tempDiskCache } from "../../../storage/tempDiskCache.js";
 
 export default class PdfDoc {
 
-    static async extract(file: LocalFile) {
-        const output = await spawnPromise("pdftotext", [
+    static async extract(file: LocalFile, outputFile: LocalFile) {
+        await spawnPromise("pdftotext", [
             file.path,
-            "-"
+            outputFile.path,
         ], {
             logData: false,
             timeout: 5*60*1000
         });
-        return output.all;
+        return outputFile;
     }
 
     static async extractAsHtmlToFile(src: LocalFile, output: LocalFile) {

package/src/server/workflows/search/IndexAppFilesWorkflow.ts

@@ -11,6 +11,8 @@ import EmailParserService from "../../smtp/services/EmailParserService.js";
 import { parse } from "path";
 import { Extract } from "../../services/extract/Extract.js";
 import { AppWorkflowContext } from "../AppWorkflowContext.js";
+import { appendFile } from "fs/promises";
+import BufferHelper from "../../../common/BufferHelper.js";
 
 export default class IndexAppFilesWorkflow extends Workflow {
 
@@ -73,80 +75,75 @@ export default class IndexAppFilesWorkflow extends Workflow {
 
     private async extractText(file: LocalFile, tempFileService: TempFileService, db: SocialMailContext) {
 
+
+        const outputFile = await tempFileService.createTempFile(".txt");
+
         const isEmail = file.contentType === "message/rfc822";
         if(isEmail) {
             const parser = ServiceProvider.resolve(this, EmailParserService);
             const parsedEmail = await parser.parse(file, true);
-            const text = [parsedEmail.text];
+            await outputFile.appendLine(parsedEmail.text);
             for (const iterator of parsedEmail.attachments) {
                 const { ext } = parse(iterator.filename || "file.dat");
                 const tf = await tempFileService.createTempFile(ext, iterator.filename, iterator.contentType);
                 await tf.writeAll(iterator.content);
                 const t = await this.extractText(tf, tempFileService, db);
                 if (t) {
-                    text.push(t);
+                    await outputFile.appendLine(await t.readAsText());
                 }
             }
-            return text.join("\n");
+
+            return outputFile;
         }
 
         if (/image\//i.test(file.contentType)) {
-            return Extract.text(file, true);
+            await Extract.text(file, outputFile, true);
+            return outputFile;
         } if (FileType.canBeCompressed(file)) {
-            return Extract.text(file);
+            await Extract.text(file, outputFile);
+            return outputFile;
         }
-
-        return "";
+        await outputFile.writeAllText("");
+        return outputFile;
     }
 
 
     private async saveTextContent(appFile: AppFile, tempFileService: TempFileService, db: SocialMailContext) {
-        let searchable = "";
-
-        // const isEmail = appFile.contentType === "message/rfc822";
-
-        // if (isEmail) {
-        //     ServiceProvider.resolve(this, MailPar)
-        //     // this is email... it will contain the attachments..
-        //     // const email = await db.emails.where(appFile, (p) => (x) => x.originalFileID === p.appFileID)
-        //     //     .include((x) => x.attachments)
-        //     //     .first();
-        //     // if (email) {
-        //     //     const text = [email.textBody];
-        //     //     if (email.attachments) {
-        //     //         for (const attachment of email.attachments) {
-        //     //             text.push(attachment.name);
-        //     //             const t = await this.saveTextContent(attachment, tempFileService, db);
-        //     //             if (t) {
-        //     //                 text.push(t);
-        //     //             }
-        //     //         }
-        //     //     }
-        //     //     searchable = text.join("\n");
-        //     // }
-        // }
 
         const changes: Partial<FileContent> = {
             textDocumentID: 0,
             fileContentID: appFile.fileContentID
         };
 
-        if (!searchable) {
-            const file = await tempFileService.downloadAppFile(appFile);
+        const file = await tempFileService.downloadAppFile(appFile);
 
-            searchable = await this.extractText(file, tempFileService, db);
-        }
-        if (searchable) {
-            const td = await db.textDocuments.statements.upsert({
-                textDocumentID: changes.fileContentID,
-                searchable
-            });
-            changes.textDocumentID = td.textDocumentID;
+        const fx = await this.extractText(file, tempFileService, db);
+
+        let searchable = "";
+
+        if (fx.contentSize < 10*1024*1024) {
+
+            // we need to first remove 00 byte sequence...
+            const newFile = await tempFileService.createTempFile("txt");
+            const empty = Buffer.from("\n");
+            for await (const buffer of fx.readBuffers()) {
+                BufferHelper.replace00(buffer, empty);
+                await appendFile(newFile.path, buffer);
+            }
+
+            searchable = await newFile.readAsText();
         }
+
+        const td = await db.textDocuments.statements.upsert({
+            textDocumentID: changes.fileContentID,
+            searchable
+        });
+        changes.textDocumentID = td.textDocumentID;
+
         await db.fileContents.statements.update(
             changes
         );
 
-        return searchable;
+        return;
     }
 }