@social-mail/social-mail-web-server 1.8.433 → 1.8.439

package/src/server/workflows/email/index/IndexEmailContentWorkflow.ts

@@ -0,0 +1,54 @@
+/* eslint-disable no-console */
+import Inject from "@entity-access/entity-access/dist/di/di.js";
+import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
+import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import IndexEmailService from "../../../services/emails/IndexEmailService.js";
+
+export default class IndexEmailContentWorkflow extends Workflow {
+
+    static taskGroup = "batch";
+
+    static queue(context: WorkflowContext) {
+        const now = DateTime.now;
+        return context.queue(IndexEmailContentWorkflow, "", {
+            id: `index-email-${now.msSinceEpoch.toString(36)}`
+        });
+    }
+
+    async run() {
+        await this.index();
+    }
+
+    @UniqueActivity
+    async index(
+        @Inject db?: SocialMailContext,
+        @Inject indexService?: IndexEmailService
+    ) {
+
+        for(let i = 0;i<1000;i++) {
+            const pending = await db.emails.where(void 0, (p) => (x) => x.searchIndexed === false
+                && x.statePostReceiveInvoked === true)
+                .limit(100)
+                .map(void 0, (p) => (x) => ({ emailID: x.emailID, textBody: x.textBody}))
+                .toArray();
+
+            if (!pending.length) {
+                return;
+            }
+
+            for (const email of pending) {
+                try {
+                    await indexService.indexEmail(email);
+                } catch (error) {
+                    console.error(error);
+                }
+                const { emailID } = email;
+                await db.emails.statements.update({ searchIndexed: true }, { emailID });
+            }
+            db.changeSet.clear();
+        }
+    }
+
+}

package/src/server/workflows/search/IndexFileContentsWorkflow.ts

@@ -1,18 +1,15 @@
 /* eslint-disable no-console */
 import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
-import Inject, { ServiceProvider } from "@entity-access/entity-access/dist/di/di.js";
+import Inject from "@entity-access/entity-access/dist/di/di.js";
 import SocialMailContext from "../../model/SocialMailContext.js";
 import TempFileService from "../../storage/TempFileService.js";
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
-import { FileType } from "../../../common/FileType.js";
 import FileContent from "../../model/entities/FileContent.js";
-import EmailParserService from "../../smtp/services/EmailParserService.js";
-import { parse } from "path";
-import { Extract } from "../../services/extract/Extract.js";
 import { AppWorkflowContext } from "../AppWorkflowContext.js";
 import { appendFile } from "fs/promises";
 import BufferHelper from "../../../common/BufferHelper.js";
 import sleep from "../../../common/sleep.js";
+import TextExtractorService from "../../services/extract/TextExtractorService.js";
 
 export default class IndexFileContentsWorkflow extends Workflow {
 
@@ -38,7 +35,8 @@ export default class IndexFileContentsWorkflow extends Workflow {
     async index(
         start,
         @Inject db?: SocialMailContext,
-        @Inject tempFileService?: TempFileService
+        @Inject tempFileService?: TempFileService,
+        @Inject te?: TextExtractorService
     ) {
 
         db.verifyFilters = false;
@@ -63,7 +61,7 @@ export default class IndexFileContentsWorkflow extends Workflow {
         for (const iterator of files) {
             start = iterator.fileContentID;
             try {
-                await this.saveTextContent(iterator, tempFileService, db);
+                await this.saveTextContent(iterator, tempFileService, db, te);
             } catch (error) {
                 console.error(error);
             }
@@ -72,42 +70,8 @@ export default class IndexFileContentsWorkflow extends Workflow {
         return start;
     }
 
-    private async extractText(file: LocalFile, tempFileService: TempFileService, db: SocialMailContext) {
 
-
-        const outputFile = await tempFileService.createTempFile(".txt");
-
-        const isEmail = file.contentType === "message/rfc822";
-        if(isEmail) {
-            const parser = ServiceProvider.resolve(this, EmailParserService);
-            const parsedEmail = await parser.parse(file, true);
-            await outputFile.appendLine(parsedEmail.text);
-            for (const iterator of parsedEmail.attachments) {
-                const { ext } = parse(iterator.filename || "file.dat");
-                const tf = await tempFileService.createTempFile(ext, iterator.filename, iterator.contentType);
-                await tf.writeAll(iterator.content);
-                const t = await this.extractText(tf, tempFileService, db);
-                if (t) {
-                    await outputFile.appendLine(await t.readAsText());
-                }
-            }
-
-            return outputFile;
-        }
-
-        if (/image\//i.test(file.contentType)) {
-            await Extract.text(file, outputFile, true);
-            return outputFile;
-        } if (FileType.canBeCompressed(file)) {
-            await Extract.text(file, outputFile);
-            return outputFile;
-        }
-        await outputFile.writeAllText("");
-        return outputFile;
-    }
-
-
-    private async saveTextContent(fileContent: FileContent, tempFileService: TempFileService, db: SocialMailContext) {
+    private async saveTextContent(fileContent: FileContent, tempFileService: TempFileService, db: SocialMailContext, te: TextExtractorService) {
 
         let searchable = "";
 
@@ -143,7 +107,7 @@ export default class IndexFileContentsWorkflow extends Workflow {
         }
 
         if (file) {
-            const fx = await this.extractText(file, tempFileService, db);
+            const fx = await te.extract(file);
 
             if (fx.contentSize < 10*1024*1024) {