@social-mail/social-mail-web-server 1.8.434 → 1.8.440

package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.d.ts

@@ -0,0 +1,11 @@
+import Workflow from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import NameTokenService from "../../../services/name-tokens/NameTokenService.js";
+import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
+export default class IndexEmailAddressWorkflow extends Workflow {
+    static taskGroup: string;
+    static queue(c: WorkflowContext): Promise<string>;
+    run(): Promise<void>;
+    index(start: number, db?: SocialMailContext, nts?: NameTokenService): Promise<number>;
+}
+//# sourceMappingURL=IndexEmailAddressWorkflow.d.ts.map

package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexEmailAddressWorkflow.d.ts","sourceRoot":"","sources":["../../../../../src/server/workflows/email/index/IndexEmailAddressWorkflow.ts"],"names":[],"mappings":"AACA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AACnG,OAAO,iBAAiB,MAAM,qCAAqC,CAAC;AACpE,OAAO,gBAAgB,MAAM,mDAAmD,CAAC;AACjF,OAAO,eAAe,MAAM,gEAAgE,CAAC;AAE7F,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,QAAQ;IAE3D,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe;IAUzB,GAAG;IAWH,KAAK,CACP,KAAK,EAAE,MAAM,EACL,EAAE,CAAC,EAAE,iBAAiB,EACtB,GAAG,CAAC,EAAE,gBAAgB;CA8BrC"}

package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.js

@@ -0,0 +1,69 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import Inject from "@entity-access/entity-access/dist/di/di.js";
+import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import NameTokenService from "../../../services/name-tokens/NameTokenService.js";
+class IndexEmailAddressWorkflow extends Workflow {
+    static queue(c) {
+        const d = new Date();
+        d.setMinutes(Math.ceil(d.getMinutes() / 5));
+        d.setSeconds(0);
+        d.setMilliseconds(0);
+        return c.queue(IndexEmailAddressWorkflow, 0, {
+            id: `index-ea-${d.getTime().toString(36)}`
+        });
+    }
+    async run() {
+        let start = 0;
+        for (;;) {
+            start = await this.index(start);
+            if (!BigInt(start)) {
+                break;
+            }
+        }
+    }
+    async index(start, db, nts) {
+        for (let i = 0; i < 100; i++) {
+            const all = await db.emailAddresses.where({ start }, (p) => (x) => x.nameIndexed === false && x.emailAddressID > p.start)
+                .limit(100)
+                .toArray();
+            if (!all.length) {
+                return 0;
+            }
+            for (const { emailAddressID, name, emailAddress } of all) {
+                start = emailAddressID;
+                if (name) {
+                    await nts.updateName(name, { emailAddressID });
+                }
+                const [username] = emailAddress.split("@", 2);
+                await nts.updateName(username, { emailAddressID }, true);
+                await db.emailAddresses.statements.update({ nameIndexed: true }, { emailAddressID });
+            }
+            db.changeSet.clear();
+        }
+        return start;
+    }
+}
+IndexEmailAddressWorkflow.taskGroup = "batch";
+export default IndexEmailAddressWorkflow;
+__decorate([
+    UniqueActivity,
+    __param(1, Inject),
+    __param(2, Inject),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number, SocialMailContext,
+        NameTokenService]),
+    __metadata("design:returntype", Promise)
+], IndexEmailAddressWorkflow.prototype, "index", null);
+//# sourceMappingURL=IndexEmailAddressWorkflow.js.map

package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexEmailAddressWorkflow.js","sourceRoot":"","sources":["../../../../../src/server/workflows/email/index/IndexEmailAddressWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,MAAM,MAAM,4CAA4C,CAAC;AAChE,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,iBAAiB,MAAM,qCAAqC,CAAC;AACpE,OAAO,gBAAgB,MAAM,mDAAmD,CAAC;AAGjF,MAAqB,yBAA0B,SAAQ,QAAQ;IAI3D,MAAM,CAAC,KAAK,CAAC,CAAkB;QAC3B,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,GAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,CAAC,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,EAAE;YACzC,EAAE,EAAE,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;SAC7C,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,SAAQ,CAAC;YACL,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,IAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACP,KAAa,EACL,EAAsB,EACtB,GAAsB;QAG9B,KAAI,IAAI,CAAC,GAAC,CAAC,EAAC,CAAC,GAAC,GAAG,EAAC,CAAC,EAAE,EAAE,CAAC;YAEpB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1D,CAAC,CAAC,WAAW,KAAK,KAAK,IAAI,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC;iBACzD,KAAK,CAAC,GAAG,CAAC;iBACV,OAAO,EAAE,CAAC;YAEf,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,CAAC;YACb,CAAC;YAED,KAAK,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,GAAG,EAAE,CAAC;gBACvD,KAAK,GAAG,cAAc,CAAC;gBACvB,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;gBACnD,CAAC;gBACD,MAAM,CAAC,QAAQ,CAAE,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,cAAc,EAAE,EAAE,IAAI,CAAC,CAAC;gBACzD,MAAM,EAAE,CAAC,cAAc,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,EAAC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;YACxF,CAAC;YAED,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;;AAtDM,mCAAS,GAAG,OAAO,CAAC;eAFV,yBAAyB;AAyBpC;IADL,cAAc;IAGV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;6CADM,iBAAiB;QAChB,gBAAgB;;sDA4BjC"}

package/dist/server/workflows/email/index/IndexEmailContentWorkflow.d.ts

@@ -0,0 +1,11 @@
+import Workflow from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import IndexEmailService from "../../../services/emails/IndexEmailService.js";
+export default class IndexEmailContentWorkflow extends Workflow {
+    static taskGroup: string;
+    static queue(context: WorkflowContext): Promise<string>;
+    run(): Promise<void>;
+    index(db?: SocialMailContext, indexService?: IndexEmailService): Promise<void>;
+}
+//# sourceMappingURL=IndexEmailContentWorkflow.d.ts.map

package/dist/server/workflows/email/index/IndexEmailContentWorkflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexEmailContentWorkflow.d.ts","sourceRoot":"","sources":["../../../../../src/server/workflows/email/index/IndexEmailContentWorkflow.ts"],"names":[],"mappings":"AAGA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AACnG,OAAO,eAAe,MAAM,gEAAgE,CAAC;AAC7F,OAAO,iBAAiB,MAAM,qCAAqC,CAAC;AACpE,OAAO,iBAAiB,MAAM,+CAA+C,CAAC;AAE9E,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,QAAQ;IAE3D,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe;IAO/B,GAAG;IAKH,KAAK,CACC,EAAE,CAAC,EAAE,iBAAiB,EACtB,YAAY,CAAC,EAAE,iBAAiB;CA2B/C"}

package/dist/server/workflows/email/index/IndexEmailContentWorkflow.js

@@ -0,0 +1,64 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+/* eslint-disable no-console */
+import Inject from "@entity-access/entity-access/dist/di/di.js";
+import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
+import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import IndexEmailService from "../../../services/emails/IndexEmailService.js";
+class IndexEmailContentWorkflow extends Workflow {
+    static queue(context) {
+        const now = DateTime.now;
+        return context.queue(IndexEmailContentWorkflow, "", {
+            id: `index-email-${now.msSinceEpoch.toString(36)}`
+        });
+    }
+    async run() {
+        await this.index();
+    }
+    async index(db, indexService) {
+        for (let i = 0; i < 1000; i++) {
+            const pending = await db.emails.where(void 0, (p) => (x) => x.searchIndexed === false
+                && x.statePostReceiveInvoked === true)
+                .limit(100)
+                .map(void 0, (p) => (x) => ({ emailID: x.emailID, textBody: x.textBody }))
+                .toArray();
+            if (!pending.length) {
+                return;
+            }
+            for (const email of pending) {
+                try {
+                    await indexService.indexEmail(email);
+                }
+                catch (error) {
+                    console.error(error);
+                }
+                const { emailID } = email;
+                await db.emails.statements.update({ searchIndexed: true }, { emailID });
+            }
+            db.changeSet.clear();
+        }
+    }
+}
+IndexEmailContentWorkflow.taskGroup = "batch";
+export default IndexEmailContentWorkflow;
+__decorate([
+    UniqueActivity,
+    __param(0, Inject),
+    __param(1, Inject),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [SocialMailContext,
+        IndexEmailService]),
+    __metadata("design:returntype", Promise)
+], IndexEmailContentWorkflow.prototype, "index", null);
+//# sourceMappingURL=IndexEmailContentWorkflow.js.map

package/dist/server/workflows/email/index/IndexEmailContentWorkflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexEmailContentWorkflow.js","sourceRoot":"","sources":["../../../../../src/server/workflows/email/index/IndexEmailContentWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+BAA+B;AAC/B,OAAO,MAAM,MAAM,4CAA4C,CAAC;AAChE,OAAO,QAAQ,MAAM,qDAAqD,CAAC;AAC3E,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,qCAAqC,CAAC;AACpE,OAAO,iBAAiB,MAAM,+CAA+C,CAAC;AAE9E,MAAqB,yBAA0B,SAAQ,QAAQ;IAI3D,MAAM,CAAC,KAAK,CAAC,OAAwB;QACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QACzB,OAAO,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,EAAE;YAChD,EAAE,EAAE,eAAe,GAAG,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;SACrD,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACC,EAAsB,EACtB,YAAgC;QAGxC,KAAI,IAAI,CAAC,GAAG,CAAC,EAAC,CAAC,GAAC,IAAI,EAAC,CAAC,EAAE,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,KAAK;mBAC9E,CAAC,CAAC,uBAAuB,KAAK,IAAI,CAAC;iBACrC,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAC,CAAC,CAAC;iBACxE,OAAO,EAAE,CAAC;YAEf,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO;YACX,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACD,MAAM,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBACzC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACzB,CAAC;gBACD,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;gBAC1B,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;YACD,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;;AAzCM,mCAAS,GAAG,OAAO,CAAC;eAFV,yBAAyB;AAgBpC;IADL,cAAc;IAEV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;qCADM,iBAAiB;QACP,iBAAiB;;sDAyB3C"}

package/dist/server/workflows/search/IndexFileContentsWorkflow.d.ts

@@ -2,12 +2,12 @@ import Workflow from "@entity-access/entity-access/dist/workflows/Workflow.js";
 import SocialMailContext from "../../model/SocialMailContext.js";
 import TempFileService from "../../storage/TempFileService.js";
 import { AppWorkflowContext } from "../AppWorkflowContext.js";
+import TextExtractorService from "../../services/extract/TextExtractorService.js";
 export default class IndexFileContentsWorkflow extends Workflow {
     static taskGroup: string;
     static startIndexing(context: AppWorkflowContext): Promise<string>;
     run(): Promise<void>;
-    index(start: any, db?: SocialMailContext, tempFileService?: TempFileService): Promise<any>;
-    private extractText;
+    index(start: any, db?: SocialMailContext, tempFileService?: TempFileService, te?: TextExtractorService): Promise<any>;
     private saveTextContent;
 }
 //# sourceMappingURL=IndexFileContentsWorkflow.d.ts.map

package/dist/server/workflows/search/IndexFileContentsWorkflow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"IndexFileContentsWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexFileContentsWorkflow.ts"],"names":[],"mappings":"AACA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAO/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAK9D,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,QAAQ;IAE3D,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,kBAAkB;IAM1C,GAAG;IAWH,KAAK,CACP,KAAK,KAAA,EACG,EAAE,CAAC,EAAE,iBAAiB,EACtB,eAAe,CAAC,EAAE,eAAe;YAkC/B,WAAW;YAmCX,eAAe;CAgEhC"}
+{"version":3,"file":"IndexFileContentsWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexFileContentsWorkflow.ts"],"names":[],"mappings":"AACA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAI9D,OAAO,oBAAoB,MAAM,gDAAgD,CAAC;AAElF,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,QAAQ;IAE3D,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,kBAAkB;IAM1C,GAAG;IAWH,KAAK,CACP,KAAK,KAAA,EACG,EAAE,CAAC,EAAE,iBAAiB,EACtB,eAAe,CAAC,EAAE,eAAe,EACjC,EAAE,CAAC,EAAE,oBAAoB;YAmCvB,eAAe;CAgEhC"}

package/dist/server/workflows/search/IndexFileContentsWorkflow.js

@@ -12,16 +12,13 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
 };
 /* eslint-disable no-console */
 import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
-import Inject, { ServiceProvider } from "@entity-access/entity-access/dist/di/di.js";
+import Inject from "@entity-access/entity-access/dist/di/di.js";
 import SocialMailContext from "../../model/SocialMailContext.js";
 import TempFileService from "../../storage/TempFileService.js";
-import { FileType } from "../../../common/FileType.js";
-import EmailParserService from "../../smtp/services/EmailParserService.js";
-import { parse } from "path";
-import { Extract } from "../../services/extract/Extract.js";
 import { appendFile } from "fs/promises";
 import BufferHelper from "../../../common/BufferHelper.js";
 import sleep from "../../../common/sleep.js";
+import TextExtractorService from "../../services/extract/TextExtractorService.js";
 class IndexFileContentsWorkflow extends Workflow {
     static startIndexing(context) {
         return context.queue(IndexFileContentsWorkflow, 0, {
@@ -37,7 +34,7 @@ class IndexFileContentsWorkflow extends Workflow {
             }
         }
     }
-    async index(start, db, tempFileService) {
+    async index(start, db, tempFileService, te) {
         db.verifyFilters = false;
         db.raiseEvents = false;
         let q = db.fileContents
@@ -55,7 +52,7 @@ class IndexFileContentsWorkflow extends Workflow {
         for (const iterator of files) {
             start = iterator.fileContentID;
             try {
-                await this.saveTextContent(iterator, tempFileService, db);
+                await this.saveTextContent(iterator, tempFileService, db, te);
             }
             catch (error) {
                 console.error(error);
@@ -63,36 +60,7 @@ class IndexFileContentsWorkflow extends Workflow {
         }
         return start;
     }
-    async extractText(file, tempFileService, db) {
-        const outputFile = await tempFileService.createTempFile(".txt");
-        const isEmail = file.contentType === "message/rfc822";
-        if (isEmail) {
-            const parser = ServiceProvider.resolve(this, EmailParserService);
-            const parsedEmail = await parser.parse(file, true);
-            await outputFile.appendLine(parsedEmail.text);
-            for (const iterator of parsedEmail.attachments) {
-                const { ext } = parse(iterator.filename || "file.dat");
-                const tf = await tempFileService.createTempFile(ext, iterator.filename, iterator.contentType);
-                await tf.writeAll(iterator.content);
-                const t = await this.extractText(tf, tempFileService, db);
-                if (t) {
-                    await outputFile.appendLine(await t.readAsText());
-                }
-            }
-            return outputFile;
-        }
-        if (/image\//i.test(file.contentType)) {
-            await Extract.text(file, outputFile, true);
-            return outputFile;
-        }
-        if (FileType.canBeCompressed(file)) {
-            await Extract.text(file, outputFile);
-            return outputFile;
-        }
-        await outputFile.writeAllText("");
-        return outputFile;
-    }
-    async saveTextContent(fileContent, tempFileService, db) {
+    async saveTextContent(fileContent, tempFileService, db, te) {
         let searchable = "";
         // check if file content already has text content to remove duplicates
         const has = await db.textDocuments.statements.select({}, { textDocumentID: fileContent.fileContentID });
@@ -122,7 +90,7 @@ class IndexFileContentsWorkflow extends Workflow {
             }
         }
         if (file) {
-            const fx = await this.extractText(file, tempFileService, db);
+            const fx = await te.extract(file);
             if (fx.contentSize < 10 * 1024 * 1024) {
                 // we need to first remove 00 byte sequence...
                 const newFile = await tempFileService.createTempFile("txt");
@@ -149,9 +117,11 @@ __decorate([
     UniqueActivity,
     __param(1, Inject),
     __param(2, Inject),
+    __param(3, Inject),
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object, SocialMailContext,
-        TempFileService]),
+        TempFileService,
+        TextExtractorService]),
     __metadata("design:returntype", Promise)
 ], IndexFileContentsWorkflow.prototype, "index", null);
 //# sourceMappingURL=IndexFileContentsWorkflow.js.map

package/dist/server/workflows/search/IndexFileContentsWorkflow.js.map

@@ -1 +1 @@
-{"version":3,"file":"IndexFileContentsWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexFileContentsWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+BAA+B;AAC/B,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,EAAE,EAAE,eAAe,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAE/D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAEvD,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,YAAY,MAAM,iCAAiC,CAAC;AAC3D,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAE7C,MAAqB,yBAA0B,SAAQ,QAAQ;IAI3D,MAAM,CAAC,aAAa,CAAC,OAA2B;QAC5C,OAAO,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,EAAE;YAC/C,EAAE,EAAE,qBAAqB;SAC5B,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;QACf,SAAQ,CAAC;YACL,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACP,KAAK,EACG,EAAsB,EACtB,eAAiC;QAGzC,EAAE,CAAC,aAAa,GAAG,KAAK,CAAC;QACzB,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY;aAClB,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC;QAE5D,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC;aAChB,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;aACxD,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACb,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC3B,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC;YAC/B,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAC9D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe,EAAE,eAAgC,EAAE,EAAqB;QAG9F,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAEhE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,KAAK,gBAAgB,CAAC;QACtD,IAAG,OAAO,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACnD,MAAM,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC7C,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9F,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;gBAC1D,IAAI,CAAC,EAAE,CAAC;oBACJ,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAED,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;YAC3C,OAAO,UAAU,CAAC;QACtB,CAAC;QAAC,IAAI,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACrC,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,MAAM,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,UAAU,CAAC;IACtB,CAAC;IAGO,KAAK,CAAC,eAAe,CAAC,WAAwB,EAAE,eAAgC,EAAE,EAAqB;QAE3G,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC,aAAa,EAAE,CAAC,CAAC;QACxG,IAAG,GAAG,EAAE,CAAC;YACL,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAyB;YAClC,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,WAAW,CAAC,aAAa;SAC3C,CAAC;QAEF,IAAI,IAAI,GAAc,IAAI,CAAC;QAE3B,KAAI,IAAI,CAAC,GAAC,CAAC,EAAC,CAAC,GAAC,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC;YAClB,IAAI,CAAC;gBAED,WAAW;gBACX,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC;gBAChH,IAAI,CAAC,OAAO,EAAE,CAAC;oBACX,MAAM;gBACV,CAAC;gBACD,IAAI,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACtD,MAAM;YACV,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,4BAA4B;gBAC5B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACrB,UAAU,GAAG,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAE7D,IAAI,EAAE,CAAC,WAAW,GAAG,EAAE,GAAC,IAAI,GAAC,IAAI,EAAE,CAAC;gBAEhC,8CAA8C;gBAC9C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC1C,YAAY,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACtC,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBAED,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChD,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,UAAU;SACb,CAAC,CAAC;QACH,OAAO,CAAC,cAAc,GAAG,EAAE,CAAC,cAAc,CAAC;QAE3C,MAAM,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CACnC,OAAO,CACV,CAAC;QAEF,OAAO;IACX,CAAC;;AA1JM,mCAAS,GAAG,OAAO,CAAC;eAFV,yBAAyB;AAqBpC;IADL,cAAc;IAGV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;6CADM,iBAAiB;QACJ,eAAe;;sDAgC5C"}
+{"version":3,"file":"IndexFileContentsWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexFileContentsWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+BAA+B;AAC/B,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,MAAM,4CAA4C,CAAC;AAChE,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAI/D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,YAAY,MAAM,iCAAiC,CAAC;AAC3D,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAC7C,OAAO,oBAAoB,MAAM,gDAAgD,CAAC;AAElF,MAAqB,yBAA0B,SAAQ,QAAQ;IAI3D,MAAM,CAAC,aAAa,CAAC,OAA2B;QAC5C,OAAO,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,EAAE;YAC/C,EAAE,EAAE,qBAAqB;SAC5B,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;QACf,SAAQ,CAAC;YACL,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACP,KAAK,EACG,EAAsB,EACtB,eAAiC,EACjC,EAAyB;QAGjC,EAAE,CAAC,aAAa,GAAG,KAAK,CAAC;QACzB,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY;aAClB,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC;QAE5D,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC;aAChB,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;aACxD,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACb,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC3B,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC;YAC/B,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAClE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAGO,KAAK,CAAC,eAAe,CAAC,WAAwB,EAAE,eAAgC,EAAE,EAAqB,EAAE,EAAwB;QAErI,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC,aAAa,EAAE,CAAC,CAAC;QACxG,IAAG,GAAG,EAAE,CAAC;YACL,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAyB;YAClC,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,WAAW,CAAC,aAAa;SAC3C,CAAC;QAEF,IAAI,IAAI,GAAc,IAAI,CAAC;QAE3B,KAAI,IAAI,CAAC,GAAC,CAAC,EAAC,CAAC,GAAC,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC;YAClB,IAAI,CAAC;gBAED,WAAW;gBACX,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,CAAC;gBAChH,IAAI,CAAC,OAAO,EAAE,CAAC;oBACX,MAAM;gBACV,CAAC;gBACD,IAAI,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACtD,MAAM;YACV,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,4BAA4B;gBAC5B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACrB,UAAU,GAAG,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAElC,IAAI,EAAE,CAAC,WAAW,GAAG,EAAE,GAAC,IAAI,GAAC,IAAI,EAAE,CAAC;gBAEhC,8CAA8C;gBAC9C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC1C,YAAY,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACtC,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBAED,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChD,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,UAAU;SACb,CAAC,CAAC;QACH,OAAO,CAAC,cAAc,GAAG,EAAE,CAAC,cAAc,CAAC;QAE3C,MAAM,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CACnC,OAAO,CACV,CAAC;QAEF,OAAO;IACX,CAAC;;AAzHM,mCAAS,GAAG,OAAO,CAAC;eAFV,yBAAyB;AAqBpC;IADL,cAAc;IAGV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;6CAFM,iBAAiB;QACJ,eAAe;QAC5B,oBAAoB;;sDAgCpC"}