@social-mail/social-mail-web-server 1.8.395 → 1.8.397

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.8.395",
+  "version": "1.8.397",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/server/workflows/SocialWorkflowContext.ts

@@ -13,7 +13,7 @@ import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
 // import UpdateEmailReferences from "./incoming/UpdateEmailReferences.js";
 import BroadcastEmail from "./channel/BroadcastEmail.js";
 import ClearEmailHtmlBody from "./email/ClearEmailHtmlBody.js";
-import IndexAppFilesWorkflow from "./search/IndexAppFilesWorkflow.js";
+import IndexFileContentsWorkflow from "./search/IndexFileContentsWorkflow.js";
 import RepairConversationWorkflow from "./repair/conversations/RepairConversationWorkflow.js";
 // import PostReceiveWorkflow from "./post-receive/PostReceiveWorkflow.js";
 import SendChannelInvitation from "./channel/invite/SendChannelInvitation.js";
@@ -71,7 +71,7 @@ export default class SocialWorkflowContext extends AppWorkflowContext {
         this.register(UpdateFolderSize);
         this.register(UpdateUserStorage);
         this.register(DailyWorkflow);
-        this.register(IndexAppFilesWorkflow);
+        this.register(IndexFileContentsWorkflow);
         this.register(BroadcastEmail);
         this.register(DispatchEmailToMX);
         this.register(DispatchNotificationToMX);

package/src/server/workflows/daily/DailyWorkflow.ts

@@ -7,7 +7,7 @@ import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
 // import IndexMailboxWorkflow from "../search/IndexMailboxWorkflow.js";
 import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
 import ClearEmailHtmlBody from "../email/ClearEmailHtmlBody.js";
-import IndexAppFilesWorkflow from "../search/IndexAppFilesWorkflow.js";
+import IndexFileContentsWorkflow from "../search/IndexFileContentsWorkflow.js";
 import DbLockService from "../../services/db-lock/DbLockService.js";
 import PendingPostReceiveWorkflow from "../pending/PendingPostReceiveWorkflow.js";
 import PendingReferencesWorkflow from "../pending/PendingReferencesWorkflow.js";
@@ -109,6 +109,6 @@ export default class DailyWorkflow extends Workflow<any,any> {
 
             await this.context.queue(ClearEmailHtmlBody, "1", { id : "clean-email-html-body" });
 
-        await IndexAppFilesWorkflow.startIndexing(this.context);
+        await IndexFileContentsWorkflow.startIndexing(this.context);
     }
 }

package/src/server/workflows/search/{IndexAppFilesWorkflow.ts → IndexFileContentsWorkflow.ts}

@@ -6,7 +6,6 @@ import TempFileService from "../../storage/TempFileService.js";
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import { FileType } from "../../../common/FileType.js";
 import FileContent from "../../model/entities/FileContent.js";
-import AppFile from "../../model/entities/AppFile.js";
 import EmailParserService from "../../smtp/services/EmailParserService.js";
 import { parse } from "path";
 import { Extract } from "../../services/extract/Extract.js";
@@ -15,13 +14,13 @@ import { appendFile } from "fs/promises";
 import BufferHelper from "../../../common/BufferHelper.js";
 import sleep from "../../../common/sleep.js";
 
-export default class IndexAppFilesWorkflow extends Workflow {
+export default class IndexFileContentsWorkflow extends Workflow {
 
     static taskGroup = "batch";
 
     static startIndexing(context: AppWorkflowContext) {
-        return context.queue(IndexAppFilesWorkflow, 0, {
-            id: `index-app-files`
+        return context.queue(IndexFileContentsWorkflow, 0, {
+            id: `index-file-contents`
         });
     }
 
@@ -45,17 +44,16 @@ export default class IndexAppFilesWorkflow extends Workflow {
         db.verifyFilters = false;
         db.raiseEvents = false;
 
-        let q = db.appFiles
-            .where(void 0, (p) => (x) => x.fileContentID !== null && x.fileContent.textDocumentID === null);
+        let q = db.fileContents
+            .where(void 0, (p) => (x) => x.textDocumentID === null);
 
         if (BigInt(start) > 0) {
-            q = q.where({ start }, (p) => (x) => x.appFileID < p.start);
+            q = q.where({ start }, (p) => (x) => x.fileContentID < p.start);
         }
 
         const files = await q
-            .orderByDescending(void 0, (p) => (x) => x.appFileID)
+            .orderByDescending(void 0, (p) => (x) => x.fileContentID)
             .limit(100)
-            .include((x) => x.fileContent)
             .toArray();
 
         if (!files.length) {
@@ -63,7 +61,7 @@ export default class IndexAppFilesWorkflow extends Workflow {
         }
 
         for (const iterator of files) {
-            start = iterator.appFileID;
+            start = iterator.fileContentID;
             try {
                 await this.saveTextContent(iterator, tempFileService, db);
             } catch (error) {
@@ -109,25 +107,28 @@ export default class IndexAppFilesWorkflow extends Workflow {
     }
 
 
-    private async saveTextContent(appFile: AppFile, tempFileService: TempFileService, db: SocialMailContext) {
+    private async saveTextContent(fileContent: FileContent, tempFileService: TempFileService, db: SocialMailContext) {
 
         let searchable = "";
 
         // check if file content already has text content to remove duplicates
-        const has = await db.textDocuments.statements.select({}, { textDocumentID: appFile.fileContentID });
+        const has = await db.textDocuments.statements.select({}, { textDocumentID: fileContent.fileContentID });
         if(has) {
             return;
         }
 
         const changes: Partial<FileContent> = {
             textDocumentID: 0,
-            fileContentID: appFile.fileContentID
+            fileContentID: fileContent.fileContentID
         };
 
         let file: LocalFile = null;
 
         for(let i=0;i<3;i++) {
             try {
+
+                // get file
+                const appFile = await db.appFiles.where(fileContent, (p) => (x) => x.fileContentID === p.fileContentID).first();
                 file = await tempFileService.downloadAppFile(appFile);
                 break;
             } catch (error) {

package/dist/server/workflows/search/IndexAppFilesWorkflow.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"IndexAppFilesWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":"AACA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAQ/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAK9D,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,QAAQ;IAEvD,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,kBAAkB;IAM1C,GAAG;IAWH,KAAK,CACP,KAAK,KAAA,EACG,EAAE,CAAC,EAAE,iBAAiB,EACtB,eAAe,CAAC,EAAE,eAAe;YAmC/B,WAAW;YAmCX,eAAe;CA0DhC"}

package/dist/server/workflows/search/IndexAppFilesWorkflow.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"IndexAppFilesWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+BAA+B;AAC/B,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,EAAE,EAAE,eAAe,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAE/D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,YAAY,MAAM,iCAAiC,CAAC;AAC3D,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAE7C,MAAqB,qBAAsB,SAAQ,QAAQ;IAIvD,MAAM,CAAC,aAAa,CAAC,OAA2B;QAC5C,OAAO,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE;YAC3C,EAAE,EAAE,iBAAiB;SACxB,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;QACf,SAAQ,CAAC;YACL,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACP,KAAK,EACG,EAAsB,EACtB,eAAiC;QAGzC,EAAE,CAAC,aAAa,GAAG,KAAK,CAAC;QACzB,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ;aACd,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC;QAEpG,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC;aAChB,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aACpD,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;aAC7B,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACb,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC3B,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;YAC3B,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAC9D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe,EAAE,eAAgC,EAAE,EAAqB;QAG9F,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAEhE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,KAAK,gBAAgB,CAAC;QACtD,IAAG,OAAO,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACnD,MAAM,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC7C,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9F,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;gBAC1D,IAAI,CAAC,EAAE,CAAC;oBACJ,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAED,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;YAC3C,OAAO,UAAU,CAAC;QACtB,CAAC;QAAC,IAAI,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACrC,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,MAAM,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,UAAU,CAAC;IACtB,CAAC;IAGO,KAAK,CAAC,eAAe,CAAC,OAAgB,EAAE,eAAgC,EAAE,EAAqB;QAEnG,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;QACpG,IAAG,GAAG,EAAE,CAAC;YACL,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAyB;YAClC,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,OAAO,CAAC,aAAa;SACvC,CAAC;QAEF,IAAI,IAAI,GAAc,IAAI,CAAC;QAE3B,KAAI,IAAI,CAAC,GAAC,CAAC,EAAC,CAAC,GAAC,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC;YAClB,IAAI,CAAC;gBACD,IAAI,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACtD,MAAM;YACV,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,4BAA4B;gBAC5B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACrB,UAAU,GAAG,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAE7D,IAAI,EAAE,CAAC,WAAW,GAAG,EAAE,GAAC,IAAI,GAAC,IAAI,EAAE,CAAC;gBAEhC,8CAA8C;gBAC9C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC1C,YAAY,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACtC,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBAED,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChD,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,UAAU;SACb,CAAC,CAAC;QACH,OAAO,CAAC,cAAc,GAAG,EAAE,CAAC,cAAc,CAAC;QAE3C,MAAM,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CACnC,OAAO,CACV,CAAC;QAEF,OAAO;IACX,CAAC;;AArJM,+BAAS,GAAG,OAAO,CAAC;eAFV,qBAAqB;AAqBhC;IADL,cAAc;IAGV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;6CADM,iBAAiB;QACJ,eAAe;;kDAiC5C"}