@social-mail/social-mail-web-server 1.8.434 → 1.8.439

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.8.434",
+  "version": "1.8.439",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",
@@ -22,7 +22,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.913.0",
     "@azure/storage-blob": "^12.29.1",
-    "@entity-access/entity-access": "^1.0.526",
+    "@entity-access/entity-access": "^1.0.528",
     "@entity-access/server-pages": "^1.1.565",
     "@parse/node-apn": "^6.5.0",
     "@paypal/paypal-server-sdk": "^1.0.0",

package/src/server/model/SocialMailContext.ts

@@ -80,6 +80,8 @@ import UserGroupMember from "./entities/UserGroupMember.js";
 import WebSiteTag from "./entities/WebSiteTag.js";
 import AppDbContext from "@entity-access/server-pages/dist/core/AppDbContext.js";
 import AppPassword from "./entities/AppPassword.js";
+import SearchToken from "./entities/SearchToken.js";
+import SearchTokenEmail from "./entities/SearchTokenEmail.js";
 
 @RegisterScoped
 export default class SocialMailContext extends AppDbContext {
@@ -178,6 +180,10 @@ export default class SocialMailContext extends AppDbContext {
 
     public subscriberActivities = this.model.register(SubscriberActivity);
 
+    public searchTokens = this.model.register(SearchToken);
+
+    public searchTokenEmails = this.model.register(SearchTokenEmail);
+
     public searchWords = this.model.register(SearchWord);
 
     public searchWordIndexes = this.model.register(SearchWordIndex);

package/src/server/model/SocialMailContextEvents.ts

@@ -146,6 +146,10 @@ import WebSiteTag from "./entities/WebSiteTag.js";
 import WebSiteTagEvents from "./events/WebSiteTagEvents.js";
 import AppPassword from "./entities/AppPassword.js";
 import AppPasswordEvents from "./events/AppPasswordEvents.js";
+import SearchToken from "./entities/SearchToken.js";
+import SearchTokenEvents from "./events/SearchTokenEvents.js";
+import SearchTokenEmail from "./entities/SearchTokenEmail.js";
+import SearchTokenEmailEvents from "./events/SearchTokenEmailEvents.js";
 
 
 @RegisterSingleton
@@ -195,6 +199,10 @@ export default class SocialMailContextEvents extends ContextEvents {
         this.register(MailboxFolder, MailboxFolderEvents);
         this.register(ChannelEmail, MailboxEmailEvents);
         this.register(NameToken, NameTokenEvents);
+
+        this.register(SearchToken, SearchTokenEvents);
+        this.register(SearchTokenEmail, SearchTokenEmailEvents);
+
         this.register(StoreItem, StoreItemEvents);
         this.register(StoreItemPrice, StoreItemPriceEvents);
         this.register(StoreAccount, StoreAccountEvents);

package/src/server/model/entities/Domain.ts

@@ -10,9 +10,10 @@ import type HostedZone from "./HostedZone.js";
 
 @Table("Domains")
 @Index({
-    name: "IX_Unique_Domains",
+    name: "IX_Unique_Domains_V2",
+    dropNames:["IX_Unique_Domains"],
     unique: true,
-    columns: [{ name: (x) => x.name, descending: false}]
+    columns: [{ name: (x) => x.name, descending: false, operatorClass: "varchar_pattern"}]
 })
 export default class Domain {
 

package/src/server/model/entities/Email.ts

@@ -18,6 +18,7 @@ import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
 import { apiPath } from "../../../common/globalEnv.js";
 import type MailboxContactEvent from "./MailboxContactEvent.js";
 import type ThreadParticipant from "./ThreadParticipant.js";
+import type SearchTokenEmail from "./SearchTokenEmail.js";
 
 const emailStatuses = [ "queued", "received", "draft", "published", "pending-approval", "approved", "scheduled", "sent", "sent-failed", "deleted", "closed"] as const;
 export type emailStatusType = typeof emailStatuses[number];
@@ -73,6 +74,11 @@ export type emailStatusType = typeof emailStatuses[number];
         && x.status !== "draft"
         && x.status !== "scheduled"
 })
+@Index({
+    name: "IX_Email_Pending_Search",
+    columns: [{ name: (x) => x.searchIndexed, descending: false }],
+    filter: (x) => x.searchIndexed === false && x.statePostReceiveInvoked === true
+})
 export class Email {
 
     @Column({ key: true, dataType: "BigInt", generated: "identity" })
@@ -260,6 +266,9 @@ export class Email {
     @IgnoreJsonProperty
     statePostReceiveInvoked: boolean;
 
+    @Column( {dataType: "Boolean", default: () => false })
+    searchIndexed: boolean;
+
     @ReadOnlyJsonProperty
     postID: string;
 
@@ -314,6 +323,7 @@ export class Email {
     mailboxContactEvents: MailboxContactEvent[];
     threadParticipants: ThreadParticipant[];
     channelEmails: ChannelEmail[];
+    searchTokenEmails: SearchTokenEmail[];
 }
 
 @Table("EmailLogs")

package/src/server/model/entities/EmailAddress.ts

@@ -16,10 +16,16 @@ import type MailboxContactEmail from "./MailboxContactEmail.js";
 
 @Table("EmailAddresses")
 @Index({
-    name: "IX_Unique_EmailAddresses",
-    columns: [{ name: (x) => x.emailAddress, descending : false }],
+    name: "IX_Unique_EmailAddresses_V2",
+    dropNames: [ "IX_Unique_EmailAddresses"],
+    columns: [{ name: (x) => x.emailAddress, descending : false, operatorClass: "varchar_pattern" }],
     unique: true
 })
+@Index({
+    name: "IX_EmailAddresses_Pending_Name_Index",
+    columns: [ { name: (x) => x.emailAddressID, descending: false }],
+    filter: (x) => x.nameIndexed === false
+})
 export default class EmailAddress {
 
     @Column({ key: true, generated: "identity",  dataType: "BigInt"})
@@ -38,6 +44,9 @@ export default class EmailAddress {
     @Column({ dataType: "Char", length: 400 })
     public name: string;
 
+    @Column({ dataType: "Boolean", default: () => false})
+    public nameIndexed: boolean;
+
     @Column({ dataType: "Boolean", default: () => false})
     public blocked: boolean;
 

package/src/server/model/entities/NameToken.ts

@@ -13,7 +13,7 @@ import type EmailAddressName from "./EmailAddressName.js";
  * Since one name will be linked with many addresses. It will require less
  * storage and faster access compared to inbuilt FTS.
  *
- * And This is the most importat part for Auto Complete.
+ * And This is the most important part for Auto Complete.
  *
  * Why store suggest column?
  * In dropdown we will only display suggestion if the name token has more than
@@ -26,15 +26,16 @@ import type EmailAddressName from "./EmailAddressName.js";
 
 @Table("NameTokens")
 @Index({
-    name: "UX_NameTokens_Unique",
+    name: "UX_NameTokens_Unique_V2",
+    dropNames: [ "UX_NameTokens_Unique"],
     unique: true,
     columns: [
-        { name: (x) => x.word, descending: false },
+        { name: (x) => x.word, descending: false, operatorClass: "varchar_pattern" },
     ]
 })
 @Index({
     name: "IX_NameTokens_Suggest",
-    columns: [ { name: (x) => x.word, descending: false }],
+    columns: [ { name: (x) => x.word, descending: false, operatorClass: "varchar_pattern" }],
     filter: (x) => x.suggest === true
 })
 export default class NameToken {

package/src/server/model/entities/SearchToken.ts

@@ -0,0 +1,26 @@
+import Column from "@entity-access/entity-access/dist/decorators/Column.js";
+import Index from "@entity-access/entity-access/dist/decorators/Index.js";
+import Table from "@entity-access/entity-access/dist/decorators/Table.js";
+import type SearchTokenEmail from "./SearchTokenEmail.js";
+
+@Table("SearchTokens")
+@Index({
+    name: "UX_SearchTokens_Word",
+    columns: [ { name: (x) => x.word, descending: false, operatorClass: "varchar_pattern" } ],
+    include: [ (x) => x.doNotIndex],
+    unique: true
+})
+export default class SearchToken {
+
+    @Column({ dataType: "BigInt", key: true, generated: "identity"})
+    tokenID: number;
+
+    @Column({ dataType:"Char", length: 400, nullable: false })
+    word: string;
+
+    @Column({ dataType: "Boolean", default: () => false })
+    doNotIndex: boolean;
+
+    tokenEmails: SearchTokenEmail[];
+
+}

package/src/server/model/entities/SearchTokenEmail.ts

@@ -0,0 +1,34 @@
+import Column from "@entity-access/entity-access/dist/decorators/Column.js";
+import { RelateTo } from "@entity-access/entity-access/dist/decorators/Relate.js";
+import Table from "@entity-access/entity-access/dist/decorators/Table.js";
+import SearchToken from "./SearchToken.js";
+import { Email } from "./Email.js";
+
+export const tokenKind = ["body", "attachment"] as const;
+export type tokenKindType = typeof tokenKind[number];
+
+@Table("SearchTokenEmails")
+export default class SearchTokenEmail {
+
+    @Column({ dataType: "BigInt", key: true})
+    @RelateTo(SearchToken, {
+        property: (x) => x.token,
+        inverseProperty: (x) => x.tokenEmails,
+    })
+    tokenID: number;
+
+    @Column({ dataType: "Char", key: true, length: 20 })
+    tokenKind: tokenKindType;
+
+    @Column({ dataType: "BigInt", key: true })
+    @RelateTo(Email, {
+        property: (x) => x.email,
+        inverseProperty:  (x) => x.searchTokenEmails
+    })
+    emailID: number;
+
+    token: SearchToken;
+
+    email: Email;
+
+}

package/src/server/model/entities/SearchWord.ts

@@ -65,7 +65,7 @@ export class SearchWordIndex {
     fileContentID: number;
 
     /**
-     * First two bytes represents the first word occurance (not character index) in the document.
+     * First two bytes represents the first word occurrence (not character index) in the document.
      * Last two bytes represents how many times it occurs in the document.
      * This is based on the fact that import words will appear in the title or the first paragraph
      * of the document.

package/src/server/model/events/NameTokenEvents.ts

@@ -1,15 +1,8 @@
-import { IEntityQuery } from "@entity-access/entity-access/dist/model/IFilterWithParameter.js";
 import NameToken from "../entities/NameToken.js";
-import AuthenticatedEvents from "./AuthenticatedEvents.js";
+import ReadOnlyEvents from "./ReadOnlyEvents.js";
 
 
-export default class NameTokenEvents extends AuthenticatedEvents<NameToken> {
+export default class NameTokenEvents extends ReadOnlyEvents<NameToken> {
 
-    filter(query: IEntityQuery<NameToken>): IEntityQuery<NameToken> {
-        if (this.verify) {
-            this.sessionUser.ensureLoggedIn();
-        }
-        return query.limit(20);
-    }
 
 }

package/src/server/model/events/ReadOnlyEvents.ts

@@ -0,0 +1,20 @@
+import { IEntityQuery } from "@entity-access/entity-access/dist/model/IFilterWithParameter.js";
+import AuthenticatedEvents from "./AuthenticatedEvents.js";
+
+export default class ReadOnlyEvents<T> extends AuthenticatedEvents<T> {
+
+    filter(query: IEntityQuery<T>): IEntityQuery<T> {
+        if (this.verify) {
+            this.sessionUser.ensureLoggedIn();
+        }
+        return query.limit(20);
+    }
+
+    includeFilter(query: IEntityQuery<T>, type?: any, key?: string): IEntityQuery<T> {
+        if (this.verify) {
+            this.sessionUser.ensureLoggedIn();
+        }
+        return query;
+    }
+
+}

package/src/server/model/events/SearchTokenEmailEvents.ts

@@ -0,0 +1,6 @@
+import SearchTokenEmail from "../entities/SearchTokenEmail.js";
+import ReadOnlyEvents from "./ReadOnlyEvents.js";
+
+export default class SearchTokenEmailEvents extends ReadOnlyEvents<SearchTokenEmail> {
+
+}

package/src/server/model/events/SearchTokenEvents.ts

@@ -0,0 +1,6 @@
+import SearchToken from "../entities/SearchToken.js";
+import ReadOnlyEvents from "./ReadOnlyEvents.js";
+
+export default class SearchTokenEvents extends ReadOnlyEvents<SearchToken> {
+
+}

package/src/server/seed/ui/seed-ui.ts

@@ -17,7 +17,7 @@ export default async function seedUI(config: DBConfig) {
     await config.saveVersion(UIPackageConfig, {
         package: "@social-mail/social-mail-client",
         view: "dist/web/AppIndex",
-        version: "1.9.88"
+        version: "1.9.90"
     });
 
     await config.saveVersion(WebComponentsPackageConfig, {

package/src/server/services/EmailAddressService.ts

@@ -5,8 +5,8 @@ import { globalServices } from "../../globalServices.js";
 import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
 import {ICacheContainer, ObjectCache, cacheFor } from "../../common/cacheFor.js";
 import Domain from "../model/entities/Domain.js";
-import RepairEmailNameTagsWorkflow from "../workflows/repair/tags/RepairEmailNameTagsWorkflow.js";
 import { AppWorkflowContext } from "../workflows/AppWorkflowContext.js";
+import IndexEmailAddressWorkflow from "../workflows/email/index/IndexEmailAddressWorkflow.js";
 
 
 
@@ -234,25 +234,14 @@ export default class EmailAddressService implements ICacheContainer<EmailAddress
                 if (name && ea.name !== name) {
                     // schedule for names tag
                     const wc = ServiceProvider.resolve(this, AppWorkflowContext);
-                    await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name });
-                    if (name !== username) {
-                        // we should try to save username as well but without stripping numbers
-                        await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name:username, includeNumbers: true });
-                    }
+                    await IndexEmailAddressWorkflow.queue(wc);
                 }
                 if(!hasMailbox && ((name && ea.name !== name) || ea.domainID !== domainID)) {
                     ea.name = name;
                     ea.domainID = domainID;
 
                     // schedule tag updates...
-
                     await db.emailAddresses.statements.update(ea);
-                    const wc = ServiceProvider.resolve(this, AppWorkflowContext);
-                    await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name });
-                    if (name !== username) {
-                        // we should try to save username as well but without stripping numbers
-                        await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name:username, includeNumbers: true });
-                    }
                     // await db.emailAddresses.saveDirect({
                     //     mode: "update",
                     //     changes: {

package/src/server/services/emails/IndexEmailService.ts

@@ -0,0 +1,65 @@
+import Inject, { RegisterScoped } from "@entity-access/entity-access/dist/di/di.js";
+import SocialMailContext from "../../model/SocialMailContext.js";
+import { Email } from "../../model/entities/Email.js";
+import SearchTokenService from "../search/SearchTokenService.js";
+import { tokenKindType } from "../../model/entities/SearchTokenEmail.js";
+import TextExtractorService from "../extract/TextExtractorService.js";
+import TempFileService from "../../storage/TempFileService.js";
+
+@RegisterScoped
+export default class IndexEmailService {
+
+    @Inject
+    db: SocialMailContext;
+
+    @Inject
+    sts: SearchTokenService;
+
+    @Inject
+    te: TextExtractorService;
+
+    @Inject
+    tempFileService: TempFileService;
+
+    async indexEmail({ emailID, textBody }: Partial<Email>) {
+
+        // delete all search email tokens if exists
+        const { db } = this;
+
+        await db.searchTokenEmails.asQuery()
+            .delete({ emailID }, (p) => (x) => x.emailID === p.emailID);
+
+        await this.insertText(emailID, textBody, "body");
+
+        // for all attachments...
+        const attachments = await db.appFiles.where({ emailID}, (p) => (x) => x.emailID === p.emailID)
+            .toArray();
+        for(const att of attachments) {
+            const tf = await this.tempFileService.downloadAppFile(att);
+            const textFile = await this.te.extract(tf);
+            for await(const line of textFile.lines()) {
+                if (!line) {
+                    continue;
+                }
+                await this.insertText(emailID, line, "attachment");
+            }
+        }
+    }
+
+
+    private async insertText(emailID: number, line: string, tokenKind: tokenKindType) {
+        const words = line.match(/\b[\w\.]{2,399}\b/g);
+        if (!words) {
+            return;
+        }
+        for (const word of words) {
+            const w = word.trim().toLowerCase();
+            if (!w) {
+                continue;
+            }
+            const token = await this.sts.getToken(w);
+            const ste = { tokenID: token.tokenID, emailID, tokenKind };
+            await this.db.searchTokenEmails.statements.selectOrInsert(ste, ste);
+        }
+    }
+}

package/src/server/services/extract/TextExtractorService.ts

@@ -0,0 +1,52 @@
+import Inject, { RegisterScoped } from "@entity-access/entity-access/dist/di/di.js";
+import TempFileService from "../../storage/TempFileService.js";
+import EmailParserService from "../../smtp/services/EmailParserService.js";
+import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
+import { parse } from "path";
+import { Extract } from "./Extract.js";
+import { FileType } from "../../../common/FileType.js";
+
+@RegisterScoped
+export default class TextExtractorService {
+
+    @Inject
+    tempFileService: TempFileService;
+
+    @Inject
+    parser: EmailParserService;
+
+    async extract(file: LocalFile) {
+
+        const { tempFileService, parser } = this;
+
+        const outputFile = await tempFileService.createTempFile(".txt");
+
+        const isEmail = file.contentType === "message/rfc822";
+        if(isEmail) {
+            const parsedEmail = await parser.parse(file, true);
+            await outputFile.appendLine(parsedEmail.text);
+            for (const iterator of parsedEmail.attachments) {
+                const { ext } = parse(iterator.filename || "file.dat");
+                const tf = await tempFileService.createTempFile(ext, iterator.filename, iterator.contentType);
+                await tf.writeAll(iterator.content);
+                const t = await this.extract(tf);
+                if (t) {
+                    await outputFile.appendLine(await t.readAsText());
+                }
+            }
+
+            return outputFile;
+        }
+
+        if (/image\//i.test(file.contentType)) {
+            await Extract.text(file, outputFile, true);
+            return outputFile;
+        } if (FileType.canBeCompressed(file)) {
+            await Extract.text(file, outputFile);
+            return outputFile;
+        }
+        await outputFile.writeAllText("");
+        return outputFile;
+    }
+
+}

package/src/server/services/search/SearchTokenService.ts

@@ -0,0 +1,33 @@
+import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
+import { RegisterSingleton, ServiceProvider } from "@entity-access/entity-access/dist/di/di.js";
+import { cacheFor, ICacheContainer, ObjectCache } from "../../../common/cacheFor.js";
+import ReplicaReader from "../replica/ReplicaReader.js";
+import SocialMailContext from "../../model/SocialMailContext.js";
+
+@RegisterSingleton
+export default class SearchTokenService implements ICacheContainer<SearchTokenService>{
+
+    cache: ObjectCache<SearchTokenService>;
+
+    async getToken(word: string) {
+        word = word.trim().toLowerCase();
+        if  (!word) {
+            throw new EntityAccessError("Word cannot be empty");
+        }
+        return this.getRegisteredToken(word);
+    }
+
+    @cacheFor()
+    private async getRegisteredToken(word: string) {
+        using scope = ServiceProvider.createScope(this);
+        const replica = scope.resolve(ReplicaReader);
+        const db = scope.resolve(SocialMailContext);
+        let existing = await replica.query((x) => x.searchTokens.statements.select({}, { word }));
+        if (existing) {
+            return existing;
+        }
+        existing = await db.searchTokens.statements.selectOrInsert({ word }, { word });
+        return existing;
+    }
+
+}

package/src/server/workflows/SocialWorkflowContext.ts

@@ -55,6 +55,8 @@ import SendLibraryContentWorkflow from "./channel/library/SendLibraryContentWork
 import CreateChannelWorkflow from "./channel/creation/CreateChannelWorkflow.js";
 import { AppWorkflowContext } from "./AppWorkflowContext.js";
 import DeleteFileHistoryWorkflow from "./files/DeleteFileHistoryWorkflow.js";
+import IndexEmailContentWorkflow from "./email/index/IndexEmailContentWorkflow.js";
+import IndexEmailAddressWorkflow from "./email/index/IndexEmailAddressWorkflow.js";
 
 @RegisterSingleton
 export default class SocialWorkflowContext extends AppWorkflowContext {
@@ -121,6 +123,9 @@ export default class SocialWorkflowContext extends AppWorkflowContext {
         this.register(SendLibraryContentWorkflow);
         this.register(CreateChannelWorkflow);
 
+        this.register(IndexEmailContentWorkflow);
+        this.register(IndexEmailAddressWorkflow);
+
         // start processing...
         // this.start().catch((error) => console.error(error));
     }

package/src/server/workflows/daily/DailyWorkflow.ts

@@ -13,6 +13,8 @@ import PendingPostReceiveWorkflow from "../pending/PendingPostReceiveWorkflow.js
 import PendingReferencesWorkflow from "../pending/PendingReferencesWorkflow.js";
 import MessagingService from "../../services/message-events/MessagingService.js";
 import FetchCheckoutResultWorkflow from "../store/FetchCheckoutResultWorkflow.js";
+import IndexEmailContentWorkflow from "../email/index/IndexEmailContentWorkflow.js";
+import IndexEmailAddressWorkflow from "../email/index/IndexEmailAddressWorkflow.js";
 
 export default class DailyWorkflow extends Workflow<any,any> {
 
@@ -55,6 +57,8 @@ export default class DailyWorkflow extends Workflow<any,any> {
 
         await PendingPostReceiveWorkflow.queue(this.context);
         await PendingReferencesWorkflow.queue(this.context);
+        await IndexEmailContentWorkflow.queue(this.context);
+        await IndexEmailAddressWorkflow.queue(this.context);
 
         const all = await db.mailboxSyncTargets
             .where({}, (p) => (x) => x.active === true && x.protocol === "imap" && x.direction === "in")

package/src/server/workflows/email/index/IndexEmailAddressWorkflow.ts

@@ -0,0 +1,65 @@
+import Inject from "@entity-access/entity-access/dist/di/di.js";
+import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import NameTokenService from "../../../services/name-tokens/NameTokenService.js";
+import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
+
+export default class IndexEmailAddressWorkflow extends Workflow {
+
+    static taskGroup = "batch";
+
+    static queue(c: WorkflowContext) {
+        const d = new Date();
+        d.setMinutes(Math.ceil(d.getMinutes()/5));
+        d.setSeconds(0);
+        d.setMilliseconds(0);
+        return c.queue(IndexEmailAddressWorkflow, 0, {
+            id: `index-ea-${d.getTime().toString(36)}`
+        });
+    }
+
+    async run() {
+        let start = 0;
+        for(;;) {
+            start = await this.index(start);
+            if(!BigInt(start)) {
+                break;
+            }
+        }
+    }
+
+    @UniqueActivity
+    async index(
+        start: number,
+        @Inject db?: SocialMailContext,
+        @Inject nts?: NameTokenService
+    ) {
+
+        for(let i=0;i<100;i++) {
+
+            const all = await db.emailAddresses.where({ start }, (p) => (x) =>
+                    x.nameIndexed === false && x.emailAddressID > p.start)
+                .limit(100)
+                .toArray();
+
+            if (!all.length) {
+                return 0;
+            }
+
+            for (const { emailAddressID, name, emailAddress } of all) {
+                start = emailAddressID;
+                if (name) {
+                    await nts.updateName(name, { emailAddressID });
+                }
+                const [username ] = emailAddress.split("@", 2);
+                await nts.updateName(username, { emailAddressID }, true);
+                await db.emailAddresses.statements.update({ nameIndexed: true}, { emailAddressID });
+            }
+
+            db.changeSet.clear();
+        }
+
+        return start;
+    }
+
+}

package/src/server/workflows/email/index/IndexEmailContentWorkflow.ts

@@ -0,0 +1,54 @@
+/* eslint-disable no-console */
+import Inject from "@entity-access/entity-access/dist/di/di.js";
+import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
+import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import IndexEmailService from "../../../services/emails/IndexEmailService.js";
+
+export default class IndexEmailContentWorkflow extends Workflow {
+
+    static taskGroup = "batch";
+
+    static queue(context: WorkflowContext) {
+        const now = DateTime.now;
+        return context.queue(IndexEmailContentWorkflow, "", {
+            id: `index-email-${now.msSinceEpoch.toString(36)}`
+        });
+    }
+
+    async run() {
+        await this.index();
+    }
+
+    @UniqueActivity
+    async index(
+        @Inject db?: SocialMailContext,
+        @Inject indexService?: IndexEmailService
+    ) {
+
+        for(let i = 0;i<1000;i++) {
+            const pending = await db.emails.where(void 0, (p) => (x) => x.searchIndexed === false
+                && x.statePostReceiveInvoked === true)
+                .limit(100)
+                .map(void 0, (p) => (x) => ({ emailID: x.emailID, textBody: x.textBody}))
+                .toArray();
+
+            if (!pending.length) {
+                return;
+            }
+
+            for (const email of pending) {
+                try {
+                    await indexService.indexEmail(email);
+                } catch (error) {
+                    console.error(error);
+                }
+                const { emailID } = email;
+                await db.emails.statements.update({ searchIndexed: true }, { emailID });
+            }
+            db.changeSet.clear();
+        }
+    }
+
+}