@social-mail/social-mail-web-server 1.7.91 → 1.7.95

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.7.91",
+  "version": "1.7.95",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/protocols/core/ReactionExtractor.ts

@@ -14,14 +14,20 @@ export const ReactionExtractor = {
         headers: string | {[key: string]: any}
     }, attachmentsLoader: () => Promise<{ contentType: string }[]>, forceFullMatch = true) {
         try {
-            let inReplyTo = null;
+            let inReplyTo = null as string;
+            let emailID = null as string;
             let engagement: string;
-            const index = email.subject.indexOf(":");
+            let index = email.subject.indexOf(":");
             if (index !== -1 && index <= 50) {
                 engagement = email.subject.substring(0, index);
                 inReplyTo = email.subject.substring(index + 1);
+                index = inReplyTo.indexOf(":");
+                if (index !== -1) {
+                    emailID = inReplyTo.substring(0, index);
+                    inReplyTo = inReplyTo.substring(index+1);
+                }
                 if (/\p{Emoji}/u.test(engagement)) {
-                    return { engagement, inReplyTo };
+                    return { engagement, inReplyTo, emailID };
                 }
             }
 
@@ -40,7 +46,7 @@ export const ReactionExtractor = {
             engagement = matches[0];
 
             if (engagement === textBody) {
-                return { engagement };
+                return { engagement, inReplyTo, emailID };
             }
 
             /**

package/src/server/model/entities/MailboxContactEmail.ts

@@ -62,9 +62,6 @@ export default class MailboxContactEmail {
     @Column({ dataType: "Char", length: 200, nullable: true})
     engagement: string;
 
-    @Column({ dataType: "BigInt", nullable: true })
-    signedFileLogID: number;
-
     email: Email;
 
     mailbox: Mailbox;

package/src/server/seed/ui/seed-ui.ts

@@ -3,7 +3,7 @@ import semver from "semver";
 
 export default async function seedUI(config: DBConfig) {
 
-    const version = "1.4.111";
+    const version = "1.4.115";
 
     const existing = await config.getValue(UIPackageConfig, false);
     if (existing) {

package/src/server/smtp/services/CachedEmailService.ts

@@ -25,11 +25,8 @@ export default class CachedEmailService {
     @Inject
     encryptionService: EncryptionService;
 
-    async getHtmlForRecipient({ emailID, emailAddressID, mailboxID, signedFileLogID }: {
-        emailID: number,
-        emailAddressID?: number,
-        signedFileLogID?: number,
-        mailboxID?: number
+    async getHtmlForRecipient({ emailID }: {
+        emailID: number
     }) {
 
         const factory = async () => {
@@ -38,32 +35,10 @@ export default class CachedEmailService {
 
             let logFile: EmailLog;
 
-            if (signedFileLogID) {
+            if (!logFile) {
                 logFile = await this.db.emailLogs
-                    .where({ signedFileLogID }, (p) => (x) => x.logID === p.signedFileLogID)
-                    .first();
-            } else {
-
-                if (emailAddressID) {
-
-                    const mce = await this.db.mailboxContactEmails.where({ emailID, emailAddressID, mailboxID},
-                        (p) => (x) => x.emailAddressID === p.emailAddressID
-                            && x.emailID === p.emailID
-                            && x.mailboxID === p.mailboxID
-                            && x.signedFileLogID !== null
-                        ).first();
-                    if (mce) {
-                        signedFileLogID = mce.signedFileLogID;
-                        logFile = await this.db.emailLogs
-                            .where({ signedFileLogID }, (p) => (x) => x.logID === p.signedFileLogID)
-                            .first();
-                    }
-                }
-                if (!logFile) {
-                    logFile = await this.db.emailLogs
-                    .where({ emailID }, (p) => (x) => x.mimeFileEmails.some((mf) => mf.emailID === p.emailID))
-                    .first();
-                }
+                .where({ emailID }, (p) => (x) => x.mimeFileEmails.some((mf) => mf.emailID === p.emailID))
+                .first();
             }
 
             let html = "";
@@ -101,7 +76,7 @@ export default class CachedEmailService {
         };
 
         const ef = await this.cacheService.get({
-            key: `cached-email-html-a9-${emailID}-${emailAddressID}-${signedFileLogID}`,
+            key: `cached-email-html-b1-${emailID}`,
             factory,
             ttl: TimeSpan.fromDays(30)
         });

package/src/server/smtp/services/EmailReferenceService.ts

@@ -364,10 +364,7 @@ export default class EmailReferenceService {
 
         const senderMailbox = email.senderEmail?.mailbox;
 
-        const saved = new Set();
-
-
-        if (senderMailbox) {
+        if (senderMailbox && !email.received) {
             for (const { emailAddressID, emailAddress } of email.recipients) {
                 if (!emailAddress) {
                     console.warn(`EmailAddressID ${emailAddressID} in Recipient of EmailID ${emailID} not registered`);
@@ -375,7 +372,6 @@ export default class EmailReferenceService {
                 }
                 const { mailbox } = emailAddress;
                 if (emailAddressID !== senderID) {
-                    saved.add(senderID);
                     // Sender's Thread - Outbox
                     await this.updateThread({
                         engagement,
@@ -399,33 +395,33 @@ export default class EmailReferenceService {
                 //     emailID
                 // });
 
-                if (mailbox) {
-
-                    // this is the case when email was sent from the system
-                    // we will insert in respective mailboxes here...
-                    // await db.mailboxEmails.statements.selectOrInsert({
-                    //     mailboxID: emailAddressID,
-                    //     emailID
-                    // });
-
-                    saved.add(emailAddressID);
-
-                    // Recipient's thread - Inbox
-                    await this.updateThread({
-                        engagement,
-                        mailbox,
-                        mailboxID: emailAddressID,
-                        emailAddressID: senderID,
-                        lastEmailID: emailID,
-                        silent,
-                        timeRead,
-                        dateUpdated,
-                        tagIDs,
-                        email,
-                        newOpenThread,
-                        newThread,
-                    });
-                }
+                // if (mailbox) {
+
+                //     // this is the case when email was sent from the system
+                //     // we will insert in respective mailboxes here...
+                //     // await db.mailboxEmails.statements.selectOrInsert({
+                //     //     mailboxID: emailAddressID,
+                //     //     emailID
+                //     // });
+
+                //     saved.add(emailAddressID);
+
+                //     // Recipient's thread - Inbox
+                //     await this.updateThread({
+                //         engagement,
+                //         mailbox,
+                //         mailboxID: emailAddressID,
+                //         emailAddressID: senderID,
+                //         lastEmailID: emailID,
+                //         silent,
+                //         timeRead,
+                //         dateUpdated,
+                //         tagIDs,
+                //         email,
+                //         newOpenThread,
+                //         newThread,
+                //     });
+                // }
             }
         } else {
 
@@ -436,7 +432,6 @@ export default class EmailReferenceService {
             for (const { emailAddressID, emailAddress: { mailbox } } of email.recipients) {
                 // this is the case when email is received from outside
                 // we will only store in mailboxes that exist on the system
-                saved.add(emailAddressID);
                 if (mailbox) {
                     // await db.mailboxEmails.statements.selectOrInsert({
                     //     mailboxID: emailAddressID,

package/src/server/smtp/services/EmailSerializerService.ts

@@ -2,7 +2,7 @@ import Inject, { RegisterScoped } from "@entity-access/entity-access/dist/di/di.
 import TempFileService from "../../storage/TempFileService.js";
 import { AddressObject } from "mailparser";
 import BaseStorage from "../../storage/BaseStorage.js";
-import { emailStatusType } from "../../model/entities/Email.js";
+import { EmailLog, emailStatusType } from "../../model/entities/Email.js";
 import EmailAddressService from "../../services/EmailAddressService.js";
 import SocialMailContext from "../../model/SocialMailContext.js";
 import type { contentDispositionTypes } from "../../model/entities/AppFile.js";
@@ -27,6 +27,7 @@ import { ReactionExtractor } from "../../../protocols/core/ReactionExtractor.js"
 import { stripBrackets } from "../../../common/stripBrackets.js";
 import SmtpError from "./SmtpError.js";
 import { SmtpStatusCodes } from "../client/SmtpStatusCodes.js";
+import { Base36 } from "../../../common/Base36.js";
 // import PostReceiveWorkflow from "../../workflows/post-receive/PostReceiveWorkflow.js";
 // import UpdateEmailReferences from "../../workflows/incoming/UpdateEmailReferences.js";
 
@@ -91,6 +92,19 @@ export interface IEmailInfo {
     localEmailID?: number;
 };
 
+export interface ISaveEmailInfo {
+    sender?: string,
+    localEmailID?: number,
+    mailboxes: Mailbox[],
+    received?: boolean,
+    silent?: boolean,
+    isDraft?: boolean,
+    ars?: any,
+    log?: string,
+    labels?: string[],
+    externalEmailID?: string
+}
+
 @RegisterScoped
 export default class EmailSerializerService {
 
@@ -186,18 +200,7 @@ export default class EmailSerializerService {
         localEmailID,
         externalEmailID,
         labels
-    }: {
-        sender?: string,
-        localEmailID?: number,
-        mailboxes: Mailbox[],
-        received?: boolean,
-        silent?: boolean,
-        isDraft?: boolean,
-        ars?: any,
-        log?: string,
-        labels?: string[],
-        externalEmailID?: string
-    }) {
+    }: ISaveEmailInfo) {
 
         const { tempFileService, db } = this;
         db.verifyFilters = false;
@@ -360,12 +363,10 @@ export default class EmailSerializerService {
             tags = labels.map((x) => x.startsWith("#") ? x : `#${x}`).join(",");
         }
 
-        // console.log(`Saving Log ${file.path}`);
-        const mimeFile = await this.emailLogService.save(0, file);
-        // console.log(`Saved Log ${file.path}`);
-
         let engagement = null;
 
+        let verifyParentID = null;
+
         if (received) {
             // check for engagements...
             const e1 = await ReactionExtractor.extractWithInReplyTo({
@@ -377,6 +378,7 @@ export default class EmailSerializerService {
             engagement = e1.engagement;
             if (!inReplyTo) {
                 inReplyTo = e1.inReplyTo;
+                verifyParentID = Base36.decode(e1.emailID) as any;
             }
         }
 
@@ -386,11 +388,22 @@ export default class EmailSerializerService {
                 .orderBy(void 0,(p) => (x) => x.emailID)
                 .first();
             if (original) {
+
+                if (verifyParentID) {
+                    if(verifyParentID !== original.emailID) {
+                        throw new SmtpError("Unable to process this email.", SmtpStatusCodes.DeliveryNotPermitted);
+                    }
+                }
+
                 parentID = original.emailID;
                 inReplyTo = null;
             }
         }
 
+        // console.log(`Saving Log ${file.path}`);
+        const mimeFile = await this.emailLogService.save(0, file);
+        // console.log(`Saved Log ${file.path}`);
+
         let emailID = void 0;
 
         if (localEmailID) {

package/src/server/workflows/pending/PendingReferencesWorkflow.ts

@@ -13,7 +13,7 @@ export default class PendingReferencesWorkflow extends Workflow {
     static taskGroup = "default";
 
     static queue(context: WorkflowContext) {
-        return context.queue(PendingReferencesWorkflow, {}, { id: `update-references`} );
+        return context.queue(PendingReferencesWorkflow, {}, { id: `update-references-${Date.now()}`, throttle: { group: "update-ref", maxPerSecond: 1 }} );
     }
 
     async run() {

package/src/server/workflows/smtp/DispatchEmailToMX.ts

@@ -18,6 +18,7 @@ import EmailReferenceService from "../../smtp/services/EmailReferenceService.js"
 import EmailPostReceiveService from "../../smtp/services/EmailPostReceiveService.js";
 import TagService from "../../services/tags/TagService.js";
 import { IMessageEnvelope } from "./IMessageEnvelope.js";
+import EmailSerializerService from "../../smtp/services/EmailSerializerService.js";
 
 export const smtpPool = new NamedObjectPool<SMTPClient>((host) => ({
     asyncFactory: () => new SMTPClient({ host }).connect(),
@@ -148,113 +149,121 @@ export default class DispatchEmailToMX extends Workflow<IMessageEnvelope> {
         attempt: number,
         @Inject els?: EmailLogService,
         @Inject eas?: EmailAddressService,
-        @Inject db?: SocialMailContext,
-        @Inject eps?: EmailPostReceiveService,
-        @Inject ts?: TagService,
-        @Inject ers?: EmailReferenceService
+        @Inject ess?: EmailSerializerService,
     ) {
 
         // if it is local...
         // save signedFileLogID to MailboxContactEmail
         const { senderID, emailID, recipientAddressID, external } = this.input;
         const { signedFileLogID } = this;
-        const mailboxID = await eas.getHostedMailboxID(this.input.to[0]);
-
-        if (!external && mailboxID) {
-
-            const dateReceived = DateTime.now;
-            try {
-
-                // internal broad casting is not working
-                // we either need to create an email or add the recipient
-
-                const emailAddressID = senderID;
-
-                const email = await db.emails.loadByKeys({ emailID });
-                await db.changeSet.getEntry(email).loadNavigationAsync((x) => x.emailTags);
-
-                const tagIDs = email.emailTags?.map((x) => x.tagID);
-
-                // await ers.updateThread({
-                //     email,
-                //     tagIDs,
-
-                // });
-
-                const folderTagID = await ts.getSystemTagID("New");
-
-                let addFolder = true;
-
-                await db.mailboxContacts.statements.upsert({
-                    mailboxID,
-                    emailAddressID,
-                    folderTagID,
-                    newRequest: true,
-                }, (x) => {
-                    addFolder = false;
-                    x.unread++;
-                    x.total++;
-                    x.dateUpdated = dateReceived;
-                    if (!email.threadID) {
-                        x.openThreads++;
-                        x.totalThreads++;
-                    }
-                    x.lastEmailID = emailID;
-                    return x;
-                }, {
-                    mailboxID,
-                    emailAddressID
-                });
-
-                if (addFolder) {
-                    await db.mailboxFolders.statements.upsert({
-                        mailboxID,
-                        tagID: folderTagID,
-                        count: 1
-                    }, (x) => {
-                        x.count++;
-                        return x;
-                    }, {
-                        mailboxID,
-                        tagID: folderTagID
-                    });
-                }
 
-                if (tagIDs) {
-                    for (const tagID of tagIDs) {
-                        await db.mailboxContactTags.statements.selectOrInsert({
-                            tagID,
-                            emailAddressID,
-                            mailboxID
-                        });
-                        await db.mailboxTags.statements.selectOrInsert({
-                            tagID,
-                            mailboxID
-                        });
-                    }
-                }
+        const mailbox = await eas.getMailbox(this.input.to[0]);
 
-                // await db.mailboxEmails.statements.selectOrInsert({
-                //     mailboxID,
-                //     emailID
-                // });
+        if (!external && mailbox) {
 
-                await db.mailboxContactEmails.statements.selectOrInsert({
-                    mailboxID,
-                    emailID,
-                    emailAddressID,
-                    signedFileLogID,
-                    dateReceived
-                });
 
+            const localFile = await els.load({ logID: signedFileLogID });
 
-                // await ers.updateReferences({ emailID, dateUpdated: dateReceived, silent: false, timeRead: null });
-                if (senderID !== emailAddressID) {
-                    await eps.execute({ senderID, emailID, recipientID: emailAddressID });
-                }
-            } catch {
+            await ess.save(localFile, {
+                mailboxes: [mailbox],
+                isDraft: false,
+                received: true,
+                sender: this.input.from
+            });
 
-            }
+            // const dateReceived = DateTime.now;
+            // try {
+
+            //     // internal broad casting is not working
+            //     // we either need to create an email or add the recipient
+
+            //     const emailAddressID = senderID;
+
+            //     const email = await db.emails.loadByKeys({ emailID });
+            //     await db.changeSet.getEntry(email).loadNavigationAsync((x) => x.emailTags);
+
+            //     const tagIDs = email.emailTags?.map((x) => x.tagID);
+
+            //     // await ers.updateThread({
+            //     //     email,
+            //     //     tagIDs,
+
+            //     // });
+
+            //     const folderTagID = await ts.getSystemTagID("New");
+
+            //     let addFolder = true;
+
+            //     await db.mailboxContacts.statements.upsert({
+            //         mailboxID,
+            //         emailAddressID,
+            //         folderTagID,
+            //         newRequest: true,
+            //     }, (x) => {
+            //         addFolder = false;
+            //         x.unread++;
+            //         x.total++;
+            //         x.dateUpdated = dateReceived;
+            //         if (!email.threadID) {
+            //             x.openThreads++;
+            //             x.totalThreads++;
+            //         }
+            //         x.lastEmailID = emailID;
+            //         return x;
+            //     }, {
+            //         mailboxID,
+            //         emailAddressID
+            //     });
+
+            //     if (addFolder) {
+            //         await db.mailboxFolders.statements.upsert({
+            //             mailboxID,
+            //             tagID: folderTagID,
+            //             count: 1
+            //         }, (x) => {
+            //             x.count++;
+            //             return x;
+            //         }, {
+            //             mailboxID,
+            //             tagID: folderTagID
+            //         });
+            //     }
+
+            //     if (tagIDs) {
+            //         for (const tagID of tagIDs) {
+            //             await db.mailboxContactTags.statements.selectOrInsert({
+            //                 tagID,
+            //                 emailAddressID,
+            //                 mailboxID
+            //             });
+            //             await db.mailboxTags.statements.selectOrInsert({
+            //                 tagID,
+            //                 mailboxID
+            //             });
+            //         }
+            //     }
+
+            //     // await db.mailboxEmails.statements.selectOrInsert({
+            //     //     mailboxID,
+            //     //     emailID
+            //     // });
+
+            //     await db.mailboxContactEmails.statements.selectOrInsert({
+            //         mailboxID,
+            //         emailID,
+            //         emailAddressID,
+            //         signedFileLogID,
+            //         dateReceived
+            //     });
+
+
+            //     // await ers.updateReferences({ emailID, dateUpdated: dateReceived, silent: false, timeRead: null });
+            //     if (senderID !== emailAddressID) {
+            //         await eps.execute({ senderID, emailID, recipientID: emailAddressID });
+            //     }
+            // } catch {
+
+            // }
 
             // await PostReceiveWorkflow.queue(this.context, {
             //     senderID: this.input.senderID,