@social-mail/social-mail-web-server 1.4.921 → 1.4.923

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.4.921",
+  "version": "1.4.923",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/common/spawnPromise.ts

@@ -24,7 +24,7 @@ export const spawnPromise = (path, args?: string[], options?: SpawnOptionsWithou
     cd.on("error", (error) => {
         const errorText = color.red(error.stack ?? error.toString());
         all.push(error.stack ?? error.toString());
-        if (logData) {
+        if (logData || logError) {
             console.error(errorText);
         }
         reject(error);

package/src/server/model/SocialMailContext.ts

@@ -43,6 +43,7 @@ import AccessToken from "./entities/AccessToken.js";
 import ServerActivity from "./entities/ServerActivity.js";
 import IncomingSmsSource from "./entities/IncomingSmsSource.js";
 import VerificationToken, { VerificationTokenAddress } from "./entities/VerificationToken.js";
+import MailboxFolder from "./entities/MailboxFolder.js";
 
 @RegisterScoped
 export default class SocialMailContext extends EntityContext {
@@ -107,6 +108,8 @@ export default class SocialMailContext extends EntityContext {
 
     public mailboxContacts = this.model.register(MailboxContact);
 
+    public mailboxFolders = this.model.register(MailboxFolder);
+
     public mailboxContactEmails = this.model.register(MailboxContactEmail);
 
     public mailboxContactTags = this.model.register(MailboxContactTag);

package/src/server/model/entities/Mailbox.ts

@@ -19,6 +19,7 @@ import { IgnoreJsonProperty, JsonProperty, ReadOnlyJsonProperty } from "@entity-
 import type ChannelConfig from "./ChannelConfig.js";
 import type { WebSiteChannel } from "./WebSiteFolder.js";
 import Domain from "./Domain.js";
+import type MailboxFolder from "./MailboxFolder.js";
 
 /**
  * MailBox ID is number because foreign key on Email will
@@ -206,6 +207,7 @@ export default class Mailbox {
     public subscriberActivities: SubscriberActivity[];
 
     public websiteChannels: WebSiteChannel[];
+    folders: MailboxFolder[];
 
 
 }

package/src/server/model/entities/MailboxContact.ts

@@ -25,9 +25,10 @@ export type subscriberStatusType = typeof subscriberStatus[number];
 
 @Table("MailboxContacts")
 @Index({
-    name: "IX_MailBox_Thread_DateUpdated",
+    name: "IX_MailBox_Thread_Tagged_DateUpdated",
     columns: [
         { name: (x) => x.mailboxID, descending: false },
+        { name: (x) => x.folderTagID, descending: false },
         { name: (x) => x.dateUpdated, descending: true }
     ]
 })
@@ -59,6 +60,9 @@ export default class MailboxContact {
     })
     public emailAddressID: number;
 
+    @Column({ dataType: "BigInt", nullable: true })
+    public folderTagID: number;
+
     @Column({ dataType: "BigInt", nullable: true})
     @RelateTo(Email, {
         property: (mt) => mt.lastEmail,
@@ -118,6 +122,12 @@ export default class MailboxContact {
     @Column({ dataType: "Boolean", default: () => false})
     public blocked: boolean;
 
+    @Column({ dataType: "Boolean", default: () => false})
+    public newRequest: boolean;
+
+    @Column({ dataType: "Boolean", default: () => false})
+    public spam: boolean;
+
     @Column({ dataType: "Boolean", default: () => false})
     public mute: boolean;
 

package/src/server/model/entities/MailboxFolder.ts

@@ -0,0 +1,37 @@
+import Column from "@entity-access/entity-access/dist/decorators/Column.js";
+import { RelateTo } from "@entity-access/entity-access/dist/decorators/Relate.js";
+import Table from "@entity-access/entity-access/dist/decorators/Table.js";
+import Mailbox from "./Mailbox.js";
+import Tag from "./Tag.js";
+
+@Table("MailboxFolders")
+export default class MailboxFolder {
+
+    @Column({ dataType: "BigInt", key: true })
+    @RelateTo(Mailbox, {
+        property: (x) => x.mailbox,
+        inverseProperty: (x) => x.folders,
+        foreignKeyConstraint: {
+            cascade: "delete"
+        }
+    })
+    mailboxID: number;
+
+    @Column({ dataType: "BigInt", key: true })
+    @RelateTo(Tag, {
+        property: (x) => x.tag,
+        inverseProperty: (x) => x.mailboxFolders,
+        foreignKeyConstraint: {
+            cascade: "restrict"
+        }
+    })
+    tagID: number;
+
+    @Column({ dataType: "Int", default: () => 0})
+    count: number;
+
+    mailbox: Mailbox;
+
+    tag: Tag;
+
+}

package/src/server/model/entities/Tag.ts

@@ -9,6 +9,7 @@ import { IgnoreJsonProperty, ReadOnlyJsonProperty } from "@entity-access/server-
 import type MailboxContactTag from "./MailboxContactTag.js";
 import type EmailTag from "./EmailTag.js";
 import type { StoreItemFolderTag, StoreItemTag, StoreTag } from "./Store.js";
+import type MailboxFolder from "./MailboxFolder.js";
 
 /**
  * The reason tags table exists in form of string,bigint pair is, we need
@@ -81,4 +82,5 @@ export default class Tag {
     public storeItemTags: StoreItemTag[];
     storeTags: StoreTag[];
     storeItemFolderTags: StoreItemFolderTag[];
+    mailboxFolders: MailboxFolder[];
 }

package/src/server/model/events/MailboxContactEvents.ts

@@ -8,6 +8,8 @@ import { ForeignKeyFilter } from "@entity-access/entity-access/dist/model/events
 import SocialMailContext from "../SocialMailContext.js";
 import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
 import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
+import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
+import TagService from "../../services/tags/TagService.js";
 
 export default class MailboxContactEvents extends AuthenticatedEvents<MailboxContact> {
 
@@ -120,6 +122,64 @@ export default class MailboxContactEvents extends AuthenticatedEvents<MailboxCon
 
         }
 
+        let tagUpdated = false;
+
+        if (entry.isModified("folderTagID")) {
+            tagUpdated = true;
+            const oldID = entry.getChange("folderTagID").oldValue;
+            await db.mailboxFolders.where({ mailboxID, oldID }, (p) => (x) => x.mailboxID === p.mailboxID && x.tagID === p.oldID)
+                .update(void 0, (p) => (x) => ({
+                    count: Sql.math.max(0, x.count-1)
+                }));
+            await db.mailboxFolders.where({ mailboxID, tagID: entity.folderTagID },
+                    (p) => (x) => x.mailboxID === p.mailboxID && x.tagID === p.tagID)
+                .update(void 0, (p) => (x) => ({
+                    count: x.count + 1
+                }));
+
+            const ts = ServiceProvider.resolve(this, TagService);
+            const spam = await ts.getSystemTagID("Spam");
+            const newRequest = await ts.getSystemTagID("New");
+            const isBlocked = await ts.getSystemTagID("Blocked");
+
+            entity.spam = spam === entity.folderTagID;
+            entity.newRequest = newRequest === entity.folderTagID;
+            entity.blocked = isBlocked === entity.folderTagID;
+            entry.detect();
+        }
+
+        if ( !tagUpdated && (entry.isModified("spam") || entry.isModified("newRequest") || entry.isModified("blocked"))) {
+            // update tag folder...
+            // only if older folder is one of the system folder...
+
+            const ts = ServiceProvider.resolve(this, TagService);
+            const systemIDs = await ts.getSystemTagIDs(["Contacts", "New", "Spam", "Blocked"]);
+            if (systemIDs.includes(entity.folderTagID)) {
+                const oldID = entity.folderTagID;
+                const tagID = await ts.getSystemTagID(entity.spam
+                    ? "Spam"
+                    : entity.newRequest
+                        ? "New"
+                        : entity.blocked
+                            ? "Blocked"
+                            : "Contacts");
+                entity.folderTagID = tagID;
+                if (tagID !== oldID) {
+                    await db.mailboxFolders
+                        .where({ mailboxID, oldID }, (p) =>
+                            (x) => x.mailboxID === p.mailboxID && x.tagID === p.oldID)
+                        .update(void 0, (p) => (x) => ({
+                            count: Sql.math.max(0, x.count-1)
+                        }));
+                    await db.mailboxFolders
+                        .where({ mailboxID, tagID }, (p) =>
+                            (x) => x.mailboxID === p.mailboxID && x.tagID === p.tagID)
+                        .update(void 0, (p) => (x) => ({
+                            count: x.count + 1
+                        }));
+                }
+            }
+        }
 
         if (entry.isModified("autoClose") || entry.isModified("blocked")) {
             if (autoClose || blocked) {

package/src/server/seed/tags/seed-tags.ts

@@ -9,7 +9,7 @@ interface IFileTag {
 
 interface ITagParent {
     name: string;
-    children: ITagLike[];
+    children?: ITagLike[];
     contentTypes?: string[];
 }
 
@@ -154,6 +154,23 @@ export default async function seedTags(context: SocialMailContext) {
         ]
     });
 
+    await addTag(context, {
+        name: "Contacts",
+    });
+
+    await addTag(context, {
+        name: "New",
+    });
+
+    await addTag(context, {
+        name: "Spam",
+    });
+
+    await addTag(context, {
+        name: "Blocked",
+    });
+
+
     await context.saveChanges();
 
     await updatePath(context);

package/src/server/services/captcha/CaptchaService.ts

@@ -94,7 +94,8 @@ export default class CaptchaService {
             "-map", "[v]",
              "-y", tf.path
         ], {
-            logData: false
+            logData: false,
+            logCommand: false
         });
 
         await spawnPromise("/ffmpeg/ffmpeg", [
@@ -102,7 +103,8 @@ export default class CaptchaService {
             "-filter:v", "scale=300:-1",
             "-y", saveAs
         ],{
-            logData: false
+            logData: false,
+            logCommand: false
         });
     }
 

package/src/server/services/tags/TagService.ts

@@ -2,10 +2,33 @@
 import { RegisterSingleton, ServiceProvider } from "@entity-access/entity-access/dist/di/di.js";
 import SocialMailContext from "../../model/SocialMailContext.js";
 import { ConsoleLogger } from "@entity-access/entity-access/dist/common/Logger.js";
+import TimedCache from "@entity-access/entity-access/dist/common/cache/TimedCache.js";
+import Tag from "../../model/entities/Tag.js";
 
 @RegisterSingleton
 export default class TagService {
 
+    map = new Map<string, number>();
+
+    async getSystemTagIDs(names: string[]) {
+        const ids = [];
+        for (const name of names) {
+            ids.push(await this.getSystemTagID(name));
+        }
+        return ids;
+    }
+
+    async getSystemTagID(name: string) {
+        name = name.trim().toLocaleLowerCase();
+        let tagID = this.map.get(name);
+        if (!tagID) {
+            const tags = await this.getTagID(name);
+            tagID = tags.pop();
+            this.map.set(name, tagID);
+        }
+        return tagID;
+    }
+
     async getTagID(name: string): Promise<number[]> {
         if (!name) {
             return [];

package/src/server/smtp/services/EmailReferenceService.ts

@@ -390,20 +390,50 @@ export default class EmailReferenceService {
         const unread = email.senderID === mailboxID || silent ? 0 : 1;
         const total = 1;
 
+        let newRequest = email.received && email.senderID !== mailboxID;
+
+        let blocked = false;
+        let spam = false;
+
+        let addFolder = true;
+
+        const name = spam
+            ? "Spam"
+            : blocked
+                ? "Blocked"
+                : newRequest
+                    ? "New"
+                    : "Contacts";
+
+        let tagID = await this.tagService.getSystemTagID(name);
+
         await db.mailboxContacts.statements.upsert({
                 emailAddressID,
                 mailboxID,
+                folderTagID: tagID,
                 openThreads: newOpenThread === 1 ? 1 : 0,
                 totalThreads: newThread === 1 ? 1 : 0,
                 lastEmailID: emailID,
+                newRequest,
                 dateUpdated,
+                blocked,
+                spam,
                 unread,
                 total
             },
             (item) => {
+                addFolder = false;
                 item.unread = item.unread ? item.unread + ( unread ? 1 : 0 ) : 0;
                 item.total = item.total ? item.total + 1 : 1;
                 item.dateUpdated = dateUpdated;
+                newRequest = item.newRequest;
+                blocked = item.blocked;
+                spam = item.spam;
+                if(!item.folderTagID) {
+                    item.folderTagID = tagID;
+                } else {
+                    tagID = item.folderTagID;
+                }
                 if (item.autoClose || item.blocked) {
                     newOpenThread = 0;
                     newThread = 0;
@@ -417,6 +447,20 @@ export default class EmailReferenceService {
             }
         );
 
+        if (addFolder) {
+            await db.mailboxFolders.statements.upsert({
+                mailboxID,
+                tagID,
+                count: 1
+            }, (x) => {
+                x.count++;
+                return x;
+            }, {
+                mailboxID,
+                tagID
+            });
+        }
+
 
         await db.mailboxes.where({ mailboxID }, (p) => (x) => x.mailboxID === p.mailboxID)
             .update({ newThread, newOpenThread, unread: unread ? 1 : 0 }, (p) => (x) => ({

package/src/server/smtp/services/EmailSerializerService.ts

@@ -105,7 +105,11 @@ const stripDuplicates = (addresses: IEmailRecipientInfo[]) => {
 };
 
 export interface IEmailInfo {
-    externalEmailID: string, labels: string[], mailbox: Mailbox, silent?: boolean,
+    externalEmailID: string,
+    labels: string[],
+    mailbox: Mailbox,
+    silent?: boolean,
+    receivedViaSMTP?: boolean,
     messageId: string
 };
 
@@ -147,7 +151,7 @@ export default class EmailSerializerService {
     }
 
     async saveRFC822File(file: LocalFile, emailInfo: IEmailInfo) {
-        const { externalEmailID, labels, mailbox, silent, messageId} = emailInfo;
+        const { externalEmailID, labels, mailbox, silent, messageId, receivedViaSMTP} = emailInfo;
         const { tempFileService, db } = this;
         // check if email already saved in same mailbox...
         const { mailboxID } = mailbox;
@@ -167,7 +171,7 @@ export default class EmailSerializerService {
         try {
             // scan for viruses....
             await AVService.scan(file);
-            await this.save(file, { mailboxes, externalEmailID, silent, labels });
+            await this.save(file, { mailboxes, externalEmailID, silent, labels, received: receivedViaSMTP });
         } catch (error) {
             if (error instanceof FileInfectedError) {
 

package/src/server/workflows/repair/mailbox-folders/RepairMailboxFoldersWorkflow.ts

@@ -0,0 +1,77 @@
+import Inject from "@entity-access/entity-access/dist/di/di.js";
+import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
+import SocialMailContext from "../../../model/SocialMailContext.js";
+import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
+import Mailbox from "../../../model/entities/Mailbox.js";
+import TagService from "../../../services/tags/TagService.js";
+
+export default class RepairMailboxFoldersWorkflow extends Workflow {
+
+    static taskGroup = "batch";
+
+    async run() {
+        let start = 0;
+        do{
+            start = await this.fixEmailTags(start);
+        } while(BigInt(start));
+    }
+
+    @UniqueActivity
+    async fixEmailTags(
+        start: number,
+        @Inject db?: SocialMailContext,
+        @Inject ts?: TagService
+    ) {
+
+        const contacts = await ts.getSystemTagID("Contacts");
+        const isNew = await ts.getSystemTagID("New");
+        const isBlocked = await ts.getSystemTagID("Blocked");
+        const isSpam = await ts.getSystemTagID("Spam");
+
+        for(let i = 0;i< 100;i++) {
+            const mailboxes = await db.mailboxes.where({ start }, (p) => (x) => x.mailboxID > p.start)
+                .orderBy(void 0, (p) => (x) => x.mailboxID)
+                .limit(10)
+                .include((x) => [x.folders])
+                .toArray();
+
+            if (!mailboxes.length) {
+                return 0;
+            }
+
+            for (const iterator of mailboxes) {
+                start = iterator.mailboxID;
+                // fix tag...
+                const counts = await db.mailboxContacts.where(iterator, (p) => (x) => x.mailboxID === p.mailboxID)
+                    .sum(void 0, (p) => (x) => ( {
+                        blocked: x.blocked === true ? 1 : 0,
+                        newRequest: x.newRequest === true ? 1 : 0,
+                        spam: x.spam === true ? 1 : 0,
+                        total: 1
+                    }));
+
+                await this.saveFolder(db, iterator, contacts, counts.total);
+                await this.saveFolder(db, iterator, isNew, counts.total);
+                await this.saveFolder(db, iterator, isBlocked, counts.total);
+                await this.saveFolder(db, iterator, isSpam , counts.total);
+
+            }
+            db.changeSet.clear();
+        }
+        return start;
+    }
+
+    async saveFolder(db: SocialMailContext, { mailboxID }: Mailbox, tagID: number, count: number) {
+        await db.mailboxFolders.statements.upsert({
+            mailboxID,
+            tagID,
+            count
+        }, (x) => {
+            x.count = count;
+            return x;
+        }, {
+            mailboxID,
+            tagID
+        });
+    }
+}

package/src/server/workflows/sync/SyncMailboxWorkflow.ts

@@ -260,6 +260,7 @@ export default class SyncMailboxWorkflow extends Workflow<any, any> {
                     messageId,
                     labels,
                     mailbox: target.mailbox,
+                    receivedViaSMTP: false,
                     silent: true
                 });
                 // console.log(`Saved ${uid}`);