@social-mail/social-mail-web-server 1.7.85 → 1.7.87

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.7.85",
+  "version": "1.7.87",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",
@@ -21,8 +21,8 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.717.0",
     "@azure/storage-blob": "^12.25.0",
-    "@entity-access/entity-access": "^1.0.401",
-    "@entity-access/server-pages": "^1.1.400",
+    "@entity-access/entity-access": "^1.0.402",
+    "@entity-access/server-pages": "^1.1.401",
     "@ffmpeg/core-mt": "^0.12.6",
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.1",

package/src/common/globalEnv.ts

@@ -16,6 +16,8 @@ if (Number.isNaN(port)) {
     port = process.env.PORT as unknown as number;
 }
 
+const serverID = process.env.SOCIAL_MAIL_SERVER_ID || "0";
+
 export const globalEnv = {
     protocol: void 0,
     strictSSL: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== "0",
@@ -48,7 +50,7 @@ export const globalEnv = {
     smtp2http: {
         publicKey: process.env.SOCIAL_MAIL_SMTP2HTTP_PUBLIC_KEY
     },
-    serverID: process.env.SOCIAL_MAIL_SERVER_ID,
+    serverID,
     parentApiToken: process.env.SOCIAL_MAIL_PARENT_API_TOKEN,
     forceHttp1: false,
     isTestMode,
@@ -103,6 +105,7 @@ export const globalEnv = {
     },
     tmpDir: process.env.SOCIAL_MAIL_TMP_PATH || tmpdir(),
     tmpSize: process.env.SOCIAL_MAIL_TMP_SIZE || "7gb",
+    cacheDir: join((process.env.SOCIAL_MAIL_CACHE_PATH || "/cache"), serverID),
     serverPackage: process.env.npm_package_name,
     serverPackageVersion: process.env.npm_package_version,
     push: {

package/src/protocols/core/ReactionExtractor.ts

@@ -1,36 +1,46 @@
 export const ReactionExtractor = {
 
-    async extract(email: {
+    extract(email: {
+        subject: string,
+        textBody: string,
+        headers: string | {[key: string]: any}
+    }, attachmentsLoader: () => Promise<{ contentType: string }[]>, forceFullMatch = true) {
+        return this.extractWithInReplyTo(email, attachmentsLoader, forceFullMatch).then(({ engagement } = { engagement: void 0 }) => engagement);
+    },
+
+    async extractWithInReplyTo(email: {
         subject: string,
         textBody: string,
         headers: string | {[key: string]: any}
     }, attachmentsLoader: () => Promise<{ contentType: string }[]>, forceFullMatch = true) {
         try {
+            let inReplyTo = null;
             let engagement: string;
             const index = email.subject.indexOf(":");
             if (index !== -1 && index <= 50) {
                 engagement = email.subject.substring(0, index);
+                inReplyTo = email.subject.substring(index + 1);
                 if (/\p{Emoji}/u.test(engagement)) {
-                    return engagement;
+                    return { engagement, inReplyTo };
                 }
             }
 
             let { textBody } = email;
             textBody = textBody?.trim();
             if (!textBody) {
-                return;
+                return {};
             }
 
             const matches = /^\p{Emoji}/u.exec(textBody);
 
             if(!matches) {
-                return;
+                return {};
             }
 
             engagement = matches[0];
 
             if (engagement === textBody) {
-                return engagement;
+                return { engagement };
             }
 
             /**
@@ -51,26 +61,27 @@ export const ReactionExtractor = {
                     : {};
             const jsonEngagement = jsonHeaders["x-engagement"] ?? jsonHeaders["x-reaction"];
             if (jsonEngagement) {
-                return jsonEngagement;
+                return { engagement: jsonEngagement };
             }
 
             const attachments = await attachmentsLoader();
             if (attachments) {
                 for (const iterator of attachments) {
                     if (/reaction/i.test(iterator.contentType) && /json/i.test(iterator.contentType)) {
-                        return engagement;
+                        return { engagement };
                     }
                 }
             }
 
             if (forceFullMatch) {
-                return null;
+                return { engagement: null };
             }
-            return engagement;
+            return { engagement };
         } catch (error) {
             // eslint-disable-next-line no-console
             console.error(error);
         }
+        return {};
     }
 
 };

package/src/server/model/entities/Mailbox.ts

@@ -164,6 +164,9 @@ export default class Mailbox {
     @Column({ dataType: "Boolean", default: () => false })
     public isPrivateChannel: boolean;
 
+    @Column({ dataType: "Boolean", default: () => false })
+    public membersCanPostTopLevel: boolean;
+
     @Column({ dataType: "Boolean", default: () => false })
     public nonMembersCanPost: boolean;
 

package/src/server/seed/ui/seed-ui.ts

@@ -3,7 +3,7 @@ import semver from "semver";
 
 export default async function seedUI(config: DBConfig) {
 
-    const version = "1.4.106";
+    const version = "1.4.107";
 
     const existing = await config.getValue(UIPackageConfig, false);
     if (existing) {

package/src/server/services/cache/DiskCacheService.ts

@@ -15,14 +15,9 @@ import sleep from "../../../common/sleep.js";
 import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
 import { parentPort } from "worker_threads";
 import { unlink } from "fs/promises";
-import { isTestMode } from "../../../globalServices.js";
-import { tmpdir} from "node:os";
 
-const cacheRoot = isTestMode
-    ? join(globalEnv.tmpDir, "tc_" + Date.now().toString())
-    :  "/cache";
-
-const serverCache = join(cacheRoot, globalEnv.serverID ?? "0");
+const cacheRoot = globalEnv.cacheDir;
+const serverCache = globalEnv.cacheDir;
 
 ensureDir(serverCache);
 

package/src/server/services/email-logs/EmailLogService.ts

@@ -44,7 +44,7 @@ export default class EmailLogService {
         await this.db.emailLogs.statements.delete({ logID });
         await tx.commit();
 
-        await this.updateStorage({ emailID: log, size: -log.fileSize});
+        await this.updateStorage({ emailID: log.emailID, size: -log.fileSize});
     }
 
     async save(emailID: number, file: LocalFile, compression: string = "gz") {

package/src/server/smtp/services/EmailPostReceiveService.ts

@@ -15,6 +15,7 @@ import CachedEmailService from "./CachedEmailService.js";
 import QueueInvokeWebHookWorkflow from "../../workflows/web-hooks/QueueInvokeWebHookWorkflow.js";
 import EmailThreadService from "../../services/threads/EmailThreadService.js";
 import ChannelModerationService from "../../services/channels/ChannelModerationService.js";
+import Mailbox from "../../model/entities/Mailbox.js";
 
 @RegisterScoped
 export default class EmailPostReceiveService {
@@ -40,14 +41,11 @@ export default class EmailPostReceiveService {
     @Inject
     ets?: EmailThreadService;
 
-    @Inject
-    cms?: ChannelModerationService;
-
     async executeAll({ senderID, emailID }) {
         const { db } = this;
         const mcs = await db.mailboxContactEmails.where({ emailID, senderID}, (p) => (x) => x.emailID === p.emailID
             && x.mailboxID !== p.senderID )
-            .select(void 0, (p) => ({ mailboxID }) => ({ mailboxID }))
+            .select(void 0, () => ({ mailboxID }) => ({ mailboxID }))
             .toArray();
 
         for (const { mailboxID } of mcs) {
@@ -61,7 +59,17 @@ export default class EmailPostReceiveService {
     }
 
     async execute(input: { senderID, emailID, recipientID }) {
-        await this.dispatchChannel(input);
+
+        const { eas } = this;
+
+        let channel = await eas.getHostedMailbox(input.recipientID);
+        channel = channel?.isChannel ? channel : void 0;
+
+        if (channel) {
+            await this.dispatchChannel(channel, input);
+        } else {
+            await this.dispatchMailbox(input);
+        }
         await this.queueWebHooks(input);
     }
 
@@ -100,9 +108,9 @@ export default class EmailPostReceiveService {
         }
     }
 
-    async dispatchChannel(input: { senderID, emailID, recipientID }) {
+    async dispatchChannel(channel: Mailbox, input: { senderID, emailID, recipientID }) {
 
-        const { db, eas, channelService, tfs, ces } = this;
+        const { db, channelService, tfs, ces } = this;
 
         const { senderID, emailID, recipientID } = input;
 
@@ -118,53 +126,45 @@ export default class EmailPostReceiveService {
             return;
         }
 
-        let channel = await eas.getHostedMailbox(recipientID);
-        channel = channel?.isChannel ? channel : void 0;
-
         if (ne.engagement) {
 
             await db.mailboxes.where(input, (p) => (x) => x.mailboxID === p.recipientID)
-                .update(void 0, (p) => (x) => ({
+                .update(void 0, () => () => ({
                     dateEngaged: Sql.date.now()
                 }));
 
             switch(ne.engagement) {
                 case "✅":
-                    if (channel) {
-                        return await channelService.approveSubscriber(emailID);
-                    }
-                    return;
+                    return await channelService.approveSubscriber(emailID);
                 case "🔔":
                 case "📈":
                     return;
                 case "🔕":
                 case "❌":
-                        // unsubscribe particular thread...
-                        await this.ets.unsubscribe(senderID, emailID);
-                        return;
+                    // unsubscribe particular thread...
+                    await this.ets.unsubscribe(senderID, emailID);
+                    return;
                 case "➕":
-                    if (channel) {
-                        await BroadcastEmail.broadcast(this.context, {
-                            mailboxID: recipientID,
-                            emailID,
-                            senderID,
-                            subscribe: true
-                        });
-                    }
+                    await BroadcastEmail.broadcast(this.context, {
+                        mailboxID: recipientID,
+                        emailID,
+                        senderID,
+                        subscribe: true
+                    });
                     return;
                 case "💬":
                     // following will make comment appear publicly
-                    if (channel && !channel.isPrivateChannel && channel.requiresCommentModeration) {
+                    if (!channel.isPrivateChannel && channel.requiresCommentModeration) {
                         await db.emails.where(input, (p) => (x) => x.emailID === p.emailID
                             && x.parent.status === "published"
                         )
-                        .update(void 0, (p) => (x) => ({ status: "pending-approval" }));
+                        .update(void 0, () => () => ({ status: "pending-approval" }));
                         return;
                     }
                     await db.emails.where(input, (p) => (x) => x.emailID === p.emailID
                         && x.parent.status === "published"
                     )
-                    .update(void 0, (p) => (x) => ({ status: "published" }));
+                    .update(void 0, () => () => ({ status: "published" }));
                     await this.ets.saveParticipant(senderID, emailID);
                     return;
                 default:
@@ -178,33 +178,14 @@ export default class EmailPostReceiveService {
                 && x.emailAddressID === p.senderID)
                 .first();
 
-        if(!channel) {
-
-            // send push notification here...
-            if (mc?.blocked) {
-                return;
-            }
+        if (!channel.isPrivateChannel && channel.requiresPostModeration && !mc.isOwner) {
 
-            const mn = ServiceProvider.resolve(this, MailboxNamespace);
-            mn.newMail(input.recipientID, input.senderID, input.emailID);
-
-            // verify if anyone is listening...
-            const hasSubscriptions = await db.userPushSubscriptions.where(input, (p) => (x) => x.mailboxID === p.recipientID).some();
-            if (hasSubscriptions) {
-                const id = `send-push-${input.emailID}-${input.recipientID}`;
-                await this.context.queue(SendWebPushNotification, {
-                    mailboxID: ne.mailboxID,
-                    emailID: ne.emailID,
-                    emailAddressID: ne.emailAddressID
-                }, { id, throwIfExists: false });
-            }
-            return;
-        }
+            await this.db.emails.statements.update({
+                status: "pending-approval"
+            }, {
+                emailID
+            });
 
-        if (!channel.isPrivateChannel && channel.requiresPostModeration) {
-            if(await this.cms.addToQueue(channel, emailID, mc)) {
-                return;
-            }
         }
 
         // can send...
@@ -261,4 +242,59 @@ export default class EmailPostReceiveService {
         });
     }
 
+    async dispatchMailbox(input: { senderID, emailID, recipientID }) {
+
+        const { db } = this;
+
+        const { senderID, emailID } = input;
+
+        // check if it is not an engagement and can send an email
+        const ne = await db.mailboxContactEmails.where(input, (p) =>
+            (x) => x.emailID === p.emailID
+                && x.mailboxID === p.recipientID
+                && x.emailAddressID === p.senderID
+                && x.email.status !== "published")
+                .first();
+
+        if (!ne) {
+            return;
+        }
+
+        await this.ets.saveParticipant(senderID, emailID);
+
+        if (ne.engagement) {
+
+            await db.mailboxes.where(input, (p) => (x) => x.mailboxID === p.recipientID)
+                .update(void 0, () => () => ({
+                    dateEngaged: Sql.date.now()
+                }));
+        }
+
+        const mc = await db.mailboxContacts.where(input, (p) =>
+            (x) => x.mailboxID === p.recipientID
+                && x.emailAddressID === p.senderID)
+                .first();
+
+
+        // send push notification here...
+        if (mc?.blocked) {
+            return;
+        }
+
+        const mn = ServiceProvider.resolve(this, MailboxNamespace);
+        mn.newMail(input.recipientID, input.senderID, input.emailID);
+
+        // verify if anyone is listening...
+        const hasSubscriptions = await db.userPushSubscriptions.where(input, (p) => (x) => x.mailboxID === p.recipientID).some();
+        if (hasSubscriptions) {
+            const id = `send-push-${input.emailID}-${input.recipientID}`;
+            await this.context.queue(SendWebPushNotification, {
+                mailboxID: ne.mailboxID,
+                emailID: ne.emailID,
+                emailAddressID: ne.emailAddressID
+            }, { id, throwIfExists: false });
+        }
+
+    }
+
 }

package/src/server/smtp/services/EmailSerializerService.ts

@@ -25,6 +25,8 @@ import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
 import EmailRecipient from "../../model/entities/EmailRecipient.js";
 import { ReactionExtractor } from "../../../protocols/core/ReactionExtractor.js";
 import { stripBrackets } from "../../../common/stripBrackets.js";
+import SmtpError from "./SmtpError.js";
+import { SmtpStatusCodes } from "../client/SmtpStatusCodes.js";
 // import PostReceiveWorkflow from "../../workflows/post-receive/PostReceiveWorkflow.js";
 // import UpdateEmailReferences from "../../workflows/incoming/UpdateEmailReferences.js";
 
@@ -349,16 +351,6 @@ export default class EmailSerializerService {
 
         // console.log("Recipients Saved");
 
-        if (inReplyTo) {
-            // find the id..
-            const original = await db.emails.where({ inReplyTo }, (p) => (x) => x.headerMessageID === p.inReplyTo)
-                .orderBy(void 0,(p) => (x) => x.emailID)
-                .first();
-            if (original) {
-                parentID = original.emailID;
-                inReplyTo = null;
-            }
-        }
 
         const dateReceived = DateTime.from(parsedMail.date ?? DateTime.now);
         const dateSent = dateReceived;
@@ -376,11 +368,27 @@ export default class EmailSerializerService {
 
         if (received) {
             // check for engagements...
-            engagement = await ReactionExtractor.extract({
+            const e1 = await ReactionExtractor.extractWithInReplyTo({
                 subject,
                 textBody,
                 headers: mailHeaders
             }, () => Promise.resolve(parsedMail.attachments), true);
+
+            engagement = e1.engagement;
+            if (!inReplyTo) {
+                inReplyTo = e1.inReplyTo;
+            }
+        }
+
+        if (inReplyTo) {
+            // find the id..
+            const original = await db.emails.where({ inReplyTo }, (p) => (x) => x.headerMessageID === p.inReplyTo)
+                .orderBy(void 0,(p) => (x) => x.emailID)
+                .first();
+            if (original) {
+                parentID = original.emailID;
+                inReplyTo = null;
+            }
         }
 
         let emailID = void 0;
@@ -395,6 +403,21 @@ export default class EmailSerializerService {
         // add recipient from mailboxes
         if (mailboxes) {
             for (const mailbox of mailboxes) {
+
+                // check if it is a channel
+                // verify
+
+                if (mailbox.isChannel && !mailbox.membersCanPostTopLevel) {
+
+                    if (!parentID) {
+                        const me = await this.db.mailboxContacts.statements.select({}, { mailboxID: mailbox.mailboxID, emailAddressID: senderID});
+                        if (!(me.canContribute || me.isOwner)) {
+                            throw new SmtpError("Only active members of channel can create new posts to channel", SmtpStatusCodes.DeliveryNotPermitted);
+                        }
+                    }
+
+                }
+
                 if(recipients.some((e) => e.emailAddressID === mailbox.mailboxID)) {
                     continue;
                 }

package/src/server/workflows/pending/PendingReferencesWorkflow.ts

@@ -5,6 +5,8 @@ import SocialMailContext from "../../model/SocialMailContext.js";
 import EmailReferenceService from "../../smtp/services/EmailReferenceService.js";
 import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
 import { globalEnv } from "../../../common/globalEnv.js";
+import PendingPostReceiveWorkflow from "./PendingPostReceiveWorkflow.js";
+import { randomUUID } from "node:crypto";
 
 export default class PendingReferencesWorkflow extends Workflow {
 
@@ -19,6 +21,8 @@ export default class PendingReferencesWorkflow extends Workflow {
         do {
             start = await this.processPendingReferences(start);
         } while(BigInt(start));
+
+        await this.context.queue(PendingPostReceiveWorkflow, "run", { id: `post-receive-${randomUUID()}`, throttle: { group: "post-receive", maxPerSecond: 1 } } );
     }
 
     @UniqueActivity

package/test.js

@@ -5,6 +5,13 @@ import { tmpdir } from "os";
 import { existsSync, mkdirSync } from "fs";
 import { spawnPromise } from "./dist/common/spawnPromise.js";
 
+const ensureDir = (d) => {
+    if(!existsSync(d)) {
+        mkdirSync(d, { recursive: true });
+    }
+    return d;
+}
+
 const host = process.env.POSTGRES_HOST ?? "localhost";
 const postGresPort = Number(process.env.POSTGRES_PORT ?? 5432);
 process.env.SOCIAL_MAIL_SECRET_KEY = "abcd123";
@@ -35,13 +42,10 @@ let start = Date.now();
 process.env.SOCIAL_MAIL_DB_SSL = "false";
 process.env.SOCIAL_MAIL_TEST = "true";
 process.env.SOCIAL_MAIL_AV = "none";
-const tmpPath = process.env.SOCIAL_MAIL_TMP_PATH = `${tmpdir()}/t/${Date.now().toString(36)}`;
+const tmpPath = process.env.SOCIAL_MAIL_TMP_PATH = ensureDir(`${tmpdir()}/t-${Date.now().toString(36)}/t`);
+process.env.SOCIAL_MAIL_CACHE_PATH = ensureDir(`${tmpdir()}/tc-${Date.now().toString(36)}/tc`);
 process.env.SOCIAL_MAIL_DB_DATABASE = "SM" + start;
 
-if(!existsSync(tmpPath)) {
-    mkdirSync(tmpPath, { recursive: true });
-}
-
 export default class TestRunner {
 
     static get drivers() {
@@ -66,10 +70,8 @@ export default class TestRunner {
         if (!testFile) {
             const dbID = start++;
             let db = `social-mail-test-${dbID}`;
-            const tPath = `${tmpdir()}/t/${dbID}`;
-            if(!existsSync(tPath)) {
-                mkdirSync(tPath, { recursive: true });
-            }
+            const tPath = ensureDir(`${tmpdir()}/t-${dbID}/t`);
+            const tcPath = ensureDir(`${tmpdir()}/t-${dbID}/tc`);
             
             const t1 = setTimeout(() => console.warn(`Test ${name} ran for more than 30 seconds`), 30000);
             try {
@@ -83,6 +85,7 @@ export default class TestRunner {
                         SOCIAL_MAIL_TEST: "true",
                         SOCIAL_MAIL_AV: "none",
                         SOCIAL_MAIL_TMP_PATH: tPath,
+                        SOCIAL_MAIL_CACHE_PATH: tcPath,
                         SOCIAL_MAIL_DB_DATABASE: db,
                         SOCIAL_MAIL_DB_USER: "postgres",
                         SOCIAL_MAIL_DB_PASSWORD: "abcd123",