@social-mail/social-mail-web-server 1.5.455 → 1.7.2

package/src/server/services/puppeteer/PuppeteerServer.ts

@@ -1,194 +1,194 @@
-/* eslint-disable no-console */
-import Inject, { RegisterSingleton } from "@entity-access/entity-access/dist/di/di.js";
-import puppeteer, { Page } from "puppeteer-core";
-import { globalEnv } from "../../../common/globalEnv.js";
-import HtmlSanitizer from "../dom/HtmlSanitizer.js";
-import GlobalEvents from "../../../common/GlobalEvents.js";
-import { rmdir, writeFile } from "node:fs/promises";
-import { existsSync, mkdirSync, unlinkSync } from "node:fs";
-import LockFile from "../../storage/LockFile.js";
-
-const options = [
-    '--disable-background-networking',
-    '--disable-background-timer-throttling',
-    '--disable-backgrounding-occluded-windows',
-    '--disable-breakpad',
-    '--disable-client-side-phishing-detection',
-    '--disable-component-update',
-    '--disable-default-apps',
-    '--disable-dev-shm-usage',
-    '--disable-domain-reliability',
-    '--disable-extensions',
-    '--disable-features=AudioServiceOutOfProcess',
-    '--disable-hang-monitor',
-    '--disable-ipc-flooding-protection',
-    '--disable-notifications',
-    '--disable-offer-store-unmasked-wallet-cards',
-    '--disable-popup-blocking',
-    '--disable-print-preview',
-    '--disable-prompt-on-repost',
-    '--disable-renderer-backgrounding',
-    '--disable-setuid-sandbox',
-    '--disable-speech-api',
-    '--disable-sync',
-    '--disk-cache-size=33554432',
-    // '--hide-scrollbars',
-    // '--ignore-gpu-blacklist',
-    // '--metrics-recording-only',
-    '--mute-audio',
-    '--no-default-browser-check',
-    '--no-first-run',
-    '--no-pings',
-    '--no-sandbox',
-    '--no-zygote',
-    '--password-store=basic',
-    '--use-mock-keychain',
-    // '--single-process',
-    '--autoplay-policy=no-user-gesture-required',
-    // '--enable-logging=stderr',
-    // '--v=1',
-    // '--disable-gpu',
-    //   '--window-size=400,800',
-];
-
-// eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents
-export type DisposablePage = Partial<Page> & AsyncDisposable ;
-
-const buffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAAtJREFUGFdjYAACAAAFAAGq1chRAAAAAElFTkSuQmCC", "base64");
-
-@RegisterSingleton
-export default class PuppeteerServer {
-
-    @Inject
-    htmlSanitizer: HtmlSanitizer;
-
-    public async htmlToPdf(content: string, { outputFile, deviceScaleFactor, pdfOptions }: { outputFile: string, pdfOptions?: string, deviceScaleFactor?: number }) {
-        try {
-            await using page = await this.newPage({ deviceScaleFactor });
-            content = this.htmlSanitizer.prepareForPreview(content);
-            if(typeof pdfOptions === "string") {
-                pdfOptions = JSON.parse(pdfOptions);
-            }
-            await page.pdf({ ... pdfOptions as any,
-                path: outputFile
-             });
-        } catch (error) {
-            await writeFile(outputFile, buffer);
-            if (/Node\shas\s0\swidth/i.test(error.message ?? error)) {
-                return;
-            }
-            console.log(content);
-            console.error(error);
-        }
-    }
-
-    public async htmlToPng(content: string, {
-        selector,
-        outputFile,
-        count,
-        deviceScaleFactor
-    }: { selector?: string; outputFile: string, count?: string, deviceScaleFactor?: number}) {
-        try {
-            await using page = await this.newPage({ deviceScaleFactor });
-            content = this.htmlSanitizer.prepareForPreview(content);
-            await page.setContent(content, {
-                waitUntil: "networkidle2"
-            });
-            const element = await page.$(selector);
-            if (!element) {
-                await writeFile(buffer, outputFile);
-                return;
-            }
-            if (count) {
-                await element.evaluate((e, c) => {
-                    const span = e.querySelector(`[data-social-mail-element="count"]`);
-                    if (span) {
-                        span.textContent = c;
-                    }
-                }, count);
-            }
-            await element.scrollIntoView();
-            const clip = await element.boundingBox();
-            let top = Math.floor(clip.y);
-            if (top < clip.y) {
-                clip.height++;
-            } else {
-                top-=2;
-                clip.height+=4;
-            }
-            clip.height = Math.ceil(clip.height);
-            clip.y = top;
-            await page.screenshot({
-                path: outputFile,
-                clip,
-                omitBackground: true
-            });
-
-            // change the dpi...
-
-
-        } catch (error) {
-            await writeFile(outputFile, buffer);
-            if (/Node\shas\s0\swidth/i.test(error.message ?? error)) {
-                return;
-            }
-            console.log(content);
-            console.error(error);
-        }
-    }
-
-    public async svgToPng(svgFile: string, outputFile: string) {
-        await using page = await this.newPage();
-        await page.goto(svgFile, { waitUntil: "networkidle2"});
-        await page.screenshot({ path: outputFile });
-    }
-
-    public async newPage({ deviceScaleFactor }: Partial<puppeteer.Viewport> = {}): Promise<DisposablePage> {
-
-        const lf = await LockFile.lock("puppeteer-html-to-image");
-
-        const userDataDir = `/tmp/${Date.now().toString(36)}`;
-
-        if(!existsSync(userDataDir)) {
-            try {
-                mkdirSync(userDataDir, { recursive: true });
-            } catch {
-                // ignore this...
-            }
-        }
-
-        const browser = await puppeteer.launch({
-                executablePath: globalEnv.puppeteerExecutablePath,
-                userDataDir,
-                args: options,
-                // dumpio: true,
-                // ignoreHTTPSErrors: true,
-                ignoreDefaultArgs: ["--mute-audio"]
-        });
-
-        const page = await browser.newPage();
-        await page.setViewport({
-            ... (page.viewport() ?? {} as any),
-            deviceScaleFactor
-        });
-
-        // we will kill browser in 15 seconds..
-        const ct = setTimeout(() => browser.close().catch(console.error), 15000);
-
-        page[Symbol.asyncDispose] = async () => {
-            clearTimeout(ct);
-            lf[Symbol.dispose]();
-            try {
-                await browser.close();
-            } catch {
-                // ignore error.
-            }
-            try {
-                await rmdir(userDataDir);
-            } catch {
-                // ignore
-            }
-        };
-        return page as any as DisposablePage;
-    }
-}
+// /* eslint-disable no-console */
+// import Inject, { RegisterSingleton } from "@entity-access/entity-access/dist/di/di.js";
+// import puppeteer, { Page } from "puppeteer-core";
+// import { globalEnv } from "../../../common/globalEnv.js";
+// import HtmlSanitizer from "../dom/HtmlSanitizer.js";
+// import GlobalEvents from "../../../common/GlobalEvents.js";
+// import { rmdir, writeFile } from "node:fs/promises";
+// import { existsSync, mkdirSync, unlinkSync } from "node:fs";
+// import LockFile from "../../storage/LockFile.js";
+
+// const options = [
+//     '--disable-background-networking',
+//     '--disable-background-timer-throttling',
+//     '--disable-backgrounding-occluded-windows',
+//     '--disable-breakpad',
+//     '--disable-client-side-phishing-detection',
+//     '--disable-component-update',
+//     '--disable-default-apps',
+//     '--disable-dev-shm-usage',
+//     '--disable-domain-reliability',
+//     '--disable-extensions',
+//     '--disable-features=AudioServiceOutOfProcess',
+//     '--disable-hang-monitor',
+//     '--disable-ipc-flooding-protection',
+//     '--disable-notifications',
+//     '--disable-offer-store-unmasked-wallet-cards',
+//     '--disable-popup-blocking',
+//     '--disable-print-preview',
+//     '--disable-prompt-on-repost',
+//     '--disable-renderer-backgrounding',
+//     '--disable-setuid-sandbox',
+//     '--disable-speech-api',
+//     '--disable-sync',
+//     '--disk-cache-size=33554432',
+//     // '--hide-scrollbars',
+//     // '--ignore-gpu-blacklist',
+//     // '--metrics-recording-only',
+//     '--mute-audio',
+//     '--no-default-browser-check',
+//     '--no-first-run',
+//     '--no-pings',
+//     '--no-sandbox',
+//     '--no-zygote',
+//     '--password-store=basic',
+//     '--use-mock-keychain',
+//     // '--single-process',
+//     '--autoplay-policy=no-user-gesture-required',
+//     // '--enable-logging=stderr',
+//     // '--v=1',
+//     // '--disable-gpu',
+//     //   '--window-size=400,800',
+// ];
+
+// // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents
+// export type DisposablePage = Partial<Page> & AsyncDisposable ;
+
+// const buffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAAtJREFUGFdjYAACAAAFAAGq1chRAAAAAElFTkSuQmCC", "base64");
+
+// @RegisterSingleton
+// export default class PuppeteerServer {
+
+//     @Inject
+//     htmlSanitizer: HtmlSanitizer;
+
+//     public async htmlToPdf(content: string, { outputFile, deviceScaleFactor, pdfOptions }: { outputFile: string, pdfOptions?: string, deviceScaleFactor?: number }) {
+//         try {
+//             await using page = await this.newPage({ deviceScaleFactor });
+//             content = this.htmlSanitizer.prepareForPreview(content);
+//             if(typeof pdfOptions === "string") {
+//                 pdfOptions = JSON.parse(pdfOptions);
+//             }
+//             await page.pdf({ ... pdfOptions as any,
+//                 path: outputFile
+//              });
+//         } catch (error) {
+//             await writeFile(outputFile, buffer);
+//             if (/Node\shas\s0\swidth/i.test(error.message ?? error)) {
+//                 return;
+//             }
+//             console.log(content);
+//             console.error(error);
+//         }
+//     }
+
+//     public async htmlToPng(content: string, {
+//         selector,
+//         outputFile,
+//         count,
+//         deviceScaleFactor
+//     }: { selector?: string; outputFile: string, count?: string, deviceScaleFactor?: number}) {
+//         try {
+//             await using page = await this.newPage({ deviceScaleFactor });
+//             content = this.htmlSanitizer.prepareForPreview(content);
+//             await page.setContent(content, {
+//                 waitUntil: "networkidle2"
+//             });
+//             const element = await page.$(selector);
+//             if (!element) {
+//                 await writeFile(buffer, outputFile);
+//                 return;
+//             }
+//             if (count) {
+//                 await element.evaluate((e, c) => {
+//                     const span = e.querySelector(`[data-social-mail-element="count"]`);
+//                     if (span) {
+//                         span.textContent = c;
+//                     }
+//                 }, count);
+//             }
+//             await element.scrollIntoView();
+//             const clip = await element.boundingBox();
+//             let top = Math.floor(clip.y);
+//             if (top < clip.y) {
+//                 clip.height++;
+//             } else {
+//                 top-=2;
+//                 clip.height+=4;
+//             }
+//             clip.height = Math.ceil(clip.height);
+//             clip.y = top;
+//             await page.screenshot({
+//                 path: outputFile,
+//                 clip,
+//                 omitBackground: true
+//             });
+
+//             // change the dpi...
+
+
+//         } catch (error) {
+//             await writeFile(outputFile, buffer);
+//             if (/Node\shas\s0\swidth/i.test(error.message ?? error)) {
+//                 return;
+//             }
+//             console.log(content);
+//             console.error(error);
+//         }
+//     }
+
+//     public async svgToPng(svgFile: string, outputFile: string) {
+//         await using page = await this.newPage();
+//         await page.goto(svgFile, { waitUntil: "networkidle2"});
+//         await page.screenshot({ path: outputFile });
+//     }
+
+//     public async newPage({ deviceScaleFactor }: Partial<puppeteer.Viewport> = {}): Promise<DisposablePage> {
+
+//         const lf = await LockFile.lock("puppeteer-html-to-image");
+
+//         const userDataDir = `/tmp/${Date.now().toString(36)}`;
+
+//         if(!existsSync(userDataDir)) {
+//             try {
+//                 mkdirSync(userDataDir, { recursive: true });
+//             } catch {
+//                 // ignore this...
+//             }
+//         }
+
+//         const browser = await puppeteer.launch({
+//                 executablePath: globalEnv.puppeteerExecutablePath,
+//                 userDataDir,
+//                 args: options,
+//                 // dumpio: true,
+//                 // ignoreHTTPSErrors: true,
+//                 ignoreDefaultArgs: ["--mute-audio"]
+//         });
+
+//         const page = await browser.newPage();
+//         await page.setViewport({
+//             ... (page.viewport() ?? {} as any),
+//             deviceScaleFactor
+//         });
+
+//         // we will kill browser in 15 seconds..
+//         const ct = setTimeout(() => browser.close().catch(console.error), 15000);
+
+//         page[Symbol.asyncDispose] = async () => {
+//             clearTimeout(ct);
+//             lf[Symbol.dispose]();
+//             try {
+//                 await browser.close();
+//             } catch {
+//                 // ignore error.
+//             }
+//             try {
+//                 await rmdir(userDataDir);
+//             } catch {
+//                 // ignore
+//             }
+//         };
+//         return page as any as DisposablePage;
+//     }
+// }

package/src/server/services/puppeteer/PuppeteerService.ts

@@ -1 +1 @@
-export { default as PuppeteerService } from "./PuppeteerServer.js";
+// export { default as PuppeteerService } from "./PuppeteerServer.js";

package/src/server/smtp/services/CachedEmailService.ts

@@ -101,7 +101,7 @@ export default class CachedEmailService {
         };
 
         const ef = await this.cacheService.get({
-            key: `cached-email-html-a8-${emailID}-${emailAddressID}-${signedFileLogID}`,
+            key: `cached-email-html-a9-${emailID}-${emailAddressID}-${signedFileLogID}`,
             factory,
             ttl: TimeSpan.fromDays(30)
         });

package/src/server/smtp/services/EmailReferenceService.ts

@@ -285,7 +285,6 @@ export default class EmailReferenceService {
             .include((x) => [
                 x.recipients.forEach((r) => r.emailAddress.mailbox),
                 x.senderEmail.mailbox,
-                x.mailboxes.forEach((r) => r.mailbox),
                 x.thread.lastReply
             ])
             .first();
@@ -395,19 +394,19 @@ export default class EmailReferenceService {
                 }
 
                 // event sent emails are part of mailbox
-                await db.mailboxEmails.statements.selectOrInsert({
-                    mailboxID: senderID,
-                    emailID
-                });
+                // await db.mailboxEmails.statements.selectOrInsert({
+                //     mailboxID: senderID,
+                //     emailID
+                // });
 
                 if (mailbox) {
 
                     // this is the case when email was sent from the system
                     // we will insert in respective mailboxes here...
-                    await db.mailboxEmails.statements.selectOrInsert({
-                        mailboxID: emailAddressID,
-                        emailID
-                    });
+                    // await db.mailboxEmails.statements.selectOrInsert({
+                    //     mailboxID: emailAddressID,
+                    //     emailID
+                    // });
 
                     saved.add(emailAddressID);
 
@@ -439,10 +438,10 @@ export default class EmailReferenceService {
                 // we will only store in mailboxes that exist on the system
                 saved.add(emailAddressID);
                 if (mailbox) {
-                    await db.mailboxEmails.statements.selectOrInsert({
-                        mailboxID: emailAddressID,
-                        emailID
-                    });
+                    // await db.mailboxEmails.statements.selectOrInsert({
+                    //     mailboxID: emailAddressID,
+                    //     emailID
+                    // });
                     const mailboxID = emailAddressID;
                     await this.updateThread({
                         engagement,
@@ -467,30 +466,30 @@ export default class EmailReferenceService {
         // or sent as bcc to hosted mailbox
         // in that case we will put mail in sender's folder
 
-        if (email.mailboxes) {
-            for (const { mailbox } of email.mailboxes) {
-
-                const mailboxID = mailbox.mailboxID;
-                if (saved.has(mailboxID)) {
-                    continue;
-                }
-
-                await this.updateThread({
-                    engagement,
-                    mailbox,
-                    mailboxID,
-                    emailAddressID: senderID,
-                    lastEmailID: emailID,
-                    silent,
-                    timeRead,
-                    dateUpdated,
-                    tagIDs,
-                    email,
-                    newOpenThread,
-                    newThread,
-                });
-            }
-        }
+        // if (email.mailboxes) {
+        //     for (const { mailbox } of email.mailboxes) {
+
+        //         const mailboxID = mailbox.mailboxID;
+        //         if (saved.has(mailboxID)) {
+        //             continue;
+        //         }
+
+        //         await this.updateThread({
+        //             engagement,
+        //             mailbox,
+        //             mailboxID,
+        //             emailAddressID: senderID,
+        //             lastEmailID: emailID,
+        //             silent,
+        //             timeRead,
+        //             dateUpdated,
+        //             tagIDs,
+        //             email,
+        //             newOpenThread,
+        //             newThread,
+        //         });
+        //     }
+        // }
 
         return email;
     }

package/src/server/smtp/services/EngagementService.ts

@@ -195,7 +195,7 @@ export default class EngagementService {
         const ea = { emailID, engagement };
 
         if (email.status === "published") {
-            await this.db.mailboxEmails.where(ea, (p) => (x) => x.emailID === p.emailID)
+            await this.db.channelEmails.where(ea, (p) => (x) => x.emailID === p.emailID)
             .update(void 0, (p) => (x) => ({
                 engagementCount: x.engagementCount + 1,
                 dateUpdated: Sql.date.now()

package/src/server/workflows/channel/BroadcastEmail.ts

@@ -9,6 +9,7 @@ import { randomUUID } from "crypto";
 import DispatchEmailToMX from "../smtp/DispatchEmailToMX.js";
 import EmailAddressService from "../../services/EmailAddressService.js";
 import TimeSpan from "@entity-access/entity-access/dist/types/TimeSpan.js";
+import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
 
 /**
  * Broadcasting Email is done in following manner.
@@ -82,22 +83,34 @@ export default class BroadcastEmail extends Workflow<IBroadcastEmail> {
         const [email] = await db.emails.where(this.input, (p) => (x) => x.emailID === p.emailID)
             .updateSelect(void 0, (p) => (x) => ({ status: "published"}));
 
+        // find first attachment
+        const types = ["video/%", "image/%"];
+
+        const firstAttachment = await db.appFiles.where({ types, emailID }, (p) => (x) =>
+            x.emailID === p.emailID
+            && Sql.text.likeAny(x.contentType, p.types))
+            .orderBy(void 0, (p) => (x) => Sql.text.like(x.contentType, "video/%"))
+            .first();
+
+        const previewFileID = firstAttachment?.appFileID;
+
         await db.mailboxes.statements.update({ datePublished }, { mailboxID });
 
         // gmail sends in-reply-to for forwarded messages.
         // ideally in-reply-to must not be set if the message was forwarded
 
         let isThread = email.parentID === null;
-        if(email.parentID) {
-            const { parentID } = email;
-            isThread = !(await db.mailboxEmails.where({ mailboxID , parentID}, (p) => (x) => x.emailID === p.parentID && x.mailboxID === p.mailboxID)
-                .some());
-        }
 
         if (isThread) {
-            await db.mailboxEmails.statements.update({ isThread: true, datePublished, dateUpdated }, { mailboxID, emailID });
+            await db.channelEmails.statements.upsert({ previewFileID, isThread: true, datePublished, dateUpdated,mailboxID, emailID }, (x) => x, { mailboxID, emailID });
         } else {
-            await db.mailboxEmails.statements.update({ datePublished, dateUpdated }, { mailboxID, emailID });
+            await db.channelEmails.statements.upsert({ previewFileID, datePublished, dateUpdated, mailboxID, emailID }, (x) => x, { mailboxID, emailID });
+        }
+
+        if(email.parentID) {
+            const { parentID } = email;
+            isThread = !(await db.channelEmails.where({ mailboxID , parentID}, (p) => (x) => x.emailID === p.parentID && x.mailboxID === p.mailboxID)
+                .some());
         }
 
         // load email..
@@ -151,7 +164,7 @@ export default class BroadcastEmail extends Workflow<IBroadcastEmail> {
                 const { activityID } = activity;
 
                 await this.context.queue(DispatchEmailToMX, {
-                    emailAddressID,
+                    recipientAddressID: emailAddressID,
                     emailID,
                     senderID,
                     external: false,

package/src/server/workflows/repair/channel/RepairChannelEmailCountsWorkflow.ts

@@ -36,7 +36,7 @@ export default class RepairChannelEmailCountsWorkflow extends Workflow {
             for (const iterator of mailboxes) {
                 start = iterator.mailboxID;
 
-                await db.mailboxEmails.where(iterator, (p) => (x) => x.mailboxID === p.mailboxID)
+                await db.channelEmails.where(iterator, (p) => (x) => x.mailboxID === p.mailboxID)
                     .update(void 0, (p) => (x) => ({
                         isThread: x.email.parentID === null ? true : false,
                         datePublished: Sql.cast.asDateTime( x.email.status === "published" ? x.email.dateSent : null ) as any,

package/src/server/workflows/repair/counts/RepairEmailCountsWorkflow.ts

@@ -56,7 +56,7 @@ export default class RepairEmailCountsWorkflow extends Workflow<{ mailboxID? }>
         const q = db.emails
                 .where({ start, mailboxID }, (p) => (x) => x.emailID > p.start
                     && x.threadID === null
-                    && x.mailboxes.some((t) => t.mailboxID === p.mailboxID)
+                    && x.mailboxContactEvents.some((t) => t.mailboxID === p.mailboxID)
                     && x.lastReplyID === null
                 );
         const list = await q

package/src/server/workflows/repair/tags/RepairEmailTagsWorkflow.ts

@@ -173,7 +173,7 @@ export default class RepairEmailTagsWorkflow extends Workflow<{ mailboxID? }> {
 
                     // get last updated...
                     const last = await db.emails.where({ tagID, mailboxID}, (p) => (x) =>
-                        x.mailboxes.some((m) => m.mailboxID === p.mailboxID)
+                        x.mailboxContactEvents.some((m) => m.mailboxID === p.mailboxID)
                         && x.emailTags.some((t) => t.tagID === p.tagID)
                     ).orderByDescending(void 0, (p) => (x) => x.dateCreated).first();