@social-mail/social-mail-web-server 1.4.800 → 1.4.802

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.4.800",
+  "version": "1.4.802",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/common/BufferStream.ts

@@ -1,18 +1,145 @@
-import { Readable } from "node:stream";
+import Queue from "../protocols/core/Queue.js";
 
-export class BufferStream extends Readable {
+const empty = Buffer.alloc(0);
 
-    public static from(buffer: Buffer) {
-        return new BufferStream(buffer);
+class BufferList {
+
+    get size() {
+        return this.length;
+    }
+
+    private list =[];
+    private length = 0;
+
+    add(buffer: Buffer) {
+        this.list.push(buffer);
+        this.length += buffer.length;
+        return this;
+    }
+
+    asBuffer() {
+        return Buffer.concat(this.list);
+    }
+
+    toString(encoding: BufferEncoding) {
+        return this.asBuffer().toString(encoding);
+    }
+
+    clear() {
+        this.length = 0;
+        this.list.length = 0;
+    }
+
+}
+
+export class BufferStream {
+
+    store: Queue<Buffer> = new Queue();
+    start: number = 0;
+    unread: Buffer = empty;
+
+    get isDataAvailable() {
+        return !!this.store.peek;
+    }
+
+    enqueue(buffer: Buffer) {
+        this.store.enqueue(buffer);
+        return this;
     }
 
-    constructor ( private buffer ){
-        super();
-        this.buffer = buffer;
+    async read(size: number, signal?: AbortSignal) {
+
+        if (size <= this.unread.length) {
+            // read this...
+            const slice = this.unread.subarray(0, size);
+            this.unread = size === this.unread.length
+                ? empty
+                : this.unread.subarray(size);
+            return slice;
+        }
+
+        const pending = new BufferList();
+        pending.add(this.unread);
+        this.unread = empty;
+
+        size -= pending.size;
+
+        for(;;) {
+
+            if (signal?.aborted) {
+                throw new Error("aborted");
+            }
+
+            const peek = await this.store.waitForPeek(signal);
+
+            const bufLen = this.start + size;
+
+            if (bufLen <= peek.length) {
+                pending.add(peek.subarray(this.start, bufLen));
+                const buffer = pending.asBuffer();
+                this.start = bufLen;
+                size -= bufLen;
+                this.unread = empty;
+                pending.clear();
+
+                if (bufLen === peek.length) {
+                    this.store.dequeue();
+                    this.start = 0;
+                }
+
+                return buffer;
+            }
+
+            const remaining = this.start ? peek.subarray(this.start) : peek;
+            pending.add(remaining);
+            size -= remaining.length;
+            this.store.dequeue();
+            this.start = 0;
+        }
     }
 
-    _read (){
-        this.push( this.buffer );
-        this.push( null );
+    async readTill(sep : string | number | Buffer = 10, encoding: BufferEncoding = "utf8", trimR = false, signal?: AbortSignal) {
+        for(;;) {
+            const peek = await this.store.waitForPeek(signal);
+
+            if (signal?.aborted) {
+                throw new Error("aborted");
+            }
+
+            const index = peek.indexOf(sep, this.start);
+            if (index === -1) {
+                const remaining = this.start ? peek.subarray(this.start) : peek;
+                this.unread = Buffer.concat([ this.unread, remaining ]);
+                this.start = 0;
+                this.store.dequeue();
+                continue;
+            }
+
+            if (index === 0) {
+                // this is a special case...
+                const last = this.unread.length - 1;
+                if (trimR && this.unread.at(last) === 13) {
+                    const t = this.unread.toString(encoding, 0, last);
+                    this.unread = empty;
+                    this.start = 1;
+                    return t;
+                }
+            }
+
+            const subarray = peek.subarray(this.start, trimR && index && peek.at(index-1) === 13 ? index -1 : index);
+            const token = this.unread.length === 0
+                ? subarray
+                : Buffer.concat([this.unread, subarray]);
+            // this is last...
+            if (index + 1 === peek.length) {
+                this.start = 0;
+                this.store.dequeue();
+            } else {
+                this.start = index + 1;
+            }
+            this.unread = empty;
+            return token.toString(encoding);
+        }
     }
+
 }

package/src/protocols/core/Queue.ts

@@ -6,15 +6,37 @@ export default class Queue<T> {
 
     public readonly eventItemAdded: EventSet<T> = new EventSet(this);
 
-
     get peek() {
         return this.store.get(this.head);
     }
 
+    get isEmpty() {
+        return this.head === this.tail;
+    }
+
+    get signal() {
+        return this.abortController.signal;
+    }
+
+    private readonly abortController: AbortController = new AbortController();
+
     private head = 0;
     private tail = 0;
     private readonly store: Map<number,T> = new Map();
 
+    public async waitForPeek(signal?: AbortSignal) {
+        if (this.isEmpty) {
+            await new Promise<any>((resolve, reject)=> {
+                this.eventItemAdded.listenOnce(resolve);
+                signal?.addEventListener("abort", () => {
+                    this.abortController.abort("abort");
+                    reject("abort");
+                });
+            });
+        }
+        return this.peek;
+    }
+
     public enqueue(item: T) {
         this.store.set(this.tail,item);
         this.tail++;

package/src/protocols/imap/client/ImapClient.ts

@@ -206,7 +206,7 @@ export default class ImapClient {
                         // read these many bytes...
 
                         // lastLine = lastLine.substring(0, index);
-                        const literal = await this.socket.readChars(length);
+                        const literal = (await this.socket.readChars(length)).toString("ascii");
                         // parser will take care of literal
                         // data.push(`${lastLine}\r\n${current}`);
                         // we will inject literal as an object with toString so

package/src/protocols/smtp/ESmtpClient.ts

@@ -63,7 +63,7 @@ export default class ESmtpClient {
         }
 
         const eSmtpClient = new ESmtpClient(client, logger);
-        await eSmtpClient.init(helo, greeting.toLocaleLowerCase().includes("esmtp"), host);
+        await eSmtpClient.init(helo, greeting.toLocaleLowerCase().includes("esmtp"), host, startTLS);
         const login = auth?.login;
         if (login) {
             await eSmtpClient.sendCommand("AUTH LOGIN",334);
@@ -148,17 +148,17 @@ export default class ESmtpClient {
         await this.sendCommand(".");
     }
 
-    private async init(name: string, eSmtp = false, host?: string) {
+    private async init(name: string, eSmtp = false, host?: string, startTLS?: boolean) {
 
         let startTLSSupported = false;
 
         if (eSmtp) {
-            await this.sendCommand(`EHLO ${name ?? os.hostname() }`, 250, (t) => startTLSSupported ||= /-STARTTLS$/i.test(t) );
+            await this.sendCommand(`EHLO ${name ?? os.hostname() }`, 250, (t) => startTLSSupported ||= /-STARTTLS/i.test(t) );
         } else {
             await this.sendCommand(`HELO ${name ?? os.hostname() }`);
         }
 
-        if (startTLSSupported) {
+        if (startTLSSupported && startTLS) {
             await this.sendCommand("STARTTLS", 220);
             await this.client.upgradeTLS( { servername: host, rejectUnauthorized: false });
             await this.sendCommand(`EHLO ${name ?? os.hostname() }`);
@@ -175,6 +175,7 @@ export default class ESmtpClient {
         const lines = [];
         do {
             const response = await this.client.readLine();
+            this.logger?.(response);
             if (/^\d+\-/.test(response)) {
                 parse?.(response);
                 lines.push(response);

package/src/protocols/socket/client/SocketClient.ts

@@ -3,6 +3,7 @@ import Queue from "../../core/Queue.js";
 import EventSet from "@entity-access/entity-access/dist/common/EventSet.js";
 import tls, { TLSSocket } from "tls";
 import { AsyncDisposableScope } from "@entity-access/entity-access/dist/common/usingAsync.js";
+import { BufferStream } from "../../../common/BufferStream.js";
 
 export default class SocketClient {
 
@@ -12,22 +13,17 @@ export default class SocketClient {
             const c = ssl
                 ? tls.connect({ port, host, rejectUnauthorized: false })
                 : host ? net.connect({ host, port }) : net.connect(port);
-            const asc = new AbortController();
             c.once("error", (error) => {
                 logger?.(error);
                 reject(error);
             });
-            c.on("close", () => {
-                logger?.(`Connection closed`);
-                asc.abort();
-            });
-            const q = new Queue<{ data: Buffer }>();
+            const bs = new BufferStream();
             c.on("data", (data) => {
-                q.enqueue({ data });
+                bs.enqueue(data);
             });
             c.on("connect", () => {
                 logger?.("Socket connected");
-                resolve(new SocketClient(c, q, asc.signal, logger));
+                resolve(new SocketClient(c, bs , logger));
             });
         });
         logger?.(`Connection established`);
@@ -39,7 +35,7 @@ export default class SocketClient {
     eventClosed = new EventSet(this);
 
     get isDataAvailable() {
-        return this.queue.peek;
+        return this.bufferStream.isDataAvailable;
     }
 
     /**
@@ -59,8 +55,8 @@ export default class SocketClient {
 
     private constructor(
         private client: net.Socket,
-        private queue: Queue<{ data: Buffer }>,
-        private signal: AbortSignal,
+        // private queue: Queue<{ data: Buffer }>,
+        private bufferStream: BufferStream,
         private logger: (data) => void = () => void 0,
     ) {
         this.client.setTimeout(30000);
@@ -111,121 +107,12 @@ export default class SocketClient {
         });
     }
 
-    async readLine() {
-
-        if (this.signal.aborted) {
-            throw new Error("Aborted");
-        }
-
-        await using scope = new AsyncDisposableScope();
-        return await new Promise<string>((resolve, reject) => {
-
-            let start = "";
-
-            this.signal.onabort = reject;
-
-            const processPeek = () => {
-                for(;;) {
-                    if (this.signal.aborted) {
-                        reject();
-                        return;
-                    }
-                    if (!this.queue.peek) {
-                        return;
-                    }
-                    const { data } = this.queue.peek;
-                    if (data.length === 0) {
-                        this.queue.dequeue();
-                        continue;
-                    }
-                    // This is edge case if \r was the last character
-                    // read and first character in buffer is \n (ASCII 10)
-                    if (start.endsWith("\r") && data.at(0) === 10 ) {
-                        start = start.substring(0, start.length-1);
-                        if (data.length === 1) {
-                            this.queue.dequeue();
-                        } else {
-                            this.queue.peek.data = data.subarray(1);
-                        }
-                        this.logger(`<< ${start}`);
-                        resolve(start);
-                        return;
-                    }
-
-                    // search for `\r\n`
-                    const index = data.indexOf('\r\n');
-                    if (index === -1) {
-
-                        start += data.toString("ascii");
-                        this.queue.dequeue();
-                        continue;
-                    }
-
-                    // it is last
-                    if (index === data.length - 2) {
-                        this.queue.dequeue();
-                        start += data.toString("ascii", 0, index);
-                        this.logger(`<< ${start}`);
-                        resolve(start);
-                        return;
-                    }
-
-                    this.queue.peek.data = data.subarray(index + 2);
-                    start += data.toString("ascii", 0, index);
-                    this.logger(`<< ${start}`);
-                    resolve(start);
-                    return;
-                }
-            };
-
-            scope.register(this.queue.eventItemAdded.listen(processPeek));
-
-            processPeek();
-
-        });
+    readLine(signal?: AbortSignal) {
+        return this.bufferStream.readTill("\n", "ascii", true, signal);
     }
 
-    public async readChars(length: number) {
-        length = Number(length);
-        let text = "";
-        await using scope = new AsyncDisposableScope();
-        return await new Promise<string>((resolve, reject) => {
-            this.signal.onabort = reject;
-            const processPeek = () => {
-                for(;;) {
-                    if (this.signal.aborted) {
-                        reject(new Error("Aborted"));
-                        return;
-                    }
-                    if (!this.queue.peek) {
-                        return;
-                    }
-                    const { data } = this.queue.peek;
-                    if (data.length === 0) {
-                        this.queue.dequeue();
-                        continue;
-                    }
-                    if (data.length > length) {
-                        text += data.toString("ascii", 0, length);
-                        this.queue.peek.data = data.subarray(length);
-                        resolve(text);
-                        return;
-                    }
-
-                    text += data.toString("ascii");
-                    this.queue.dequeue();
-                    length -= data.length;
-                    if (length === 0) {
-                        resolve(text);
-                        return;
-                    }
-                }
-            };
-
-            scope.register(this.queue.eventItemAdded.listen(processPeek));
-
-            processPeek();
-        });
+    public async readChars(length: number, signal?: AbortSignal) {
+        return this.bufferStream.read(length, signal);
     }
 
     public async  upgradeTLS(options: tls.ConnectionOptions = {  }) {
@@ -250,12 +137,8 @@ export default class SocketClient {
         ts.setTimeout(this.client.timeout);
         this.client.setTimeout(0);
         this.client = ts;
-        this.queue = new Queue<{ data: Buffer}>();
-        this.client.on("data", (data) => this.queue.enqueue({ data }));
-        const ac = new AbortController();
-        this.client.on("close", () => ac.abort());
-        this.signal = ac.signal;
-
+        this.bufferStream = new BufferStream();
+        this.client.on("data", (data) => this.bufferStream.enqueue(data));
         this.client.on("close", this.eventClosed.dispatcher);
         this.client.on("timeout", this._onTimeout);
         this.client.on("error", this.eventError.dispatcher);

package/src/server/model/events/EmailLogEvents.ts

@@ -1,11 +1,7 @@
 import { IEntityQuery } from "@entity-access/entity-access/dist/model/IFilterWithParameter.js";
 import BaseEvents from "./BaseEvents.js";
-import ChangeEntry from "@entity-access/entity-access/dist/model/changes/ChangeEntry.js";
 import Inject from "@entity-access/entity-access/dist/di/di.js";
 import BaseStorage from "../../storage/BaseStorage.js";
-import { Readable, Stream } from "stream";
-import { BufferStream } from "../../../common/BufferStream.js";
-import DateTime from "@entity-access/entity-access/dist/types/DateTime.js";
 import TempFileService from "../../storage/TempFileService.js";
 import { EmailLog } from "../entities/Email.js";
 

package/src/server/seed/ui/seed-ui.ts

@@ -3,7 +3,7 @@ import semver from "semver";
 
 export default async function seedUI(config: DBConfig) {
 
-    const version = "1.2.312";
+    const version = "1.2.314";
 
     const existing = await config.get(UIPackageConfig, false);
     if (existing) {

package/src/tests/core/buffer/buffer-stream.test.ts

@@ -0,0 +1,106 @@
+import assert from "assert";
+import { BufferStream } from "../../../common/BufferStream.js";
+
+export default async function() {
+
+    await readSizes();
+
+    await readLines();
+}
+
+async function readLines() {
+    const bf = new BufferStream();
+
+    setTimeout(() => bf.enqueue(Buffer.from("1234")), 10);
+
+    setTimeout(() => bf.enqueue(Buffer.from("5678")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("90\n12")), 10);
+
+    let line = await bf.readTill("\n");
+    assert.equal("1234567890", line);
+
+    setTimeout(() => bf.enqueue(Buffer.from("12\n34")), 10);
+
+    setTimeout(() => bf.enqueue(Buffer.from("5678")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("90\n12")), 10);
+
+    line = await bf.readTill("\n");
+    assert.equal("1212", line);
+    line = await bf.readTill("\n");
+    assert.equal("34567890", line);
+
+    line = (await bf.read(2)).toString("utf8");
+    assert.equal("12", line);
+
+    setTimeout(() => bf.enqueue(Buffer.from("12\n1234\n12345")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("6\n")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("123456\r\n")), 10);
+
+    line = await bf.readTill("\n");
+    assert.equal("12", line);
+    line = await bf.readTill("\n");
+    assert.equal("1234", line);
+    line = await bf.readTill("\n");
+    assert.equal("123456", line);
+
+    line = await bf.readTill("\n", "ascii", true);
+    assert.equal("123456", line);
+    setTimeout(() => bf.enqueue(Buffer.from("123456\r")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("\n123456\r\n")), 10);
+    line = await bf.readTill("\n", "ascii", true);
+    assert.equal("123456", line);
+    line = await bf.readTill("\n", "ascii", true);
+    assert.equal("123456", line);
+}
+
+async function readSizes() {
+    const bf = new BufferStream();
+
+    setTimeout(() => bf.enqueue(Buffer.from("1234")), 10);
+
+    let b1 = await bf.read(1);
+    assert.equal("1", b1.toString("utf8"));
+
+    b1 = await bf.read(2);
+    assert.equal("23", b1.toString("utf8"));
+
+    b1 = await bf.read(1);
+    assert.equal("4", b1.toString("utf8"));
+
+    setTimeout(() => bf.enqueue(Buffer.from("1234")), 10);
+
+    b1 = await bf.read(1);
+    assert.equal("1", b1.toString("utf8"));
+
+    setTimeout(() => bf.enqueue(Buffer.from("1234")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("1234")), 10);
+
+    b1 = await bf.read(2);
+    assert.equal("23", b1.toString("utf8"));
+
+    b1 = await bf.read(2);
+    assert.equal("41", b1.toString("utf8"));
+
+    b1 = await bf.read(7);
+    assert.equal("2341234", b1.toString("utf8"));
+
+    setTimeout(() => bf.enqueue(Buffer.from("1234567890")), 10);
+    setTimeout(() => bf.enqueue(Buffer.from("1234567890")), 10);
+
+    b1 = await bf.read(1);
+    assert.equal("1", b1.toString("utf8"));
+
+    b1 = await bf.read(2);
+    assert.equal("23", b1.toString("utf8"));
+
+    b1 = await bf.read(1);
+    assert.equal("4", b1.toString("utf8"));
+
+    b1 = await bf.read(10);
+    assert.equal("5678901234", b1.toString("utf8"));
+
+    b1 = await bf.read(6);
+    assert.equal("567890", b1.toString("utf8"));
+
+    assert.equal(0, bf.start);
+}

package/src/tests/protocols/parsers/email-parser.test.ts

@@ -3,42 +3,6 @@ import { MimeNode, ReadableLineStream, LineStream } from "@social-mail/shared/in
 import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
 import { tmpName } from "tmp-promise";
 
-class FileLineStream extends LineStream {
-
-    constructor(private file: LocalFile) {
-        super();
-    }
-
-    async *lines() {
-        let line = "";
-        const trimEndR = (t: string) => {
-            if (t.endsWith("\r")) {
-                return t.substring(0, t.length - 1);
-            }
-            return t;
-        };
-        for await(const buffer of this.file.readBuffers(4 * 1024 * 1024)) {
-            let start = 0;
-            do {
-
-                const index = buffer.indexOf("\n", start);
-                if (index === -1) {
-                    line += buffer.toString("utf8", start);
-                    break;
-                }
-
-                yield trimEndR(line + buffer.toString("utf8", start, index));
-                start = index + 1;
-                line = "";
-            } while (true);
-        }
-        line = trimEndR(line);
-        if (line) {
-            yield line;
-        }
-    }
-}
-
 export default async function() {
 
     const tmp = await tmpName();

package/src/tests/services/smtp/auth.ts

@@ -3,7 +3,7 @@ import ESmtpClient from "../../../protocols/smtp/ESmtpClient.js";
 
 export default async function () {
 
-    return true;
+    // return true;
 
     using c = await ESmtpClient.connect({
         port: 25,
@@ -11,8 +11,8 @@ export default async function () {
         helo: "localhost",
         auth: {
             login: {
-                username: "",
-                password: ""
+                username: "a",
+                password: "b"
             }
         },
         logger: console.log