@chronicle.app/email 0.3.0 → 0.4.0

package/README.md

@@ -63,14 +63,21 @@ The extractor parses the following email components:
 
 ## Schema
 
-- Each email is a `MessageAction` by the sender whose object is a `Message`.
-  The message's `name` is the subject, its `description` is the body, and its
-  `recipient`s are the To, CC and BCC addresses.
-- Each address is an `Agent` whose `handle` is the email address and whose
-  `name` is the display name, or the address when there is none.
+Messages are built by [@chronicle.app/email-core](../../core/email/README.md), the same way the Gmail plugin builds them:
 
-The source is `email`. A message with a Message-ID uses it as its `sourceId`;
-one without is identified by its sender, date and subject.
+- Each email is a `MessageAction` by the sender whose object is a `Message`.
+  The message's `name` is the subject, its `recipient`s are the To, Cc and Bcc
+  addresses, and it is `inReplyTo` the message its In-Reply-To names.
+- Its `body` is what the sender wrote, as plain text or Markdown: no HTML,
+  quoted history, or signature.
+- Each address is an `Agent` whose `handle` is the email address, lowercased.
+  A display name is an `alternateName`, not its `name`: the sender or their
+  address book chose it, and it changes from message to message. A display
+  name that's only the address is left out.
+
+A message is keyed by its Message-ID in the `email` namespace, so the same email read
+from an mbox and from Gmail is one node. A person is their address, lowercased,
+in the `email` namespace other plugins link an address to.
 
 ## Tests
 

package/dist/connectors/EmailTransformer.d.ts

@@ -2,7 +2,5 @@ import { ChronicleTransformer, Record } from '@chronicle.app/etl';
 import { ActionAndChildren } from '@chronicle.app/schema';
 export default class EmailTransformer extends ChronicleTransformer {
     transform(record: Record): Promise<ActionAndChildren[]>;
-    private buildEmailActions;
-    private buildPersonFromEmail;
 }
 //# sourceMappingURL=EmailTransformer.d.ts.map

package/dist/connectors/EmailTransformer.js

@@ -1,91 +1,9 @@
 import { ChronicleTransformer } from '@chronicle.app/etl';
-import { createRequire } from 'node:module';
-const require = createRequire(import.meta.url);
-const parseEmailAddresses = require('emailjs-addressparser').default;
+import { messageAction } from '@chronicle.app/email-core';
 export default class EmailTransformer extends ChronicleTransformer {
     async transform(record) {
-        const actions = [];
-        if (record.extraction.recordType === 'emails') {
-            actions.push(...this.buildEmailActions(record));
-        }
-        return actions;
-    }
-    buildEmailActions(record) {
-        const actions = [];
-        const emailData = record.data;
-        // Build sender person with email address as handle
-        const sender = this.buildPersonFromEmail(emailData.from);
-        // Build recipient persons
-        const recipients = [
-            ...emailData.to.map((email) => this.buildPersonFromEmail(email)),
-            ...emailData.cc.map((email) => this.buildPersonFromEmail(email)),
-            ...emailData.bcc.map((email) => this.buildPersonFromEmail(email)),
-        ];
-        // Every action requires a valid agent
-        if (!sender) {
-            throw new Error(`Invalid or missing sender in email ${emailData.messageId ?? '(no Message-ID)'}: ${emailData.from}`);
-        }
-        // Identity. The Message-ID is the source's own id and stays `sourceId`. A
-        // message without one (common in early-2000s mail) is keyed on the natural
-        // composite of who sent it, when, and under what subject — the same three
-        // real headers the extractor's keyOf composes, read here off the
-        // transformed fields: the MessageAction carries them directly, the Message
-        // reaches the first two through its carrying action (`action.*`). Dotted
-        // paths to scalars only; a whole nested node in a key couples identity to
-        // every mutable attribute of that node.
-        const messageId = emailData.messageId || null;
-        const timestamp = emailData.date ? new Date(emailData.date) : new Date();
-        if (!messageId && (!emailData.date || Number.isNaN(timestamp.getTime()))) {
-            throw new Error(`Email without a Message-ID needs a parseable Date to be identified: ${emailData.from} / ${emailData.subject}`);
-        }
-        // Build email message entity
-        const emailMessage = {
-            '@type': 'Message',
-            source: 'email',
-            ...(messageId && { sourceId: messageId }),
-            name: emailData.subject,
-            description: emailData.body,
-            recipient: recipients.filter(Boolean),
-            author: [sender],
-            '@key': messageId
-                ? ['@type', 'source', 'sourceId']
-                : ['@type', 'source', 'action.timestamp', 'action.agent.handle', 'name'],
-        };
-        const messageAction = {
-            '@type': 'MessageAction',
-            timestamp,
-            source: 'email',
-            ...(messageId && { sourceId: messageId }),
-            '@key': messageId
-                ? ['source', 'sourceId', '@type']
-                : ['@type', 'source', 'timestamp', 'agent.handle', 'object.name'],
-            agent: sender,
-            object: emailMessage,
-        };
-        actions.push(messageAction);
-        return actions;
-    }
-    buildPersonFromEmail(emailString) {
-        if (!emailString?.trim())
-            return null;
-        try {
-            const parsed = parseEmailAddresses(emailString);
-            if (!parsed || parsed.length === 0)
-                return null;
-            // Take the first parsed address
-            const address = parsed[0];
-            return {
-                '@type': 'Agent',
-                source: 'email',
-                handle: address.address, // Clean email address
-                name: address.name || address.address, // Use name if available, fallback to email
-                '@key': ['@type', 'source', 'handle'],
-            };
-        }
-        catch {
-            // Log and skip malformed addresses
-            console.warn(`Failed to parse email address: ${emailString}`);
-            return null;
-        }
+        if (record.extraction.recordType !== 'emails')
+            return [];
+        return [messageAction(record.data)];
     }
 }

package/dist/connectors/mbox/EmailMboxExtractor.d.ts

@@ -28,27 +28,13 @@ export declare class EmailMboxExtractor extends Extractor<typeof EmailMboxExtrac
     }>;
     /**
      * The RFC 5322 Message-ID header — the message's own id, identical in any mail
-     * tool. A message without one is keyed on the natural composite of its From,
-     * Date, and Subject headers: real source fields, never an invented id. That
-     * needs From and a Date that parses; a message with neither has nothing to be
-     * identified by, and parseMbox never emits it. EmailTransformer keys the same
-     * records on the same three facts (sender handle, sent time, subject).
+     * tool. A message without one is keyed on the natural composite of its
+     * sender, sent time, and subject: real source fields, never an invented id.
+     * A message with neither has nothing to be identified by, and is skipped.
+     * The transformer keys the same records on the same three facts.
      */
     keyOf(record: Record): string | null;
-    /**
-     * Message-ID when present; otherwise `From␟Date␟Subject` over the raw
-     * headers (Subject may be empty and still takes part); null when there is no
-     * From or no parseable Date to compose.
-     */
-    static identityOf(headers: {
-        [key: string]: string;
-    }): string | null;
     extract(): AsyncGenerator<Record>;
-    private parseMbox;
-    private parseMessage;
-    private extractTextContent;
-    private decodeMimeHeader;
-    private parseEmailList;
     determineCount(): Promise<number | null>;
 }
 //# sourceMappingURL=EmailMboxExtractor.d.ts.map

package/dist/connectors/mbox/EmailMboxExtractor.js

@@ -1,17 +1,7 @@
 import { Extractor } from '@chronicle.app/etl';
-import { readFile } from 'node:fs/promises';
+import { countMbox, identityOf, parseMessage, readMbox, } from '@chronicle.app/email-core';
 import { z } from 'zod';
-import { createRequire } from 'node:module';
 import EmailTransformer from '../EmailTransformer.js';
-const require = createRequire(import.meta.url);
-const { Libmime } = require('libmime');
-const libmime = new Libmime();
-/**
- * Joins the parts of a composite key. A control character that RFC 5322 header
- * text cannot carry, so no From/Date/Subject combination can collide with
- * another by concatenation.
- */
-const COMPOSITE_DELIMITER = '\u001F';
 export class EmailMboxExtractor extends Extractor {
     static source = 'email';
     static description = 'Messages from an mbox file';
@@ -25,214 +15,64 @@ export class EmailMboxExtractor extends Extractor {
     });
     /**
      * The RFC 5322 Message-ID header — the message's own id, identical in any mail
-     * tool. A message without one is keyed on the natural composite of its From,
-     * Date, and Subject headers: real source fields, never an invented id. That
-     * needs From and a Date that parses; a message with neither has nothing to be
-     * identified by, and parseMbox never emits it. EmailTransformer keys the same
-     * records on the same three facts (sender handle, sent time, subject).
+     * tool. A message without one is keyed on the natural composite of its
+     * sender, sent time, and subject: real source fields, never an invented id.
+     * A message with neither has nothing to be identified by, and is skipped.
+     * The transformer keys the same records on the same three facts.
      */
     keyOf(record) {
-        return EmailMboxExtractor.identityOf(record.data.headers);
-    }
-    /**
-     * Message-ID when present; otherwise `From␟Date␟Subject` over the raw
-     * headers (Subject may be empty and still takes part); null when there is no
-     * From or no parseable Date to compose.
-     */
-    static identityOf(headers) {
-        const messageId = headers['message-id'];
-        if (messageId)
-            return messageId;
-        const { from, date } = headers;
-        if (!from || !date || Number.isNaN(new Date(date).getTime()))
-            return null;
-        return [from, date, headers.subject ?? ''].join(COMPOSITE_DELIMITER);
+        return identityOf(record.data);
     }
     async *extract() {
         const config = this.config;
-        try {
-            const content = await readFile(config.input, 'utf-8');
-            const emails = this.parseMbox(content);
-            let count = 0;
-            for (const email of emails) {
-                // Apply date filtering if configured
-                if (config.since || config.until) {
-                    const emailDate = new Date(email.date);
-                    if (config.since && emailDate < config.since)
-                        continue;
-                    if (config.until && emailDate > config.until)
-                        continue;
-                }
-                // Apply limit if configured
-                if (this.shouldStopExtracting(count))
-                    break;
-                yield this.createRecord(email, {
-                    messageId: email.messageId,
-                    from: email.from,
-                    subject: email.subject,
-                    strategy: 'mbox',
-                });
-                count++;
-            }
-        }
-        catch (error) {
-            throw new Error(`Failed to read mbox file: ${error}`);
-        }
-    }
-    parseMbox(content) {
-        const emails = [];
+        let count = 0;
         let unidentified = 0;
-        const messages = content.split(/^From /m);
-        // Skip the first element if it's empty (before the first "From " line)
-        const validMessages = messages.slice(1);
-        for (const message of validMessages) {
-            const email = this.parseMessage('From ' + message);
-            if (!email)
+        let undated = 0;
+        for await (const { raw, envelopeDate } of readMbox(config.input)) {
+            const email = await parseMessage(raw);
+            // No Date header: when its `From ` line says it arrived.
+            email.date ??= envelopeDate;
+            if (!email.date) {
+                undated++;
                 continue;
-            if (EmailMboxExtractor.identityOf(email.headers) === null) {
+            }
+            if (identityOf(email) === null) {
                 // No Message-ID, and no From + parseable Date to compose a natural key
                 // from: the source gives nothing real to identify this message by, and
                 // a fabricated id would mint a new entity on every run. Skip and say so.
                 unidentified++;
                 continue;
             }
-            emails.push(email);
-        }
-        if (unidentified > 0) {
-            this.logger.warn(`Skipped ${unidentified} message(s) with no Message-ID and no From + Date to key on`);
-        }
-        return emails;
-    }
-    parseMessage(message) {
-        const lines = message.split('\n');
-        const headers = {};
-        let headerSection = true;
-        const bodyLines = [];
-        for (const line of lines) {
-            if (headerSection) {
-                if (line.trim() === '') {
-                    headerSection = false;
+            if (config.since || config.until) {
+                const sent = email.date ? new Date(email.date) : undefined;
+                if (!sent)
                     continue;
-                }
-                // Handle multi-line headers
-                if (line.startsWith(' ') || line.startsWith('\t')) {
-                    const lastHeader = Object.keys(headers).pop();
-                    if (lastHeader) {
-                        headers[lastHeader] += ' ' + line.trim();
-                    }
+                if (config.since && sent < config.since)
+                    continue;
+                if (config.until && sent > config.until)
                     continue;
-                }
-                const colonIndex = line.indexOf(':');
-                if (colonIndex > 0) {
-                    const key = line.slice(0, Math.max(0, colonIndex)).trim().toLowerCase();
-                    const value = line.slice(Math.max(0, colonIndex + 1)).trim();
-                    headers[key] = value;
-                }
-            }
-            else {
-                bodyLines.push(line);
-            }
-        }
-        // Extract required fields
-        const from = headers.from || '';
-        const to = this.parseEmailList(headers.to || '');
-        const cc = this.parseEmailList(headers.cc || '');
-        const bcc = this.parseEmailList(headers.bcc || '');
-        const subject = this.decodeMimeHeader(headers.subject || '');
-        const date = headers.date || '';
-        const messageId = headers['message-id'] ?? null;
-        const body = this.extractTextContent(bodyLines, headers);
-        if (!from && !subject) {
-            return null; // Skip invalid messages
-        }
-        return {
-            from,
-            to,
-            cc,
-            bcc,
-            subject,
-            date,
-            messageId,
-            body,
-            headers,
-        };
-    }
-    extractTextContent(bodyLines, headers) {
-        const contentType = headers['content-type'] || '';
-        // If not multipart, return the body as-is
-        if (!contentType.toLowerCase().includes('multipart')) {
-            return bodyLines.join('\n').trim();
-        }
-        // Extract boundary from Content-Type header
-        const boundaryMatch = contentType.match(/boundary=["']?([^"';]+)["']?/i);
-        if (!boundaryMatch) {
-            return bodyLines.join('\n').trim(); // Fallback if no boundary found
-        }
-        const boundary = boundaryMatch[1];
-        const bodyText = bodyLines.join('\n');
-        // Split by boundary
-        const parts = bodyText.split(`--${boundary}`);
-        const textParts = [];
-        for (const part of parts) {
-            if (!part.trim() || part.trim() === '--')
-                continue;
-            // Parse each part's headers and content
-            const partLines = part.split('\n');
-            const partHeaders = {};
-            let partHeaderSection = true;
-            const partBodyLines = [];
-            for (const line of partLines) {
-                if (partHeaderSection) {
-                    if (line.trim() === '') {
-                        partHeaderSection = false;
-                        continue;
-                    }
-                    const colonIndex = line.indexOf(':');
-                    if (colonIndex > 0) {
-                        const key = line.slice(0, colonIndex).trim().toLowerCase();
-                        const value = line.slice(colonIndex + 1).trim();
-                        partHeaders[key] = value;
-                    }
-                }
-                else {
-                    partBodyLines.push(line);
-                }
-            }
-            // Only include text/plain parts
-            const partContentType = partHeaders['content-type'] || '';
-            if (partContentType.toLowerCase().includes('text/plain')) {
-                textParts.push(partBodyLines.join('\n').trim());
             }
+            if (this.shouldStopExtracting(count))
+                break;
+            yield this.createRecord(email, {
+                messageId: email.messageId,
+                from: email.from?.address,
+                subject: email.subject,
+                strategy: 'mbox',
+            });
+            count++;
         }
-        // Join all text parts
-        return textParts.join('\n\n').trim();
-    }
-    decodeMimeHeader(headerValue) {
-        if (!headerValue)
-            return '';
-        try {
-            return libmime.decodeWords(headerValue);
+        if (undated > 0) {
+            this.logger.warn(`Skipped ${undated} message(s) with no date to place them by`);
         }
-        catch {
-            // If decoding fails, return the original value
-            this.logger.warn('Failed to decode MIME header', { headerValue });
-            return headerValue;
+        if (unidentified > 0) {
+            this.logger.warn(`Skipped ${unidentified} message(s) with no Message-ID and no From + Date to key on`);
         }
     }
-    parseEmailList(emailString) {
-        if (!emailString.trim())
-            return [];
-        return emailString
-            .split(',')
-            .map(email => email.trim())
-            .filter(email => email.length > 0);
-    }
     async determineCount() {
         const config = this.config;
         try {
-            const content = await readFile(config.input, 'utf-8');
-            const messages = content.split(/^From /m);
-            return Math.max(0, messages.length - 1); // Subtract 1 for the content before first "From "
+            return await countMbox(config.input);
         }
         catch {
             return null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/email",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -11,7 +11,8 @@
     "build": "tsc -b tsconfig.lib.json",
     "typecheck": "tsc -p tsconfig.json",
     "lint": "eslint src --ext .ts,.js",
-    "test": "node --test src/*.test.js"
+    "test": "node --test src/*.test.js",
+    "shapes": "node src/shapes.test.js --update"
   },
   "keywords": [
     "chronicle",
@@ -22,9 +23,9 @@
   "type": "module",
   "description": "Chronicle plugin for email data from mbox format files",
   "devDependencies": {
-    "@chronicle.app/etl": "0.3.0",
-    "@chronicle.app/schema": "0.3.0",
-    "@chronicle.app/tsconfig": "0.3.0",
+    "@chronicle.app/etl": "0.4.0",
+    "@chronicle.app/schema": "0.4.0",
+    "@chronicle.app/tsconfig": "0.4.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },
@@ -39,8 +40,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "emailjs-addressparser": "^2.0.3",
-    "libmime": "^5.3.7",
+    "@chronicle.app/email-core": "0.4.0",
     "zod": "^3.24.1"
   },
   "peerDependencies": {