@postalsys/email-text-tools 1.3.2 → 1.4.1

package/.ncurc.js

@@ -2,7 +2,6 @@ module.exports = {
     upgrade: true,
     reject: [
         // Block package upgrades that moved to ESM
-        'node-fetch',
         'nanoid'
     ]
 };

package/examples/risk.js

@@ -21,7 +21,12 @@ async function analyzeEmail(path) {
             html: parsed.html,
             text: parsed.text
         },
-        process.env.OPENAI_API_KEY
+        process.env.OPENAI_API_KEY,
+        {
+            //gptModel: 'gpt-3.5-turbo'
+            gptModel: 'gpt-4',
+            maxTokens: 6000
+        }
     );
 
     result.file = Path.basename(path);

package/examples/summary.js

@@ -3,6 +3,7 @@
 const fs = require('fs').promises;
 const generateSummary = require('../lib/generate-summary');
 const simpleParser = require('mailparser').simpleParser;
+const libmime = require('libmime');
 
 async function main() {
     const eml = await fs.readFile(process.argv[2]);
@@ -11,12 +12,17 @@ async function main() {
 
     const summary = await generateSummary(
         {
-            from: parsed.from.value,
-            subject: parsed.subject,
+            headers: parsed.headerLines.map(header => libmime.decodeHeader(header.line)),
+            attachments: parsed.attachments,
             html: parsed.html,
             text: parsed.text
         },
-        process.env.OPENAI_API_KEY
+        process.env.OPENAI_API_KEY,
+        {
+            //gptModel: 'gpt-3.5-turbo'
+            gptModel: 'gpt-4',
+            maxTokens: 6000
+        }
     );
 
     console.log(summary);

package/lib/generate-summary.js

@@ -2,10 +2,15 @@
 
 const packageData = require('../package.json');
 const htmlToText = require('../lib/html-to-text');
-const nodeFetch = require('node-fetch');
-const fetchCmd = global.fetch || nodeFetch;
+const GPT3Tokenizer = require('gpt3-tokenizer').default;
 
-const MAX_TEXT_LENGTH = 4 * 1024;
+const { fetch: fetchCmd, Agent } = require('undici');
+const fetchAgent = new Agent({ connect: { timeout: 90 * 1000 } });
+
+const tokenizer = new GPT3Tokenizer({ type: 'gpt3' });
+
+const MAX_ALLOWED_TEXT_LENGTH = 32 * 1024;
+const MAX_ALLOWED_TOKENS = 4000;
 const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
 
 const SYSTEM_PROMPT = `
@@ -16,38 +21,89 @@ You do not comment or explain anything.
     .trim();
 
 const USER_PROMPT = `
-Describe the sentiment of the email using one word. Use either "positive", "neutral" or "negative". Prefix the sentiment with "Sentiment:".
-On a separate line, write a one-sentence summary of the email. Prefix the summary with "Summary:".
-I want you to only reply with the sentiment and summary, do not write explanations. 
-All the text that follows from now on is the email message:
-`
-    .replace(/\s+/g, ' ')
-    .trim();
-
-async function generateSummary(message, apiToken) {
-    let text = message.text;
-    if (message.html && (!text || message.html.length > text.length * 10)) {
+Instructions:
+- You are an executive assistant scanning incoming emails to inform your superiors about what is important and what is not.
+- Your task is to monitor and analyze incoming emails, which consist of the message headers, a list of attachments, and text content
+- Describe the sentiment of the email using one word. Use either "positive", "neutral" or "negative". Include this value in the response as a "sentiment" property.
+- Generate a one-sentence summary of the email. Include this value in the response as a "summary" property.
+- Does it seem like the sender of the email would expect a reply to this email? Include this information in the response as a "shouldReply" property with the value "true" if they expect it and "false" if not.
+- If this email is a reply to a previous email, then extract the text content that only the email's sender wrote, and include this as a "replyText" property in the response.
+- Do not include message signatures in the extracted reply text
+- Respond with a JSON formatted structure. Do not write any other explanations.
+
+Facts:
+- An email consists of message headers, an attachments list, and text content
+- The email to analyze is formatted in a JSON format
+- An email is a reply to a previous email only if it includes a "in-reply-to" header
+- The email structure includes a property "headers" that contains an array of header values.
+- Each header contains two properties, "key" as the header field key name and "value" as the header value without the key prefix
+- The email structure includes a property "attachments" that contains an array of attachments.
+- Each attachment includes a "filename" property that describes the file name, a "contentType" property that describes the Content-Type value of the attachment
+- The email includes a "text" property for the text content
+
+Analyze the following email:
+`.trim();
+
+async function generateSummary(message, apiToken, opts) {
+    opts = opts || {};
+
+    let maxAllowedTokens = opts.maxTokens || MAX_ALLOWED_TOKENS;
+    let gptModel = opts.gptModel || 'gpt-3.5-turbo';
+
+    let text = message.text || '';
+    if (message.html && (!text || message.html.length > text.length * 2)) {
         text = htmlToText(message.html);
     }
 
-    text = text.trim();
-    if (text.length > MAX_TEXT_LENGTH) {
-        text = text.substring(0, MAX_TEXT_LENGTH);
-    }
+    let prompt;
 
-    let header = [];
-    if (message.subject) {
-        header.push(`Subject: ${message.subject}`);
-    }
-    if (message.from) {
-        let from = [].concat(message.from || []).shift();
-        header.push(`From: ${from.name || from.address}`);
+    const content = {
+        headers: [].concat(message.headers || []).filter(
+            // use a whitelist
+            header => ['from', 'to', 'cc', 'bcc', 'subject', 'mime-version', 'date', 'content-type', 'list-id'].includes(header.key)
+        ),
+        attachments: [].concat(message.attachments || []).map(attachment => ({ filename: attachment.filename, contentType: attachment.contentType }))
+    };
+
+    let charactersRemoved = 0;
+    let promptText = text;
+
+    if (promptText.length > MAX_ALLOWED_TEXT_LENGTH) {
+        charactersRemoved += promptText.length - MAX_ALLOWED_TEXT_LENGTH;
+        promptText = promptText.substr(0, MAX_ALLOWED_TEXT_LENGTH);
     }
 
-    if (header.length) {
-        text = `${header.join('\n')}
+    while (promptText.length) {
+        content.text = promptText;
+        prompt = `${USER_PROMPT}
 
-${text}`;
+${JSON.stringify(content)}`;
+
+        let tokens = tokenizer.encode(prompt);
+        if (tokens.text.length <= maxAllowedTokens) {
+            break;
+        }
+        if (promptText.length > 2 * 1024 * 1024) {
+            promptText = promptText.substring(0, promptText.length - 1024 * 1024);
+            charactersRemoved += 1024 * 1024;
+        } else if (promptText.length > 2 * 1024) {
+            promptText = promptText.substring(0, promptText.length - 1024);
+            charactersRemoved += 1024;
+        } else if (promptText.length > 2 * 256) {
+            promptText = promptText.substring(0, promptText.length - 256);
+            charactersRemoved += 255;
+        } else if (promptText.length > 2 * 100) {
+            promptText = promptText.substring(0, promptText.length - 100);
+            charactersRemoved += 100;
+        } else if (promptText.length > 2 * 10) {
+            promptText = promptText.substring(0, promptText.length - 10);
+            charactersRemoved += 10;
+        } else if (promptText.length > 1) {
+            promptText = promptText.substring(0, promptText.length - 1);
+            charactersRemoved += 1;
+        } else {
+            throw new Error(`Prompt too long. Removed ${charactersRemoved} characters.`);
+        }
     }
 
     let headers = {
@@ -57,7 +113,7 @@ ${text}`;
     };
 
     let payload = {
-        model: 'gpt-3.5-turbo',
+        model: gptModel,
         messages: [
             {
                 role: 'system',
@@ -65,42 +121,58 @@ ${text}`;
             },
             {
                 role: 'user',
-                content: `${USER_PROMPT}
-
-${text}`
+                content: prompt
             }
         ]
     };
 
-    let res = await fetchCmd(OPENAI_API_URL, {
-        method: 'post',
-        headers,
-        body: JSON.stringify(payload)
-    });
+    let res;
+    let data;
+    let retries = 0;
+
+    let run = async () => {
+        res = await fetchCmd(OPENAI_API_URL, {
+            method: 'post',
+            headers,
+            body: JSON.stringify(payload),
+            dispatcher: fetchAgent
+        });
+
+        data = await res.json();
+
+        if (!res.ok) {
+            if (res.status === 429 && ++retries < 5) {
+                // try again
+                await new Promise(r => setTimeout(r, 1000));
+                return await run();
+            }
 
-    let data = await res.json();
+            if (data && data.error) {
+                let error = new Error(data.error.message || data.error);
+                if (data.error.code) {
+                    error.code = data.error.code;
+                }
 
-    if (!res.ok) {
-        if (data && data.error) {
-            let error = new Error(data.error.message || data.error);
-            if (data.error.code) {
-                error.code = data.error.code;
+                error.statusCode = res.status;
+                throw error;
             }
 
+            let error = new Error('Failed to run API request');
             error.statusCode = res.status;
             throw error;
         }
 
-        let error = new Error('Failed to run API request');
-        error.statusCode = res.status;
-        throw error;
-    }
+        if (!data) {
+            throw new Error(`Failed to POST API request`);
+        }
+    };
 
-    if (!data) {
-        throw new Error(`Failed to POST API request`);
-    }
+    const reqStartTime = Date.now();
+    await run();
+    const reqEndTime = Date.now();
 
-    let output =
+    let values;
+    const output =
         data &&
         data.choices &&
         data.choices
@@ -112,25 +184,29 @@ ${text}`
             .replace(/^\s*"|"\s*$/g, '')
             .trim();
 
-    let structured = {};
-
-    let summaryMatch = output.match(/\bsummary:\s*(.*)$/im);
-    if (summaryMatch) {
-        structured.summary = summaryMatch[1].replace(/^["'\s]+|["'\s]+$/g, '');
+    try {
+        values = JSON.parse(output);
+        if (typeof values.shouldReply === 'string') {
+            values.shouldReply = values.shouldReply === 'true';
+        }
+        if (typeof values.shouldReply !== 'undefined' && typeof values.shouldReply !== 'boolean') {
+            delete values.shouldReply;
+        }
+    } catch (err) {
+        let error = new Error('Failed to parse output from OpenAI API');
+        error.textContent = output;
+        throw error;
     }
 
-    let sentimentMatch = output.match(/\bsentiment:\s*(.*)$/im);
-    if (sentimentMatch) {
-        structured.sentiment = sentimentMatch[1].replace(/^["'\s]+|["'\s]+$/g, '').toLowerCase();
-    }
+    const response = Object.assign({ id: null, tokens: null, model: null }, values, {
+        id: data && data.id,
+        tokens: data && data.usage && data.usage.total_tokens,
+        model: gptModel,
 
-    const response = Object.assign(
-        {
-            id: data && data.id,
-            tokens: data && data.usage && data.usage.total_tokens
-        },
-        structured
-    );
+        _text: content.text,
+        _time: reqEndTime - reqStartTime,
+        _cr: charactersRemoved
+    });
 
     return response;
 }

package/lib/risk-analysis.js

@@ -2,9 +2,10 @@
 
 const packageData = require('../package.json');
 const htmlToText = require('../lib/html-to-text');
-const nodeFetch = require('node-fetch');
 const GPT3Tokenizer = require('gpt3-tokenizer').default;
-const fetchCmd = global.fetch || nodeFetch;
+
+const { fetch: fetchCmd, Agent } = require('undici');
+const fetchAgent = new Agent({ connect: { timeout: 90 * 1000 } });
 
 const tokenizer = new GPT3Tokenizer({ type: 'gpt3' });
 
@@ -60,7 +61,12 @@ Facts:
 Analyze the following email:
 `.trim();
 
-async function riskAnalysis(message, apiToken) {
+async function riskAnalysis(message, apiToken, opts) {
+    opts = opts || {};
+
+    let maxAllowedTokens = opts.maxTokens || MAX_ALLOWED_TOKENS;
+    let gptModel = opts.gptModel || 'gpt-3.5-turbo';
+
     let text = message.text || '';
     if (message.html && (!text || message.html.length > text.length * 2)) {
         text = htmlToText(message.html);
@@ -105,7 +111,7 @@ async function riskAnalysis(message, apiToken) {
 ${JSON.stringify(content)}`;
 
         let tokens = tokenizer.encode(prompt);
-        if (tokens.text.length <= MAX_ALLOWED_TOKENS) {
+        if (tokens.text.length <= maxAllowedTokens) {
             break;
         }
         if (promptText.length > 2 * 1024 * 1024) {
@@ -138,7 +144,7 @@ ${JSON.stringify(content)}`;
     };
 
     let payload = {
-        model: 'gpt-3.5-turbo',
+        model: gptModel,
         messages: [
             {
                 role: 'system',
@@ -151,37 +157,50 @@ ${JSON.stringify(content)}`;
         ]
     };
 
-    const reqStartTime = Date.now();
-
-    let res = await fetchCmd(OPENAI_API_URL, {
-        method: 'post',
-        headers,
-        body: JSON.stringify(payload)
-    });
-
-    const reqEndTime = Date.now();
+    let res;
+    let data;
+    let retries = 0;
+
+    let run = async () => {
+        res = await fetchCmd(OPENAI_API_URL, {
+            method: 'post',
+            headers,
+            body: JSON.stringify(payload),
+            dispatcher: fetchAgent
+        });
+
+        data = await res.json();
+
+        if (!res.ok) {
+            if (res.status === 429 && ++retries < 5) {
+                // try again
+                await new Promise(r => setTimeout(r, 1000));
+                return await run();
+            }
 
-    let data = await res.json();
+            if (data && data.error) {
+                let error = new Error(data.error.message || data.error);
+                if (data.error.code) {
+                    error.code = data.error.code;
+                }
 
-    if (!res.ok) {
-        if (data && data.error) {
-            let error = new Error(data.error.message || data.error);
-            if (data.error.code) {
-                error.code = data.error.code;
+                error.statusCode = res.status;
+                throw error;
             }
 
+            let error = new Error('Failed to run API request');
             error.statusCode = res.status;
             throw error;
         }
 
-        let error = new Error('Failed to run API request');
-        error.statusCode = res.status;
-        throw error;
-    }
+        if (!data) {
+            throw new Error(`Failed to POST API request`);
+        }
+    };
 
-    if (!data) {
-        throw new Error(`Failed to POST API request`);
-    }
+    const reqStartTime = Date.now();
+    await run();
+    const reqEndTime = Date.now();
 
     let values;
     const output =
@@ -205,9 +224,10 @@ ${JSON.stringify(content)}`;
         throw error;
     }
 
-    const response = Object.assign({ id: null, tokens: null }, values, {
+    const response = Object.assign({ id: null, tokens: null, model: null }, values, {
         id: data && data.id,
         tokens: data && data.usage && data.usage.total_tokens,
+        model: gptModel,
 
         _text: content.text,
         _time: reqEndTime - reqStartTime,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@postalsys/email-text-tools",
-    "version": "1.3.2",
+    "version": "1.4.1",
     "description": "Email text tools",
     "main": "index.js",
     "scripts": {
@@ -23,7 +23,7 @@
         "mailparser": "3.6.4"
     },
     "dependencies": {
-        "dompurify": "3.0.1",
+        "dompurify": "3.0.2",
         "gpt3-tokenizer": "1.1.5",
         "handlebars": "4.7.7",
         "he": "1.2.0",
@@ -33,8 +33,8 @@
         "juice": "9.0.0",
         "linkify-it": "4.0.1",
         "moment-timezone": "0.5.43",
-        "node-fetch": "2.6.7",
         "node-html-parser": "6.1.5",
-        "tlds": "1.238.0"
+        "tlds": "1.238.0",
+        "undici": "5.22.0"
     }
 }