@postalsys/email-text-tools 1.3.1 → 1.4.0

package/examples/risk.js

@@ -21,7 +21,12 @@ async function analyzeEmail(path) {
             html: parsed.html,
             text: parsed.text
         },
-        process.env.OPENAI_API_KEY
+        process.env.OPENAI_API_KEY,
+        {
+            //gptModel: 'gpt-3.5-turbo'
+            gptModel: 'gpt-4',
+            maxTokens: 6000
+        }
     );
 
     result.file = Path.basename(path);

package/examples/summary.js

@@ -3,6 +3,7 @@
 const fs = require('fs').promises;
 const generateSummary = require('../lib/generate-summary');
 const simpleParser = require('mailparser').simpleParser;
+const libmime = require('libmime');
 
 async function main() {
     const eml = await fs.readFile(process.argv[2]);
@@ -11,12 +12,17 @@ async function main() {
 
     const summary = await generateSummary(
         {
-            from: parsed.from.value,
-            subject: parsed.subject,
+            headers: parsed.headerLines.map(header => libmime.decodeHeader(header.line)),
+            attachments: parsed.attachments,
             html: parsed.html,
             text: parsed.text
         },
-        process.env.OPENAI_API_KEY
+        process.env.OPENAI_API_KEY,
+        {
+            //gptModel: 'gpt-3.5-turbo'
+            gptModel: 'gpt-4',
+            maxTokens: 6000
+        }
     );
 
     console.log(summary);

package/lib/generate-summary.js

@@ -3,9 +3,14 @@
 const packageData = require('../package.json');
 const htmlToText = require('../lib/html-to-text');
 const nodeFetch = require('node-fetch');
+const GPT3Tokenizer = require('gpt3-tokenizer').default;
+
 const fetchCmd = global.fetch || nodeFetch;
 
-const MAX_TEXT_LENGTH = 4 * 1024;
+const tokenizer = new GPT3Tokenizer({ type: 'gpt3' });
+
+const MAX_ALLOWED_TEXT_LENGTH = 32 * 1024;
+const MAX_ALLOWED_TOKENS = 4000;
 const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
 
 const SYSTEM_PROMPT = `
@@ -16,38 +21,89 @@ You do not comment or explain anything.
     .trim();
 
 const USER_PROMPT = `
-Describe the sentiment of the email using one word. Use either "positive", "neutral" or "negative". Prefix the sentiment with "Sentiment:".
-On a separate line, write a one-sentence summary of the email. Prefix the summary with "Summary:".
-I want you to only reply with the sentiment and summary, do not write explanations. 
-All the text that follows from now on is the email message:
-`
-    .replace(/\s+/g, ' ')
-    .trim();
-
-async function generateSummary(message, apiToken) {
-    let text = message.text;
-    if (message.html && (!text || message.html.length > text.length * 10)) {
+Instructions:
+- You are an executive assistant scanning incoming emails to inform your superiors about what is important and what is not.
+- Your task is to monitor and analyze incoming emails, which consist of the message headers, a list of attachments, and text content
+- Describe the sentiment of the email using one word. Use either "positive", "neutral" or "negative". Include this value in the response as a "sentiment" property.
+- Generate a one-sentence summary of the email. Include this value in the response as a "summary" property.
+- Does it seem like the sender of the email would expect a reply to this email? Include this information in the response as a "shouldReply" property with the value "true" if they expect it and "false" if not.
+- If this email is a reply to a previous email, then extract the text content that only the email's sender wrote, and include this as a "replyText" property in the response.
+- Do not include message signatures in the extracted reply text
+- Respond with a JSON formatted structure. Do not write any other explanations.
+
+Facts:
+- An email consists of message headers, an attachments list, and text content
+- The email to analyze is formatted in a JSON format
+- An email is a reply to a previous email only if it includes a "in-reply-to" header
+- The email structure includes a property "headers" that contains an array of header values.
+- Each header contains two properties, "key" as the header field key name and "value" as the header value without the key prefix
+- The email structure includes a property "attachments" that contains an array of attachments.
+- Each attachment includes a "filename" property that describes the file name, a "contentType" property that describes the Content-Type value of the attachment
+- The email includes a "text" property for the text content
+
+Analyze the following email:
+`.trim();
+
+async function generateSummary(message, apiToken, opts) {
+    opts = opts || {};
+
+    let maxAllowedTokens = opts.maxTokens || MAX_ALLOWED_TOKENS;
+    let gptModel = opts.gptModel || 'gpt-3.5-turbo';
+
+    let text = message.text || '';
+    if (message.html && (!text || message.html.length > text.length * 2)) {
         text = htmlToText(message.html);
     }
 
-    text = text.trim();
-    if (text.length > MAX_TEXT_LENGTH) {
-        text = text.substring(0, MAX_TEXT_LENGTH);
-    }
+    let prompt;
 
-    let header = [];
-    if (message.subject) {
-        header.push(`Subject: ${message.subject}`);
-    }
-    if (message.from) {
-        let from = [].concat(message.from || []).shift();
-        header.push(`From: ${from.name || from.address}`);
+    const content = {
+        headers: [].concat(message.headers || []).filter(
+            // use a whitelist
+            header => ['from', 'to', 'cc', 'bcc', 'subject', 'mime-version', 'date', 'content-type', 'list-id'].includes(header.key)
+        ),
+        attachments: [].concat(message.attachments || []).map(attachment => ({ filename: attachment.filename, contentType: attachment.contentType }))
+    };
+
+    let charactersRemoved = 0;
+    let promptText = text;
+
+    if (promptText.length > MAX_ALLOWED_TEXT_LENGTH) {
+        charactersRemoved += promptText.length - MAX_ALLOWED_TEXT_LENGTH;
+        promptText = promptText.substr(0, MAX_ALLOWED_TEXT_LENGTH);
     }
 
-    if (header.length) {
-        text = `${header.join('\n')}
+    while (promptText.length) {
+        content.text = promptText;
+        prompt = `${USER_PROMPT}
 
-${text}`;
+${JSON.stringify(content)}`;
+
+        let tokens = tokenizer.encode(prompt);
+        if (tokens.text.length <= maxAllowedTokens) {
+            break;
+        }
+        if (promptText.length > 2 * 1024 * 1024) {
+            promptText = promptText.substring(0, promptText.length - 1024 * 1024);
+            charactersRemoved += 1024 * 1024;
+        } else if (promptText.length > 2 * 1024) {
+            promptText = promptText.substring(0, promptText.length - 1024);
+            charactersRemoved += 1024;
+        } else if (promptText.length > 2 * 256) {
+            promptText = promptText.substring(0, promptText.length - 256);
+            charactersRemoved += 255;
+        } else if (promptText.length > 2 * 100) {
+            promptText = promptText.substring(0, promptText.length - 100);
+            charactersRemoved += 100;
+        } else if (promptText.length > 2 * 10) {
+            promptText = promptText.substring(0, promptText.length - 10);
+            charactersRemoved += 10;
+        } else if (promptText.length > 1) {
+            promptText = promptText.substring(0, promptText.length - 1);
+            charactersRemoved += 1;
+        } else {
+            throw new Error(`Prompt too long. Removed ${charactersRemoved} characters.`);
+        }
     }
 
     let headers = {
@@ -57,7 +113,7 @@ ${text}`;
     };
 
     let payload = {
-        model: 'gpt-3.5-turbo',
+        model: gptModel,
         messages: [
             {
                 role: 'system',
@@ -65,19 +121,21 @@ ${text}`;
             },
             {
                 role: 'user',
-                content: `${USER_PROMPT}
-
-${text}`
+                content: prompt
             }
         ]
     };
 
+    const reqStartTime = Date.now();
+
     let res = await fetchCmd(OPENAI_API_URL, {
         method: 'post',
         headers,
         body: JSON.stringify(payload)
     });
 
+    const reqEndTime = Date.now();
+
     let data = await res.json();
 
     if (!res.ok) {
@@ -100,7 +158,8 @@ ${text}`
         throw new Error(`Failed to POST API request`);
     }
 
-    let output =
+    let values;
+    const output =
         data &&
         data.choices &&
         data.choices
@@ -112,25 +171,26 @@ ${text}`
             .replace(/^\s*"|"\s*$/g, '')
             .trim();
 
-    let structured = {};
-
-    let summaryMatch = output.match(/\bsummary:\s*(.*)$/im);
-    if (summaryMatch) {
-        structured.summary = summaryMatch[1].replace(/^["'\s]+|["'\s]+$/g, '');
+    try {
+        values = JSON.parse(output);
+        if (typeof values.shouldReply === 'string') {
+            values.shouldReply = values.shouldReply === 'true';
+        }
+    } catch (err) {
+        let error = new Error('Failed to parse output from OpenAI API');
+        error.textContent = output;
+        throw error;
     }
 
-    let sentimentMatch = output.match(/\bsentiment:\s*(.*)$/im);
-    if (sentimentMatch) {
-        structured.sentiment = sentimentMatch[1].replace(/^["'\s]+|["'\s]+$/g, '').toLowerCase();
-    }
+    const response = Object.assign({ id: null, tokens: null, model: null }, values, {
+        id: data && data.id,
+        tokens: data && data.usage && data.usage.total_tokens,
+        model: gptModel,
 
-    const response = Object.assign(
-        {
-            id: data && data.id,
-            tokens: data && data.usage && data.usage.total_tokens
-        },
-        structured
-    );
+        _text: content.text,
+        _time: reqEndTime - reqStartTime,
+        _cr: charactersRemoved
+    });
 
     return response;
 }

package/lib/risk-analysis.js

@@ -4,6 +4,7 @@ const packageData = require('../package.json');
 const htmlToText = require('../lib/html-to-text');
 const nodeFetch = require('node-fetch');
 const GPT3Tokenizer = require('gpt3-tokenizer').default;
+
 const fetchCmd = global.fetch || nodeFetch;
 
 const tokenizer = new GPT3Tokenizer({ type: 'gpt3' });
@@ -20,39 +21,39 @@ I want you to act as are an IT security expert that monitors emails for suspicio
 
 const USER_PROMPT = `
 Instructions:
-- You are an IT security expert.
-- Your task is to monitor and analyze incoming emails which consist of the message headers, a list of attachments, and text content
-- Your analysis should contain (but is not limited to) the following:
+- You are an IT security expert scanning incoming emails to detect and block fraud attempts.
+- Your task is to monitor and analyze incoming emails, which consist of the message headers, a list of attachments, and text content
+- Your analysis should contain (but is not limited to) the following risk factors:
   - Does the email include links with domain names that contain typos or homoglyphs that might mislead the user about the actual target of the link
-  - Does the sender address of the email match the persona or organization the sender email claims to be
-  - Does the email promise the user extremely good financial outcome
+  - Does the sender address of the email not seem to match the persona or organization the sender email claims to be
+  - Does the email promise the user an extremely good financial outcome
   - Does the email promise unclaimed money or goods
   - Does the email offer a vague business opportunity with no specific details
   - Does the email suggest there is a severe penalty if the user does not act as requested
   - Does the email claim that there are technical issues with the user's email account
   - Does the email offer services or activities that are not suitable for underage persons
-  - Does the sender email look like a throwaway address
-  - Does the sender email look like it might be spoofed
+  - Does the sender's email look like a throwaway address
+  - Does the sender's email look like it might be spoofed
   - Does the email contain attachments where the name of the attachment might be misleading and suggest a different file format than is actually used
   - Does the email contain attachments that might be executable files
-- Provide a risk score for the email using the following scale: 1 - 5 (where 1 is low rist, and 5 is high risk), take into account what may happen if a user acts by the instructions given in the email.
+- Provide a risk score for the email using the following scale: 1 - 5 (where 1 is low risk, and 5 is high risk), taking into account what may happen if a user acts by the instructions given in the email.
 - Provide a short single-sentence assessment text that includes details about issues that increase the risk score, do not disclose details that decrease the risk score or do not affect it.
-- Respond with a JSON formatted structure with numeric risk score as "risk" property, and the assessment as the "assessment" property. Do not write any other explanations.
+- Respond with a JSON formatted structure with a numeric risk score as "risk" property and the assessment as the "assessment" property. Do not write any other explanations.
 
 Facts:
-- An email consists of message headers, attachments list, and text content
-- An email might contain an "authentication-results" result header that indicate if the message passed SPF, DKIM, ARC and DMARC checks
-- Having a valid SPF, DKIM and DMARC increases the chances that the sender domain name is not spoofed
-- For valid SPF the "authentication-results" header must include "spf=pass"
-- For valid DKIM the "authentication-results" header must include "dkim=pass"
-- For valid DMARC the "authentication-results" header must include "dmarc=pass"
-- For valid ARC the "authentication-results" header must include "arc=pass"
+- An email consists of message headers, an attachments list, and text content
+- An email might contain an "authentication-results" result header that indicates if the message passed SPF, DKIM, ARC, and DMARC checks
+- Having a valid SPF, DKIM, and DMARC increases the chances that the sender domain name is not spoofed
+- For valid SPF, the "authentication-results" header must include "spf=pass"
+- For valid DKIM, the "authentication-results" header must include "dkim=pass"
+- For valid DMARC, the "authentication-results" header must include "dmarc=pass"
+- For valid ARC, the "authentication-results" header must include "arc=pass"
 - Throwaway email addresses might contain a word or a name and more than one number in the username of the email address
 - Throwaway email addresses might use a randomly generated or hex text string as the username of the email address
-- Throwaway email addresses might use free email services like gmail.com, outlook.com, hotmail.com, yahoo.com, aol.com etc
+- Throwaway email addresses might use free email services like gmail.com, outlook.com, hotmail.com, yahoo.com, aol.com, etc
 - The email to analyze is formatted in a JSON format
 - The email structure includes a property "headers" that contains an array of header values.
-- Each header contains of two properties, "key" as the header field key name, and "value" as the header value without the key prefix
+- Each header contains two properties, "key" as the header field key name and "value" as the header value without the key prefix
 - The email structure includes a property "attachments" that contains an array of attachments.
 - Each attachment includes a "filename" property that describes the file name, "contentType" property that describes the Content-Type value of the attachment
 - The email includes a "text" property for the text content
@@ -60,7 +61,12 @@ Facts:
 Analyze the following email:
 `.trim();
 
-async function riskAnalysis(message, apiToken) {
+async function riskAnalysis(message, apiToken, opts) {
+    opts = opts || {};
+
+    let maxAllowedTokens = opts.maxTokens || MAX_ALLOWED_TOKENS;
+    let gptModel = opts.gptModel || 'gpt-3.5-turbo';
+
     let text = message.text || '';
     if (message.html && (!text || message.html.length > text.length * 2)) {
         text = htmlToText(message.html);
@@ -105,7 +111,7 @@ async function riskAnalysis(message, apiToken) {
 ${JSON.stringify(content)}`;
 
         let tokens = tokenizer.encode(prompt);
-        if (tokens.text.length <= MAX_ALLOWED_TOKENS) {
+        if (tokens.text.length <= maxAllowedTokens) {
             break;
         }
         if (promptText.length > 2 * 1024 * 1024) {
@@ -138,7 +144,7 @@ ${JSON.stringify(content)}`;
     };
 
     let payload = {
-        model: 'gpt-3.5-turbo',
+        model: gptModel,
         messages: [
             {
                 role: 'system',
@@ -205,11 +211,12 @@ ${JSON.stringify(content)}`;
         throw error;
     }
 
-    const response = Object.assign({ id: null, tokens: null }, values, {
+    const response = Object.assign({ id: null, tokens: null, model: null }, values, {
         id: data && data.id,
         tokens: data && data.usage && data.usage.total_tokens,
-        text: content.text,
+        model: gptModel,
 
+        _text: content.text,
         _time: reqEndTime - reqStartTime,
         _cr: charactersRemoved
     });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@postalsys/email-text-tools",
-    "version": "1.3.1",
+    "version": "1.4.0",
     "description": "Email text tools",
     "main": "index.js",
     "scripts": {
@@ -23,7 +23,7 @@
         "mailparser": "3.6.4"
     },
     "dependencies": {
-        "dompurify": "3.0.1",
+        "dompurify": "3.0.2",
         "gpt3-tokenizer": "1.1.5",
         "handlebars": "4.7.7",
         "he": "1.2.0",