@postalsys/email-text-tools 1.3.0 → 1.3.2

package/examples/risk.js

@@ -4,9 +4,13 @@ const fs = require('fs').promises;
 const { riskAnalysis } = require('..');
 const simpleParser = require('mailparser').simpleParser;
 const libmime = require('libmime');
+const Path = require('path');
 
-async function main() {
-    const eml = await fs.readFile(process.argv[2]);
+const BASE_PATH = process.argv[2] || '.';
+const MAX_ENTRIES = Math.abs(Number(process.argv[3]) || 0);
+
+async function analyzeEmail(path) {
+    const eml = await fs.readFile(path);
 
     const parsed = await simpleParser(eml);
 
@@ -20,7 +24,57 @@ async function main() {
         process.env.OPENAI_API_KEY
     );
 
-    console.log(result);
+    result.file = Path.basename(path);
+
+    console.log(JSON.stringify(result));
+}
+
+function shuffleArray(array) {
+    for (let i = array.length - 1; i > 0; i--) {
+        let j = Math.floor(Math.random() * (i + 1));
+        let temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+    return array;
+}
+
+async function main() {
+    const stat = await fs.stat(BASE_PATH);
+    if (stat.isDirectory()) {
+        // process as directory
+
+        let processed = 0;
+        let list = await fs.readdir(BASE_PATH);
+
+        for (let file of shuffleArray(list)) {
+            if (Path.extname(file).toLowerCase() === '.eml') {
+                if (processed) {
+                    process.stdout.write(',');
+                }
+                try {
+                    await analyzeEmail(Path.join(BASE_PATH, file));
+                    console.error(`Processed ${file} [${processed + 1}]`);
+                } catch (err) {
+                    console.log(JSON.stringify({ file, error: err.message }));
+                    console.error(`Failed processing ${file} [${processed + 1}]`);
+                    console.error(err);
+                }
+                processed++;
+
+                if (MAX_ENTRIES && processed >= MAX_ENTRIES) {
+                    break;
+                }
+            }
+        }
+        console.error(`Processed ${processed} files from ${BASE_PATH}`);
+    } else {
+        // process as file
+        await analyzeEmail(BASE_PATH);
+    }
 }
 
-main();
+console.log('[');
+main().then(() => {
+    console.log(']');
+});

package/lib/risk-analysis.js

@@ -3,9 +3,13 @@
 const packageData = require('../package.json');
 const htmlToText = require('../lib/html-to-text');
 const nodeFetch = require('node-fetch');
+const GPT3Tokenizer = require('gpt3-tokenizer').default;
 const fetchCmd = global.fetch || nodeFetch;
 
-const MAX_TEXT_LENGTH = 4 * 1024;
+const tokenizer = new GPT3Tokenizer({ type: 'gpt3' });
+
+const MAX_ALLOWED_TEXT_LENGTH = 32 * 1024;
+const MAX_ALLOWED_TOKENS = 4000;
 const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
 
 const SYSTEM_PROMPT = `
@@ -16,29 +20,39 @@ I want you to act as are an IT security expert that monitors emails for suspicio
 
 const USER_PROMPT = `
 Instructions:
-- You are an IT security expert.
-- Your task is to monitor and analyze incoming emails which consist of the message headers, a list of attachments, and text content
-- Your analysis should contain (but is not limited to) the following:
+- You are an IT security expert scanning incoming emails to detect and block fraud attempts.
+- Your task is to monitor and analyze incoming emails, which consist of the message headers, a list of attachments, and text content
+- Your analysis should contain (but is not limited to) the following risk factors:
   - Does the email include links with domain names that contain typos or homoglyphs that might mislead the user about the actual target of the link
-  - Does the sender address of the email match the persona or organization the sender email claims to be
-  - Does the email promise the user extremely good financial outcome
+  - Does the sender address of the email not seem to match the persona or organization the sender email claims to be
+  - Does the email promise the user an extremely good financial outcome
   - Does the email promise unclaimed money or goods
+  - Does the email offer a vague business opportunity with no specific details
   - Does the email suggest there is a severe penalty if the user does not act as requested
   - Does the email claim that there are technical issues with the user's email account
   - Does the email offer services or activities that are not suitable for underage persons
-  - Does the sender email look like a throwaway address
+  - Does the sender's email look like a throwaway address
+  - Does the sender's email look like it might be spoofed
   - Does the email contain attachments where the name of the attachment might be misleading and suggest a different file format than is actually used
-- Provide a risk score for the email using the following scale: 1 - 5 (where 1 is low rist, and 5 is high risk), take into account what may happen if a user acts by the instructions given in the email.
-- Respond with a JSON formatted structure with numeric risk score as "risk" property, and a single-paragraph assessment as the "assessment" property. Do not write any other explanations.
+  - Does the email contain attachments that might be executable files
+- Provide a risk score for the email using the following scale: 1 - 5 (where 1 is low risk, and 5 is high risk), taking into account what may happen if a user acts by the instructions given in the email.
+- Provide a short single-sentence assessment text that includes details about issues that increase the risk score, do not disclose details that decrease the risk score or do not affect it.
+- Respond with a JSON formatted structure with a numeric risk score as "risk" property and the assessment as the "assessment" property. Do not write any other explanations.
 
 Facts:
-- An email consists of message headers, attachments list, and text content
+- An email consists of message headers, an attachments list, and text content
+- An email might contain an "authentication-results" result header that indicates if the message passed SPF, DKIM, ARC, and DMARC checks
+- Having a valid SPF, DKIM, and DMARC increases the chances that the sender domain name is not spoofed
+- For valid SPF, the "authentication-results" header must include "spf=pass"
+- For valid DKIM, the "authentication-results" header must include "dkim=pass"
+- For valid DMARC, the "authentication-results" header must include "dmarc=pass"
+- For valid ARC, the "authentication-results" header must include "arc=pass"
 - Throwaway email addresses might contain a word or a name and more than one number in the username of the email address
 - Throwaway email addresses might use a randomly generated or hex text string as the username of the email address
-- Throwaway email addresses might use free email services like gmail.com, outlook.com, hotmail.com, yahoo.com, aol.com etc
+- Throwaway email addresses might use free email services like gmail.com, outlook.com, hotmail.com, yahoo.com, aol.com, etc
 - The email to analyze is formatted in a JSON format
 - The email structure includes a property "headers" that contains an array of header values.
-- Each header contains of two properties, "key" as the header field key name, and "value" as the header value without the key prefix
+- Each header contains two properties, "key" as the header field key name and "value" as the header value without the key prefix
 - The email structure includes a property "attachments" that contains an array of attachments.
 - Each attachment includes a "filename" property that describes the file name, "contentType" property that describes the Content-Type value of the attachment
 - The email includes a "text" property for the text content
@@ -47,33 +61,21 @@ Analyze the following email:
 `.trim();
 
 async function riskAnalysis(message, apiToken) {
-    let text = message.text;
+    let text = message.text || '';
     if (message.html && (!text || message.html.length > text.length * 2)) {
         text = htmlToText(message.html);
     }
 
-    text = text.trim();
-    if (text.length > MAX_TEXT_LENGTH) {
-        text = text.substring(0, MAX_TEXT_LENGTH);
-    }
+    let prompt;
 
     const headerSeen = new Set();
     const content = {
         headers: []
             .concat(message.headers || [])
             .filter(
-                // remove unneeded
+                // use a whitelist
                 header =>
-                    ![
-                        'x-received',
-                        'received',
-                        'dkim-signature',
-                        'domainkey-signature',
-                        'arc-seal',
-                        'arc-message-signature',
-                        'arc-authentication-results',
-                        'x-google-dkim-signature'
-                    ].includes(header.key)
+                    ['from', 'to', 'cc', 'bcc', 'subject', 'mime-version', 'authentication-results', 'date', 'content-type', 'list-id'].includes(header.key)
             )
             .filter(header => {
                 // only keept the latest
@@ -85,10 +87,50 @@ async function riskAnalysis(message, apiToken) {
                 }
                 return true;
             }),
-        attachments: [].concat(message.attachments || []).map(attachment => ({ filename: attachment.filename, contentType: attachment.contentType })),
-        text
+        attachments: [].concat(message.attachments || []).map(attachment => ({ filename: attachment.filename, contentType: attachment.contentType }))
     };
 
+    let charactersRemoved = 0;
+    let promptText = text;
+
+    if (promptText.length > MAX_ALLOWED_TEXT_LENGTH) {
+        charactersRemoved += promptText.length - MAX_ALLOWED_TEXT_LENGTH;
+        promptText = promptText.substr(0, MAX_ALLOWED_TEXT_LENGTH);
+    }
+
+    while (promptText.length) {
+        content.text = promptText;
+        prompt = `${USER_PROMPT}
+
+${JSON.stringify(content)}`;
+
+        let tokens = tokenizer.encode(prompt);
+        if (tokens.text.length <= MAX_ALLOWED_TOKENS) {
+            break;
+        }
+        if (promptText.length > 2 * 1024 * 1024) {
+            promptText = promptText.substring(0, promptText.length - 1024 * 1024);
+            charactersRemoved += 1024 * 1024;
+        } else if (promptText.length > 2 * 1024) {
+            promptText = promptText.substring(0, promptText.length - 1024);
+            charactersRemoved += 1024;
+        } else if (promptText.length > 2 * 256) {
+            promptText = promptText.substring(0, promptText.length - 256);
+            charactersRemoved += 255;
+        } else if (promptText.length > 2 * 100) {
+            promptText = promptText.substring(0, promptText.length - 100);
+            charactersRemoved += 100;
+        } else if (promptText.length > 2 * 10) {
+            promptText = promptText.substring(0, promptText.length - 10);
+            charactersRemoved += 10;
+        } else if (promptText.length > 1) {
+            promptText = promptText.substring(0, promptText.length - 1);
+            charactersRemoved += 1;
+        } else {
+            throw new Error(`Prompt too long. Removed ${charactersRemoved} characters.`);
+        }
+    }
+
     let headers = {
         'User-Agent': `${packageData.name}/${packageData.version}`,
         Authorization: `Bearer ${apiToken}`,
@@ -104,19 +146,21 @@ async function riskAnalysis(message, apiToken) {
             },
             {
                 role: 'user',
-                content: `${USER_PROMPT}
-
-${JSON.stringify(content)}`
+                content: prompt
             }
         ]
     };
 
+    const reqStartTime = Date.now();
+
     let res = await fetchCmd(OPENAI_API_URL, {
         method: 'post',
         headers,
         body: JSON.stringify(payload)
     });
 
+    const reqEndTime = Date.now();
+
     let data = await res.json();
 
     if (!res.ok) {
@@ -163,7 +207,11 @@ ${JSON.stringify(content)}`
 
     const response = Object.assign({ id: null, tokens: null }, values, {
         id: data && data.id,
-        tokens: data && data.usage && data.usage.total_tokens
+        tokens: data && data.usage && data.usage.total_tokens,
+
+        _text: content.text,
+        _time: reqEndTime - reqStartTime,
+        _cr: charactersRemoved
     });
 
     return response;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@postalsys/email-text-tools",
-    "version": "1.3.0",
+    "version": "1.3.2",
     "description": "Email text tools",
     "main": "index.js",
     "scripts": {
@@ -24,6 +24,7 @@
     },
     "dependencies": {
         "dompurify": "3.0.1",
+        "gpt3-tokenizer": "1.1.5",
         "handlebars": "4.7.7",
         "he": "1.2.0",
         "html-to-text": "9.0.5",