cyberchef 9.37.0 → 9.37.3

package/nightwatch.json

@@ -10,11 +10,12 @@
         "start_process": true,
         "server_path": "./node_modules/.bin/chromedriver",
         "port": 9515,
-        "log_path": false
+        "log_path": "tests/browser/output"
       },
       "desiredCapabilities": {
         "browserName": "chrome"
-      }
+      },
+      "enable_fail_fast": true
     },
 
     "dev": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.37.0",
+  "version": "9.37.3",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

package/src/core/Utils.mjs

@@ -723,8 +723,8 @@ class Utils {
         }
 
         if (removeScriptAndStyle) {
-            htmlStr = recursiveRemove(/<script[^>]*>.*?<\/script>/gi, htmlStr);
-            htmlStr = recursiveRemove(/<style[^>]*>.*?<\/style>/gi, htmlStr);
+            htmlStr = recursiveRemove(/<script[^>]*>.*?<\/script[^>]*>/gi, htmlStr);
+            htmlStr = recursiveRemove(/<style[^>]*>.*?<\/style[^>]*>/gi, htmlStr);
         }
         return htmlStr.replace(/<[^>]+>/g, "");
     }

package/src/core/config/OperationConfig.json

@@ -4946,7 +4946,17 @@
             {
                 "name": "Display total",
                 "type": "boolean",
-                "value": true
+                "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -4973,6 +4983,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -4999,6 +5019,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -5091,6 +5121,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -5174,6 +5214,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -5228,6 +5278,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -9875,9 +9935,9 @@
                 ]
             },
             {
-                "name": "Number of bytes to drop",
+                "name": "Number of dwords to drop",
                 "type": "number",
-                "value": 768
+                "value": 192
             }
         ]
     },
@@ -12712,6 +12772,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -16039,6 +16109,11 @@
                     "Colon",
                     "Nothing (separate chars)"
                 ]
+            },
+            {
+                "name": "Display count",
+                "type": "boolean",
+                "value": false
             }
         ]
     },

package/src/core/lib/Extract.mjs

@@ -12,15 +12,15 @@
  *
  * @param {string} input
  * @param {RegExp} searchRegex
- * @param {RegExp} removeRegex - A regular expression defining results to remove from the
+ * @param {RegExp} [removeRegex=null] - A regular expression defining results to remove from the
  *      final list
- * @param {boolean} includeTotal - Whether or not to include the total number of results
+ * @param {Function} [sortBy=null] - The sorting comparison function to apply
+ * @param {boolean} [unique=false] - Whether to unique the results
  * @returns {string}
  */
-export function search (input, searchRegex, removeRegex, includeTotal) {
-    let output = "",
-        total = 0,
-        match;
+export function search(input, searchRegex, removeRegex=null, sortBy=null, unique=false) {
+    let results = [];
+    let match;
 
     while ((match = searchRegex.exec(input))) {
         // Moves pointer when an empty string is matched (prevents infinite loop)
@@ -30,14 +30,19 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
         if (removeRegex && removeRegex.test(match[0]))
             continue;
-        total++;
-        output += match[0] + "\n";
+
+        results.push(match[0]);
+    }
+
+    if (sortBy) {
+        results = results.sort(sortBy);
     }
 
-    if (includeTotal)
-        output = "Total found: " + total + "\n\n" + output;
+    if (unique) {
+        results = results.unique();
+    }
 
-    return output;
+    return results;
 }
 
 

package/src/core/lib/Sort.mjs

@@ -0,0 +1,105 @@
+/**
+ * Sorting functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * Comparison operation for sorting of strings ignoring case.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function caseInsensitiveSort(a, b) {
+    return a.toLowerCase().localeCompare(b.toLowerCase());
+}
+
+
+/**
+ * Comparison operation for sorting of IPv4 addresses.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function ipSort(a, b) {
+    let a_ = a.split("."),
+        b_ = b.split(".");
+
+    a_ = a_[0] * 0x1000000 + a_[1] * 0x10000 + a_[2] * 0x100 + a_[3] * 1;
+    b_ = b_[0] * 0x1000000 + b_[1] * 0x10000 + b_[2] * 0x100 + b_[3] * 1;
+
+    if (isNaN(a_) && !isNaN(b_)) return 1;
+    if (!isNaN(a_) && isNaN(b_)) return -1;
+    if (isNaN(a_) && isNaN(b_)) return a.localeCompare(b);
+
+    return a_ - b_;
+}
+
+/**
+ * Comparison operation for sorting of numeric values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function numericSort(a, b) {
+    const a_ = a.split(/([^\d]+)/),
+        b_ = b.split(/([^\d]+)/);
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}
+
+/**
+ * Comparison operation for sorting of hexadecimal values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function hexadecimalSort(a, b) {
+    let a_ = a.split(/([^\da-f]+)/i),
+        b_ = b.split(/([^\da-f]+)/i);
+
+    a_ = a_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    b_ = b_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}

package/src/core/operations/ExtractDates.mjs

@@ -44,7 +44,13 @@ class ExtractDates extends Operation {
             date3 = "(?:0[1-9]|1[012])[- /.](?:0[1-9]|[12][0-9]|3[01])[- /.](?:19|20)\\d\\d", // mm/dd/yyyy
             regex = new RegExp(date1 + "|" + date2 + "|" + date3, "ig");
 
-        return search(input, regex, null, displayTotal);
+        const results = search(input, regex);
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractDomains.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search, DOMAIN_REGEX } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract domains operation
@@ -25,9 +26,19 @@ class ExtractDomains extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": true
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -38,8 +49,21 @@ class ExtractDomains extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0];
-        return search(input, DOMAIN_REGEX, null, displayTotal);
+        const [displayTotal, sort, unique] = args;
+
+        const results = search(
+            input,
+            DOMAIN_REGEX,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractEmailAddresses.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract email addresses operation
@@ -25,9 +26,19 @@ class ExtractEmailAddresses extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -38,10 +49,23 @@ class ExtractEmailAddresses extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
+        const [displayTotal, sort, unique] = args,
             // email regex from: https://www.regextester.com/98066
             regex = /(?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9](?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9-]*[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9])?\.)+[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9](?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9-]*[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}\])/ig;
-        return search(input, regex, null, displayTotal);
+
+        const results = search(
+            input,
+            regex,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractFilePaths.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract file paths operation
@@ -25,19 +26,29 @@ class ExtractFilePaths extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Windows",
-                "type": "boolean",
-                "value": true
+                name: "Windows",
+                type: "boolean",
+                value: true
             },
             {
-                "name": "UNIX",
-                "type": "boolean",
-                "value": true
+                name: "UNIX",
+                type: "boolean",
+                value: true
             },
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -48,7 +59,7 @@ class ExtractFilePaths extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const [includeWinPath, includeUnixPath, displayTotal] = args,
+        const [includeWinPath, includeUnixPath, displayTotal, sort, unique] = args,
             winDrive = "[A-Z]:\\\\",
             winName = "[A-Z\\d][A-Z\\d\\- '_\\(\\)~]{0,61}",
             winExt = "[A-Z\\d]{1,6}",
@@ -65,12 +76,25 @@ class ExtractFilePaths extends Operation {
             filePaths = unixPath;
         }
 
-        if (filePaths) {
-            const regex = new RegExp(filePaths, "ig");
-            return search(input, regex, null, displayTotal);
-        } else {
+        if (!filePaths) {
             return "";
         }
+
+        const regex = new RegExp(filePaths, "ig");
+        const results = search(
+            input,
+            regex,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
+
     }
 
 }

package/src/core/operations/ExtractIPAddresses.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { ipSort } from "../lib/Sort.mjs";
 
 /**
  * Extract IP addresses operation
@@ -25,24 +26,34 @@ class ExtractIPAddresses extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "IPv4",
-                "type": "boolean",
-                "value": true
+                name: "IPv4",
+                type: "boolean",
+                value: true
             },
             {
-                "name": "IPv6",
-                "type": "boolean",
-                "value": false
+                name: "IPv6",
+                type: "boolean",
+                value: false
             },
             {
-                "name": "Remove local IPv4 addresses",
-                "type": "boolean",
-                "value": false
+                name: "Remove local IPv4 addresses",
+                type: "boolean",
+                value: false
             },
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -53,7 +64,7 @@ class ExtractIPAddresses extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const [includeIpv4, includeIpv6, removeLocal, displayTotal] = args,
+        const [includeIpv4, includeIpv6, removeLocal, displayTotal, sort, unique] = args,
             ipv4 = "(?:(?:\\d|[01]?\\d\\d|2[0-4]\\d|25[0-5])\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d|\\d)(?:\\/\\d{1,2})?",
             ipv6 = "((?=.*::)(?!.*::.+::)(::)?([\\dA-F]{1,4}:(:|\\b)|){5}|([\\dA-F]{1,4}:){6})((([\\dA-F]{1,4}((?!\\3)::|:\\b|(?![\\dA-F])))|(?!\\2\\3)){2}|(((2[0-4]|1\\d|[1-9])?\\d|25[0-5])\\.?\\b){4})";
         let ips  = "";
@@ -66,23 +77,29 @@ class ExtractIPAddresses extends Operation {
             ips = ipv6;
         }
 
-        if (ips) {
-            const regex = new RegExp(ips, "ig");
+        if (!ips) return "";
 
-            if (removeLocal) {
-                const ten = "10\\..+",
-                    oneninetwo = "192\\.168\\..+",
-                    oneseventwo = "172\\.(?:1[6-9]|2\\d|3[01])\\..+",
-                    onetwoseven = "127\\..+",
-                    removeRegex = new RegExp("^(?:" + ten + "|" + oneninetwo +
-                        "|" + oneseventwo + "|" + onetwoseven + ")");
+        const regex = new RegExp(ips, "ig");
 
-                return search(input, regex, removeRegex, displayTotal);
-            } else {
-                return search(input, regex, null, displayTotal);
-            }
+        const ten = "10\\..+",
+            oneninetwo = "192\\.168\\..+",
+            oneseventwo = "172\\.(?:1[6-9]|2\\d|3[01])\\..+",
+            onetwoseven = "127\\..+",
+            removeRegex = new RegExp("^(?:" + ten + "|" + oneninetwo +
+                "|" + oneseventwo + "|" + onetwoseven + ")");
+
+        const results = search(
+            input,
+            regex,
+            removeLocal ? removeRegex : null,
+            sort ? ipSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
         } else {
-            return "";
+            return results.join("\n");
         }
     }