cyberchef 9.37.2 → 9.38.1

package/src/core/lib/Sort.mjs

@@ -0,0 +1,105 @@
+/**
+ * Sorting functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * Comparison operation for sorting of strings ignoring case.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function caseInsensitiveSort(a, b) {
+    return a.toLowerCase().localeCompare(b.toLowerCase());
+}
+
+
+/**
+ * Comparison operation for sorting of IPv4 addresses.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function ipSort(a, b) {
+    let a_ = a.split("."),
+        b_ = b.split(".");
+
+    a_ = a_[0] * 0x1000000 + a_[1] * 0x10000 + a_[2] * 0x100 + a_[3] * 1;
+    b_ = b_[0] * 0x1000000 + b_[1] * 0x10000 + b_[2] * 0x100 + b_[3] * 1;
+
+    if (isNaN(a_) && !isNaN(b_)) return 1;
+    if (!isNaN(a_) && isNaN(b_)) return -1;
+    if (isNaN(a_) && isNaN(b_)) return a.localeCompare(b);
+
+    return a_ - b_;
+}
+
+/**
+ * Comparison operation for sorting of numeric values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function numericSort(a, b) {
+    const a_ = a.split(/([^\d]+)/),
+        b_ = b.split(/([^\d]+)/);
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}
+
+/**
+ * Comparison operation for sorting of hexadecimal values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function hexadecimalSort(a, b) {
+    let a_ = a.split(/([^\da-f]+)/i),
+        b_ = b.split(/([^\da-f]+)/i);
+
+    a_ = a_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    b_ = b_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}

package/src/core/lib/Stream.mjs

@@ -27,15 +27,17 @@ export default class Stream {
     }
 
     /**
-     * Get a number of bytes from the current position.
+     * Get a number of bytes from the current position, or all remaining bytes.
      *
-     * @param {number} numBytes
+     * @param {number} [numBytes=null]
      * @returns {Uint8Array}
      */
-    getBytes(numBytes) {
+    getBytes(numBytes=null) {
         if (this.position > this.length) return undefined;
 
-        const newPosition = this.position + numBytes;
+        const newPosition = numBytes !== null ?
+            this.position + numBytes :
+            this.length;
         const bytes = this.bytes.slice(this.position, newPosition);
         this.position = newPosition;
         this.bitPos = 0;
@@ -91,34 +93,40 @@ export default class Stream {
     }
 
     /**
-     * Reads a number of bits from the buffer.
-     *
-     * @TODO Add endianness
+     * Reads a number of bits from the buffer in big or little endian.
      *
      * @param {number} numBits
+     * @param {string} [endianness="be"]
      * @returns {number}
      */
-    readBits(numBits) {
+    readBits(numBits, endianness="be") {
         if (this.position > this.length) return undefined;
 
         let bitBuf = 0,
             bitBufLen = 0;
 
         // Add remaining bits from current byte
-        bitBuf = (this.bytes[this.position++] & bitMask(this.bitPos)) >>> this.bitPos;
+        bitBuf = this.bytes[this.position++] & bitMask(this.bitPos);
+        if (endianness !== "be") bitBuf >>>= this.bitPos;
         bitBufLen = 8 - this.bitPos;
         this.bitPos = 0;
 
         // Not enough bits yet
         while (bitBufLen < numBits) {
-            bitBuf |= this.bytes[this.position++] << bitBufLen;
+            if (endianness === "be")
+                bitBuf = (bitBuf << bitBufLen) | this.bytes[this.position++];
+            else
+                bitBuf |= this.bytes[this.position++] << bitBufLen;
             bitBufLen += 8;
         }
 
         // Reverse back to numBits
         if (bitBufLen > numBits) {
             const excess = bitBufLen - numBits;
-            bitBuf &= (1 << numBits) - 1;
+            if (endianness === "be")
+                bitBuf >>>= excess;
+            else
+                bitBuf &= (1 << numBits) - 1;
             bitBufLen -= excess;
             this.position--;
             this.bitPos = 8 - excess;
@@ -133,7 +141,9 @@ export default class Stream {
          * @returns {number} The bit mask
          */
         function bitMask(bitPos) {
-            return 256 - (1 << bitPos);
+            return endianness === "be" ?
+                (1 << (8 - bitPos)) - 1 :
+                256 - (1 << bitPos);
         }
     }
 

package/src/core/operations/ExtractDates.mjs

@@ -44,7 +44,13 @@ class ExtractDates extends Operation {
             date3 = "(?:0[1-9]|1[012])[- /.](?:0[1-9]|[12][0-9]|3[01])[- /.](?:19|20)\\d\\d", // mm/dd/yyyy
             regex = new RegExp(date1 + "|" + date2 + "|" + date3, "ig");
 
-        return search(input, regex, null, displayTotal);
+        const results = search(input, regex);
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractDomains.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search, DOMAIN_REGEX } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract domains operation
@@ -25,9 +26,19 @@ class ExtractDomains extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": true
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -38,8 +49,21 @@ class ExtractDomains extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0];
-        return search(input, DOMAIN_REGEX, null, displayTotal);
+        const [displayTotal, sort, unique] = args;
+
+        const results = search(
+            input,
+            DOMAIN_REGEX,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractEmailAddresses.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract email addresses operation
@@ -25,9 +26,19 @@ class ExtractEmailAddresses extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -38,10 +49,23 @@ class ExtractEmailAddresses extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
+        const [displayTotal, sort, unique] = args,
             // email regex from: https://www.regextester.com/98066
             regex = /(?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9](?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9-]*[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9])?\.)+[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9](?:[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9-]*[\u00A0-\uD7FF\uE000-\uFFFFa-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}\])/ig;
-        return search(input, regex, null, displayTotal);
+
+        const results = search(
+            input,
+            regex,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractFilePaths.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract file paths operation
@@ -25,19 +26,29 @@ class ExtractFilePaths extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Windows",
-                "type": "boolean",
-                "value": true
+                name: "Windows",
+                type: "boolean",
+                value: true
             },
             {
-                "name": "UNIX",
-                "type": "boolean",
-                "value": true
+                name: "UNIX",
+                type: "boolean",
+                value: true
             },
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -48,7 +59,7 @@ class ExtractFilePaths extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const [includeWinPath, includeUnixPath, displayTotal] = args,
+        const [includeWinPath, includeUnixPath, displayTotal, sort, unique] = args,
             winDrive = "[A-Z]:\\\\",
             winName = "[A-Z\\d][A-Z\\d\\- '_\\(\\)~]{0,61}",
             winExt = "[A-Z\\d]{1,6}",
@@ -65,12 +76,25 @@ class ExtractFilePaths extends Operation {
             filePaths = unixPath;
         }
 
-        if (filePaths) {
-            const regex = new RegExp(filePaths, "ig");
-            return search(input, regex, null, displayTotal);
-        } else {
+        if (!filePaths) {
             return "";
         }
+
+        const regex = new RegExp(filePaths, "ig");
+        const results = search(
+            input,
+            regex,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
+
     }
 
 }

package/src/core/operations/ExtractIPAddresses.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { ipSort } from "../lib/Sort.mjs";
 
 /**
  * Extract IP addresses operation
@@ -25,24 +26,34 @@ class ExtractIPAddresses extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "IPv4",
-                "type": "boolean",
-                "value": true
+                name: "IPv4",
+                type: "boolean",
+                value: true
             },
             {
-                "name": "IPv6",
-                "type": "boolean",
-                "value": false
+                name: "IPv6",
+                type: "boolean",
+                value: false
             },
             {
-                "name": "Remove local IPv4 addresses",
-                "type": "boolean",
-                "value": false
+                name: "Remove local IPv4 addresses",
+                type: "boolean",
+                value: false
             },
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -53,7 +64,7 @@ class ExtractIPAddresses extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const [includeIpv4, includeIpv6, removeLocal, displayTotal] = args,
+        const [includeIpv4, includeIpv6, removeLocal, displayTotal, sort, unique] = args,
             ipv4 = "(?:(?:\\d|[01]?\\d\\d|2[0-4]\\d|25[0-5])\\.){3}(?:25[0-5]|2[0-4]\\d|[01]?\\d\\d|\\d)(?:\\/\\d{1,2})?",
             ipv6 = "((?=.*::)(?!.*::.+::)(::)?([\\dA-F]{1,4}:(:|\\b)|){5}|([\\dA-F]{1,4}:){6})((([\\dA-F]{1,4}((?!\\3)::|:\\b|(?![\\dA-F])))|(?!\\2\\3)){2}|(((2[0-4]|1\\d|[1-9])?\\d|25[0-5])\\.?\\b){4})";
         let ips  = "";
@@ -66,23 +77,29 @@ class ExtractIPAddresses extends Operation {
             ips = ipv6;
         }
 
-        if (ips) {
-            const regex = new RegExp(ips, "ig");
+        if (!ips) return "";
 
-            if (removeLocal) {
-                const ten = "10\\..+",
-                    oneninetwo = "192\\.168\\..+",
-                    oneseventwo = "172\\.(?:1[6-9]|2\\d|3[01])\\..+",
-                    onetwoseven = "127\\..+",
-                    removeRegex = new RegExp("^(?:" + ten + "|" + oneninetwo +
-                        "|" + oneseventwo + "|" + onetwoseven + ")");
+        const regex = new RegExp(ips, "ig");
 
-                return search(input, regex, removeRegex, displayTotal);
-            } else {
-                return search(input, regex, null, displayTotal);
-            }
+        const ten = "10\\..+",
+            oneninetwo = "192\\.168\\..+",
+            oneseventwo = "172\\.(?:1[6-9]|2\\d|3[01])\\..+",
+            onetwoseven = "127\\..+",
+            removeRegex = new RegExp("^(?:" + ten + "|" + oneninetwo +
+                "|" + oneseventwo + "|" + onetwoseven + ")");
+
+        const results = search(
+            input,
+            regex,
+            removeLocal ? removeRegex : null,
+            sort ? ipSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
         } else {
-            return "";
+            return results.join("\n");
         }
     }
 

package/src/core/operations/ExtractMACAddresses.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search } from "../lib/Extract.mjs";
+import { hexadecimalSort } from "../lib/Sort.mjs";
 
 /**
  * Extract MAC addresses operation
@@ -25,9 +26,19 @@ class ExtractMACAddresses extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -38,10 +49,21 @@ class ExtractMACAddresses extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0],
-            regex = /[A-F\d]{2}(?:[:-][A-F\d]{2}){5}/ig;
+        const [displayTotal, sort, unique] = args,
+            regex = /[A-F\d]{2}(?:[:-][A-F\d]{2}){5}/ig,
+            results = search(
+                input,
+                regex,
+                null,
+                sort ? hexadecimalSort : null,
+                unique
+            );
 
-        return search(input, regex, null, displayTotal);
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }

package/src/core/operations/ExtractURLs.mjs

@@ -6,6 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import { search, URL_REGEX } from "../lib/Extract.mjs";
+import { caseInsensitiveSort } from "../lib/Sort.mjs";
 
 /**
  * Extract URLs operation
@@ -25,9 +26,19 @@ class ExtractURLs extends Operation {
         this.outputType = "string";
         this.args = [
             {
-                "name": "Display total",
-                "type": "boolean",
-                "value": false
+                name: "Display total",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Sort",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Unique",
+                type: "boolean",
+                value: false
             }
         ];
     }
@@ -38,8 +49,20 @@ class ExtractURLs extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const displayTotal = args[0];
-        return search(input, URL_REGEX, null, displayTotal);
+        const [displayTotal, sort, unique] = args;
+        const results = search(
+            input,
+            URL_REGEX,
+            null,
+            sort ? caseInsensitiveSort : null,
+            unique
+        );
+
+        if (displayTotal) {
+            return `Total found: ${results.length}\n\n${results.join("\n")}`;
+        } else {
+            return results.join("\n");
+        }
     }
 
 }