cyberchef 9.37.1 → 9.38.0

package/CHANGELOG.md

@@ -13,6 +13,9 @@ All major and minor version changes will be documented in this file. Details of
 
 ## Details
 
+### [9.38.0] - 2022-05-30
+- Added 'Parse TCP' operation [@n1474335] | [a895d1d]
+
 ### [9.37.0] - 2022-03-29
 - 'SM4 Encrypt' and 'SM4 Decrypt' operations added [@swesven] | [#1189]
 - NoPadding options added for CBC and ECB modes in AES, DES and Triple DES Decrypt operations [@swesven] | [#1189]
@@ -135,7 +138,7 @@ All major and minor version changes will be documented in this file. Details of
 
 <details>
     <summary>Click to expand v8 minor versions</summary>
-    
+
 ### [8.38.0] - 2019-07-03
 - 'Streebog' and 'GOST hash' operations added [@MShwed] [@n1474335] | [#530]
 
@@ -288,6 +291,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[9.38.0]: https://github.com/gchq/CyberChef/releases/tag/v9.38.0
 [9.37.0]: https://github.com/gchq/CyberChef/releases/tag/v9.37.0
 [9.36.0]: https://github.com/gchq/CyberChef/releases/tag/v9.36.0
 [9.35.0]: https://github.com/gchq/CyberChef/releases/tag/v9.35.0
@@ -416,6 +420,7 @@ All major and minor version changes will be documented in this file. Details of
 [289a417]: https://github.com/gchq/CyberChef/commit/289a417dfb5923de5e1694354ec42a08d9395bfe
 [e9ca4dc]: https://github.com/gchq/CyberChef/commit/e9ca4dc9caf98f33fd986431cd400c88082a42b8
 [dd18e52]: https://github.com/gchq/CyberChef/commit/dd18e529939078b89867297b181a584e8b2cc7da
+[a895d1d]: https://github.com/gchq/CyberChef/commit/a895d1d82a2f92d440a0c5eca2bc7c898107b737
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -502,4 +507,4 @@ All major and minor version changes will be documented in this file. Details of
 [#1242]: https://github.com/gchq/CyberChef/pull/1242
 [#1244]: https://github.com/gchq/CyberChef/pull/1244
 [#1313]: https://github.com/gchq/CyberChef/pull/1313
-[#1326]: https://github.com/gchq/CyberChef/pull/1326
+[#1326]: https://github.com/gchq/CyberChef/pull/1326

package/Gruntfile.js

@@ -217,7 +217,8 @@ module.exports = function (grunt) {
                     client: {
                         logging: "error",
                         overlay: true
-                    }
+                    },
+                    hot: "only"
                 },
                 plugins: [
                     new webpack.DefinePlugin(BUILD_CONSTANTS),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.37.1",
+  "version": "9.38.0",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -48,7 +48,7 @@
     "autoprefixer": "^10.4.4",
     "babel-loader": "^8.2.4",
     "babel-plugin-dynamic-import-node": "^2.3.3",
-    "chromedriver": "^99.0.0",
+    "chromedriver": "^101.0.0",
     "cli-progress": "^3.10.0",
     "colors": "^1.4.0",
     "copy-webpack-plugin": "^10.2.4",

package/src/core/Utils.mjs

@@ -723,8 +723,8 @@ class Utils {
         }
 
         if (removeScriptAndStyle) {
-            htmlStr = recursiveRemove(/<script[^>]*>.*?<\/script>/gi, htmlStr);
-            htmlStr = recursiveRemove(/<style[^>]*>.*?<\/style>/gi, htmlStr);
+            htmlStr = recursiveRemove(/<script[^>]*>.*?<\/script[^>]*>/gi, htmlStr);
+            htmlStr = recursiveRemove(/<style[^>]*>.*?<\/style[^>]*>/gi, htmlStr);
         }
         return htmlStr.replace(/<[^>]+>/g, "");
     }

package/src/core/config/Categories.json

@@ -190,6 +190,7 @@
             "Parse IP range",
             "Parse IPv6 address",
             "Parse IPv4 header",
+            "Parse TCP",
             "Parse UDP",
             "Parse SSH Host Key",
             "Parse URI",

package/src/core/config/OperationConfig.json

@@ -4946,7 +4946,17 @@
             {
                 "name": "Display total",
                 "type": "boolean",
-                "value": true
+                "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -4973,6 +4983,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -4999,6 +5019,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -5091,6 +5121,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -5174,6 +5214,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -5228,6 +5278,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -9458,6 +9518,25 @@
             }
         ]
     },
+    "Parse TCP": {
+        "module": "Default",
+        "description": "Parses a TCP header and payload (if present).",
+        "infoURL": "https://wikipedia.org/wiki/Transmission_Control_Protocol",
+        "inputType": "string",
+        "outputType": "html",
+        "flowControl": false,
+        "manualBake": false,
+        "args": [
+            {
+                "name": "Input format",
+                "type": "option",
+                "value": [
+                    "Hex",
+                    "Raw"
+                ]
+            }
+        ]
+    },
     "Parse TLV": {
         "module": "Default",
         "description": "Converts a Type-Length-Value (TLV) encoded string into a JSON object.  Can optionally include a <code>Key</code> / <code>Type</code> entry. <br><br>Tags: Key-Length-Value, KLV, Length-Value, LV",
@@ -9488,11 +9567,20 @@
         "module": "Default",
         "description": "Parses a UDP header and payload (if present).",
         "infoURL": "https://wikipedia.org/wiki/User_Datagram_Protocol",
-        "inputType": "ArrayBuffer",
+        "inputType": "string",
         "outputType": "html",
         "flowControl": false,
         "manualBake": false,
-        "args": []
+        "args": [
+            {
+                "name": "Input format",
+                "type": "option",
+                "value": [
+                    "Hex",
+                    "Raw"
+                ]
+            }
+        ]
     },
     "Parse UNIX file permissions": {
         "module": "Default",
@@ -12712,6 +12800,16 @@
                 "name": "Display total",
                 "type": "boolean",
                 "value": false
+            },
+            {
+                "name": "Sort",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Unique",
+                "type": "boolean",
+                "value": false
             }
         ]
     },
@@ -16039,6 +16137,11 @@
                     "Colon",
                     "Nothing (separate chars)"
                 ]
+            },
+            {
+                "name": "Display count",
+                "type": "boolean",
+                "value": false
             }
         ]
     },

package/src/core/config/modules/Default.mjs

@@ -97,6 +97,7 @@ import ParseIPRange from "../../operations/ParseIPRange.mjs";
 import ParseIPv4Header from "../../operations/ParseIPv4Header.mjs";
 import ParseIPv6Address from "../../operations/ParseIPv6Address.mjs";
 import ParseSSHHostKey from "../../operations/ParseSSHHostKey.mjs";
+import ParseTCP from "../../operations/ParseTCP.mjs";
 import ParseTLV from "../../operations/ParseTLV.mjs";
 import ParseUDP from "../../operations/ParseUDP.mjs";
 import ParseUNIXFilePermissions from "../../operations/ParseUNIXFilePermissions.mjs";
@@ -263,6 +264,7 @@ OpModules.Default = {
     "Parse IPv4 header": ParseIPv4Header,
     "Parse IPv6 address": ParseIPv6Address,
     "Parse SSH Host Key": ParseSSHHostKey,
+    "Parse TCP": ParseTCP,
     "Parse TLV": ParseTLV,
     "Parse UDP": ParseUDP,
     "Parse UNIX file permissions": ParseUNIXFilePermissions,

package/src/core/lib/Binary.mjs

@@ -13,7 +13,7 @@ import OperationError from "../errors/OperationError.mjs";
 /**
  * Convert a byte array into a binary string.
  *
- * @param {Uint8Array|byteArray} data
+ * @param {Uint8Array|byteArray|number} data
  * @param {string} [delim="Space"]
  * @param {number} [padding=8]
  * @returns {string}
@@ -26,13 +26,17 @@ import OperationError from "../errors/OperationError.mjs";
  * toBinary([10,20,30], ":");
  */
 export function toBinary(data, delim="Space", padding=8) {
-    if (!data) return "";
-
     delim = Utils.charRep(delim);
     let output = "";
 
-    for (let i = 0; i < data.length; i++) {
-        output += data[i].toString(2).padStart(padding, "0") + delim;
+    if (data.length) { // array
+        for (let i = 0; i < data.length; i++) {
+            output += data[i].toString(2).padStart(padding, "0") + delim;
+        }
+    } else if (typeof data === "number") { // Single value
+        return data.toString(2).padStart(padding, "0");
+    } else {
+        return "";
     }
 
     if (delim.length) {

package/src/core/lib/Extract.mjs

@@ -12,15 +12,15 @@
  *
  * @param {string} input
  * @param {RegExp} searchRegex
- * @param {RegExp} removeRegex - A regular expression defining results to remove from the
+ * @param {RegExp} [removeRegex=null] - A regular expression defining results to remove from the
  *      final list
- * @param {boolean} includeTotal - Whether or not to include the total number of results
+ * @param {Function} [sortBy=null] - The sorting comparison function to apply
+ * @param {boolean} [unique=false] - Whether to unique the results
  * @returns {string}
  */
-export function search (input, searchRegex, removeRegex, includeTotal) {
-    let output = "",
-        total = 0,
-        match;
+export function search(input, searchRegex, removeRegex=null, sortBy=null, unique=false) {
+    let results = [];
+    let match;
 
     while ((match = searchRegex.exec(input))) {
         // Moves pointer when an empty string is matched (prevents infinite loop)
@@ -30,14 +30,19 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
         if (removeRegex && removeRegex.test(match[0]))
             continue;
-        total++;
-        output += match[0] + "\n";
+
+        results.push(match[0]);
+    }
+
+    if (sortBy) {
+        results = results.sort(sortBy);
     }
 
-    if (includeTotal)
-        output = "Total found: " + total + "\n\n" + output;
+    if (unique) {
+        results = results.unique();
+    }
 
-    return output;
+    return results;
 }
 
 

package/src/core/lib/FileSignatures.mjs

@@ -3778,8 +3778,8 @@ function parseDEFLATE(stream) {
 
     while (!finalBlock) {
         // Read header
-        finalBlock = stream.readBits(1);
-        const blockType = stream.readBits(2);
+        finalBlock = stream.readBits(1, "le");
+        const blockType = stream.readBits(2, "le");
 
         if (blockType === 0) {
             /* No compression */
@@ -3798,16 +3798,16 @@ function parseDEFLATE(stream) {
             /* Dynamic Huffman */
 
             // Read the number of liternal and length codes
-            const hlit = stream.readBits(5) + 257;
+            const hlit = stream.readBits(5, "le") + 257;
             // Read the number of distance codes
-            const hdist = stream.readBits(5) + 1;
+            const hdist = stream.readBits(5, "le") + 1;
             // Read the number of code lengths
-            const hclen = stream.readBits(4) + 4;
+            const hclen = stream.readBits(4, "le") + 4;
 
             // Parse code lengths
             const codeLengths = new Uint8Array(huffmanOrder.length);
             for (let i = 0; i < hclen; i++) {
-                codeLengths[huffmanOrder[i]] = stream.readBits(3);
+                codeLengths[huffmanOrder[i]] = stream.readBits(3, "le");
             }
 
             // Parse length table
@@ -3819,16 +3819,16 @@ function parseDEFLATE(stream) {
                 code = readHuffmanCode(stream, codeLengthsTable);
                 switch (code) {
                     case 16:
-                        repeat = 3 + stream.readBits(2);
+                        repeat = 3 + stream.readBits(2, "le");
                         while (repeat--) lengthTable[i++] = prev;
                         break;
                     case 17:
-                        repeat = 3 + stream.readBits(3);
+                        repeat = 3 + stream.readBits(3, "le");
                         while (repeat--) lengthTable[i++] = 0;
                         prev = 0;
                         break;
                     case 18:
-                        repeat = 11 + stream.readBits(7);
+                        repeat = 11 + stream.readBits(7, "le");
                         while (repeat--) lengthTable[i++] = 0;
                         prev = 0;
                         break;
@@ -3886,11 +3886,11 @@ function parseHuffmanBlock(stream, litTab, distTab) {
         if (code < 256) continue;
 
         // Length code
-        stream.readBits(lengthExtraTable[code - 257]);
+        stream.readBits(lengthExtraTable[code - 257], "le");
 
         // Dist code
         code = readHuffmanCode(stream, distTab);
-        stream.readBits(distanceExtraTable[code]);
+        stream.readBits(distanceExtraTable[code], "le");
     }
 }
 
@@ -3948,7 +3948,7 @@ function readHuffmanCode(stream, table) {
     const [codeTable, maxCodeLength] = table;
 
     // Read max length
-    const bitsBuf = stream.readBits(maxCodeLength);
+    const bitsBuf = stream.readBits(maxCodeLength, "le");
     const codeWithLength = codeTable[bitsBuf & ((1 << maxCodeLength) - 1)];
     const codeLength = codeWithLength >>> 16;
 

package/src/core/lib/Protocol.mjs

@@ -0,0 +1,47 @@
+/**
+ * Protocol parsing functions.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ */
+
+import BigNumber from "bignumber.js";
+import {toHexFast} from "../lib/Hex.mjs";
+
+/**
+ * Recursively displays a JSON object as an HTML table
+ *
+ * @param {Object} obj
+ * @returns string
+ */
+export function objToTable(obj, nested=false) {
+    let html = `<table
+        class='table table-sm table-nonfluid ${nested ? "mb-0 table-borderless" : "table-bordered"}'
+        style='table-layout: fixed; ${nested ? "margin: -1px !important;" : ""}'>`;
+    if (!nested)
+        html += `<tr>
+            <th>Field</th>
+            <th>Value</th>
+        </tr>`;
+
+    for (const key in obj) {
+        html += `<tr><td style='word-wrap: break-word'>${key}</td>`;
+        if (typeof obj[key] === "object")
+            html += `<td style='padding: 0'>${objToTable(obj[key], true)}</td>`;
+        else
+            html += `<td>${obj[key]}</td>`;
+        html += "</tr>";
+    }
+    html += "</table>";
+    return html;
+}
+
+/**
+ * Converts bytes into a BigNumber string
+ * @param {Uint8Array} bs
+ * @returns {string}
+ */
+export function bytesToLargeNumber(bs) {
+    return BigNumber(toHexFast(bs), 16).toString();
+}

package/src/core/lib/Sort.mjs

@@ -0,0 +1,105 @@
+/**
+ * Sorting functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * Comparison operation for sorting of strings ignoring case.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function caseInsensitiveSort(a, b) {
+    return a.toLowerCase().localeCompare(b.toLowerCase());
+}
+
+
+/**
+ * Comparison operation for sorting of IPv4 addresses.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function ipSort(a, b) {
+    let a_ = a.split("."),
+        b_ = b.split(".");
+
+    a_ = a_[0] * 0x1000000 + a_[1] * 0x10000 + a_[2] * 0x100 + a_[3] * 1;
+    b_ = b_[0] * 0x1000000 + b_[1] * 0x10000 + b_[2] * 0x100 + b_[3] * 1;
+
+    if (isNaN(a_) && !isNaN(b_)) return 1;
+    if (!isNaN(a_) && isNaN(b_)) return -1;
+    if (isNaN(a_) && isNaN(b_)) return a.localeCompare(b);
+
+    return a_ - b_;
+}
+
+/**
+ * Comparison operation for sorting of numeric values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function numericSort(a, b) {
+    const a_ = a.split(/([^\d]+)/),
+        b_ = b.split(/([^\d]+)/);
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}
+
+/**
+ * Comparison operation for sorting of hexadecimal values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function hexadecimalSort(a, b) {
+    let a_ = a.split(/([^\da-f]+)/i),
+        b_ = b.split(/([^\da-f]+)/i);
+
+    a_ = a_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    b_ = b_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}

package/src/core/lib/Stream.mjs

@@ -27,15 +27,17 @@ export default class Stream {
     }
 
     /**
-     * Get a number of bytes from the current position.
+     * Get a number of bytes from the current position, or all remaining bytes.
      *
-     * @param {number} numBytes
+     * @param {number} [numBytes=null]
      * @returns {Uint8Array}
      */
-    getBytes(numBytes) {
+    getBytes(numBytes=null) {
         if (this.position > this.length) return undefined;
 
-        const newPosition = this.position + numBytes;
+        const newPosition = numBytes !== null ?
+            this.position + numBytes :
+            this.length;
         const bytes = this.bytes.slice(this.position, newPosition);
         this.position = newPosition;
         this.bitPos = 0;
@@ -91,34 +93,40 @@ export default class Stream {
     }
 
     /**
-     * Reads a number of bits from the buffer.
-     *
-     * @TODO Add endianness
+     * Reads a number of bits from the buffer in big or little endian.
      *
      * @param {number} numBits
+     * @param {string} [endianness="be"]
      * @returns {number}
      */
-    readBits(numBits) {
+    readBits(numBits, endianness="be") {
         if (this.position > this.length) return undefined;
 
         let bitBuf = 0,
             bitBufLen = 0;
 
         // Add remaining bits from current byte
-        bitBuf = (this.bytes[this.position++] & bitMask(this.bitPos)) >>> this.bitPos;
+        bitBuf = this.bytes[this.position++] & bitMask(this.bitPos);
+        if (endianness !== "be") bitBuf >>>= this.bitPos;
         bitBufLen = 8 - this.bitPos;
         this.bitPos = 0;
 
         // Not enough bits yet
         while (bitBufLen < numBits) {
-            bitBuf |= this.bytes[this.position++] << bitBufLen;
+            if (endianness === "be")
+                bitBuf = (bitBuf << bitBufLen) | this.bytes[this.position++];
+            else
+                bitBuf |= this.bytes[this.position++] << bitBufLen;
             bitBufLen += 8;
         }
 
         // Reverse back to numBits
         if (bitBufLen > numBits) {
             const excess = bitBufLen - numBits;
-            bitBuf &= (1 << numBits) - 1;
+            if (endianness === "be")
+                bitBuf >>>= excess;
+            else
+                bitBuf &= (1 << numBits) - 1;
             bitBufLen -= excess;
             this.position--;
             this.bitPos = 8 - excess;
@@ -133,7 +141,9 @@ export default class Stream {
          * @returns {number} The bit mask
          */
         function bitMask(bitPos) {
-            return 256 - (1 << bitPos);
+            return endianness === "be" ?
+                (1 << (8 - bitPos)) - 1 :
+                256 - (1 << bitPos);
         }
     }