cyberchef 9.37.3 → 9.38.2

package/CHANGELOG.md

@@ -13,6 +13,9 @@ All major and minor version changes will be documented in this file. Details of
 
 ## Details
 
+### [9.38.0] - 2022-05-30
+- Added 'Parse TCP' operation [@n1474335] | [a895d1d]
+
 ### [9.37.0] - 2022-03-29
 - 'SM4 Encrypt' and 'SM4 Decrypt' operations added [@swesven] | [#1189]
 - NoPadding options added for CBC and ECB modes in AES, DES and Triple DES Decrypt operations [@swesven] | [#1189]
@@ -135,7 +138,7 @@ All major and minor version changes will be documented in this file. Details of
 
 <details>
     <summary>Click to expand v8 minor versions</summary>
-    
+
 ### [8.38.0] - 2019-07-03
 - 'Streebog' and 'GOST hash' operations added [@MShwed] [@n1474335] | [#530]
 
@@ -288,6 +291,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[9.38.0]: https://github.com/gchq/CyberChef/releases/tag/v9.38.0
 [9.37.0]: https://github.com/gchq/CyberChef/releases/tag/v9.37.0
 [9.36.0]: https://github.com/gchq/CyberChef/releases/tag/v9.36.0
 [9.35.0]: https://github.com/gchq/CyberChef/releases/tag/v9.35.0
@@ -416,6 +420,7 @@ All major and minor version changes will be documented in this file. Details of
 [289a417]: https://github.com/gchq/CyberChef/commit/289a417dfb5923de5e1694354ec42a08d9395bfe
 [e9ca4dc]: https://github.com/gchq/CyberChef/commit/e9ca4dc9caf98f33fd986431cd400c88082a42b8
 [dd18e52]: https://github.com/gchq/CyberChef/commit/dd18e529939078b89867297b181a584e8b2cc7da
+[a895d1d]: https://github.com/gchq/CyberChef/commit/a895d1d82a2f92d440a0c5eca2bc7c898107b737
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
@@ -502,4 +507,4 @@ All major and minor version changes will be documented in this file. Details of
 [#1242]: https://github.com/gchq/CyberChef/pull/1242
 [#1244]: https://github.com/gchq/CyberChef/pull/1244
 [#1313]: https://github.com/gchq/CyberChef/pull/1313
-[#1326]: https://github.com/gchq/CyberChef/pull/1326
+[#1326]: https://github.com/gchq/CyberChef/pull/1326

package/Gruntfile.js

@@ -217,7 +217,8 @@ module.exports = function (grunt) {
                     client: {
                         logging: "error",
                         overlay: true
-                    }
+                    },
+                    hot: "only"
                 },
                 plugins: [
                     new webpack.DefinePlugin(BUILD_CONSTANTS),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.37.3",
+  "version": "9.38.2",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -48,7 +48,7 @@
     "autoprefixer": "^10.4.4",
     "babel-loader": "^8.2.4",
     "babel-plugin-dynamic-import-node": "^2.3.3",
-    "chromedriver": "^99.0.0",
+    "chromedriver": "^101.0.0",
     "cli-progress": "^3.10.0",
     "colors": "^1.4.0",
     "copy-webpack-plugin": "^10.2.4",
@@ -69,6 +69,7 @@
     "html-webpack-plugin": "^5.5.0",
     "imports-loader": "^3.1.1",
     "mini-css-extract-plugin": "2.6.0",
+    "modify-source-webpack-plugin": "^3.0.0",
     "nightwatch": "^2.0.10",
     "postcss": "^8.4.12",
     "postcss-css-variables": "^0.18.0",
@@ -174,6 +175,7 @@
     "lint": "npx grunt lint",
     "postinstall": "npx grunt exec:fixCryptoApiImports",
     "newop": "node --experimental-modules --experimental-json-modules src/core/config/scripts/newOperation.mjs",
+    "minor": "node --experimental-modules --experimental-json-modules src/core/config/scripts/newMinorVersion.mjs",
     "getheapsize": "node -e 'console.log(`node heap limit = ${require(\"v8\").getHeapStatistics().heap_size_limit / (1024 * 1024)} Mb`)'",
     "setheapsize": "export NODE_OPTIONS=--max_old_space_size=2048"
   }

package/src/core/config/Categories.json

@@ -190,6 +190,7 @@
             "Parse IP range",
             "Parse IPv6 address",
             "Parse IPv4 header",
+            "Parse TCP",
             "Parse UDP",
             "Parse SSH Host Key",
             "Parse URI",

package/src/core/config/OperationConfig.json

@@ -9518,6 +9518,25 @@
             }
         ]
     },
+    "Parse TCP": {
+        "module": "Default",
+        "description": "Parses a TCP header and payload (if present).",
+        "infoURL": "https://wikipedia.org/wiki/Transmission_Control_Protocol",
+        "inputType": "string",
+        "outputType": "html",
+        "flowControl": false,
+        "manualBake": false,
+        "args": [
+            {
+                "name": "Input format",
+                "type": "option",
+                "value": [
+                    "Hex",
+                    "Raw"
+                ]
+            }
+        ]
+    },
     "Parse TLV": {
         "module": "Default",
         "description": "Converts a Type-Length-Value (TLV) encoded string into a JSON object.  Can optionally include a <code>Key</code> / <code>Type</code> entry. <br><br>Tags: Key-Length-Value, KLV, Length-Value, LV",
@@ -9548,11 +9567,20 @@
         "module": "Default",
         "description": "Parses a UDP header and payload (if present).",
         "infoURL": "https://wikipedia.org/wiki/User_Datagram_Protocol",
-        "inputType": "ArrayBuffer",
+        "inputType": "string",
         "outputType": "html",
         "flowControl": false,
         "manualBake": false,
-        "args": []
+        "args": [
+            {
+                "name": "Input format",
+                "type": "option",
+                "value": [
+                    "Hex",
+                    "Raw"
+                ]
+            }
+        ]
     },
     "Parse UNIX file permissions": {
         "module": "Default",

package/src/core/config/modules/Default.mjs

@@ -97,6 +97,7 @@ import ParseIPRange from "../../operations/ParseIPRange.mjs";
 import ParseIPv4Header from "../../operations/ParseIPv4Header.mjs";
 import ParseIPv6Address from "../../operations/ParseIPv6Address.mjs";
 import ParseSSHHostKey from "../../operations/ParseSSHHostKey.mjs";
+import ParseTCP from "../../operations/ParseTCP.mjs";
 import ParseTLV from "../../operations/ParseTLV.mjs";
 import ParseUDP from "../../operations/ParseUDP.mjs";
 import ParseUNIXFilePermissions from "../../operations/ParseUNIXFilePermissions.mjs";
@@ -263,6 +264,7 @@ OpModules.Default = {
     "Parse IPv4 header": ParseIPv4Header,
     "Parse IPv6 address": ParseIPv6Address,
     "Parse SSH Host Key": ParseSSHHostKey,
+    "Parse TCP": ParseTCP,
     "Parse TLV": ParseTLV,
     "Parse UDP": ParseUDP,
     "Parse UNIX file permissions": ParseUNIXFilePermissions,

package/src/core/config/scripts/newMinorVersion.mjs

@@ -0,0 +1,144 @@
+/**
+ * This script updates the CHANGELOG when a new minor version is created.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ */
+
+/* eslint no-console: ["off"] */
+
+import prompt from "prompt";
+import colors from "colors";
+import path from "path";
+import fs  from "fs";
+import process from "process";
+
+const dir = path.join(process.cwd() + "/src/core/config/");
+if (!fs.existsSync(dir)) {
+    console.log("\nCWD: " + process.cwd());
+    console.log("Error: newMinorVersion.mjs should be run from the project root");
+    console.log("Example> node --experimental-modules src/core/config/scripts/newMinorVersion.mjs");
+    process.exit(1);
+}
+
+let changelogData = fs.readFileSync(path.join(process.cwd(), "CHANGELOG.md"), "utf8");
+const lastVersion = changelogData.match(/## Details\s+### \[(\d+)\.(\d+)\.(\d+)\]/);
+const newVersion = [
+    parseInt(lastVersion[1], 10),
+    parseInt(lastVersion[2], 10) + 1,
+    0
+];
+
+let knownContributors = changelogData.match(/^\[@([^\]]+)\]/gm);
+knownContributors = knownContributors.map(c => c.slice(2, -1));
+
+const date = (new Date()).toISOString().split("T")[0];
+
+const schema = {
+    properties: {
+        message: {
+            description: "A short but descriptive summary of a feature in this version",
+            example: "Added 'Op name' operation",
+            prompt: "Feature description",
+            type: "string",
+            required: true,
+        },
+        author: {
+            description: "The author of the feature (only one supported, edit manually to add more)",
+            example: "n1474335",
+            prompt: "Author",
+            type: "string",
+            default: "n1474335"
+        },
+        id: {
+            description: "The PR number or full commit hash for this feature.",
+            example: "1200",
+            prompt: "Pull request or commit ID",
+            type: "string"
+        },
+        another: {
+            description: "y/n",
+            example: "y",
+            prompt: "Add another feature?",
+            type: "string",
+            pattern: /^[yn]$/,
+        }
+    }
+};
+
+// Build schema
+for (const prop in schema.properties) {
+    const p = schema.properties[prop];
+    p.description = "\n" + colors.white(p.description) + colors.cyan("\nExample: " + p.example) + "\n" + colors.green(p.prompt);
+}
+
+prompt.message = "";
+prompt.delimiter = ":".green;
+
+const features = [];
+const authors = [];
+const prIDs = [];
+const commitIDs = [];
+
+prompt.start();
+
+const getFeature = function() {
+    prompt.get(schema, (err, result) => {
+        if (err) {
+            console.log("\nExiting script.");
+            process.exit(0);
+        }
+
+        features.push(result);
+
+        if (result.another === "y") {
+            getFeature();
+        } else {
+            let message = `### [${newVersion[0]}.${newVersion[1]}.${newVersion[2]}] - ${date}\n`;
+
+            features.forEach(feature => {
+                const id = feature.id.length > 10 ? feature.id.slice(0, 7) : "#" + feature.id;
+                message += `- ${feature.message} [@${feature.author}] | [${id}]\n`;
+
+                if (!knownContributors.includes(feature.author)) {
+                    authors.push(`[@${feature.author}]: https://github.com/${feature.author}`);
+                }
+
+                if (feature.id.length > 10) {
+                    commitIDs.push(`[${id}]: https://github.com/gchq/CyberChef/commit/${feature.id}`);
+                } else {
+                    prIDs.push(`[#${feature.id}]: https://github.com/gchq/CyberChef/pull/${feature.id}`);
+                }
+            });
+
+            // Message
+            changelogData = changelogData.replace(/## Details\n\n/, "## Details\n\n" + message + "\n");
+
+            // Tag
+            const newTag = `[${newVersion[0]}.${newVersion[1]}.${newVersion[2]}]: https://github.com/gchq/CyberChef/releases/tag/v${newVersion[0]}.${newVersion[1]}.${newVersion[2]}\n`;
+            changelogData = changelogData.replace(/\n\n(\[\d+\.\d+\.\d+\]: https)/, "\n\n" + newTag + "$1");
+
+            // Author
+            authors.forEach(author => {
+                changelogData = changelogData.replace(/(\n\[@[^\]]+\]: https:\/\/github\.com\/[^\n]+\n)\n/, "$1" + author + "\n\n");
+            });
+
+            // Commit IDs
+            commitIDs.forEach(commitID => {
+                changelogData = changelogData.replace(/(\n\[[^\].]+\]: https:\/\/github.com\/gchq\/CyberChef\/commit\/[^\n]+\n)\n/, "$1" + commitID + "\n\n");
+            });
+
+            // PR IDs
+            prIDs.forEach(prID => {
+                changelogData = changelogData.replace(/(\n\[#[^\]]+\]: https:\/\/github.com\/gchq\/CyberChef\/pull\/[^\n]+\n)\n/, "$1" + prID + "\n\n");
+            });
+
+            fs.writeFileSync(path.join(process.cwd(), "CHANGELOG.md"), changelogData);
+
+            console.log("Written CHANGELOG.md");
+        }
+    });
+};
+
+getFeature();

package/src/core/lib/Binary.mjs

@@ -13,7 +13,7 @@ import OperationError from "../errors/OperationError.mjs";
 /**
  * Convert a byte array into a binary string.
  *
- * @param {Uint8Array|byteArray} data
+ * @param {Uint8Array|byteArray|number} data
  * @param {string} [delim="Space"]
  * @param {number} [padding=8]
  * @returns {string}
@@ -26,13 +26,17 @@ import OperationError from "../errors/OperationError.mjs";
  * toBinary([10,20,30], ":");
  */
 export function toBinary(data, delim="Space", padding=8) {
-    if (!data) return "";
-
     delim = Utils.charRep(delim);
     let output = "";
 
-    for (let i = 0; i < data.length; i++) {
-        output += data[i].toString(2).padStart(padding, "0") + delim;
+    if (data.length) { // array
+        for (let i = 0; i < data.length; i++) {
+            output += data[i].toString(2).padStart(padding, "0") + delim;
+        }
+    } else if (typeof data === "number") { // Single value
+        return data.toString(2).padStart(padding, "0");
+    } else {
+        return "";
     }
 
     if (delim.length) {

package/src/core/lib/FileSignatures.mjs

@@ -3778,8 +3778,8 @@ function parseDEFLATE(stream) {
 
     while (!finalBlock) {
         // Read header
-        finalBlock = stream.readBits(1);
-        const blockType = stream.readBits(2);
+        finalBlock = stream.readBits(1, "le");
+        const blockType = stream.readBits(2, "le");
 
         if (blockType === 0) {
             /* No compression */
@@ -3798,16 +3798,16 @@ function parseDEFLATE(stream) {
             /* Dynamic Huffman */
 
             // Read the number of liternal and length codes
-            const hlit = stream.readBits(5) + 257;
+            const hlit = stream.readBits(5, "le") + 257;
             // Read the number of distance codes
-            const hdist = stream.readBits(5) + 1;
+            const hdist = stream.readBits(5, "le") + 1;
             // Read the number of code lengths
-            const hclen = stream.readBits(4) + 4;
+            const hclen = stream.readBits(4, "le") + 4;
 
             // Parse code lengths
             const codeLengths = new Uint8Array(huffmanOrder.length);
             for (let i = 0; i < hclen; i++) {
-                codeLengths[huffmanOrder[i]] = stream.readBits(3);
+                codeLengths[huffmanOrder[i]] = stream.readBits(3, "le");
             }
 
             // Parse length table
@@ -3819,16 +3819,16 @@ function parseDEFLATE(stream) {
                 code = readHuffmanCode(stream, codeLengthsTable);
                 switch (code) {
                     case 16:
-                        repeat = 3 + stream.readBits(2);
+                        repeat = 3 + stream.readBits(2, "le");
                         while (repeat--) lengthTable[i++] = prev;
                         break;
                     case 17:
-                        repeat = 3 + stream.readBits(3);
+                        repeat = 3 + stream.readBits(3, "le");
                         while (repeat--) lengthTable[i++] = 0;
                         prev = 0;
                         break;
                     case 18:
-                        repeat = 11 + stream.readBits(7);
+                        repeat = 11 + stream.readBits(7, "le");
                         while (repeat--) lengthTable[i++] = 0;
                         prev = 0;
                         break;
@@ -3886,11 +3886,11 @@ function parseHuffmanBlock(stream, litTab, distTab) {
         if (code < 256) continue;
 
         // Length code
-        stream.readBits(lengthExtraTable[code - 257]);
+        stream.readBits(lengthExtraTable[code - 257], "le");
 
         // Dist code
         code = readHuffmanCode(stream, distTab);
-        stream.readBits(distanceExtraTable[code]);
+        stream.readBits(distanceExtraTable[code], "le");
     }
 }
 
@@ -3948,7 +3948,7 @@ function readHuffmanCode(stream, table) {
     const [codeTable, maxCodeLength] = table;
 
     // Read max length
-    const bitsBuf = stream.readBits(maxCodeLength);
+    const bitsBuf = stream.readBits(maxCodeLength, "le");
     const codeWithLength = codeTable[bitsBuf & ((1 << maxCodeLength) - 1)];
     const codeLength = codeWithLength >>> 16;
 

package/src/core/lib/Protocol.mjs

@@ -0,0 +1,47 @@
+/**
+ * Protocol parsing functions.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ */
+
+import BigNumber from "bignumber.js";
+import {toHexFast} from "../lib/Hex.mjs";
+
+/**
+ * Recursively displays a JSON object as an HTML table
+ *
+ * @param {Object} obj
+ * @returns string
+ */
+export function objToTable(obj, nested=false) {
+    let html = `<table
+        class='table table-sm table-nonfluid ${nested ? "mb-0 table-borderless" : "table-bordered"}'
+        style='table-layout: fixed; ${nested ? "margin: -1px !important;" : ""}'>`;
+    if (!nested)
+        html += `<tr>
+            <th>Field</th>
+            <th>Value</th>
+        </tr>`;
+
+    for (const key in obj) {
+        html += `<tr><td style='word-wrap: break-word'>${key}</td>`;
+        if (typeof obj[key] === "object")
+            html += `<td style='padding: 0'>${objToTable(obj[key], true)}</td>`;
+        else
+            html += `<td>${obj[key]}</td>`;
+        html += "</tr>";
+    }
+    html += "</table>";
+    return html;
+}
+
+/**
+ * Converts bytes into a BigNumber string
+ * @param {Uint8Array} bs
+ * @returns {string}
+ */
+export function bytesToLargeNumber(bs) {
+    return BigNumber(toHexFast(bs), 16).toString();
+}

package/src/core/lib/Stream.mjs

@@ -27,15 +27,17 @@ export default class Stream {
     }
 
     /**
-     * Get a number of bytes from the current position.
+     * Get a number of bytes from the current position, or all remaining bytes.
      *
-     * @param {number} numBytes
+     * @param {number} [numBytes=null]
      * @returns {Uint8Array}
      */
-    getBytes(numBytes) {
+    getBytes(numBytes=null) {
         if (this.position > this.length) return undefined;
 
-        const newPosition = this.position + numBytes;
+        const newPosition = numBytes !== null ?
+            this.position + numBytes :
+            this.length;
         const bytes = this.bytes.slice(this.position, newPosition);
         this.position = newPosition;
         this.bitPos = 0;
@@ -91,34 +93,40 @@ export default class Stream {
     }
 
     /**
-     * Reads a number of bits from the buffer.
-     *
-     * @TODO Add endianness
+     * Reads a number of bits from the buffer in big or little endian.
      *
      * @param {number} numBits
+     * @param {string} [endianness="be"]
      * @returns {number}
      */
-    readBits(numBits) {
+    readBits(numBits, endianness="be") {
         if (this.position > this.length) return undefined;
 
         let bitBuf = 0,
             bitBufLen = 0;
 
         // Add remaining bits from current byte
-        bitBuf = (this.bytes[this.position++] & bitMask(this.bitPos)) >>> this.bitPos;
+        bitBuf = this.bytes[this.position++] & bitMask(this.bitPos);
+        if (endianness !== "be") bitBuf >>>= this.bitPos;
         bitBufLen = 8 - this.bitPos;
         this.bitPos = 0;
 
         // Not enough bits yet
         while (bitBufLen < numBits) {
-            bitBuf |= this.bytes[this.position++] << bitBufLen;
+            if (endianness === "be")
+                bitBuf = (bitBuf << bitBufLen) | this.bytes[this.position++];
+            else
+                bitBuf |= this.bytes[this.position++] << bitBufLen;
             bitBufLen += 8;
         }
 
         // Reverse back to numBits
         if (bitBufLen > numBits) {
             const excess = bitBufLen - numBits;
-            bitBuf &= (1 << numBits) - 1;
+            if (endianness === "be")
+                bitBuf >>>= excess;
+            else
+                bitBuf &= (1 << numBits) - 1;
             bitBufLen -= excess;
             this.position--;
             this.bitPos = 8 - excess;
@@ -133,7 +141,9 @@ export default class Stream {
          * @returns {number} The bit mask
          */
         function bitMask(bitPos) {
-            return 256 - (1 << bitPos);
+            return endianness === "be" ?
+                (1 << (8 - bitPos)) - 1 :
+                256 - (1 << bitPos);
         }
     }