cyberchef 9.39.3 → 9.39.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "9.39.3",
+  "version": "9.39.6",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",

package/src/core/config/OperationConfig.json

@@ -5652,6 +5652,11 @@
                 "name": "Alphabet",
                 "type": "string",
                 "value": "0-9A-Z $%*+\\-./:"
+            },
+            {
+                "name": "Remove non-alphabet chars",
+                "type": "boolean",
+                "value": true
             }
         ]
     },
@@ -5995,6 +6000,31 @@
                         "value": "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~"
                     }
                 ]
+            },
+            {
+                "name": "Remove non-alphabet chars",
+                "type": "boolean",
+                "value": true
+            }
+        ],
+        "checks": [
+            {
+                "pattern": "^\\s*(?:<~)?[\\s!-uz]*[!-uz]{15}[\\s!-uz]*(?:~>)?\\s*$",
+                "args": [
+                    "!-u"
+                ]
+            },
+            {
+                "pattern": "^[\\s0-9a-zA-Z.\\-:+=^!/*?&<>()[\\]{}@%$#]*[0-9a-zA-Z.\\-:+=^!/*?&<>()[\\]{}@%$#]{15}[\\s0-9a-zA-Z.\\-:+=^!/*?&<>()[\\]{}@%$#]*$",
+                "args": [
+                    "0-9a-zA-Z.\\-:+=^!/*?&<>()[]{}@%$#"
+                ]
+            },
+            {
+                "pattern": "^[\\s0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~]*[0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~]{15}[\\s0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~]*$",
+                "args": [
+                    "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~"
+                ]
             }
         ]
     },
@@ -8340,13 +8370,19 @@
     },
     "Merge": {
         "module": "Default",
-        "description": "Consolidate all branches back into a single trunk. The opposite of Fork.",
+        "description": "Consolidate all branches back into a single trunk. The opposite of Fork. Unticking the Merge All checkbox will only consolidate all branches up to the nearest Fork/Subsection.",
         "infoURL": null,
         "inputType": "string",
         "outputType": "string",
         "flowControl": true,
         "manualBake": false,
-        "args": []
+        "args": [
+            {
+                "name": "Merge All",
+                "type": "boolean",
+                "value": true
+            }
+        ]
     },
     "Microsoft Script Decoder": {
         "module": "Default",

package/src/core/lib/Base85.mjs

@@ -1,3 +1,5 @@
+import Utils from "../Utils.mjs";
+
 /**
  * Base85 resources.
  *
@@ -32,13 +34,12 @@ export const ALPHABET_OPTIONS = [
  * @returns {string}
  */
 export function alphabetName(alphabet) {
-    alphabet = alphabet.replace(/'/g, "'");
-    alphabet = alphabet.replace(/"/g, """);
-    alphabet = alphabet.replace(/\\/g, "\");
+    alphabet = escape(alphabet);
     let name;
 
     ALPHABET_OPTIONS.forEach(function(a) {
-        if (escape(alphabet) === escape(a.value)) name = a.name;
+        const expanded = Utils.expandAlphRange(a.value).join("");
+        if (alphabet === escape(expanded)) name = a.name;
     });
 
     return name;

package/src/core/operations/Fork.mjs

@@ -65,12 +65,21 @@ class Fork extends Operation {
         if (input)
             inputs = input.split(splitDelim);
 
+        // Set to 1 as if we are here, then there is one, the current one.
+        let numOp = 1;
         // Create subOpList for each tranche to operate on
-        // (all remaining operations unless we encounter a Merge)
+        // all remaining operations unless we encounter a Merge
         for (i = state.progress + 1; i < opList.length; i++) {
             if (opList[i].name === "Merge" && !opList[i].disabled) {
-                break;
+                numOp--;
+                if (numOp === 0 || opList[i].ingValues[0])
+                    break;
+                else
+                    // Not this Fork's Merge.
+                    subOpList.push(opList[i]);
             } else {
+                if (opList[i].name === "Fork" || opList[i].name === "Subsection")
+                    numOp++;
                 subOpList.push(opList[i]);
             }
         }

package/src/core/operations/FromBase45.mjs

@@ -32,7 +32,12 @@ class FromBase45 extends Operation {
                 name: "Alphabet",
                 type: "string",
                 value: ALPHABET
-            }
+            },
+            {
+                name: "Remove non-alphabet chars",
+                type: "boolean",
+                value: true
+            },
         ];
 
         this.highlight = highlightFromBase45;
@@ -46,10 +51,17 @@ class FromBase45 extends Operation {
      */
     run(input, args) {
         if (!input) return [];
-        const alphabet = Utils.expandAlphRange(args[0]);
+        const alphabet = Utils.expandAlphRange(args[0]).join("");
+        const removeNonAlphChars = args[1];
 
         const res = [];
 
+        // Remove non-alphabet characters
+        if (removeNonAlphChars) {
+            const re = new RegExp("[^" + alphabet.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g");
+            input = input.replace(re, "");
+        }
+
         for (const triple of Utils.chunked(input, 3)) {
             triple.reverse();
             let b = 0;

package/src/core/operations/FromBase85.mjs

@@ -32,6 +32,40 @@ class FromBase85 extends Operation {
                 type: "editableOption",
                 value: ALPHABET_OPTIONS
             },
+            {
+                name: "Remove non-alphabet chars",
+                type: "boolean",
+                value: true
+            },
+        ];
+        this.checks = [
+            {
+                pattern:
+                    "^\\s*(?:<~)?" + // Optional whitespace and starting marker
+                    "[\\s!-uz]*" +   // Any amount of base85 characters and whitespace
+                    "[!-uz]{15}" +   // At least 15 continoues base85 characters without whitespace
+                    "[\\s!-uz]*" +   // Any amount of base85 characters and whitespace
+                    "(?:~>)?\\s*$",  // Optional ending marker and whitespace
+                args: ["!-u"],
+            },
+            {
+                pattern:
+                    "^" +
+                    "[\\s0-9a-zA-Z.\\-:+=^!/*?&<>()[\\]{}@%$#]*" +
+                    "[0-9a-zA-Z.\\-:+=^!/*?&<>()[\\]{}@%$#]{15}" + // At least 15 continoues base85 characters without whitespace
+                    "[\\s0-9a-zA-Z.\\-:+=^!/*?&<>()[\\]{}@%$#]*" +
+                    "$",
+                args: ["0-9a-zA-Z.\\-:+=^!/*?&<>()[]{}@%$#"],
+            },
+            {
+                pattern:
+                    "^" +
+                    "[\\s0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~]*" +
+                    "[0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~]{15}" + // At least 15 continoues base85 characters without whitespace
+                    "[\\s0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~]*" +
+                    "$",
+                args: ["0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~"],
+            },
         ];
     }
 
@@ -43,6 +77,7 @@ class FromBase85 extends Operation {
     run(input, args) {
         const alphabet = Utils.expandAlphRange(args[0]).join(""),
             encoding = alphabetName(alphabet),
+            removeNonAlphChars = args[1],
             result = [];
 
         if (alphabet.length !== 85 ||
@@ -50,11 +85,18 @@ class FromBase85 extends Operation {
             throw new OperationError("Alphabet must be of length 85");
         }
 
-        if (input.length === 0) return [];
-
-        const matches = input.match(/<~(.+?)~>/);
+        // Remove delimiters if present
+        const matches = input.match(/^<~(.+?)~>$/);
         if (matches !== null) input = matches[1];
 
+        // Remove non-alphabet characters
+        if (removeNonAlphChars) {
+            const re = new RegExp("[^" + alphabet.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g");
+            input = input.replace(re, "");
+        }
+
+        if (input.length === 0) return [];
+
         let i = 0;
         let block, blockBytes;
         while (i < input.length) {
@@ -69,7 +111,7 @@ class FromBase85 extends Operation {
                     .map((chr, idx) => {
                         const digit = alphabet.indexOf(chr);
                         if (digit < 0 || digit > 84) {
-                            throw `Invalid character '${chr}' at index ${idx}`;
+                            throw `Invalid character '${chr}' at index ${i + idx}`;
                         }
                         return digit;
                     });

package/src/core/operations/Merge.mjs

@@ -20,10 +20,16 @@ class Merge extends Operation {
         this.name = "Merge";
         this.flowControl = true;
         this.module = "Default";
-        this.description = "Consolidate all branches back into a single trunk. The opposite of Fork.";
+        this.description = "Consolidate all branches back into a single trunk. The opposite of Fork. Unticking the Merge All checkbox will only consolidate all branches up to the nearest Fork/Subsection.";
         this.inputType = "string";
         this.outputType = "string";
-        this.args = [];
+        this.args = [
+            {
+                name: "Merge All",
+                type: "boolean",
+                value: true,
+            }
+        ];
     }
 
     /**

package/src/core/operations/Subsection.mjs

@@ -67,12 +67,21 @@ class Subsection extends Operation {
             subOpList   = [];
 
         if (input && section !== "") {
+            // Set to 1 as if we are here, then there is one, the current one.
+            let numOp = 1;
             // Create subOpList for each tranche to operate on
             // all remaining operations unless we encounter a Merge
             for (let i = state.progress + 1; i < opList.length; i++) {
                 if (opList[i].name === "Merge" && !opList[i].disabled) {
-                    break;
+                    numOp--;
+                    if (numOp === 0 || opList[i].ingValues[0])
+                        break;
+                    else
+                        // Not this subsection's Merge.
+                        subOpList.push(opList[i]);
                 } else {
+                    if (opList[i].name === "Fork" || opList[i].name === "Subsection")
+                        numOp++;
                     subOpList.push(opList[i]);
                 }
             }

package/tests/lib/TestRegister.mjs

@@ -12,6 +12,7 @@
 import Chef from "../../src/core/Chef.mjs";
 import Utils from "../../src/core/Utils.mjs";
 import cliProgress from "cli-progress";
+import log from "loglevel";
 
 /**
  * Object to store and run the list of tests.
@@ -50,6 +51,9 @@ class TestRegister {
      * Runs all the tests in the register.
      */
     async runTests () {
+        // Turn off logging to avoid messy errors
+        log.setLevel("silent", false);
+
         const progBar = new cliProgress.SingleBar({
             format: formatter,
             stopOnComplete: true
@@ -84,7 +88,17 @@ class TestRegister {
 
             if (result.error) {
                 if (test.expectedError) {
-                    ret.status = "passing";
+                    if (result.error.displayStr === test.expectedOutput) {
+                        ret.status = "passing";
+                    } else {
+                        ret.status = "failing";
+                        ret.output = [
+                            "Expected",
+                            "\t" + test.expectedOutput.replace(/\n/g, "\n\t"),
+                            "Received",
+                            "\t" + result.error.displayStr.replace(/\n/g, "\n\t"),
+                        ].join("\n");
+                    }
                 } else {
                     ret.status = "erroring";
                     ret.output = result.error.displayStr;
@@ -118,6 +132,9 @@ class TestRegister {
             progBar.increment();
         }
 
+        // Turn logging back on
+        log.setLevel("info", false);
+
         return testResults;
     }
 

package/tests/operations/index.mjs

@@ -24,6 +24,7 @@ import "./tests/Base45.mjs";
 import "./tests/Base58.mjs";
 import "./tests/Base64.mjs";
 import "./tests/Base62.mjs";
+import "./tests/Base85.mjs";
 import "./tests/BitwiseOp.mjs";
 import "./tests/ByteRepr.mjs";
 import "./tests/CartesianProduct.mjs";
@@ -114,6 +115,7 @@ import "./tests/HASSH.mjs";
 import "./tests/GetAllCasings.mjs";
 import "./tests/SIGABA.mjs";
 import "./tests/ELFInfo.mjs";
+import "./tests/Subsection.mjs";
 
 
 // Cannot test operations that use the File type yet

package/tests/operations/tests/Base85.mjs

@@ -0,0 +1,48 @@
+/**
+ * Base85 tests
+ *
+ * @author john19696
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+// Example from Wikipedia
+const wpExample = "Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.";
+// Escape newline, quote & backslash
+const wpOutput = "9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKF<GL>Cj@.4Gp$d7F!,L7@<6@)/0JDEF<G%<+EV:2F!,O<\
+DJ+*.@<*K0@<6L(Df-\\0Ec5e;DffZ(EZee.Bl.9pF\"AGXBPCsi+DGm>@3BB/F*&OCAfu2/AKYi(\
+DIb:@FD,*)+C]U=@3BN#EcYf8ATD3s@q?d$AftVqCh[NqF<G:8+EV:.+Cf>-FD5W8ARlolDIal(\
+DId<j@<?3r@:F%a+D58'ATD4$Bl@l3De:,-DJs`8ARoFb/0JMK@qB4^F!,R<AKZ&-DfTqBG%G>u\
+D.RTpAKYo'+CT/5+Cei#DII?(E,9)oF*2M7/c";
+
+TestRegister.addTests([
+    {
+        name: "To Base85",
+        input: wpExample,
+        expectedOutput: wpOutput,
+        recipeConfig: [
+            { "op": "To Base85",
+                "args": ["!-u"] }
+        ]
+    },
+    {
+        name: "From Base85",
+        input: wpOutput + "\n",
+        expectedOutput: wpExample,
+        recipeConfig: [
+            { "op": "From Base85",
+                "args": ["!-u", true] }
+        ]
+    },
+    {
+        name: "From Base85",
+        input: wpOutput + "v",
+        expectedError: true,
+        expectedOutput: "From Base85 - Invalid character 'v' at index 337",
+        recipeConfig: [
+            { "op": "From Base85",
+                "args": ["!-u", false] }
+        ]
+    },
+]);

package/tests/operations/tests/Fork.mjs

@@ -31,7 +31,7 @@ TestRegister.addTests([
             },
             {
                 op: "Merge",
-                args: [],
+                args: [true],
             },
         ],
     },
@@ -50,7 +50,7 @@ TestRegister.addTests([
             },
             {
                 op: "Merge",
-                args: [],
+                args: [true],
             },
         ],
     },
@@ -66,5 +66,16 @@ TestRegister.addTests([
             {"op": "Label", "args": ["skipReturn"]},
             {"op": "To Base64", "args": ["A-Za-z0-9+/="]}
         ]
-    }
+    },
+    {
+        name: "Fork, Partial Merge",
+        input: "Hello World",
+        expectedOutput: "48656c6c6f 576f726c64",
+        recipeConfig: [
+            { "op": "Fork",   "args": [" ", " ", false] },
+            { "op": "Fork",   "args": ["l", "l", false] },
+            { "op": "Merge",  "args": [false] },
+            { "op": "To Hex", "args": ["None", 0] },
+        ]
+    },
 ]);

package/tests/operations/tests/Subsection.mjs

@@ -0,0 +1,102 @@
+/**
+ * Subsection Tests.
+ *
+ * @author n1073645 [n1073645@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        name: "Subsection: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                "op": "Subsection",
+                "args": ["", true, true, false],
+            },
+        ],
+    },
+    {
+        name: "Subsection, Full Merge: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                "op": "Subsection",
+                "args": ["", true, true, false],
+            },
+            {
+                "op": "Merge",
+                "args": [true],
+            },
+        ],
+    },
+    {
+        name: "Subsection, Partial Merge: nothing",
+        input: "",
+        expectedOutput: "",
+        recipeConfig: [
+            {
+                "op": "Subsection",
+                "args": ["", true, true, false],
+            },
+            {
+                "op": "Merge",
+                "args": [false],
+            },
+        ],
+    },
+    {
+        name: "Subsection, Full Merge: Base64 with Hex",
+        input: "SGVsbG38675629ybGQ=",
+        expectedOutput: "Hello World",
+        recipeConfig: [
+            {
+                "op": "Subsection",
+                "args": ["386756", true, true, false],
+            },
+            {
+                "op": "From Hex",
+                "args": ["Auto"],
+            },
+            {
+                "op": "Merge",
+                "args": [true],
+            },
+            {
+                "op": "From Base64",
+                "args": ["A-Za-z0-9+/=", true, false],
+            },
+        ],
+    },
+    {
+        name: "Subsection, Partial Merge: Base64 with Hex surrounded by binary data.",
+        input: "000000000SGVsbG38675629ybGQ=0000000000",
+        expectedOutput: "000000000Hello World0000000000",
+        recipeConfig: [
+            {
+                "op": "Subsection",
+                "args": ["SGVsbG38675629ybGQ=", true, true, false],
+            },
+            {
+                "op": "Subsection",
+                "args": ["386756", true, true, false],
+            },
+            {
+                "op": "From Hex",
+                "args": ["Auto"],
+            },
+            {
+                "op": "Merge",
+                "args": [false],
+            },
+            {
+                "op": "From Base64",
+                "args": ["A-Za-z0-9+/=", true, false],
+            },
+        ],
+    },
+]);