cborg 1.6.1 → 1.8.0

package/esm/lib/diagnostic.js

@@ -1,7 +1,13 @@
 import { Tokeniser } from './decode.js';
-import { toHex } from './byte-utils.js';
+import {
+  toHex,
+  fromHex
+} from './byte-utils.js';
+import { uintBoundaries } from './0uint.js';
+const utf8Encoder = new TextEncoder();
+const utf8Decoder = new TextDecoder();
 function* tokensToDiagnostic(inp, width = 100) {
-  const tokeniser = new Tokeniser(inp);
+  const tokeniser = new Tokeniser(inp, { retainStringBytes: true });
   let pos = 0;
   const indent = [];
   const slc = (start, length) => {
@@ -12,6 +18,7 @@ function* tokensToDiagnostic(inp, width = 100) {
     let margin = ''.padStart(indent.length * 2, ' ');
     let vLength = token.encodedLength - 1;
     let v = String(token.value);
+    let outp = `${ margin }${ slc(0, 1) }`;
     const str = token.type.name === 'bytes' || token.type.name === 'string';
     if (token.type.name === 'string') {
       v = v.length;
@@ -20,7 +27,29 @@ function* tokensToDiagnostic(inp, width = 100) {
       v = token.value.length;
       vLength -= v;
     }
-    let outp = `${ margin }${ slc(0, 1) } ${ slc(1, vLength) }`;
+    let multilen;
+    switch (token.type.name) {
+    case 'string':
+    case 'bytes':
+    case 'map':
+    case 'array':
+      multilen = token.type.name === 'string' ? utf8Encoder.encode(token.value).length : token.value.length;
+      if (multilen >= uintBoundaries[0]) {
+        if (multilen < uintBoundaries[1]) {
+          outp += ` ${ slc(1, 1) }`;
+        } else if (multilen < uintBoundaries[2]) {
+          outp += ` ${ slc(1, 2) }`;
+        } else if (multilen < uintBoundaries[3]) {
+          outp += ` ${ slc(1, 4) }`;
+        } else if (multilen < uintBoundaries[4]) {
+          outp += ` ${ slc(1, 8) }`;
+        }
+      }
+      break;
+    default:
+      outp += ` ${ slc(1, vLength) }`;
+      break;
+    }
     outp = outp.padEnd(width / 2, ' ');
     outp += `# ${ margin }${ token.type.name }`;
     if (token.type.name !== v) {
@@ -28,15 +57,22 @@ function* tokensToDiagnostic(inp, width = 100) {
     }
     yield outp;
     if (str) {
+      let asString = token.type.name === 'string';
       margin += '  ';
-      const repr = token.type.name === 'bytes' ? token.value : new TextEncoder().encode(token.value);
+      let repr = asString ? utf8Encoder.encode(token.value) : token.value;
+      if (asString && token.byteValue !== undefined) {
+        if (repr.length !== token.byteValue.length) {
+          repr = token.byteValue;
+          asString = false;
+        }
+      }
       const wh = (width / 2 - margin.length - 1) / 2;
       let snip = 0;
       while (repr.length - snip > 0) {
         const piece = repr.slice(snip, snip + wh);
         snip += piece.length;
-        const st = token.type.name === 'string' ? new TextDecoder().decode(piece) : piece.reduce((p, c) => {
-          if (c < 32 || c === 127) {
+        const st = asString ? utf8Decoder.decode(piece) : piece.reduce((p, c) => {
+          if (c < 32 || c >= 127 && c < 161 || c === 173) {
             return `${ p }\\x${ c.toString(16).padStart(2, '0') }`;
           }
           return `${ p }${ String.fromCharCode(c) }`;
@@ -44,17 +80,16 @@ function* tokensToDiagnostic(inp, width = 100) {
         yield `${ margin }${ toHex(piece) }`.padEnd(width / 2, ' ') + `# ${ margin }"${ st }"`;
       }
     }
+    if (indent.length) {
+      indent[indent.length - 1]--;
+    }
     if (!token.type.terminal) {
       switch (token.type.name) {
       case 'map':
-        if (token.value) {
-          indent.push(token.value * 2);
-        }
+        indent.push(token.value * 2);
         break;
       case 'array':
-        if (token.value) {
-          indent.push(token.value);
-        }
+        indent.push(token.value);
         break;
       case 'tag':
         indent.push(1);
@@ -62,17 +97,24 @@ function* tokensToDiagnostic(inp, width = 100) {
       default:
         throw new Error(`Unknown token type '${ token.type.name }'`);
       }
-    } else {
-      if (indent.length) {
-        indent[indent.length - 1]--;
-        if (indent[indent.length - 1] === 0) {
-          indent.pop();
-        }
-      }
+    }
+    while (indent.length && indent[indent.length - 1] <= 0) {
+      indent.pop();
     }
     pos += token.encodedLength;
   }
 }
+function fromDiag(input) {
+  if (typeof input !== 'string') {
+    throw new TypeError('Expected string input');
+  }
+  input = input.replace(/#.*?$/mg, '').replace(/[\s\r\n]+/mg, '');
+  if (/[^a-f0-9]/i.test(input)) {
+    throw new TypeError('Input string was not CBOR diagnostic format');
+  }
+  return fromHex(input);
+}
 export {
-  tokensToDiagnostic
+  tokensToDiagnostic,
+  fromDiag
 };

package/esm/lib/token.js

@@ -31,6 +31,7 @@ class Token {
     this.value = value;
     this.encodedLength = encodedLength;
     this.encodedBytes = undefined;
+    this.byteValue = undefined;
   }
   toString() {
     return `Token[${ this.type }].${ this.value }`;

package/esm/node-test/node-test-bin.js

@@ -2,9 +2,56 @@ import chai from 'chai';
 import { exec } from 'child_process';
 import process from 'process';
 import path from 'path';
+import { platform } from 'os';
 import { fileURLToPath } from 'url';
 import bin from '../lib/bin.js';
 const {assert} = chai;
+const fixture1JsonString = '{"a":1,"b":[2,3],"smile":"\uD83D\uDE00"}';
+const fixture1JsonPrettyString = `{
+  "a": 1,
+  "b": [
+    2,
+    3
+  ],
+  "smile": "😀"
+}
+`;
+const fixture1HexString = 'a3616101616282020365736d696c6564f09f9880';
+const fixture1Bin = fromHex(fixture1HexString);
+const fixture1BinString = new TextDecoder().decode(fixture1Bin);
+const fixture1DiagnosticString = `a3                                                # map(3)
+  61                                              #   string(1)
+    61                                            #     "a"
+  01                                              #   uint(1)
+  61                                              #   string(1)
+    62                                            #     "b"
+  82                                              #   array(2)
+    02                                            #     uint(2)
+    03                                            #     uint(3)
+  65                                              #   string(5)
+    736d696c65                                    #     "smile"
+  64                                              #   string(2)
+    f09f9880                                      #     "😀"
+`;
+const fixture2HexString = 'a4616101616282020363627566440102036165736d696c6564f09f9880';
+const fixture2DiagnosticString = `a4                                                # map(4)
+  61                                              #   string(1)
+    61                                            #     "a"
+  01                                              #   uint(1)
+  61                                              #   string(1)
+    62                                            #     "b"
+  82                                              #   array(2)
+    02                                            #     uint(2)
+    03                                            #     uint(3)
+  63                                              #   string(3)
+    627566                                        #     "buf"
+  44                                              #   bytes(4)
+    01020361                                      #     "\\x01\\x02\\x03a"
+  65                                              #   string(5)
+    736d696c65                                    #     "smile"
+  64                                              #   string(2)
+    f09f9880                                      #     "😀"
+`;
 const binPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '../lib/bin.js');
 function fromHex(hex) {
   return new Uint8Array(hex.split('').map((c, i, d) => i % 2 === 0 ? `0x${ c }${ d[i + 1] }` : '').filter(Boolean).map(e => parseInt(e, 16)));
@@ -39,15 +86,18 @@ describe('Bin', () => {
       assert.strictEqual(e.stdout, '');
       assert.strictEqual(e.stderr, `Usage: cborg <command> <args>
 Valid commands:
-\thex2diag [hex input]
-\thex2bin [hex input]
-\thex2json [--pretty] [hex input]
-\tbin2hex [binary input]
 \tbin2diag [binary input]
+\tbin2hex [binary input]
 \tbin2json [--pretty] [binary input]
-\tjson2hex '[json input]'
-\tjson2diag '[json input]'
+\tdiag2bin [diagnostic input]
+\tdiag2hex [diagnostic input]
+\tdiag2json [--pretty] [diagnostic input]
+\thex2bin [hex input]
+\thex2diag [hex input]
+\thex2json [--pretty] [hex input]
 \tjson2bin '[json input]'
+\tjson2diag '[json input]'
+\tjson2hex '[json input]'
 Input may either be supplied as an argument or piped via stdin
 `);
     }
@@ -61,15 +111,18 @@ Input may either be supplied as an argument or piped via stdin
       assert.strictEqual(e.stderr, `Unknown command: 'blip'
 Usage: cborg <command> <args>
 Valid commands:
-\thex2diag [hex input]
-\thex2bin [hex input]
-\thex2json [--pretty] [hex input]
-\tbin2hex [binary input]
 \tbin2diag [binary input]
+\tbin2hex [binary input]
 \tbin2json [--pretty] [binary input]
-\tjson2hex '[json input]'
-\tjson2diag '[json input]'
+\tdiag2bin [diagnostic input]
+\tdiag2hex [diagnostic input]
+\tdiag2json [--pretty] [diagnostic input]
+\thex2bin [hex input]
+\thex2diag [hex input]
+\thex2json [--pretty] [hex input]
 \tjson2bin '[json input]'
+\tjson2diag '[json input]'
+\tjson2hex '[json input]'
 Input may either be supplied as an argument or piped via stdin
 `);
     }
@@ -79,135 +132,209 @@ Input may either be supplied as an argument or piped via stdin
     assert.strictEqual(stdout, '');
     assert.strictEqual(stderr, `Usage: cborg <command> <args>
 Valid commands:
-\thex2diag [hex input]
-\thex2bin [hex input]
-\thex2json [--pretty] [hex input]
-\tbin2hex [binary input]
 \tbin2diag [binary input]
+\tbin2hex [binary input]
 \tbin2json [--pretty] [binary input]
-\tjson2hex '[json input]'
-\tjson2diag '[json input]'
+\tdiag2bin [diagnostic input]
+\tdiag2hex [diagnostic input]
+\tdiag2json [--pretty] [diagnostic input]
+\thex2bin [hex input]
+\thex2diag [hex input]
+\thex2json [--pretty] [hex input]
 \tjson2bin '[json input]'
+\tjson2diag '[json input]'
+\tjson2hex '[json input]'
 Input may either be supplied as an argument or piped via stdin
 `);
   });
+  it('bin2diag (stdin)', async () => {
+    const {stdout, stderr} = await execBin('bin2diag', fixture1Bin);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, fixture1DiagnosticString);
+  });
+  it('bin2hex (stdin)', async () => {
+    const {stdout, stderr} = await execBin('bin2hex', fixture1Bin);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `${ fixture1HexString }\n`);
+  });
+  it('bin2json (stdin)', async () => {
+    const {stdout, stderr} = await execBin('bin2json', fixture1Bin);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `${ fixture1JsonString }\n`);
+  });
+  it('bin2json pretty (stdin)', async () => {
+    const {stdout, stderr} = await execBin('bin2json --pretty', fixture1Bin);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, fixture1JsonPrettyString);
+  });
   for (const stdin of [
       true,
       false
     ]) {
-    it(`hex2json${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('hex2json a3616101616282020365736d696c6564f09f9880') : await execBin('hex2json', 'a3616101616282020365736d696c6564f09f9880');
-      assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, '{"a":1,"b":[2,3],"smile":"\uD83D\uDE00"}\n');
-    });
-    it(`hex2json pretty${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('hex2json --pretty a3616101616282020365736d696c6564f09f9880') : await execBin('hex2json --pretty', 'a3616101616282020365736d696c6564f09f9880');
+    if (platform() !== 'win32' || stdin) {
+      it(`diag2bin${ stdin ? ' (stdin)' : '' }`, async () => {
+        const {stdout, stderr} = !stdin ? await execBin(`diag2bin '${ fixture1DiagnosticString }'`) : await execBin('diag2bin', fixture1DiagnosticString);
+        assert.strictEqual(stderr, '');
+        assert.strictEqual(stdout, fixture1BinString);
+      });
+      it(`diag2hex${ stdin ? ' (stdin)' : '' }`, async () => {
+        const {stdout, stderr} = !stdin ? await execBin(`diag2hex '${ fixture1DiagnosticString }'`) : await execBin('diag2hex', fixture1DiagnosticString);
+        assert.strictEqual(stderr, '');
+        assert.strictEqual(stdout, `${ fixture1HexString }\n`);
+      });
+      it(`diag2json${ stdin ? ' (stdin)' : '' }`, async () => {
+        const {stdout, stderr} = !stdin ? await execBin(`diag2json '${ fixture1DiagnosticString }'`) : await execBin('diag2json', fixture1DiagnosticString);
+        assert.strictEqual(stderr, '');
+        assert.strictEqual(stdout, `${ fixture1JsonString }\n`);
+      });
+      it(`diag2json pretty${ stdin ? ' (stdin)' : '' }`, async () => {
+        const {stdout, stderr} = !stdin ? await execBin(`diag2json --pretty '${ fixture1DiagnosticString }'`) : await execBin('diag2json --pretty', fixture1DiagnosticString);
+        assert.strictEqual(stderr, '');
+        assert.strictEqual(stdout, fixture1JsonPrettyString);
+      });
+    }
+    it(`hex2bin${ stdin ? ' (stdin)' : '' }`, async () => {
+      const {stdout, stderr} = !stdin ? await execBin(`hex2bin ${ fixture1HexString }`) : await execBin('hex2bin', fixture1HexString);
       assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, `{
-  "a": 1,
-  "b": [
-    2,
-    3
-  ],
-  "smile": "😀"
-}
-`);
+      assert.strictEqual(stdout, fixture1BinString);
     });
     it(`hex2diag${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('hex2diag a4616101616282020363627566440102036165736d696c6564f09f9880') : await execBin('hex2diag', 'a4616101616282020363627566440102036165736d696c6564f09f9880');
+      const {stdout, stderr} = !stdin ? await execBin(`hex2diag ${ fixture2HexString }`) : await execBin('hex2diag', fixture2HexString);
       assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, `a4                                                # map(4)
-  61                                              #   string(1)
-    61                                            #     "a"
-  01                                              #   uint(1)
-  61                                              #   string(1)
-    62                                            #     "b"
-  82                                              #   array(2)
-    02                                            #     uint(2)
-    03                                            #     uint(3)
-  63                                              #   string(3)
-    627566                                        #     "buf"
-  44                                              #   bytes(4)
-    01020361                                      #     "\\x01\\x02\\x03a"
-  65                                              #   string(5)
-    736d696c65                                    #     "smile"
-  64 f09f                                         #   string(2)
-    f09f9880                                      #     "😀"
-`);
+      assert.strictEqual(stdout, fixture2DiagnosticString);
     });
-    it(`hex2bin${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('hex2bin a3616101616282020365736d696c6564f09f9880') : await execBin('hex2bin', 'a3616101616282020365736d696c6564f09f9880');
+    it(`hex2json${ stdin ? ' (stdin)' : '' }`, async () => {
+      const {stdout, stderr} = !stdin ? await execBin(`hex2json ${ fixture1HexString }`) : await execBin('hex2json', fixture1HexString);
       assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, new TextDecoder().decode(fromHex('a3616101616282020365736d696c6564f09f9880')));
+      assert.strictEqual(stdout, `${ fixture1JsonString }\n`);
     });
-    it(`json2hex${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('json2hex "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"\uD83D\uDE00\\"}"') : await execBin('json2hex', '{"a":1,"b":[2,3],"smile":"\uD83D\uDE00"}');
+    it(`hex2json pretty${ stdin ? ' (stdin)' : '' }`, async () => {
+      const {stdout, stderr} = !stdin ? await execBin(`hex2json --pretty ${ fixture1HexString }`) : await execBin('hex2json --pretty', fixture1HexString);
       assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, 'a3616101616282020365736d696c6564f09f9880\n');
+      assert.strictEqual(stdout, fixture1JsonPrettyString);
     });
     it(`json2bin${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('json2bin "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"\uD83D\uDE00\\"}"') : await execBin('json2bin', '{"a":1,"b":[2,3],"smile":"\uD83D\uDE00"}');
+      const {stdout, stderr} = !stdin ? await execBin('json2bin "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"\uD83D\uDE00\\"}"') : await execBin('json2bin', fixture1JsonString);
       assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, new TextDecoder().decode(fromHex('a3616101616282020365736d696c6564f09f9880')));
+      assert.strictEqual(stdout, fixture1BinString);
     });
     it(`json2diag${ stdin ? ' (stdin)' : '' }`, async () => {
-      const {stdout, stderr} = stdin ? await execBin('json2diag "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"\uD83D\uDE00\\"}"') : await execBin('json2diag', '{"a":1,"b":[2,3],"smile":"\uD83D\uDE00"}');
+      const {stdout, stderr} = !stdin ? await execBin('json2diag "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"\uD83D\uDE00\\"}"') : await execBin('json2diag', fixture1JsonString);
       assert.strictEqual(stderr, '');
-      assert.strictEqual(stdout, `a3                                                # map(3)
-  61                                              #   string(1)
-    61                                            #     "a"
-  01                                              #   uint(1)
-  61                                              #   string(1)
-    62                                            #     "b"
-  82                                              #   array(2)
-    02                                            #     uint(2)
-    03                                            #     uint(3)
-  65                                              #   string(5)
-    736d696c65                                    #     "smile"
-  64 f09f                                         #   string(2)
-    f09f9880                                      #     "😀"
-`);
+      assert.strictEqual(stdout, fixture1DiagnosticString);
+    });
+    it(`json2hex${ stdin ? ' (stdin)' : '' }`, async () => {
+      const {stdout, stderr} = !stdin ? await execBin(`json2hex "${ fixture1JsonString.replace(/"/g, '\\"') }"`) : await execBin('json2hex', fixture1JsonString);
+      assert.strictEqual(stderr, '');
+      assert.strictEqual(stdout, `${ fixture1HexString }\n`);
     });
   }
-  it('bin2diag (stdin)', async () => {
-    const {stdout, stderr} = await execBin('bin2diag', fromHex('a3616101616282020365736d696c6564f09f9880'));
+  it('diag indenting', async () => {
+    const {stdout, stderr} = await execBin('json2diag', '{"a":[],"b":{},"c":{"a":1,"b":{"a":{"a":{}}}},"d":{"a":{"a":{"a":1},"b":2,"c":[]}},"e":[[[[{"a":{}}]]]],"f":1}');
     assert.strictEqual(stderr, '');
-    assert.strictEqual(stdout, `a3                                                # map(3)
+    assert.strictEqual(stdout, `a6                                                # map(6)
   61                                              #   string(1)
     61                                            #     "a"
-  01                                              #   uint(1)
+  80                                              #   array(0)
   61                                              #   string(1)
     62                                            #     "b"
-  82                                              #   array(2)
-    02                                            #     uint(2)
-    03                                            #     uint(3)
-  65                                              #   string(5)
-    736d696c65                                    #     "smile"
-  64 f09f                                         #   string(2)
-    f09f9880                                      #     "😀"
+  a0                                              #   map(0)
+  61                                              #   string(1)
+    63                                            #     "c"
+  a2                                              #   map(2)
+    61                                            #     string(1)
+      61                                          #       "a"
+    01                                            #     uint(1)
+    61                                            #     string(1)
+      62                                          #       "b"
+    a1                                            #     map(1)
+      61                                          #       string(1)
+        61                                        #         "a"
+      a1                                          #       map(1)
+        61                                        #         string(1)
+          61                                      #           "a"
+        a0                                        #         map(0)
+  61                                              #   string(1)
+    64                                            #     "d"
+  a1                                              #   map(1)
+    61                                            #     string(1)
+      61                                          #       "a"
+    a3                                            #     map(3)
+      61                                          #       string(1)
+        61                                        #         "a"
+      a1                                          #       map(1)
+        61                                        #         string(1)
+          61                                      #           "a"
+        01                                        #         uint(1)
+      61                                          #       string(1)
+        62                                        #         "b"
+      02                                          #       uint(2)
+      61                                          #       string(1)
+        63                                        #         "c"
+      80                                          #       array(0)
+  61                                              #   string(1)
+    65                                            #     "e"
+  81                                              #   array(1)
+    81                                            #     array(1)
+      81                                          #       array(1)
+        81                                        #         array(1)
+          a1                                      #           map(1)
+            61                                    #             string(1)
+              61                                  #               "a"
+            a0                                    #             map(0)
+  61                                              #   string(1)
+    66                                            #     "f"
+  01                                              #   uint(1)
 `);
   });
-  it('bin2json (stdin)', async () => {
-    const {stdout, stderr} = await execBin('bin2json', fromHex('a3616101616282020365736d696c6564f09f9880'));
-    assert.strictEqual(stderr, '');
-    assert.strictEqual(stdout, '{"a":1,"b":[2,3],"smile":"\uD83D\uDE00"}\n');
+  describe('diag length bytes', () => {
+    it('compact', async () => {
+      const {stdout, stderr} = await execBin('json2diag', '"aaaaaaaaaaaaaaaaaaaaaaa"');
+      assert.strictEqual(stderr, '');
+      assert.strictEqual(stdout, `77                                                # string(23)
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+`);
+    });
+    it('1-byte', async () => {
+      const {stdout, stderr} = await execBin('json2diag', '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"');
+      assert.strictEqual(stderr, '');
+      assert.strictEqual(stdout, `78 23                                             # string(35)
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  616161616161616161616161                        #   "aaaaaaaaaaaa"
+`);
+    });
+    it('2-byte', async () => {
+      const {stdout, stderr} = await execBin('json2diag', '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"');
+      assert.strictEqual(stderr, '');
+      assert.strictEqual(stdout, `79 0100                                           # string(256)
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  616161                                          #   "aaa"
+`);
+    });
   });
-  it('bin2json pretty (stdin)', async () => {
-    const {stdout, stderr} = await execBin('bin2json --pretty', fromHex('a3616101616282020365736d696c6564f09f9880'));
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35';
+    let {stdout, stderr} = await execBin(`hex2diag ${ input }`);
     assert.strictEqual(stderr, '');
-    assert.strictEqual(stdout, `{
-  "a": 1,
-  "b": [
-    2,
-    3
-  ],
-  "smile": "😀"
-}
+    assert.strictEqual(stdout, `78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
 `);
-  });
-  it('bin2hex (stdin)', async () => {
-    const {stdout, stderr} = await execBin('bin2hex', fromHex('a3616101616282020365736d696c6564f09f9880'));
+    ({stdout, stderr} = await execBin('diag2hex', stdout));
     assert.strictEqual(stderr, '');
-    assert.strictEqual(stdout, 'a3616101616282020365736d696c6564f09f9880\n');
+    assert.strictEqual(stdout, `${ input }\n`);
   });
 });

package/interface.ts

@@ -39,6 +39,7 @@ export interface DecodeOptions {
   allowBigInt?: boolean
   strict?: boolean
   useMaps?: boolean
+  retainStringBytes?: boolean
   tags?: TagDecoder[],
   tokenizer?: DecodeTokenizer
 }

package/lib/3string.js

@@ -2,7 +2,7 @@ import { Token, Type } from './token.js'
 import { assertEnoughData, decodeErrPrefix } from './common.js'
 import * as uint from './0uint.js'
 import { encodeBytes } from './2bytes.js'
-import { toString } from './byte-utils.js'
+import { toString, slice } from './byte-utils.js'
 
 /**
  * @typedef {import('./bl.js').Bl} Bl
@@ -14,23 +14,28 @@ import { toString } from './byte-utils.js'
  * @param {number} pos
  * @param {number} prefix
  * @param {number} length
+ * @param {DecodeOptions} options
  * @returns {Token}
  */
-function toToken (data, pos, prefix, length) {
+function toToken (data, pos, prefix, length, options) {
   const totLength = prefix + length
   assertEnoughData(data, pos, totLength)
-  return new Token(Type.string, toString(data, pos + prefix, pos + totLength), totLength)
+  const tok = new Token(Type.string, toString(data, pos + prefix, pos + totLength), totLength)
+  if (options.retainStringBytes === true) {
+    tok.byteValue = slice(data, pos + prefix, pos + totLength)
+  }
+  return tok
 }
 
 /**
  * @param {Uint8Array} data
  * @param {number} pos
  * @param {number} minor
- * @param {DecodeOptions} _options
+ * @param {DecodeOptions} options
  * @returns {Token}
  */
-export function decodeStringCompact (data, pos, minor, _options) {
-  return toToken(data, pos, 1, minor)
+export function decodeStringCompact (data, pos, minor, options) {
+  return toToken(data, pos, 1, minor, options)
 }
 
 /**
@@ -41,7 +46,7 @@ export function decodeStringCompact (data, pos, minor, _options) {
  * @returns {Token}
  */
 export function decodeString8 (data, pos, _minor, options) {
-  return toToken(data, pos, 2, uint.readUint8(data, pos + 1, options))
+  return toToken(data, pos, 2, uint.readUint8(data, pos + 1, options), options)
 }
 
 /**
@@ -52,7 +57,7 @@ export function decodeString8 (data, pos, _minor, options) {
  * @returns {Token}
  */
 export function decodeString16 (data, pos, _minor, options) {
-  return toToken(data, pos, 3, uint.readUint16(data, pos + 1, options))
+  return toToken(data, pos, 3, uint.readUint16(data, pos + 1, options), options)
 }
 
 /**
@@ -63,7 +68,7 @@ export function decodeString16 (data, pos, _minor, options) {
  * @returns {Token}
  */
 export function decodeString32 (data, pos, _minor, options) {
-  return toToken(data, pos, 5, uint.readUint32(data, pos + 1, options))
+  return toToken(data, pos, 5, uint.readUint32(data, pos + 1, options), options)
 }
 
 // TODO: maybe we shouldn't support this ..
@@ -79,7 +84,7 @@ export function decodeString64 (data, pos, _minor, options) {
   if (typeof l === 'bigint') {
     throw new Error(`${decodeErrPrefix} 64-bit integer string lengths not supported`)
   }
-  return toToken(data, pos, 9, l)
+  return toToken(data, pos, 9, l, options)
 }
 
 export const encodeString = encodeBytes