cborg 1.6.2 → 1.8.1

package/lib/diagnostic.js

@@ -1,12 +1,16 @@
 import { Tokeniser } from './decode.js'
-import { toHex } from './byte-utils.js'
+import { toHex, fromHex } from './byte-utils.js'
+import { uintBoundaries } from './0uint.js'
+
+const utf8Encoder = new TextEncoder()
+const utf8Decoder = new TextDecoder()
 
 /**
  * @param {Uint8Array} inp
  * @param {number} [width]
  */
 function * tokensToDiagnostic (inp, width = 100) {
-  const tokeniser = new Tokeniser(inp)
+  const tokeniser = new Tokeniser(inp, { retainStringBytes: true })
   let pos = 0
   const indent = []
 
@@ -26,6 +30,7 @@ function * tokensToDiagnostic (inp, width = 100) {
     let vLength = token.encodedLength - 1
     /** @type {string|number} */
     let v = String(token.value)
+    let outp = `${margin}${slc(0, 1)}`
     const str = token.type.name === 'bytes' || token.type.name === 'string'
     if (token.type.name === 'string') {
       v = v.length
@@ -36,7 +41,33 @@ function * tokensToDiagnostic (inp, width = 100) {
       vLength -= v
     }
 
-    let outp = `${margin}${slc(0, 1)} ${slc(1, vLength)}`
+    let multilen
+    switch (token.type.name) {
+      case 'string':
+      case 'bytes':
+      case 'map':
+      case 'array':
+        // for bytes and string, we want to print out the length part of the value prefix if it
+        // exists - it exists for short lengths (<24) but does for longer lengths
+        multilen = token.type.name === 'string' ? utf8Encoder.encode(token.value).length : token.value.length
+        if (multilen >= uintBoundaries[0]) {
+          if (multilen < uintBoundaries[1]) {
+            outp += ` ${slc(1, 1)}`
+          } else if (multilen < uintBoundaries[2]) {
+            outp += ` ${slc(1, 2)}`
+            /* c8 ignore next 5 */
+          } else if (multilen < uintBoundaries[3]) { // sus
+            outp += ` ${slc(1, 4)}`
+          } else if (multilen < uintBoundaries[4]) { // orly?
+            outp += ` ${slc(1, 8)}`
+          }
+        }
+        break
+      default:
+        // print the value if it's not compacted into the first byte
+        outp += ` ${slc(1, vLength)}`
+        break
+    }
 
     outp = outp.padEnd(width / 2, ' ')
     outp += `# ${margin}${token.type.name}`
@@ -46,19 +77,26 @@ function * tokensToDiagnostic (inp, width = 100) {
     yield outp
 
     if (str) {
+      let asString = token.type.name === 'string'
       margin += '  '
-      const repr = token.type.name === 'bytes' ? token.value : new TextEncoder().encode(token.value)
+      let repr = asString ? utf8Encoder.encode(token.value) : token.value
+      if (asString && token.byteValue !== undefined) {
+        if (repr.length !== token.byteValue.length) {
+          // bail on printing this as a string, it's probably not utf8, so treat it as bytes
+          // (you can probably blame a Go programmer for this)
+          repr = token.byteValue
+          asString = false
+        }
+      }
       const wh = ((width / 2) - margin.length - 1) / 2
       let snip = 0
       while (repr.length - snip > 0) {
         const piece = repr.slice(snip, snip + wh)
         snip += piece.length
-        // the assumption that we can utf8 a byte-sliced version is a stretch,
-        // we could be slicing in the middle of a multi-byte character
-        const st = token.type.name === 'string'
-          ? new TextDecoder().decode(piece)
+        const st = asString
+          ? utf8Decoder.decode(piece)
           : piece.reduce((/** @type {string} */ p, /** @type {number} */ c) => {
-            if (c < 0x20 || c === 0x7f) {
+            if (c < 0x20 || (c >= 0x7f && c < 0xa1) || c === 0xad) {
               return `${p}\\x${c.toString(16).padStart(2, '0')}`
             }
             return `${p}${String.fromCharCode(c)}`
@@ -95,4 +133,22 @@ function * tokensToDiagnostic (inp, width = 100) {
   }
 }
 
-export { tokensToDiagnostic }
+/**
+ * Convert an input string formatted as CBOR diagnostic output into binary CBOR form.
+ * @param {string} input
+ * @returns {Uint8Array}
+ */
+function fromDiag (input) {
+  /* c8 ignore next 3 */
+  if (typeof input !== 'string') {
+    throw new TypeError('Expected string input')
+  }
+  input = input.replace(/#.*?$/mg, '').replace(/[\s\r\n]+/mg, '')
+  /* c8 ignore next 3 */
+  if (/[^a-f0-9]/i.test(input)) {
+    throw new TypeError('Input string was not CBOR diagnostic format')
+  }
+  return fromHex(input)
+}
+
+export { tokensToDiagnostic, fromDiag }

package/lib/token.js

@@ -54,6 +54,8 @@ class Token {
     this.encodedLength = encodedLength
     /** @type {Uint8Array|undefined} */
     this.encodedBytes = undefined
+    /** @type {Uint8Array|undefined} */
+    this.byteValue = undefined
   }
 
   /* c8 ignore next 3 */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cborg",
-  "version": "1.6.2",
+  "version": "1.8.1",
   "description": "Fast CBOR with a focus on strictness",
   "main": "./cjs/cborg.js",
   "bin": {
@@ -39,7 +39,7 @@
     "mocha": "^9.1.3",
     "polendina": "~2.0.1",
     "standard": "^16.0.4",
-    "typescript": "~4.5.3"
+    "typescript": "~4.6.2"
   },
   "exports": {
     ".": {

package/test/node-test-bin.js

@@ -4,12 +4,63 @@ import chai from 'chai'
 import { exec } from 'child_process'
 import process from 'process'
 import path from 'path'
+import { platform } from 'os'
 import { fileURLToPath } from 'url'
 // included here for ipjs compile tree
 import bin from '../lib/bin.js' // eslint-disable-line
 
 const { assert } = chai
 
+const fixture1JsonString = '{"a":1,"b":[2,3],"smile":"😀"}'
+const fixture1JsonPrettyString =
+`{
+  "a": 1,
+  "b": [
+    2,
+    3
+  ],
+  "smile": "😀"
+}
+`
+const fixture1HexString = 'a3616101616282020365736d696c6564f09f9880'
+const fixture1Bin = fromHex(fixture1HexString)
+const fixture1BinString = new TextDecoder().decode(fixture1Bin)
+const fixture1DiagnosticString =
+`a3                                                # map(3)
+  61                                              #   string(1)
+    61                                            #     "a"
+  01                                              #   uint(1)
+  61                                              #   string(1)
+    62                                            #     "b"
+  82                                              #   array(2)
+    02                                            #     uint(2)
+    03                                            #     uint(3)
+  65                                              #   string(5)
+    736d696c65                                    #     "smile"
+  64                                              #   string(2)
+    f09f9880                                      #     "😀"
+`
+const fixture2HexString = 'a4616101616282020363627566440102036165736d696c6564f09f9880'
+const fixture2DiagnosticString =
+`a4                                                # map(4)
+  61                                              #   string(1)
+    61                                            #     "a"
+  01                                              #   uint(1)
+  61                                              #   string(1)
+    62                                            #     "b"
+  82                                              #   array(2)
+    02                                            #     uint(2)
+    03                                            #     uint(3)
+  63                                              #   string(3)
+    627566                                        #     "buf"
+  44                                              #   bytes(4)
+    01020361                                      #     "\\x01\\x02\\x03a"
+  65                                              #   string(5)
+    736d696c65                                    #     "smile"
+  64                                              #   string(2)
+    f09f9880                                      #     "😀"
+`
+
 const binPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '../lib/bin.js')
 
 function fromHex (hex) {
@@ -48,15 +99,18 @@ describe('Bin', () => {
       assert.strictEqual(e.stderr,
 `Usage: cborg <command> <args>
 Valid commands:
-\thex2diag [hex input]
-\thex2bin [hex input]
-\thex2json [--pretty] [hex input]
-\tbin2hex [binary input]
 \tbin2diag [binary input]
+\tbin2hex [binary input]
 \tbin2json [--pretty] [binary input]
-\tjson2hex '[json input]'
-\tjson2diag '[json input]'
+\tdiag2bin [diagnostic input]
+\tdiag2hex [diagnostic input]
+\tdiag2json [--pretty] [diagnostic input]
+\thex2bin [hex input]
+\thex2diag [hex input]
+\thex2json [--pretty] [hex input]
 \tjson2bin '[json input]'
+\tjson2diag '[json input]'
+\tjson2hex '[json input]'
 Input may either be supplied as an argument or piped via stdin
 `)
     }
@@ -72,15 +126,18 @@ Input may either be supplied as an argument or piped via stdin
 `Unknown command: 'blip'
 Usage: cborg <command> <args>
 Valid commands:
-\thex2diag [hex input]
-\thex2bin [hex input]
-\thex2json [--pretty] [hex input]
-\tbin2hex [binary input]
 \tbin2diag [binary input]
+\tbin2hex [binary input]
 \tbin2json [--pretty] [binary input]
-\tjson2hex '[json input]'
-\tjson2diag '[json input]'
+\tdiag2bin [diagnostic input]
+\tdiag2hex [diagnostic input]
+\tdiag2json [--pretty] [diagnostic input]
+\thex2bin [hex input]
+\thex2diag [hex input]
+\thex2json [--pretty] [hex input]
 \tjson2bin '[json input]'
+\tjson2diag '[json input]'
+\tjson2hex '[json input]'
 Input may either be supplied as an argument or piped via stdin
 `)
     }
@@ -92,164 +149,137 @@ Input may either be supplied as an argument or piped via stdin
     assert.strictEqual(stderr,
 `Usage: cborg <command> <args>
 Valid commands:
-\thex2diag [hex input]
-\thex2bin [hex input]
-\thex2json [--pretty] [hex input]
-\tbin2hex [binary input]
 \tbin2diag [binary input]
+\tbin2hex [binary input]
 \tbin2json [--pretty] [binary input]
-\tjson2hex '[json input]'
-\tjson2diag '[json input]'
+\tdiag2bin [diagnostic input]
+\tdiag2hex [diagnostic input]
+\tdiag2json [--pretty] [diagnostic input]
+\thex2bin [hex input]
+\thex2diag [hex input]
+\thex2json [--pretty] [hex input]
 \tjson2bin '[json input]'
+\tjson2diag '[json input]'
+\tjson2hex '[json input]'
 Input may either be supplied as an argument or piped via stdin
 `)
   })
 
+  it('bin2diag (stdin)', async () => {
+    const { stdout, stderr } = await execBin('bin2diag', fixture1Bin)
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout, fixture1DiagnosticString)
+  })
+
+  it('bin2hex (stdin)', async () => {
+    const { stdout, stderr } = await execBin('bin2hex', fixture1Bin)
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout, `${fixture1HexString}\n`)
+  })
+
+  it('bin2json (stdin)', async () => {
+    const { stdout, stderr } = await execBin('bin2json', fixture1Bin)
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout, `${fixture1JsonString}\n`)
+  })
+
+  it('bin2json pretty (stdin)', async () => {
+    const { stdout, stderr } = await execBin('bin2json --pretty', fixture1Bin)
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout, fixture1JsonPrettyString)
+  })
+
   for (const stdin of [true, false]) {
-    it(`hex2json${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
-        ? await execBin('hex2json a3616101616282020365736d696c6564f09f9880')
-        : await execBin('hex2json', 'a3616101616282020365736d696c6564f09f9880')
-      assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout, '{"a":1,"b":[2,3],"smile":"😀"}\n')
-    })
+    if (platform() !== 'win32' || stdin) { // multiline CLI input is hard (impossible?) on Windows
+      it(`diag2bin${stdin ? ' (stdin)' : ''}`, async () => {
+        const { stdout, stderr } = !stdin
+          ? await execBin(`diag2bin '${fixture1DiagnosticString}'`)
+          : await execBin('diag2bin', fixture1DiagnosticString)
+        assert.strictEqual(stderr, '')
+        assert.strictEqual(stdout, fixture1BinString)
+      })
 
-    it(`hex2json pretty${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
-        ? await execBin('hex2json --pretty a3616101616282020365736d696c6564f09f9880')
-        : await execBin('hex2json --pretty', 'a3616101616282020365736d696c6564f09f9880')
+      it(`diag2hex${stdin ? ' (stdin)' : ''}`, async () => {
+        const { stdout, stderr } = !stdin
+          ? await execBin(`diag2hex '${fixture1DiagnosticString}'`)
+          : await execBin('diag2hex', fixture1DiagnosticString)
+        assert.strictEqual(stderr, '')
+        assert.strictEqual(stdout, `${fixture1HexString}\n`)
+      })
+
+      it(`diag2json${stdin ? ' (stdin)' : ''}`, async () => {
+        const { stdout, stderr } = !stdin
+          ? await execBin(`diag2json '${fixture1DiagnosticString}'`)
+          : await execBin('diag2json', fixture1DiagnosticString)
+        assert.strictEqual(stderr, '')
+        assert.strictEqual(stdout, `${fixture1JsonString}\n`)
+      })
+
+      it(`diag2json pretty${stdin ? ' (stdin)' : ''}`, async () => {
+        const { stdout, stderr } = !stdin
+          ? await execBin(`diag2json --pretty '${fixture1DiagnosticString}'`)
+          : await execBin('diag2json --pretty', fixture1DiagnosticString)
+        assert.strictEqual(stderr, '')
+        assert.strictEqual(stdout, fixture1JsonPrettyString)
+      })
+    }
+
+    it(`hex2bin${stdin ? ' (stdin)' : ''}`, async () => {
+      const { stdout, stderr } = !stdin
+        ? await execBin(`hex2bin ${fixture1HexString}`)
+        : await execBin('hex2bin', fixture1HexString)
       assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout,
-`{
-  "a": 1,
-  "b": [
-    2,
-    3
-  ],
-  "smile": "😀"
-}
-`)
+      assert.strictEqual(stdout, fixture1BinString)
     })
 
     it(`hex2diag${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
-        ? await execBin('hex2diag a4616101616282020363627566440102036165736d696c6564f09f9880')
-        : await execBin('hex2diag', 'a4616101616282020363627566440102036165736d696c6564f09f9880')
+      const { stdout, stderr } = !stdin
+        ? await execBin(`hex2diag ${fixture2HexString}`)
+        : await execBin('hex2diag', fixture2HexString)
       assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout,
-`a4                                                # map(4)
-  61                                              #   string(1)
-    61                                            #     "a"
-  01                                              #   uint(1)
-  61                                              #   string(1)
-    62                                            #     "b"
-  82                                              #   array(2)
-    02                                            #     uint(2)
-    03                                            #     uint(3)
-  63                                              #   string(3)
-    627566                                        #     "buf"
-  44                                              #   bytes(4)
-    01020361                                      #     "\\x01\\x02\\x03a"
-  65                                              #   string(5)
-    736d696c65                                    #     "smile"
-  64 f09f                                         #   string(2)
-    f09f9880                                      #     "😀"
-`)
+      assert.strictEqual(stdout, fixture2DiagnosticString)
     })
 
-    it(`hex2bin${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
-        ? await execBin('hex2bin a3616101616282020365736d696c6564f09f9880')
-        : await execBin('hex2bin', 'a3616101616282020365736d696c6564f09f9880')
+    it(`hex2json${stdin ? ' (stdin)' : ''}`, async () => {
+      const { stdout, stderr } = !stdin
+        ? await execBin(`hex2json ${fixture1HexString}`)
+        : await execBin('hex2json', fixture1HexString)
       assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout, new TextDecoder().decode(fromHex('a3616101616282020365736d696c6564f09f9880')))
+      assert.strictEqual(stdout, `${fixture1JsonString}\n`)
     })
 
-    it(`json2hex${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
-        ? await execBin('json2hex "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"😀\\"}"')
-        : await execBin('json2hex', '{"a":1,"b":[2,3],"smile":"😀"}')
+    it(`hex2json pretty${stdin ? ' (stdin)' : ''}`, async () => {
+      const { stdout, stderr } = !stdin
+        ? await execBin(`hex2json --pretty ${fixture1HexString}`)
+        : await execBin('hex2json --pretty', fixture1HexString)
       assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout, 'a3616101616282020365736d696c6564f09f9880\n')
+      assert.strictEqual(stdout, fixture1JsonPrettyString)
     })
 
     it(`json2bin${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
+      const { stdout, stderr } = !stdin
         ? await execBin('json2bin "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"😀\\"}"')
-        : await execBin('json2bin', '{"a":1,"b":[2,3],"smile":"😀"}')
+        : await execBin('json2bin', fixture1JsonString)
       assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout, new TextDecoder().decode(fromHex('a3616101616282020365736d696c6564f09f9880')))
+      assert.strictEqual(stdout, fixture1BinString)
     })
 
     it(`json2diag${stdin ? ' (stdin)' : ''}`, async () => {
-      const { stdout, stderr } = stdin
+      const { stdout, stderr } = !stdin
         ? await execBin('json2diag "{\\"a\\":1,\\"b\\":[2,3],\\"smile\\":\\"😀\\"}"')
-        : await execBin('json2diag', '{"a":1,"b":[2,3],"smile":"😀"}')
+        : await execBin('json2diag', fixture1JsonString)
       assert.strictEqual(stderr, '')
-      assert.strictEqual(stdout,
-`a3                                                # map(3)
-  61                                              #   string(1)
-    61                                            #     "a"
-  01                                              #   uint(1)
-  61                                              #   string(1)
-    62                                            #     "b"
-  82                                              #   array(2)
-    02                                            #     uint(2)
-    03                                            #     uint(3)
-  65                                              #   string(5)
-    736d696c65                                    #     "smile"
-  64 f09f                                         #   string(2)
-    f09f9880                                      #     "😀"
-`)
+      assert.strictEqual(stdout, fixture1DiagnosticString)
     })
-  }
 
-  it('bin2diag (stdin)', async () => {
-    const { stdout, stderr } = await execBin('bin2diag', fromHex('a3616101616282020365736d696c6564f09f9880'))
-    assert.strictEqual(stderr, '')
-    assert.strictEqual(stdout,
-`a3                                                # map(3)
-  61                                              #   string(1)
-    61                                            #     "a"
-  01                                              #   uint(1)
-  61                                              #   string(1)
-    62                                            #     "b"
-  82                                              #   array(2)
-    02                                            #     uint(2)
-    03                                            #     uint(3)
-  65                                              #   string(5)
-    736d696c65                                    #     "smile"
-  64 f09f                                         #   string(2)
-    f09f9880                                      #     "😀"
-`)
-  })
-
-  it('bin2json (stdin)', async () => {
-    const { stdout, stderr } = await execBin('bin2json', fromHex('a3616101616282020365736d696c6564f09f9880'))
-    assert.strictEqual(stderr, '')
-    assert.strictEqual(stdout, '{"a":1,"b":[2,3],"smile":"😀"}\n')
-  })
-
-  it('bin2json pretty (stdin)', async () => {
-    const { stdout, stderr } = await execBin('bin2json --pretty', fromHex('a3616101616282020365736d696c6564f09f9880'))
-    assert.strictEqual(stderr, '')
-    assert.strictEqual(stdout,
-`{
-  "a": 1,
-  "b": [
-    2,
-    3
-  ],
-  "smile": "😀"
-}
-`)
-  })
-
-  it('bin2hex (stdin)', async () => {
-    const { stdout, stderr } = await execBin('bin2hex', fromHex('a3616101616282020365736d696c6564f09f9880'))
-    assert.strictEqual(stderr, '')
-    assert.strictEqual(stdout, 'a3616101616282020365736d696c6564f09f9880\n')
-  })
+    it(`json2hex${stdin ? ' (stdin)' : ''}`, async () => {
+      const { stdout, stderr } = !stdin
+        ? await execBin(`json2hex "${fixture1JsonString.replace(/"/g, '\\"')}"`)
+        : await execBin('json2hex', fixture1JsonString)
+      assert.strictEqual(stderr, '')
+      assert.strictEqual(stdout, `${fixture1HexString}\n`)
+    })
+  }
 
   // complicated nesting to test indenting algorithm
   it('diag indenting', async () => {
@@ -311,4 +341,64 @@ Input may either be supplied as an argument or piped via stdin
   01                                              #   uint(1)
 `)
   })
+
+  describe('diag length bytes', () => {
+    it('compact', async () => {
+      const { stdout, stderr } = await execBin('json2diag', '"aaaaaaaaaaaaaaaaaaaaaaa"')
+      assert.strictEqual(stderr, '')
+      assert.strictEqual(stdout,
+`77                                                # string(23)
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+`)
+    })
+
+    it('1-byte', async () => {
+      const { stdout, stderr } = await execBin('json2diag', '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"')
+      assert.strictEqual(stderr, '')
+      assert.strictEqual(stdout,
+`78 23                                             # string(35)
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  616161616161616161616161                        #   "aaaaaaaaaaaa"
+`)
+    })
+
+    it('2-byte', async () => {
+      const { stdout, stderr } = await execBin('json2diag', '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"')
+      assert.strictEqual(stderr, '')
+      assert.strictEqual(stdout,
+`79 0100                                           # string(256)
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  6161616161616161616161616161616161616161616161  #   "aaaaaaaaaaaaaaaaaaaaaaa"
+  616161                                          #   "aaa"
+`)
+    })
+  })
+
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35'
+    let { stdout, stderr } = await execBin(`hex2diag ${input}`)
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout,
+`78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
+`)
+
+    // round-trip
+    ;({ stdout, stderr } = await execBin('diag2hex', stdout))
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout, `${input}\n`)
+  })
 })

package/types/interface.d.ts

@@ -29,6 +29,7 @@ export interface DecodeOptions {
     allowBigInt?: boolean;
     strict?: boolean;
     useMaps?: boolean;
+    retainStringBytes?: boolean;
     tags?: TagDecoder[];
     tokenizer?: DecodeTokenizer;
 }

package/types/interface.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,EAAE,EAAE,MAAM,UAAU,CAAA;AAE7B,oBAAY,mBAAmB,GAAG,KAAK,GAAG,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAA;AAEzE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,SAAS,GAAG,SAAS,CAAA;IAC7B,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;IACnB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAA;CACvC;AAED,oBAAY,mBAAmB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,GAAG,IAAI,CAAA;AAEtI,oBAAY,iBAAiB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,CAAA;AAE7H,oBAAY,gBAAgB,GAAG;IAC7B,CAAC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACvD,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,GAAG,MAAM,CAAC;IAC5C,WAAW,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC7D,CAAA;AAED,oBAAY,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,KAAK,MAAM,CAAA;AAEpF,oBAAY,gBAAgB,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,UAAU,GAAG,SAAS,CAAA;AAEvE,MAAM,WAAW,eAAe;IAC9B,IAAI,IAAI,OAAO,CAAC;IAChB,IAAI,IAAI,KAAK,CAAA;CACd;AAED,oBAAY,UAAU,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,GAAG,CAAA;AAE5C,MAAM,WAAW,aAAa;IAC5B,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,YAAY,CAAC,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,mBAAmB,CAAA;KAAE,CAAA;CAC3D"}
+{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,EAAE,EAAE,MAAM,UAAU,CAAA;AAE7B,oBAAY,mBAAmB,GAAG,KAAK,GAAG,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAA;AAEzE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,SAAS,GAAG,SAAS,CAAA;IAC7B,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;IACnB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAA;CACvC;AAED,oBAAY,mBAAmB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,GAAG,IAAI,CAAA;AAEtI,oBAAY,iBAAiB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,CAAA;AAE7H,oBAAY,gBAAgB,GAAG;IAC7B,CAAC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACvD,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,GAAG,MAAM,CAAC;IAC5C,WAAW,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC7D,CAAA;AAED,oBAAY,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,KAAK,MAAM,CAAA;AAEpF,oBAAY,gBAAgB,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,UAAU,GAAG,SAAS,CAAA;AAEvE,MAAM,WAAW,eAAe;IAC9B,IAAI,IAAI,OAAO,CAAC;IAChB,IAAI,IAAI,KAAK,CAAA;CACd;AAED,oBAAY,UAAU,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,GAAG,CAAA;AAE5C,MAAM,WAAW,aAAa;IAC5B,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,YAAY,CAAC,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,mBAAmB,CAAA;KAAE,CAAA;CAC3D"}

package/types/lib/3string.d.ts

@@ -2,10 +2,10 @@
  * @param {Uint8Array} data
  * @param {number} pos
  * @param {number} minor
- * @param {DecodeOptions} _options
+ * @param {DecodeOptions} options
  * @returns {Token}
  */
-export function decodeStringCompact(data: Uint8Array, pos: number, minor: number, _options: DecodeOptions): Token;
+export function decodeStringCompact(data: Uint8Array, pos: number, minor: number, options: DecodeOptions): Token;
 /**
  * @param {Uint8Array} data
  * @param {number} pos

package/types/lib/3string.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"3string.d.ts","sourceRoot":"","sources":["../../lib/3string.js"],"names":[],"mappings":"AAwBA;;;;;;GAMG;AACH,0CANW,UAAU,OACV,MAAM,SACN,MAAM,YACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,oCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAGD;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAQjB;AAED,8CAAuC;iBA7E1B,OAAO,SAAS,EAAE,EAAE;4BACpB,OAAO,cAAc,EAAE,aAAa"}
+{"version":3,"file":"3string.d.ts","sourceRoot":"","sources":["../../lib/3string.js"],"names":[],"mappings":"AA6BA;;;;;;GAMG;AACH,0CANW,UAAU,OACV,MAAM,SACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,oCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAGD;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAQjB;AAED,8CAAuC;iBAlF1B,OAAO,SAAS,EAAE,EAAE;4BACpB,OAAO,cAAc,EAAE,aAAa"}

package/types/lib/diagnostic.d.ts

@@ -3,4 +3,10 @@
  * @param {number} [width]
  */
 export function tokensToDiagnostic(inp: Uint8Array, width?: number | undefined): Generator<string, void, unknown>;
+/**
+ * Convert an input string formatted as CBOR diagnostic output into binary CBOR form.
+ * @param {string} input
+ * @returns {Uint8Array}
+ */
+export function fromDiag(input: string): Uint8Array;
 //# sourceMappingURL=diagnostic.d.ts.map

package/types/lib/diagnostic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"diagnostic.d.ts","sourceRoot":"","sources":["../../lib/diagnostic.js"],"names":[],"mappings":"AAGA;;;GAGG;AACH,wCAHW,UAAU,gEA2FpB"}
+{"version":3,"file":"diagnostic.d.ts","sourceRoot":"","sources":["../../lib/diagnostic.js"],"names":[],"mappings":"AAOA;;;GAGG;AACH,wCAHW,UAAU,gEA6HpB;AAED;;;;GAIG;AACH,gCAHW,MAAM,GACJ,UAAU,CAatB"}

package/types/lib/token.d.ts

@@ -47,6 +47,8 @@ export class Token {
     encodedLength: number | undefined;
     /** @type {Uint8Array|undefined} */
     encodedBytes: Uint8Array | undefined;
+    /** @type {Uint8Array|undefined} */
+    byteValue: Uint8Array | undefined;
     toString(): string;
 }
 //# sourceMappingURL=token.d.ts.map

package/types/lib/token.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../lib/token.js"],"names":[],"mappings":"AAAA;IACE;;;;OAIG;IACH,mBAJW,MAAM,QACN,MAAM,YACN,OAAO,EAOjB;IAJC,cAAkB;IAClB,qBAA8B;IAC9B,aAAgB;IAChB,kBAAwB;IAI1B,mBAEC;IAED;;;OAGG;IACH,aAHW,IAAI,GACF,MAAM,CAKlB;CACF;;;;;;;;;;;;;;;;;;;;AAkBD;IACE;;;;OAIG;IACH,kBAJW,IAAI,UACJ,GAAG,sCASb;IALC,WAAgB;IAChB,WAAkB;IAClB,kCAAkC;IAClC,mCAAmC;IACnC,cADW,UAAU,GAAC,SAAS,CACF;IAI/B,mBAEC;CACF"}
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../lib/token.js"],"names":[],"mappings":"AAAA;IACE;;;;OAIG;IACH,mBAJW,MAAM,QACN,MAAM,YACN,OAAO,EAOjB;IAJC,cAAkB;IAClB,qBAA8B;IAC9B,aAAgB;IAChB,kBAAwB;IAI1B,mBAEC;IAED;;;OAGG;IACH,aAHW,IAAI,GACF,MAAM,CAKlB;CACF;;;;;;;;;;;;;;;;;;;;AAkBD;IACE;;;;OAIG;IACH,kBAJW,IAAI,UACJ,GAAG,sCAWb;IAPC,WAAgB;IAChB,WAAkB;IAClB,kCAAkC;IAClC,mCAAmC;IACnC,cADW,UAAU,GAAC,SAAS,CACF;IAC7B,mCAAmC;IACnC,WADW,UAAU,GAAC,SAAS,CACL;IAI5B,mBAEC;CACF"}