cborg 1.7.0 → 1.8.0

package/README.md

@@ -246,6 +246,7 @@ Decode valid CBOR bytes from a `Uint8Array` (or `Buffer`) and return a JavaScrip
 * `strict` (boolean, default `false`): when decoding integers, including for lengths (arrays, maps, strings, bytes), values will be checked to see whether they were encoded in their smallest possible form. If not, an error will be thrown.
   * Currently, this form of deterministic strictness cannot be enforced for float representations, or map key ordering (pull requests _very_ welcome).
 * `useMaps` (boolean, default `false`): when decoding major 5 (map) entries, use a `Map` rather than a plain `Object`. This will nest for any encountered map. During encode, a `Map` will be interpreted as an `Object` and will round-trip as such unless `useMaps` is supplied, in which case, all `Map`s and `Object`s will round-trip as `Map`s. There is no way to retain the distinction during round-trip without using a custom tag.
+* `retainStringBytes` (boolean, default `false`): when decoding strings, retain the original bytes on the `Token` object as `byteValue`. Since it is possible to encode non-UTF-8 characters in strings in CBOR, and JavaScript doesn't properly handle non-UTF-8 in its conversion from bytes (`TextEncoder` or `Buffer`), this can result in a loss of data (and an inability to round-trip). Where this is important, a token stream should be consumed instead of a plain `decode()` and the `byteValue` property on string tokens can be inspected (see [lib/diagnostic.js](lib/diagnostic.js) for an example of its use.)
 * `tags` (array): a mapping of tag number to tag decoder function. By default no tags are supported. See [Tag decoders](#tag-decoders).
 * `tokenizer` (object): an object with two methods, `next()` which returns a `Token` and `done()` which returns a `boolean`. Can be used to implement custom input decoding. See the source code for examples.
 

package/cjs/browser-test/node-test-bin.js

@@ -331,4 +331,19 @@ Input may either be supplied as an argument or piped via stdin
 `);
     });
   });
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35';
+    let {stdout, stderr} = await execBin(`hex2diag ${ input }`);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
+`);
+    ({stdout, stderr} = await execBin('diag2hex', stdout));
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `${ input }\n`);
+  });
 });

package/cjs/lib/3string.js

@@ -8,29 +8,33 @@ var _0uint = require('./0uint.js');
 var _2bytes = require('./2bytes.js');
 var byteUtils = require('./byte-utils.js');
 
-function toToken(data, pos, prefix, length) {
+function toToken(data, pos, prefix, length, options) {
   const totLength = prefix + length;
   common.assertEnoughData(data, pos, totLength);
-  return new token.Token(token.Type.string, byteUtils.toString(data, pos + prefix, pos + totLength), totLength);
+  const tok = new token.Token(token.Type.string, byteUtils.toString(data, pos + prefix, pos + totLength), totLength);
+  if (options.retainStringBytes === true) {
+    tok.byteValue = byteUtils.slice(data, pos + prefix, pos + totLength);
+  }
+  return tok;
 }
-function decodeStringCompact(data, pos, minor, _options) {
-  return toToken(data, pos, 1, minor);
+function decodeStringCompact(data, pos, minor, options) {
+  return toToken(data, pos, 1, minor, options);
 }
 function decodeString8(data, pos, _minor, options) {
-  return toToken(data, pos, 2, _0uint.readUint8(data, pos + 1, options));
+  return toToken(data, pos, 2, _0uint.readUint8(data, pos + 1, options), options);
 }
 function decodeString16(data, pos, _minor, options) {
-  return toToken(data, pos, 3, _0uint.readUint16(data, pos + 1, options));
+  return toToken(data, pos, 3, _0uint.readUint16(data, pos + 1, options), options);
 }
 function decodeString32(data, pos, _minor, options) {
-  return toToken(data, pos, 5, _0uint.readUint32(data, pos + 1, options));
+  return toToken(data, pos, 5, _0uint.readUint32(data, pos + 1, options), options);
 }
 function decodeString64(data, pos, _minor, options) {
   const l = _0uint.readUint64(data, pos + 1, options);
   if (typeof l === 'bigint') {
     throw new Error(`${ common.decodeErrPrefix } 64-bit integer string lengths not supported`);
   }
-  return toToken(data, pos, 9, l);
+  return toToken(data, pos, 9, l, options);
 }
 const encodeString = _2bytes.encodeBytes;
 

package/cjs/lib/diagnostic.js

@@ -9,7 +9,7 @@ var _0uint = require('./0uint.js');
 const utf8Encoder = new TextEncoder();
 const utf8Decoder = new TextDecoder();
 function* tokensToDiagnostic(inp, width = 100) {
-  const tokeniser = new decode.Tokeniser(inp);
+  const tokeniser = new decode.Tokeniser(inp, { retainStringBytes: true });
   let pos = 0;
   const indent = [];
   const slc = (start, length) => {
@@ -59,15 +59,22 @@ function* tokensToDiagnostic(inp, width = 100) {
     }
     yield outp;
     if (str) {
+      let asString = token.type.name === 'string';
       margin += '  ';
-      const repr = token.type.name === 'bytes' ? token.value : utf8Encoder.encode(token.value);
+      let repr = asString ? utf8Encoder.encode(token.value) : token.value;
+      if (asString && token.byteValue !== undefined) {
+        if (repr.length !== token.byteValue.length) {
+          repr = token.byteValue;
+          asString = false;
+        }
+      }
       const wh = (width / 2 - margin.length - 1) / 2;
       let snip = 0;
       while (repr.length - snip > 0) {
         const piece = repr.slice(snip, snip + wh);
         snip += piece.length;
-        const st = token.type.name === 'string' ? utf8Decoder.decode(piece) : piece.reduce((p, c) => {
-          if (c < 32 || c === 127) {
+        const st = asString ? utf8Decoder.decode(piece) : piece.reduce((p, c) => {
+          if (c < 32 || c >= 127 && c < 161 || c === 173) {
             return `${ p }\\x${ c.toString(16).padStart(2, '0') }`;
           }
           return `${ p }${ String.fromCharCode(c) }`;

package/cjs/lib/token.js

@@ -35,6 +35,7 @@ class Token {
     this.value = value;
     this.encodedLength = encodedLength;
     this.encodedBytes = undefined;
+    this.byteValue = undefined;
   }
   toString() {
     return `Token[${ this.type }].${ this.value }`;

package/cjs/node-test/node-test-bin.js

@@ -331,4 +331,19 @@ Input may either be supplied as an argument or piped via stdin
 `);
     });
   });
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35';
+    let {stdout, stderr} = await execBin(`hex2diag ${ input }`);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
+`);
+    ({stdout, stderr} = await execBin('diag2hex', stdout));
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `${ input }\n`);
+  });
 });

package/esm/browser-test/node-test-bin.js

@@ -322,4 +322,19 @@ Input may either be supplied as an argument or piped via stdin
 `);
     });
   });
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35';
+    let {stdout, stderr} = await execBin(`hex2diag ${ input }`);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
+`);
+    ({stdout, stderr} = await execBin('diag2hex', stdout));
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `${ input }\n`);
+  });
 });

package/esm/lib/3string.js

@@ -8,29 +8,36 @@ import {
 } from './common.js';
 import * as uint from './0uint.js';
 import { encodeBytes } from './2bytes.js';
-import { toString } from './byte-utils.js';
-function toToken(data, pos, prefix, length) {
+import {
+  toString,
+  slice
+} from './byte-utils.js';
+function toToken(data, pos, prefix, length, options) {
   const totLength = prefix + length;
   assertEnoughData(data, pos, totLength);
-  return new Token(Type.string, toString(data, pos + prefix, pos + totLength), totLength);
+  const tok = new Token(Type.string, toString(data, pos + prefix, pos + totLength), totLength);
+  if (options.retainStringBytes === true) {
+    tok.byteValue = slice(data, pos + prefix, pos + totLength);
+  }
+  return tok;
 }
-export function decodeStringCompact(data, pos, minor, _options) {
-  return toToken(data, pos, 1, minor);
+export function decodeStringCompact(data, pos, minor, options) {
+  return toToken(data, pos, 1, minor, options);
 }
 export function decodeString8(data, pos, _minor, options) {
-  return toToken(data, pos, 2, uint.readUint8(data, pos + 1, options));
+  return toToken(data, pos, 2, uint.readUint8(data, pos + 1, options), options);
 }
 export function decodeString16(data, pos, _minor, options) {
-  return toToken(data, pos, 3, uint.readUint16(data, pos + 1, options));
+  return toToken(data, pos, 3, uint.readUint16(data, pos + 1, options), options);
 }
 export function decodeString32(data, pos, _minor, options) {
-  return toToken(data, pos, 5, uint.readUint32(data, pos + 1, options));
+  return toToken(data, pos, 5, uint.readUint32(data, pos + 1, options), options);
 }
 export function decodeString64(data, pos, _minor, options) {
   const l = uint.readUint64(data, pos + 1, options);
   if (typeof l === 'bigint') {
     throw new Error(`${ decodeErrPrefix } 64-bit integer string lengths not supported`);
   }
-  return toToken(data, pos, 9, l);
+  return toToken(data, pos, 9, l, options);
 }
 export const encodeString = encodeBytes;

package/esm/lib/diagnostic.js

@@ -7,7 +7,7 @@ import { uintBoundaries } from './0uint.js';
 const utf8Encoder = new TextEncoder();
 const utf8Decoder = new TextDecoder();
 function* tokensToDiagnostic(inp, width = 100) {
-  const tokeniser = new Tokeniser(inp);
+  const tokeniser = new Tokeniser(inp, { retainStringBytes: true });
   let pos = 0;
   const indent = [];
   const slc = (start, length) => {
@@ -57,15 +57,22 @@ function* tokensToDiagnostic(inp, width = 100) {
     }
     yield outp;
     if (str) {
+      let asString = token.type.name === 'string';
       margin += '  ';
-      const repr = token.type.name === 'bytes' ? token.value : utf8Encoder.encode(token.value);
+      let repr = asString ? utf8Encoder.encode(token.value) : token.value;
+      if (asString && token.byteValue !== undefined) {
+        if (repr.length !== token.byteValue.length) {
+          repr = token.byteValue;
+          asString = false;
+        }
+      }
       const wh = (width / 2 - margin.length - 1) / 2;
       let snip = 0;
       while (repr.length - snip > 0) {
         const piece = repr.slice(snip, snip + wh);
         snip += piece.length;
-        const st = token.type.name === 'string' ? utf8Decoder.decode(piece) : piece.reduce((p, c) => {
-          if (c < 32 || c === 127) {
+        const st = asString ? utf8Decoder.decode(piece) : piece.reduce((p, c) => {
+          if (c < 32 || c >= 127 && c < 161 || c === 173) {
             return `${ p }\\x${ c.toString(16).padStart(2, '0') }`;
           }
           return `${ p }${ String.fromCharCode(c) }`;

package/esm/lib/token.js

@@ -31,6 +31,7 @@ class Token {
     this.value = value;
     this.encodedLength = encodedLength;
     this.encodedBytes = undefined;
+    this.byteValue = undefined;
   }
   toString() {
     return `Token[${ this.type }].${ this.value }`;

package/esm/node-test/node-test-bin.js

@@ -322,4 +322,19 @@ Input may either be supplied as an argument or piped via stdin
 `);
     });
   });
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35';
+    let {stdout, stderr} = await execBin(`hex2diag ${ input }`);
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
+`);
+    ({stdout, stderr} = await execBin('diag2hex', stdout));
+    assert.strictEqual(stderr, '');
+    assert.strictEqual(stdout, `${ input }\n`);
+  });
 });

package/interface.ts

@@ -39,6 +39,7 @@ export interface DecodeOptions {
   allowBigInt?: boolean
   strict?: boolean
   useMaps?: boolean
+  retainStringBytes?: boolean
   tags?: TagDecoder[],
   tokenizer?: DecodeTokenizer
 }

package/lib/3string.js

@@ -2,7 +2,7 @@ import { Token, Type } from './token.js'
 import { assertEnoughData, decodeErrPrefix } from './common.js'
 import * as uint from './0uint.js'
 import { encodeBytes } from './2bytes.js'
-import { toString } from './byte-utils.js'
+import { toString, slice } from './byte-utils.js'
 
 /**
  * @typedef {import('./bl.js').Bl} Bl
@@ -14,23 +14,28 @@ import { toString } from './byte-utils.js'
  * @param {number} pos
  * @param {number} prefix
  * @param {number} length
+ * @param {DecodeOptions} options
  * @returns {Token}
  */
-function toToken (data, pos, prefix, length) {
+function toToken (data, pos, prefix, length, options) {
   const totLength = prefix + length
   assertEnoughData(data, pos, totLength)
-  return new Token(Type.string, toString(data, pos + prefix, pos + totLength), totLength)
+  const tok = new Token(Type.string, toString(data, pos + prefix, pos + totLength), totLength)
+  if (options.retainStringBytes === true) {
+    tok.byteValue = slice(data, pos + prefix, pos + totLength)
+  }
+  return tok
 }
 
 /**
  * @param {Uint8Array} data
  * @param {number} pos
  * @param {number} minor
- * @param {DecodeOptions} _options
+ * @param {DecodeOptions} options
  * @returns {Token}
  */
-export function decodeStringCompact (data, pos, minor, _options) {
-  return toToken(data, pos, 1, minor)
+export function decodeStringCompact (data, pos, minor, options) {
+  return toToken(data, pos, 1, minor, options)
 }
 
 /**
@@ -41,7 +46,7 @@ export function decodeStringCompact (data, pos, minor, _options) {
  * @returns {Token}
  */
 export function decodeString8 (data, pos, _minor, options) {
-  return toToken(data, pos, 2, uint.readUint8(data, pos + 1, options))
+  return toToken(data, pos, 2, uint.readUint8(data, pos + 1, options), options)
 }
 
 /**
@@ -52,7 +57,7 @@ export function decodeString8 (data, pos, _minor, options) {
  * @returns {Token}
  */
 export function decodeString16 (data, pos, _minor, options) {
-  return toToken(data, pos, 3, uint.readUint16(data, pos + 1, options))
+  return toToken(data, pos, 3, uint.readUint16(data, pos + 1, options), options)
 }
 
 /**
@@ -63,7 +68,7 @@ export function decodeString16 (data, pos, _minor, options) {
  * @returns {Token}
  */
 export function decodeString32 (data, pos, _minor, options) {
-  return toToken(data, pos, 5, uint.readUint32(data, pos + 1, options))
+  return toToken(data, pos, 5, uint.readUint32(data, pos + 1, options), options)
 }
 
 // TODO: maybe we shouldn't support this ..
@@ -79,7 +84,7 @@ export function decodeString64 (data, pos, _minor, options) {
   if (typeof l === 'bigint') {
     throw new Error(`${decodeErrPrefix} 64-bit integer string lengths not supported`)
   }
-  return toToken(data, pos, 9, l)
+  return toToken(data, pos, 9, l, options)
 }
 
 export const encodeString = encodeBytes

package/lib/diagnostic.js

@@ -10,7 +10,7 @@ const utf8Decoder = new TextDecoder()
  * @param {number} [width]
  */
 function * tokensToDiagnostic (inp, width = 100) {
-  const tokeniser = new Tokeniser(inp)
+  const tokeniser = new Tokeniser(inp, { retainStringBytes: true })
   let pos = 0
   const indent = []
 
@@ -77,19 +77,26 @@ function * tokensToDiagnostic (inp, width = 100) {
     yield outp
 
     if (str) {
+      let asString = token.type.name === 'string'
       margin += '  '
-      const repr = token.type.name === 'bytes' ? token.value : utf8Encoder.encode(token.value)
+      let repr = asString ? utf8Encoder.encode(token.value) : token.value
+      if (asString && token.byteValue !== undefined) {
+        if (repr.length !== token.byteValue.length) {
+          // bail on printing this as a string, it's probably not utf8, so treat it as bytes
+          // (you can probably blame a Go programmer for this)
+          repr = token.byteValue
+          asString = false
+        }
+      }
       const wh = ((width / 2) - margin.length - 1) / 2
       let snip = 0
       while (repr.length - snip > 0) {
         const piece = repr.slice(snip, snip + wh)
         snip += piece.length
-        // the assumption that we can utf8 a byte-sliced version is a stretch,
-        // we could be slicing in the middle of a multi-byte character
-        const st = token.type.name === 'string'
+        const st = asString
           ? utf8Decoder.decode(piece)
           : piece.reduce((/** @type {string} */ p, /** @type {number} */ c) => {
-            if (c < 0x20 || c === 0x7f) {
+            if (c < 0x20 || (c >= 0x7f && c < 0xa1) || c === 0xad) {
               return `${p}\\x${c.toString(16).padStart(2, '0')}`
             }
             return `${p}${String.fromCharCode(c)}`

package/lib/token.js

@@ -54,6 +54,8 @@ class Token {
     this.encodedLength = encodedLength
     /** @type {Uint8Array|undefined} */
     this.encodedBytes = undefined
+    /** @type {Uint8Array|undefined} */
+    this.byteValue = undefined
   }
 
   /* c8 ignore next 3 */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cborg",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "Fast CBOR with a focus on strictness",
   "main": "./cjs/cborg.js",
   "bin": {

package/test/node-test-bin.js

@@ -382,4 +382,23 @@ Input may either be supplied as an argument or piped via stdin
 `)
     })
   })
+
+  it('diag non-utf8 and non-printable ascii', async () => {
+    const input = '7864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35'
+    let { stdout, stderr } = await execBin(`hex2diag ${input}`)
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout,
+`78 64                                             # string(86)
+  f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b  #   "õ_øñ%\\x08¶>ò¿ì§Uzé\\x0dö1\\x1a^Ác\\x1b"
+  4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c  #   "J\\x1f¨C1\\x0bÙç\\x10ê¬å¡½×*пàIw\\x1c"
+  11e756338bd93865e645f1adec9b9c99ef407fbd4fc685  #   "\\x11çV3\\x8bÙ8eæEñ\\xadì\\x9b\\x9c\\x99ï@\\x7f½OÆ\\x85"
+  9e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82  #   "\\x9ey\\x04Å\\xad}ɽ\\x10¥Ì\\x16\\x97=[(ì\\x1amÔ=\\x9f\\x82"
+  f9f18c3d03418e35                                #   "ùñ\\x8c=\\x03A\\x8e5"
+`)
+
+    // round-trip
+    ;({ stdout, stderr } = await execBin('diag2hex', stdout))
+    assert.strictEqual(stderr, '')
+    assert.strictEqual(stdout, `${input}\n`)
+  })
 })

package/types/interface.d.ts

@@ -29,6 +29,7 @@ export interface DecodeOptions {
     allowBigInt?: boolean;
     strict?: boolean;
     useMaps?: boolean;
+    retainStringBytes?: boolean;
     tags?: TagDecoder[];
     tokenizer?: DecodeTokenizer;
 }

package/types/interface.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,EAAE,EAAE,MAAM,UAAU,CAAA;AAE7B,oBAAY,mBAAmB,GAAG,KAAK,GAAG,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAA;AAEzE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,SAAS,GAAG,SAAS,CAAA;IAC7B,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;IACnB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAA;CACvC;AAED,oBAAY,mBAAmB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,GAAG,IAAI,CAAA;AAEtI,oBAAY,iBAAiB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,CAAA;AAE7H,oBAAY,gBAAgB,GAAG;IAC7B,CAAC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACvD,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,GAAG,MAAM,CAAC;IAC5C,WAAW,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC7D,CAAA;AAED,oBAAY,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,KAAK,MAAM,CAAA;AAEpF,oBAAY,gBAAgB,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,UAAU,GAAG,SAAS,CAAA;AAEvE,MAAM,WAAW,eAAe;IAC9B,IAAI,IAAI,OAAO,CAAC;IAChB,IAAI,IAAI,KAAK,CAAA;CACd;AAED,oBAAY,UAAU,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,GAAG,CAAA;AAE5C,MAAM,WAAW,aAAa;IAC5B,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,YAAY,CAAC,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,mBAAmB,CAAA;KAAE,CAAA;CAC3D"}
+{"version":3,"file":"interface.d.ts","sourceRoot":"","sources":["../interface.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,EAAE,EAAE,MAAM,UAAU,CAAA;AAE7B,oBAAY,mBAAmB,GAAG,KAAK,GAAG,KAAK,EAAE,GAAG,mBAAmB,EAAE,CAAA;AAEzE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,SAAS,GAAG,SAAS,CAAA;IAC7B,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;IACnB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAA;CACvC;AAED,oBAAY,mBAAmB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,GAAG,IAAI,CAAA;AAEtI,oBAAY,iBAAiB,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,SAAS,KAAK,mBAAmB,CAAA;AAE7H,oBAAY,gBAAgB,GAAG;IAC7B,CAAC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACvD,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,GAAG,MAAM,CAAC;IAC5C,WAAW,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC7D,CAAA;AAED,oBAAY,SAAS,GAAG,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,EAAE,KAAK,MAAM,CAAA;AAEpF,oBAAY,gBAAgB,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,UAAU,GAAG,SAAS,CAAA;AAEvE,MAAM,WAAW,eAAe;IAC9B,IAAI,IAAI,OAAO,CAAC;IAChB,IAAI,IAAI,KAAK,CAAA;CACd;AAED,oBAAY,UAAU,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,GAAG,CAAA;AAE5C,MAAM,WAAW,aAAa;IAC5B,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B,IAAI,CAAC,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,YAAY,CAAC,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,mBAAmB,CAAA;KAAE,CAAA;CAC3D"}

package/types/lib/3string.d.ts

@@ -2,10 +2,10 @@
  * @param {Uint8Array} data
  * @param {number} pos
  * @param {number} minor
- * @param {DecodeOptions} _options
+ * @param {DecodeOptions} options
  * @returns {Token}
  */
-export function decodeStringCompact(data: Uint8Array, pos: number, minor: number, _options: DecodeOptions): Token;
+export function decodeStringCompact(data: Uint8Array, pos: number, minor: number, options: DecodeOptions): Token;
 /**
  * @param {Uint8Array} data
  * @param {number} pos

package/types/lib/3string.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"3string.d.ts","sourceRoot":"","sources":["../../lib/3string.js"],"names":[],"mappings":"AAwBA;;;;;;GAMG;AACH,0CANW,UAAU,OACV,MAAM,SACN,MAAM,YACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,oCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAGD;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAQjB;AAED,8CAAuC;iBA7E1B,OAAO,SAAS,EAAE,EAAE;4BACpB,OAAO,cAAc,EAAE,aAAa"}
+{"version":3,"file":"3string.d.ts","sourceRoot":"","sources":["../../lib/3string.js"],"names":[],"mappings":"AA6BA;;;;;;GAMG;AACH,0CANW,UAAU,OACV,MAAM,SACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,oCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAED;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAIjB;AAGD;;;;;;GAMG;AACH,qCANW,UAAU,OACV,MAAM,UACN,MAAM,WACN,aAAa,GACX,KAAK,CAQjB;AAED,8CAAuC;iBAlF1B,OAAO,SAAS,EAAE,EAAE;4BACpB,OAAO,cAAc,EAAE,aAAa"}

package/types/lib/diagnostic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"diagnostic.d.ts","sourceRoot":"","sources":["../../lib/diagnostic.js"],"names":[],"mappings":"AAOA;;;GAGG;AACH,wCAHW,UAAU,gEAsHpB;AAED;;;;GAIG;AACH,gCAHW,MAAM,GACJ,UAAU,CAatB"}
+{"version":3,"file":"diagnostic.d.ts","sourceRoot":"","sources":["../../lib/diagnostic.js"],"names":[],"mappings":"AAOA;;;GAGG;AACH,wCAHW,UAAU,gEA6HpB;AAED;;;;GAIG;AACH,gCAHW,MAAM,GACJ,UAAU,CAatB"}

package/types/lib/token.d.ts

@@ -47,6 +47,8 @@ export class Token {
     encodedLength: number | undefined;
     /** @type {Uint8Array|undefined} */
     encodedBytes: Uint8Array | undefined;
+    /** @type {Uint8Array|undefined} */
+    byteValue: Uint8Array | undefined;
     toString(): string;
 }
 //# sourceMappingURL=token.d.ts.map

package/types/lib/token.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../lib/token.js"],"names":[],"mappings":"AAAA;IACE;;;;OAIG;IACH,mBAJW,MAAM,QACN,MAAM,YACN,OAAO,EAOjB;IAJC,cAAkB;IAClB,qBAA8B;IAC9B,aAAgB;IAChB,kBAAwB;IAI1B,mBAEC;IAED;;;OAGG;IACH,aAHW,IAAI,GACF,MAAM,CAKlB;CACF;;;;;;;;;;;;;;;;;;;;AAkBD;IACE;;;;OAIG;IACH,kBAJW,IAAI,UACJ,GAAG,sCASb;IALC,WAAgB;IAChB,WAAkB;IAClB,kCAAkC;IAClC,mCAAmC;IACnC,cADW,UAAU,GAAC,SAAS,CACF;IAI/B,mBAEC;CACF"}
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../lib/token.js"],"names":[],"mappings":"AAAA;IACE;;;;OAIG;IACH,mBAJW,MAAM,QACN,MAAM,YACN,OAAO,EAOjB;IAJC,cAAkB;IAClB,qBAA8B;IAC9B,aAAgB;IAChB,kBAAwB;IAI1B,mBAEC;IAED;;;OAGG;IACH,aAHW,IAAI,GACF,MAAM,CAKlB;CACF;;;;;;;;;;;;;;;;;;;;AAkBD;IACE;;;;OAIG;IACH,kBAJW,IAAI,UACJ,GAAG,sCAWb;IAPC,WAAgB;IAChB,WAAkB;IAClB,kCAAkC;IAClC,mCAAmC;IACnC,cADW,UAAU,GAAC,SAAS,CACF;IAC7B,mCAAmC;IACnC,WADW,UAAU,GAAC,SAAS,CACL;IAI5B,mBAEC;CACF"}