@solana/codecs-strings 2.0.0-experimental.efe6f4d → 2.0.0-experimental.f16a625

package/README.md

@@ -16,10 +16,200 @@
 
 This package contains codecs for strings of different sizes and encodings. It can be used standalone, but it is also exported as part of the Solana JavaScript SDK [`@solana/web3.js@experimental`](https://github.com/solana-labs/solana-web3.js/tree/master/packages/library).
 
-## Types
+This package is also part of the [`@solana/codecs` package](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs) which acts as an entry point for all codec packages as well as for their documentation.
 
-TODO
+## String helper codec
 
-## Functions
+The `getStringCodec` function returns a `Codec<string>` that can be used to encode strings using various encodings and size strategies. It contains the following options:
 
-TODO
+-   `encoding`: A `VariableSizeCodec<string>` responsible for encoding and decoding a string in a specific way without worrying about its size. Examples are UTF-8, base58, base64, etc. You can see all available encodings below in this documentation.
+-   `size`: This option tells the codec how long the string goes on for in the byte array. It can be one of the following three strategies:
+    -   `Codec<number>`: When a number codec is provided, that codec will be used to encode and decode a size prefix for that string. This prefix allows us to know when to stop reading the string when decoding a given byte array.
+    -   `number`: When a fixed number is provided, a `FixedSizeCodec` of that size will be returned such that exactly that amount of bytes will be used to encode and decode the string.
+    -   `"variable"`: When the string `"variable"` is passed as a size, a `VariableSizeCodec` will be returned without any size boundary. That is, when providing a byte array to decode, the entire byte array will be decoded as a string.
+
+When using `getStringCodec` without any options, the default encoding used is UTF-8 and the default size strategy used is a `u32` prefix codec.
+
+```ts
+const bytes = getStringCodec().encode('hello');
+// 0x0500000068656c6c6f
+//   |       └-- The 5 bytes of content.
+//   └-- 4-byte prefix telling us to read 5 bytes.
+
+const value = getStringCodec().decode(bytes);
+// "hello"
+```
+
+We can use the `size` option to provide a different integer codec for the prefix.
+
+```ts
+getStringCodec({ size: getU8Codec() }).encode('hello');
+// 0x0568656c6c6f
+//   | └-- The 5 bytes of content.
+//   └-- 1-byte prefix telling us to read 5 bytes.
+```
+
+Or to provide a fixed size such that any string longer or smaller than that size will be truncated or padded respectively.
+
+```ts
+getStringCodec({ size: 5 }).encode('hello');
+// 0x68656c6c6f
+//   └-- The exact 5 bytes of content.
+
+getStringCodec({ size: 5 }).encode('hello world');
+// 0x68656c6c6f
+//   └-- The truncated 5 bytes of content.
+
+getStringCodec({ size: 5 }).encode('hell');
+// 0x68656c6c00
+//   └-- The padded 5 bytes of content.
+```
+
+Or to tell the codec we do not want to create a size boundary for our string.
+
+```ts
+getStringCodec({ size: 'variable' }).encode('hello');
+// 0x68656c6c6f
+//   └-- Any bytes necessary to encode our content.
+```
+
+On top of customizing the size, we may provide a custom `encoding` option like so.
+
+```ts
+getStringCodec({ encoding: getUtf8Codec() }).encode('hello');
+// 0x0500000068656c6c6f (Default encoding).
+
+getStringCodec({ encoding: getBase64Codec() }).encode('hello');
+// 0x0300000085e965
+
+getStringCodec({ encoding: getBase58Codec() }).encode('heLLo');
+// 0x040000001b6a3070
+```
+
+Finally, separate `getStringEncoder` and `getStringDecoder` functions are also available.
+
+```ts
+const bytes = getStringEncoder().encode('hello');
+const value = getStringDecoder().decode(bytes);
+```
+
+## Utf8 codec
+
+The `getUtf8Codec` function encodes and decodes a UTF-8 string to and from a byte array.
+
+```ts
+const bytes = getUtf8Codec().encode('hello'); // 0x68656c6c6f
+const value = getUtf8Codec().decode(bytes); // "hello"
+```
+
+As usual, separate `getUtf8Encoder` and `getUtf8Decoder` functions are also available.
+
+```ts
+const bytes = getUtf8Encoder().encode('hello'); // 0x68656c6c6f
+const value = getUtf8Decoder().decode(bytes); // "hello"
+```
+
+## Base 64 codec
+
+The `getBase64Codec` function encodes and decodes a base-64 string to and from a byte array.
+
+```ts
+const bytes = getBase64Codec().encode('hello+world'); // 0x85e965a3ec28ae57
+const value = getBase64Codec().decode(bytes); // "hello+world"
+```
+
+As usual, separate `getBase64Encoder` and `getBase64Decoder` functions are also available.
+
+```ts
+const bytes = getBase64Encoder().encode('hello+world'); // 0x85e965a3ec28ae57
+const value = getBase64Decoder().decode(bytes); // "hello+world"
+```
+
+## Base 58 codec
+
+The `getBase58Codec` function encodes and decodes a base-58 string to and from a byte array.
+
+```ts
+const bytes = getBase58Codec().encode('heLLo'); // 0x1b6a3070
+const value = getBase58Codec().decode(bytes); // "heLLo"
+```
+
+As usual, separate `getBase58Encoder` and `getBase58Decoder` functions are also available.
+
+```ts
+const bytes = getBase58Encoder().encode('heLLo'); // 0x1b6a3070
+const value = getBase58Decoder().decode(bytes); // "heLLo"
+```
+
+## Base 16 codec
+
+The `getBase16Codec` function encodes and decodes a base-16 string to and from a byte array.
+
+```ts
+const bytes = getBase16Codec().encode('deadface'); // 0xdeadface
+const value = getBase16Codec().decode(bytes); // "deadface"
+```
+
+As usual, separate `getBase16Encoder` and `getBase16Decoder` functions are also available.
+
+```ts
+const bytes = getBase16Encoder().encode('deadface'); // 0xdeadface
+const value = getBase16Decoder().decode(bytes); // "deadface"
+```
+
+## Base 10 codec
+
+The `getBase10Codec` function encodes and decodes a base-10 string to and from a byte array.
+
+```ts
+const bytes = getBase10Codec().encode('1024'); // 0x0400
+const value = getBase10Codec().decode(bytes); // "1024"
+```
+
+As usual, separate `getBase10Encoder` and `getBase10Decoder` functions are also available.
+
+```ts
+const bytes = getBase10Encoder().encode('1024'); // 0x0400
+const value = getBase10Decoder().decode(bytes); // "1024"
+```
+
+## Base X codec
+
+The `getBaseXCodec` accepts a custom `alphabet` of `X` characters and creates a base-X codec using that alphabet. It does so by iteratively dividing by `X` and handling leading zeros.
+
+The base-10 and base-58 codecs use this base-x codec under the hood.
+
+```ts
+const alphabet = '0ehlo';
+const bytes = getBaseXCodec(alphabet).encode('hello'); // 0x05bd
+const value = getBaseXCodec(alphabet).decode(bytes); // "hello"
+```
+
+As usual, separate `getBaseXEncoder` and `getBaseXDecoder` functions are also available.
+
+```ts
+const bytes = getBaseXEncoder(alphabet).encode('hello'); // 0x05bd
+const value = getBaseXDecoder(alphabet).decode(bytes); // "hello"
+```
+
+## Re-slicing base X codec
+
+The `getBaseXResliceCodec` also creates a base-x codec but uses a different strategy. It re-slices bytes into custom chunks of bits that are then mapped to a provided `alphabet`. The number of bits per chunk is also provided and should typically be set to `log2(alphabet.length)`.
+
+This is typically used to create codecs whose alphabet’s length is a power of 2 such as base-16 or base-64.
+
+```ts
+const bytes = getBaseXResliceCodec('elho', 2).encode('hellolol'); // 0x4aee
+const value = getBaseXResliceCodec('elho', 2).decode(bytes); // "hellolol"
+```
+
+As usual, separate `getBaseXResliceEncoder` and `getBaseXResliceDecoder` functions are also available.
+
+```ts
+const bytes = getBaseXResliceEncoder('elho', 2).encode('hellolol'); // 0x4aee
+const value = getBaseXResliceDecoder('elho', 2).decode(bytes); // "hellolol"
+```
+
+---
+
+To read more about the available codecs and how to use them, check out the documentation of the main [`@solana/codecs` package](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs).

package/dist/index.browser.cjs

@@ -10,43 +10,38 @@ function assertValidBaseString(alphabet4, testValue, givenValue = testValue) {
   }
 }
 var getBaseXEncoder = (alphabet4) => {
-  const base = alphabet4.length;
-  const baseBigInt = BigInt(base);
-  return {
-    description: `base${base}`,
-    encode(value) {
+  return codecsCore.createEncoder({
+    getSizeFromValue: (value) => {
+      const [leadingZeroes, tailChars] = partitionLeadingZeroes(value, alphabet4[0]);
+      if (!tailChars)
+        return value.length;
+      const base10Number = getBigIntFromBaseX(tailChars, alphabet4);
+      return leadingZeroes.length + Math.ceil(base10Number.toString(16).length / 2);
+    },
+    write(value, bytes, offset) {
       assertValidBaseString(alphabet4, value);
       if (value === "")
-        return new Uint8Array();
-      const chars = [...value];
-      let trailIndex = chars.findIndex((c) => c !== alphabet4[0]);
-      trailIndex = trailIndex === -1 ? chars.length : trailIndex;
-      const leadingZeroes = Array(trailIndex).fill(0);
-      if (trailIndex === chars.length)
-        return Uint8Array.from(leadingZeroes);
-      const tailChars = chars.slice(trailIndex);
-      let base10Number = 0n;
-      let baseXPower = 1n;
-      for (let i = tailChars.length - 1; i >= 0; i -= 1) {
-        base10Number += baseXPower * BigInt(alphabet4.indexOf(tailChars[i]));
-        baseXPower *= baseBigInt;
+        return offset;
+      const [leadingZeroes, tailChars] = partitionLeadingZeroes(value, alphabet4[0]);
+      if (!tailChars) {
+        bytes.set(new Uint8Array(leadingZeroes.length).fill(0), offset);
+        return offset + leadingZeroes.length;
       }
+      let base10Number = getBigIntFromBaseX(tailChars, alphabet4);
       const tailBytes = [];
       while (base10Number > 0n) {
         tailBytes.unshift(Number(base10Number % 256n));
         base10Number /= 256n;
       }
-      return Uint8Array.from(leadingZeroes.concat(tailBytes));
-    },
-    fixedSize: null,
-    maxSize: null
-  };
+      const bytesToAdd = [...Array(leadingZeroes.length).fill(0), ...tailBytes];
+      bytes.set(bytesToAdd, offset);
+      return offset + bytesToAdd.length;
+    }
+  });
 };
 var getBaseXDecoder = (alphabet4) => {
-  const base = alphabet4.length;
-  const baseBigInt = BigInt(base);
-  return {
-    decode(rawBytes, offset = 0) {
+  return codecsCore.createDecoder({
+    read(rawBytes, offset) {
       const bytes = offset === 0 ? rawBytes : rawBytes.slice(offset);
       if (bytes.length === 0)
         return ["", 0];
@@ -55,45 +50,57 @@ var getBaseXDecoder = (alphabet4) => {
       const leadingZeroes = alphabet4[0].repeat(trailIndex);
       if (trailIndex === bytes.length)
         return [leadingZeroes, rawBytes.length];
-      let base10Number = bytes.slice(trailIndex).reduce((sum, byte) => sum * 256n + BigInt(byte), 0n);
-      const tailChars = [];
-      while (base10Number > 0n) {
-        tailChars.unshift(alphabet4[Number(base10Number % baseBigInt)]);
-        base10Number /= baseBigInt;
-      }
-      return [leadingZeroes + tailChars.join(""), rawBytes.length];
-    },
-    description: `base${base}`,
-    fixedSize: null,
-    maxSize: null
-  };
+      const base10Number = bytes.slice(trailIndex).reduce((sum, byte) => sum * 256n + BigInt(byte), 0n);
+      const tailChars = getBaseXFromBigInt(base10Number, alphabet4);
+      return [leadingZeroes + tailChars, rawBytes.length];
+    }
+  });
 };
 var getBaseXCodec = (alphabet4) => codecsCore.combineCodec(getBaseXEncoder(alphabet4), getBaseXDecoder(alphabet4));
+function partitionLeadingZeroes(value, zeroCharacter) {
+  const [leadingZeros, tailChars] = value.split(new RegExp(`((?!${zeroCharacter}).*)`));
+  return [leadingZeros, tailChars];
+}
+function getBigIntFromBaseX(value, alphabet4) {
+  const base = BigInt(alphabet4.length);
+  let sum = 0n;
+  for (const char of value) {
+    sum *= base;
+    sum += BigInt(alphabet4.indexOf(char));
+  }
+  return sum;
+}
+function getBaseXFromBigInt(value, alphabet4) {
+  const base = BigInt(alphabet4.length);
+  const tailChars = [];
+  while (value > 0n) {
+    tailChars.unshift(alphabet4[Number(value % base)]);
+    value /= base;
+  }
+  return tailChars.join("");
+}
 
 // src/base10.ts
 var alphabet = "0123456789";
 var getBase10Encoder = () => getBaseXEncoder(alphabet);
 var getBase10Decoder = () => getBaseXDecoder(alphabet);
 var getBase10Codec = () => getBaseXCodec(alphabet);
-var getBase16Encoder = () => ({
-  description: "base16",
-  encode(value) {
+var getBase16Encoder = () => codecsCore.createEncoder({
+  getSizeFromValue: (value) => Math.ceil(value.length / 2),
+  write(value, bytes, offset) {
     const lowercaseValue = value.toLowerCase();
     assertValidBaseString("0123456789abcdef", lowercaseValue, value);
     const matches = lowercaseValue.match(/.{1,2}/g);
-    return Uint8Array.from(matches ? matches.map((byte) => parseInt(byte, 16)) : []);
-  },
-  fixedSize: null,
-  maxSize: null
+    const hexBytes = matches ? matches.map((byte) => parseInt(byte, 16)) : [];
+    bytes.set(hexBytes, offset);
+    return hexBytes.length + offset;
+  }
 });
-var getBase16Decoder = () => ({
-  decode(bytes, offset = 0) {
+var getBase16Decoder = () => codecsCore.createDecoder({
+  read(bytes, offset) {
     const value = bytes.slice(offset).reduce((str, byte) => str + byte.toString(16).padStart(2, "0"), "");
     return [value, bytes.length];
-  },
-  description: "base16",
-  fixedSize: null,
-  maxSize: null
+  }
 });
 var getBase16Codec = () => codecsCore.combineCodec(getBase16Encoder(), getBase16Decoder());
 
@@ -102,29 +109,26 @@ var alphabet2 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
 var getBase58Encoder = () => getBaseXEncoder(alphabet2);
 var getBase58Decoder = () => getBaseXDecoder(alphabet2);
 var getBase58Codec = () => getBaseXCodec(alphabet2);
-var getBaseXResliceEncoder = (alphabet4, bits) => ({
-  description: `base${alphabet4.length}`,
-  encode(value) {
+var getBaseXResliceEncoder = (alphabet4, bits) => codecsCore.createEncoder({
+  getSizeFromValue: (value) => Math.floor(value.length * bits / 8),
+  write(value, bytes, offset) {
     assertValidBaseString(alphabet4, value);
     if (value === "")
-      return new Uint8Array();
+      return offset;
     const charIndices = [...value].map((c) => alphabet4.indexOf(c));
-    return new Uint8Array(reslice(charIndices, bits, 8, false));
-  },
-  fixedSize: null,
-  maxSize: null
+    const reslicedBytes = reslice(charIndices, bits, 8, false);
+    bytes.set(reslicedBytes, offset);
+    return reslicedBytes.length + offset;
+  }
 });
-var getBaseXResliceDecoder = (alphabet4, bits) => ({
-  decode(rawBytes, offset = 0) {
+var getBaseXResliceDecoder = (alphabet4, bits) => codecsCore.createDecoder({
+  read(rawBytes, offset = 0) {
     const bytes = offset === 0 ? rawBytes : rawBytes.slice(offset);
     if (bytes.length === 0)
       return ["", rawBytes.length];
     const charIndices = reslice([...bytes], 8, bits, true);
     return [charIndices.map((i) => alphabet4[i]).join(""), rawBytes.length];
-  },
-  description: `base${alphabet4.length}`,
-  fixedSize: null,
-  maxSize: null
+  }
 });
 var getBaseXResliceCodec = (alphabet4, bits) => codecsCore.combineCodec(getBaseXResliceEncoder(alphabet4, bits), getBaseXResliceDecoder(alphabet4, bits));
 function reslice(input, inputBits, outputBits, useRemainder) {
@@ -147,33 +151,35 @@ function reslice(input, inputBits, outputBits, useRemainder) {
 }
 var getBase64Encoder = () => {
   {
-    return {
-      description: `base64`,
-      encode(value) {
+    return codecsCore.createEncoder({
+      getSizeFromValue: (value) => {
         try {
-          const bytes = atob(value).split("").map((c) => c.charCodeAt(0));
-          return new Uint8Array(bytes);
+          return atob(value).length;
         } catch (e2) {
           throw new Error(`Expected a string of base 64, got [${value}].`);
         }
       },
-      fixedSize: null,
-      maxSize: null
-    };
+      write(value, bytes, offset) {
+        try {
+          const bytesToAdd = atob(value).split("").map((c) => c.charCodeAt(0));
+          bytes.set(bytesToAdd, offset);
+          return bytesToAdd.length + offset;
+        } catch (e2) {
+          throw new Error(`Expected a string of base 64, got [${value}].`);
+        }
+      }
+    });
   }
 };
 var getBase64Decoder = () => {
   {
-    return {
-      decode(bytes, offset = 0) {
+    return codecsCore.createDecoder({
+      read(bytes, offset = 0) {
         const slice = bytes.slice(offset);
         const value = btoa(String.fromCharCode(...slice));
         return [value, bytes.length];
-      },
-      description: `base64`,
-      fixedSize: null,
-      maxSize: null
-    };
+      }
+    });
   }
 };
 var getBase64Codec = () => codecsCore.combineCodec(getBase64Encoder(), getBase64Decoder());
@@ -192,79 +198,73 @@ var o = globalThis.TextEncoder;
 // src/utf8.ts
 var getUtf8Encoder = () => {
   let textEncoder;
-  return {
-    description: "utf8",
-    encode: (value) => new Uint8Array((textEncoder || (textEncoder = new o())).encode(value)),
-    fixedSize: null,
-    maxSize: null
-  };
+  return codecsCore.createEncoder({
+    getSizeFromValue: (value) => (textEncoder ||= new o()).encode(value).length,
+    write: (value, bytes, offset) => {
+      const bytesToAdd = (textEncoder ||= new o()).encode(value);
+      bytes.set(bytesToAdd, offset);
+      return offset + bytesToAdd.length;
+    }
+  });
 };
 var getUtf8Decoder = () => {
   let textDecoder;
-  return {
-    decode(bytes, offset = 0) {
-      const value = (textDecoder || (textDecoder = new e())).decode(bytes.slice(offset));
+  return codecsCore.createDecoder({
+    read(bytes, offset) {
+      const value = (textDecoder ||= new e()).decode(bytes.slice(offset));
       return [removeNullCharacters(value), bytes.length];
-    },
-    description: "utf8",
-    fixedSize: null,
-    maxSize: null
-  };
+    }
+  });
 };
 var getUtf8Codec = () => codecsCore.combineCodec(getUtf8Encoder(), getUtf8Decoder());
 
 // src/string.ts
-var getStringEncoder = (config = {}) => {
+function getStringEncoder(config = {}) {
   const size = config.size ?? codecsNumbers.getU32Encoder();
   const encoding = config.encoding ?? getUtf8Encoder();
-  const description = config.description ?? `string(${encoding.description}; ${getSizeDescription(size)})`;
   if (size === "variable") {
-    return { ...encoding, description };
+    return encoding;
   }
   if (typeof size === "number") {
-    return codecsCore.fixEncoder(encoding, size, description);
+    return codecsCore.fixEncoder(encoding, size);
   }
-  return {
-    description,
-    encode: (value) => {
-      const contentBytes = encoding.encode(value);
-      const lengthBytes = size.encode(contentBytes.length);
-      return codecsCore.mergeBytes([lengthBytes, contentBytes]);
+  return codecsCore.createEncoder({
+    getSizeFromValue: (value) => {
+      const contentSize = codecsCore.getEncodedSize(value, encoding);
+      return codecsCore.getEncodedSize(contentSize, size) + contentSize;
     },
-    fixedSize: null,
-    maxSize: null
-  };
-};
-var getStringDecoder = (config = {}) => {
+    write: (value, bytes, offset) => {
+      const contentSize = codecsCore.getEncodedSize(value, encoding);
+      offset = size.write(contentSize, bytes, offset);
+      return encoding.write(value, bytes, offset);
+    }
+  });
+}
+function getStringDecoder(config = {}) {
   const size = config.size ?? codecsNumbers.getU32Decoder();
   const encoding = config.encoding ?? getUtf8Decoder();
-  const description = config.description ?? `string(${encoding.description}; ${getSizeDescription(size)})`;
   if (size === "variable") {
-    return { ...encoding, description };
+    return encoding;
   }
   if (typeof size === "number") {
-    return codecsCore.fixDecoder(encoding, size, description);
+    return codecsCore.fixDecoder(encoding, size);
   }
-  return {
-    decode: (bytes, offset = 0) => {
+  return codecsCore.createDecoder({
+    read: (bytes, offset = 0) => {
       codecsCore.assertByteArrayIsNotEmptyForCodec("string", bytes, offset);
-      const [lengthBigInt, lengthOffset] = size.decode(bytes, offset);
+      const [lengthBigInt, lengthOffset] = size.read(bytes, offset);
       const length = Number(lengthBigInt);
       offset = lengthOffset;
       const contentBytes = bytes.slice(offset, offset + length);
       codecsCore.assertByteArrayHasEnoughBytesForCodec("string", length, contentBytes);
-      const [value, contentOffset] = encoding.decode(contentBytes);
+      const [value, contentOffset] = encoding.read(contentBytes, 0);
       offset += contentOffset;
       return [value, offset];
-    },
-    description,
-    fixedSize: null,
-    maxSize: null
-  };
-};
-var getStringCodec = (config = {}) => codecsCore.combineCodec(getStringEncoder(config), getStringDecoder(config));
-function getSizeDescription(size) {
-  return typeof size === "object" ? size.description : `${size}`;
+    }
+  });
+}
+function getStringCodec(config = {}) {
+  return codecsCore.combineCodec(getStringEncoder(config), getStringDecoder(config));
 }
 
 exports.assertValidBaseString = assertValidBaseString;