@solana/codecs-strings 6.3.1 → 6.3.2-canary-20260313143218

package/src/index.ts

@@ -0,0 +1,210 @@
+/**
+ * This package contains codecs for strings of different sizes and encodings.
+ * It can be used standalone, but it is also exported as part of Kit
+ * [`@solana/kit`](https://github.com/anza-xyz/kit/tree/main/packages/kit).
+ *
+ * This package is also part of the [`@solana/codecs` package](https://github.com/anza-xyz/kit/tree/main/packages/codecs)
+ * which acts as an entry point for all codec packages as well as for their documentation.
+ *
+ * ## Sizing string codecs
+ *
+ * The `@solana/codecs-strings` package offers a variety of string codecs such as `utf8`, `base58`, `base64`, etc —
+ * which we will discuss in more detail below. However, before digging into the available string codecs,
+ * it's important to understand the different sizing strategies available for string codecs.
+ *
+ * By default, all available string codecs will return a `VariableSizeCodec<string>` meaning that:
+ *
+ * - When encoding a string, all bytes necessary to encode the string will be used.
+ * - When decoding a byte array at a given offset, all bytes starting from that offset will be decoded as a string.
+ *
+ * For instance, here's how you can encode/decode `utf8` strings without any size boundary:
+ *
+ * ```ts
+ * const codec = getUtf8Codec();
+ *
+ * codec.encode('hello');
+ * // 0x68656c6c6f
+ * //   └-- Any bytes necessary to encode our content.
+ *
+ * codec.decode(new Uint8Array([0x68, 0x65, 0x6c, 0x6c, 0x6f]));
+ * // 'hello'
+ * ```
+ *
+ * This might be what you want — e.g. when having a string at the end of a data structure — but in many cases,
+ * you might want to have a size boundary for your string. You may achieve this by composing your string codec
+ * with the [`fixCodecSize`](https://github.com/anza-xyz/kit/tree/main/packages/codecs-core#fixing-the-size-of-codecs)
+ * or [`addCodecSizePrefix`](https://github.com/anza-xyz/kit/tree/main/packages/codecs-core#prefixing-the-size-of-codecs) functions.
+ *
+ * The `fixCodecSize` function accepts a fixed byte length and returns a `FixedSizeCodec<string>` that will always use
+ * that amount of bytes to encode and decode a string. Any string longer or smaller than that size will be truncated
+ * or padded respectively. Here's how you can use it with a `utf8` codec:
+ *
+ * ```ts
+ * const codec = fixCodecSize(getUtf8Codec(), 5);
+ *
+ * codec.encode('hello');
+ * // 0x68656c6c6f
+ * //   └-- The exact 5 bytes of content.
+ *
+ * codec.encode('hello world');
+ * // 0x68656c6c6f
+ * //   └-- The truncated 5 bytes of content.
+ *
+ * codec.encode('hell');
+ * // 0x68656c6c00
+ * //   └-- The padded 5 bytes of content.
+ *
+ * codec.decode(new Uint8Array([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0xff, 0xff, 0xff, 0xff]));
+ * // 'hello'
+ * ```
+ *
+ * The `addCodecSizePrefix` function accepts an additional number codec that will be used to encode and
+ * decode a size prefix for the string. This prefix allows us to know when to stop reading the string when
+ * decoding a given byte array. Here's how you can use it with a `utf8` codec:
+ *
+ * ```ts
+ * const codec = addCodecSizePrefix(getUtf8Codec(), getU32Codec());
+ *
+ * codec.encode('hello');
+ * // 0x0500000068656c6c6f
+ * //   |       └-- The 5 bytes of content.
+ * //   └-- 4-byte prefix telling us to read 5 bytes.
+ *
+ * codec.decode(new Uint8Array([0x05, 0x00, 0x00, 0x00, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0xff, 0xff, 0xff, 0xff]));
+ * // "hello"
+ * ```
+ *
+ * Now, let's take a look at the available string encodings. Just remember that you can use
+ * the `fixSizeCodec` or `prefixSizeCodec` functions on any of these encodings to add a size boundary to them.
+ *
+ * ## Utf8 codec
+ *
+ * The `getUtf8Codec` function encodes and decodes a UTF-8 string to and from a byte array.
+ *
+ * ```ts
+ * const bytes = getUtf8Codec().encode('hello'); // 0x68656c6c6f
+ * const value = getUtf8Codec().decode(bytes); // "hello"
+ * ```
+ *
+ * As usual, separate `getUtf8Encoder` and `getUtf8Decoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getUtf8Encoder().encode('hello'); // 0x68656c6c6f
+ * const value = getUtf8Decoder().decode(bytes); // "hello"
+ * ```
+ *
+ * ## Base 64 codec
+ *
+ * The `getBase64Codec` function encodes and decodes a base-64 string to and from a byte array.
+ *
+ * ```ts
+ * const bytes = getBase64Codec().encode('hello+world'); // 0x85e965a3ec28ae57
+ * const value = getBase64Codec().decode(bytes); // "hello+world"
+ * ```
+ *
+ * As usual, separate `getBase64Encoder` and `getBase64Decoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getBase64Encoder().encode('hello+world'); // 0x85e965a3ec28ae57
+ * const value = getBase64Decoder().decode(bytes); // "hello+world"
+ * ```
+ *
+ * ## Base 58 codec
+ *
+ * The `getBase58Codec` function encodes and decodes a base-58 string to and from a byte array.
+ *
+ * ```ts
+ * const bytes = getBase58Codec().encode('heLLo'); // 0x1b6a3070
+ * const value = getBase58Codec().decode(bytes); // "heLLo"
+ * ```
+ *
+ * As usual, separate `getBase58Encoder` and `getBase58Decoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getBase58Encoder().encode('heLLo'); // 0x1b6a3070
+ * const value = getBase58Decoder().decode(bytes); // "heLLo"
+ * ```
+ *
+ * ## Base 16 codec
+ *
+ * The `getBase16Codec` function encodes and decodes a base-16 string to and from a byte array.
+ *
+ * ```ts
+ * const bytes = getBase16Codec().encode('deadface'); // 0xdeadface
+ * const value = getBase16Codec().decode(bytes); // "deadface"
+ * ```
+ *
+ * As usual, separate `getBase16Encoder` and `getBase16Decoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getBase16Encoder().encode('deadface'); // 0xdeadface
+ * const value = getBase16Decoder().decode(bytes); // "deadface"
+ * ```
+ *
+ * ## Base 10 codec
+ *
+ * The `getBase10Codec` function encodes and decodes a base-10 string to and from a byte array.
+ *
+ * ```ts
+ * const bytes = getBase10Codec().encode('1024'); // 0x0400
+ * const value = getBase10Codec().decode(bytes); // "1024"
+ * ```
+ *
+ * As usual, separate `getBase10Encoder` and `getBase10Decoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getBase10Encoder().encode('1024'); // 0x0400
+ * const value = getBase10Decoder().decode(bytes); // "1024"
+ * ```
+ *
+ * ## Base X codec
+ *
+ * The `getBaseXCodec` accepts a custom `alphabet` of `X` characters and creates a base-X codec using that alphabet.
+ * It does so by iteratively dividing by `X` and handling leading zeros.
+ *
+ * The base-10 and base-58 codecs use this base-x codec under the hood.
+ *
+ * ```ts
+ * const alphabet = '0ehlo';
+ * const bytes = getBaseXCodec(alphabet).encode('hello'); // 0x05bd
+ * const value = getBaseXCodec(alphabet).decode(bytes); // "hello"
+ * ```
+ *
+ * As usual, separate `getBaseXEncoder` and `getBaseXDecoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getBaseXEncoder(alphabet).encode('hello'); // 0x05bd
+ * const value = getBaseXDecoder(alphabet).decode(bytes); // "hello"
+ * ```
+ *
+ * ## Re-slicing base X codec
+ *
+ * The `getBaseXResliceCodec` also creates a base-x codec but uses a different strategy.
+ * It re-slices bytes into custom chunks of bits that are then mapped to a provided `alphabet`.
+ * The number of bits per chunk is also provided and should typically be set to `log2(alphabet.length)`.
+ *
+ * This is typically used to create codecs whose alphabet’s length is a power of 2 such as base-16 or base-64.
+ *
+ * ```ts
+ * const bytes = getBaseXResliceCodec('elho', 2).encode('hellolol'); // 0x4aee
+ * const value = getBaseXResliceCodec('elho', 2).decode(bytes); // "hellolol"
+ * ```
+ *
+ * As usual, separate `getBaseXResliceEncoder` and `getBaseXResliceDecoder` functions are also available.
+ *
+ * ```ts
+ * const bytes = getBaseXResliceEncoder('elho', 2).encode('hellolol'); // 0x4aee
+ * const value = getBaseXResliceDecoder('elho', 2).decode(bytes); // "hellolol"
+ * ```
+ *
+ * @packageDocumentation
+ */
+export * from './assertions';
+export * from './base10';
+export * from './base16';
+export * from './base58';
+export * from './base64';
+export * from './baseX';
+export * from './baseX-reslice';
+export * from './null-characters';
+export * from './utf8';

package/src/null-characters.ts

@@ -0,0 +1,36 @@
+/**
+ * Removes all null characters (`\u0000`) from a string.
+ *
+ * This function cleans a string by stripping out any null characters,
+ * which are often used as padding in fixed-size string encodings.
+ *
+ * @param value - The string to process.
+ * @returns The input string with all null characters removed.
+ *
+ * @example
+ * Removing null characters from a string.
+ * ```ts
+ * removeNullCharacters('hello\u0000\u0000'); // "hello"
+ * ```
+ */
+export const removeNullCharacters = (value: string) =>
+    // eslint-disable-next-line no-control-regex
+    value.replace(/\u0000/g, '');
+
+/**
+ * Pads a string with null characters (`\u0000`) at the end to reach a fixed length.
+ *
+ * If the input string is shorter than the specified length, it is padded with null characters
+ * until it reaches the desired size. If it is already long enough, it remains unchanged.
+ *
+ * @param value - The string to pad.
+ * @param chars - The total length of the resulting string, including padding.
+ * @returns The input string padded with null characters up to the specified length.
+ *
+ * @example
+ * Padding a string with null characters.
+ * ```ts
+ * padNullCharacters('hello', 8); // "hello\u0000\u0000\u0000"
+ * ```
+ */
+export const padNullCharacters = (value: string, chars: number) => value.padEnd(chars, '\u0000');

package/src/utf8.ts

@@ -0,0 +1,114 @@
+import {
+    combineCodec,
+    createDecoder,
+    createEncoder,
+    VariableSizeCodec,
+    VariableSizeDecoder,
+    VariableSizeEncoder,
+} from '@solana/codecs-core';
+import { TextDecoder, TextEncoder } from '@solana/text-encoding-impl';
+
+import { removeNullCharacters } from './null-characters';
+
+/**
+ * Returns an encoder for UTF-8 strings.
+ *
+ * This encoder serializes strings using UTF-8 encoding.
+ * The encoded output contains as many bytes as needed to represent the string.
+ *
+ * For more details, see {@link getUtf8Codec}.
+ *
+ * @returns A `VariableSizeEncoder<string>` for encoding UTF-8 strings.
+ *
+ * @example
+ * Encoding a UTF-8 string.
+ * ```ts
+ * const encoder = getUtf8Encoder();
+ * const bytes = encoder.encode('hello'); // 0x68656c6c6f
+ * ```
+ *
+ * @see {@link getUtf8Codec}
+ */
+export const getUtf8Encoder = (): VariableSizeEncoder<string> => {
+    let textEncoder: TextEncoder;
+    return createEncoder({
+        getSizeFromValue: value => (textEncoder ||= new TextEncoder()).encode(value).length,
+        write: (value: string, bytes, offset) => {
+            const bytesToAdd = (textEncoder ||= new TextEncoder()).encode(value);
+            bytes.set(bytesToAdd, offset);
+            return offset + bytesToAdd.length;
+        },
+    });
+};
+
+/**
+ * Returns a decoder for UTF-8 strings.
+ *
+ * This decoder deserializes UTF-8 encoded strings from a byte array.
+ * It reads all available bytes starting from the given offset.
+ *
+ * For more details, see {@link getUtf8Codec}.
+ *
+ * @returns A `VariableSizeDecoder<string>` for decoding UTF-8 strings.
+ *
+ * @example
+ * Decoding a UTF-8 string.
+ * ```ts
+ * const decoder = getUtf8Decoder();
+ * const value = decoder.decode(new Uint8Array([0x68, 0x65, 0x6c, 0x6c, 0x6f])); // "hello"
+ * ```
+ *
+ * @see {@link getUtf8Codec}
+ */
+export const getUtf8Decoder = (): VariableSizeDecoder<string> => {
+    let textDecoder: TextDecoder;
+    return createDecoder({
+        read(bytes, offset) {
+            const value = (textDecoder ||= new TextDecoder()).decode(bytes.slice(offset));
+            return [removeNullCharacters(value), bytes.length];
+        },
+    });
+};
+
+/**
+ * Returns a codec for encoding and decoding UTF-8 strings.
+ *
+ * This codec serializes strings using UTF-8 encoding.
+ * The encoded output contains as many bytes as needed to represent the string.
+ *
+ * @returns A `VariableSizeCodec<string>` for encoding and decoding UTF-8 strings.
+ *
+ * @example
+ * Encoding and decoding a UTF-8 string.
+ * ```ts
+ * const codec = getUtf8Codec();
+ * const bytes = codec.encode('hello'); // 0x68656c6c6f
+ * const value = codec.decode(bytes);   // "hello"
+ * ```
+ *
+ * @remarks
+ * This codec does not enforce a size boundary. It will encode and decode all bytes necessary to represent the string.
+ *
+ * If you need a fixed-size UTF-8 codec, consider using {@link fixCodecSize}.
+ *
+ * ```ts
+ * const codec = fixCodecSize(getUtf8Codec(), 5);
+ * ```
+ *
+ * If you need a size-prefixed UTF-8 codec, consider using {@link addCodecSizePrefix}.
+ *
+ * ```ts
+ * const codec = addCodecSizePrefix(getUtf8Codec(), getU32Codec());
+ * ```
+ *
+ * Separate {@link getUtf8Encoder} and {@link getUtf8Decoder} functions are available.
+ *
+ * ```ts
+ * const bytes = getUtf8Encoder().encode('hello');
+ * const value = getUtf8Decoder().decode(bytes);
+ * ```
+ *
+ * @see {@link getUtf8Encoder}
+ * @see {@link getUtf8Decoder}
+ */
+export const getUtf8Codec = (): VariableSizeCodec<string> => combineCodec(getUtf8Encoder(), getUtf8Decoder());