rbs 4.0.0.dev.4 → 4.0.0

data/src/string.c

@@ -5,54 +5,6 @@
 #include <stdio.h>
 #include <ctype.h>
 
-unsigned int rbs_utf8_string_to_codepoint(const rbs_string_t string) {
-    unsigned int codepoint = 0;
-    int remaining_bytes = 0;
-
-    const char *s = string.start;
-    const char *end = string.end;
-
-    if (s >= end) return 0; // End of string
-
-    if ((*s & 0x80) == 0) {
-        // Single byte character (0xxxxxxx)
-        return *s;
-    } else if ((*s & 0xE0) == 0xC0) {
-        // Two byte character (110xxxxx 10xxxxxx)
-        codepoint = *s & 0x1F;
-        remaining_bytes = 1;
-    } else if ((*s & 0xF0) == 0xE0) {
-        // Three byte character (1110xxxx 10xxxxxx 10xxxxxx)
-        codepoint = *s & 0x0F;
-        remaining_bytes = 2;
-    } else if ((*s & 0xF8) == 0xF0) {
-        // Four byte character (11110xxx 10xxxxxx 10xxxxxx 10xxxxxx)
-        codepoint = *s & 0x07;
-        remaining_bytes = 3;
-    } else {
-        // Invalid UTF-8 sequence
-        return 0xFFFD; // Unicode replacement character
-    }
-
-    s++;
-    while (remaining_bytes > 0 && s < end) {
-        if ((*s & 0xC0) != 0x80) {
-            // Invalid continuation byte
-            return 0xFFFD;
-        }
-        codepoint = (codepoint << 6) | (*s & 0x3F);
-        s++;
-        remaining_bytes--;
-    }
-
-    if (remaining_bytes > 0) {
-        // Incomplete sequence
-        return 0xFFFD;
-    }
-
-    return codepoint;
-}
-
 rbs_string_t rbs_string_new(const char *start, const char *end) {
     return (rbs_string_t) {
         .start = start,

data/src/util/rbs_allocator.c

@@ -3,6 +3,14 @@
  *
  *  A simple arena allocator that can be freed all at once.
  *
+*  This allocator maintains a linked list of pages, which come in two flavours:
+ *      1. Small allocation pages, which are the same size as the system page size.
+ *      2. Large allocation pages, which are the exact size requested, for sizes greater than the small page size.
+ *
+ *  Small allocations always fit into the unused space at the end of the "head" page. If there isn't enough room, a new
+ *  page is allocated, and the small allocation is placed at its start. This approach wastes that unused slack at the
+ *  end of the previous page, but it means that allocations are instant and never scan the linked list to find a gap.
+ *
  *  This allocator doesn't support freeing individual allocations. Only the whole arena can be freed at once at the end.
  */
 
@@ -20,16 +28,19 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/mman.h>
+#include <fcntl.h>
 #endif
 
-#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__sun)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
+typedef struct rbs_allocator_page {
+    // The previously allocated page, or NULL if this is the first page.
+    struct rbs_allocator_page *next;
 
-struct rbs_allocator {
-    uintptr_t heap_ptr;
-    uintptr_t size;
-};
+    // The size of the payload in bytes.
+    size_t size;
+
+    // The offset of the next available byte.
+    size_t used;
+} rbs_allocator_page_t;
 
 static size_t get_system_page_size(void) {
 #ifdef _WIN32
@@ -43,73 +54,43 @@ static size_t get_system_page_size(void) {
 #endif
 }
 
-static void *map_memory(size_t size) {
-#ifdef _WIN32
-    LPVOID result = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
-    rbs_assert(result != NULL, "VirtualAlloc failed");
-#else
-    void *result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-    rbs_assert(result != MAP_FAILED, "mmap failed");
-#endif
-    return result;
+static inline uintptr_t rbs_align_up_uintptr(uintptr_t value, size_t alignment) {
+    // alignment must be a non-zero power of two
+    RBS_ASSERT(alignment != 0 && (alignment & (alignment - 1)) == 0, "alignment must be a non-zero power of two. alignment: %zu", alignment);
+    return (value + (alignment - 1)) & ~(uintptr_t) (alignment - 1);
 }
 
-static void destroy_memory(void *memory, size_t size) {
-#ifdef _WIN32
-    VirtualFree(memory, 0, MEM_RELEASE);
-#else
-    munmap(memory, size);
-#endif
-}
+static rbs_allocator_page_t *rbs_allocator_page_new(size_t payload_size) {
+    const size_t page_header_size = sizeof(rbs_allocator_page_t);
 
-static void guard_page(void *memory, size_t page_size) {
-#ifdef _WIN32
-    DWORD old_protect_;
-    BOOL result = VirtualProtect(memory, page_size, PAGE_NOACCESS, &old_protect_);
-    rbs_assert(result != 0, "VirtualProtect failed");
-#else
-    int result = mprotect(memory, page_size, PROT_NONE);
-    rbs_assert(result == 0, "mprotect failed");
-#endif
-}
+    rbs_allocator_page_t *page = (rbs_allocator_page_t *) malloc(page_header_size + payload_size);
+    page->size = payload_size;
+    page->used = 0;
 
-static size_t rbs_allocator_default_mem(void) {
-    size_t kib = 1024;
-    size_t mib = kib * 1024;
-    size_t gib = mib * 1024;
-    return 4 * gib;
+    return page;
 }
 
-static inline bool is_power_of_two(uintptr_t value) {
-    return value > 0 && (value & (value - 1)) == 0;
-}
+rbs_allocator_t *rbs_allocator_init(void) {
+    rbs_allocator_t *allocator = (rbs_allocator_t *) malloc(sizeof(rbs_allocator_t));
 
-// Align `val' to nearest multiple of `alignment'.
-static uintptr_t align(uintptr_t size, uintptr_t alignment) {
-    rbs_assert(is_power_of_two(alignment), "alignment is not a power of two");
-    return (size + alignment - 1) & ~(alignment - 1);
-}
+    const size_t system_page_size = get_system_page_size();
 
-rbs_allocator_t *rbs_allocator_init(void) {
-    size_t size = rbs_allocator_default_mem();
-    size_t page_size = get_system_page_size();
-    size = align(size, page_size);
-    void *mem = map_memory(size + page_size);
-    // Guard page; remove range checks in alloc fast path and hard fail if we
-    // consume all memory
-    void *last_page = (char *) mem + size;
-    guard_page(last_page, page_size);
-    uintptr_t start = (uintptr_t) mem;
-    rbs_allocator_t header = (rbs_allocator_t) {
-        .heap_ptr = start + sizeof header,
-        .size = size + page_size,
-    };
-    memcpy(mem, &header, sizeof header);
-    return (rbs_allocator_t *) mem;
+    allocator->default_page_payload_size = system_page_size - sizeof(rbs_allocator_page_t);
+
+    allocator->page = rbs_allocator_page_new(allocator->default_page_payload_size);
+    allocator->page->next = NULL;
+
+    return allocator;
 }
 
 void rbs_allocator_free(rbs_allocator_t *allocator) {
-    destroy_memory((void *) allocator, allocator->size);
+    rbs_allocator_page_t *page = allocator->page;
+    while (page) {
+        rbs_allocator_page_t *next = page->next;
+        free(page);
+        page = next;
+    }
+    free(allocator);
 }
 
 // Allocates `new_size` bytes from `allocator`, aligned to an `alignment`-byte boundary.
@@ -123,21 +104,58 @@ void *rbs_allocator_realloc_impl(rbs_allocator_t *allocator, void *ptr, size_t o
 
 // Allocates `size` bytes from `allocator`, aligned to an `alignment`-byte boundary.
 void *rbs_allocator_malloc_impl(rbs_allocator_t *allocator, size_t size, size_t alignment) {
-    rbs_assert(size % alignment == 0, "size must be a multiple of the alignment. size: %zu, alignment: %zu", size, alignment);
-    uintptr_t aligned = align(allocator->heap_ptr, alignment);
-    allocator->heap_ptr = aligned + size;
-    return (void *) aligned;
+    if (allocator->default_page_payload_size < size) { // Big allocation, give it its own page.
+        // Add padding to ensure we can align the start pointer within this page
+        rbs_allocator_page_t *new_page = rbs_allocator_page_new(size + (alignment - 1));
+
+        // This simple allocator can only put small allocations into the head page.
+        // Naively prepending this large allocation page to the head of the allocator before the previous head page
+        // would waste the remaining space in the head page.
+        // So instead, we'll splice in the large page *after* the head page.
+        //
+        // +-------+    +-----------+        +-----------+
+        // | arena |    | head page |        | new_page  |
+        // |-------|    |-----------+        |-----------+
+        // | *page |--->|  size     |   +--->|  size     |   +---> ... previous tail
+        // +-------+    |  offset   |   |    |  offset   |   |
+        //              | *next ----+---+    | *next ----+---+
+        //              |    ...    |        |    ...    |
+        //              +-----------+        +-----------+
+        //
+        new_page->next = allocator->page->next;
+        allocator->page->next = new_page;
+
+        uintptr_t base = (uintptr_t) new_page + sizeof(rbs_allocator_page_t);
+        uintptr_t aligned_ptr = rbs_align_up_uintptr(base, alignment);
+        return (void *) aligned_ptr;
+    }
+
+    rbs_allocator_page_t *page = allocator->page;
+    uintptr_t base = (uintptr_t) page + sizeof(rbs_allocator_page_t);
+
+    // Compute aligned offset within the payload
+    size_t used_aligned = (size_t) (rbs_align_up_uintptr(base + page->used, alignment) - base);
+
+    if (used_aligned + size > page->size) {
+        // Not enough space. Allocate a new small page and prepend it to the allocator's linked list.
+        rbs_allocator_page_t *new_page = rbs_allocator_page_new(allocator->default_page_payload_size);
+        new_page->next = allocator->page;
+        allocator->page = new_page;
+        page = new_page;
+        base = (uintptr_t) page + sizeof(rbs_allocator_page_t);
+        used_aligned = (size_t) (rbs_align_up_uintptr(base, alignment) - base); // start of fresh page (usually 0 if header is aligned)
+    }
+
+    uintptr_t pointer = base + used_aligned;
+    page->used = used_aligned + size;
+    return (void *) pointer;
 }
 
 // Note: This will eagerly fill with zeroes, unlike `calloc()` which can map a page in a page to be zeroed lazily.
 //       It's assumed that callers to this function will immediately write to the allocated memory, anyway.
 void *rbs_allocator_calloc_impl(rbs_allocator_t *allocator, size_t count, size_t size, size_t alignment) {
     void *p = rbs_allocator_malloc_many_impl(allocator, count, size, alignment);
-#if defined(__linux__)
-    // mmap with MAP_ANONYMOUS gives zero-filled pages.
-#else
     memset(p, 0, count * size);
-#endif
     return p;
 }
 

data/src/util/rbs_assert.c

@@ -5,7 +5,7 @@
 #include <stdlib.h>
 #include <stdbool.h>
 
-void rbs_assert(bool condition, const char *fmt, ...) {
+void rbs_assert_impl(bool condition, const char *fmt, ...) {
     if (condition) {
         return;
     }

data/src/util/rbs_buffer.c

@@ -25,7 +25,7 @@ void rbs_buffer_append_string(rbs_allocator_t *allocator, rbs_buffer_t *buffer,
     if (next_length > buffer->capacity) {
         size_t old_capacity = buffer->capacity;
 
-        rbs_assert(old_capacity != 0, "Precondition: capacity must be at least 1. Got %zu", old_capacity);
+        RBS_ASSERT(old_capacity != 0, "Precondition: capacity must be at least 1. Got %zu", old_capacity);
 
         size_t new_capacity = buffer->capacity * 2;
 
@@ -34,7 +34,7 @@ void rbs_buffer_append_string(rbs_allocator_t *allocator, rbs_buffer_t *buffer,
         }
 
         char *new_value = rbs_allocator_realloc(allocator, buffer->value, old_capacity, new_capacity, char);
-        rbs_assert(new_value != NULL, "Failed to append to buffer. Old capacity: %zu, new capacity: %zu", old_capacity, new_capacity);
+        RBS_ASSERT(new_value != NULL, "Failed to append to buffer. Old capacity: %zu, new capacity: %zu", old_capacity, new_capacity);
 
         buffer->value = new_value;
         buffer->capacity = new_capacity;

data/src/util/rbs_constant_pool.c

@@ -37,7 +37,7 @@ next_power_of_two(uint32_t v) {
     return v;
 }
 
-static bool is_power_of_two(uint32_t size) {
+RBS_ATTRIBUTE_UNUSED static bool is_power_of_two(uint32_t size) {
     return (size & (size - 1)) == 0;
 }
 
@@ -46,7 +46,7 @@ static bool is_power_of_two(uint32_t size) {
  */
 static inline bool
 rbs_constant_pool_resize(rbs_constant_pool_t *pool) {
-    rbs_assert(is_power_of_two(pool->capacity), "pool->capacity is not a power of two. Got %i", pool->capacity);
+    RBS_ASSERT(is_power_of_two(pool->capacity), "pool->capacity is not a power of two. Got %i", pool->capacity);
 
     uint32_t next_capacity = pool->capacity * 2;
     if (next_capacity < pool->capacity) return false;
@@ -57,8 +57,8 @@ rbs_constant_pool_resize(rbs_constant_pool_t *pool) {
     void *next = calloc(next_capacity, element_size);
     if (next == NULL) return false;
 
-    rbs_constant_pool_bucket_t *next_buckets = next;
-    rbs_constant_t *next_constants = (void *) (((char *) next) + next_capacity * sizeof(rbs_constant_pool_bucket_t));
+    rbs_constant_pool_bucket_t *next_buckets = (rbs_constant_pool_bucket_t *) next;
+    rbs_constant_t *next_constants = (rbs_constant_t *) (((char *) next) + next_capacity * sizeof(rbs_constant_pool_bucket_t));
 
     // For each bucket in the current constant pool, find the index in the
     // next constant pool, and insert it.
@@ -95,10 +95,6 @@ rbs_constant_pool_resize(rbs_constant_pool_t *pool) {
     return true;
 }
 
-// This storage is initialized by `Init_rbs_extension()` in `main.c`.
-static rbs_constant_pool_t RBS_GLOBAL_CONSTANT_POOL_STORAGE = { 0 };
-rbs_constant_pool_t *RBS_GLOBAL_CONSTANT_POOL = &RBS_GLOBAL_CONSTANT_POOL_STORAGE;
-
 /**
  * Initialize a new constant pool with a given capacity.
  */
@@ -111,8 +107,8 @@ bool rbs_constant_pool_init(rbs_constant_pool_t *pool, uint32_t capacity) {
     void *memory = calloc(capacity, element_size);
     if (memory == NULL) return false;
 
-    pool->buckets = memory;
-    pool->constants = (void *) (((char *) memory) + capacity * sizeof(rbs_constant_pool_bucket_t));
+    pool->buckets = (rbs_constant_pool_bucket_t *) memory;
+    pool->constants = (rbs_constant_t *) (((char *) memory) + capacity * sizeof(rbs_constant_pool_bucket_t));
     pool->size = 0;
     pool->capacity = capacity;
     return true;
@@ -123,7 +119,7 @@ bool rbs_constant_pool_init(rbs_constant_pool_t *pool, uint32_t capacity) {
  */
 rbs_constant_t *
 rbs_constant_pool_id_to_constant(const rbs_constant_pool_t *pool, rbs_constant_id_t constant_id) {
-    rbs_assert(constant_id != RBS_CONSTANT_ID_UNSET && constant_id <= pool->size, "constant_id is not valid. Got %i, pool->size: %i", constant_id, pool->size);
+    RBS_ASSERT(constant_id != RBS_CONSTANT_ID_UNSET && constant_id <= pool->size, "constant_id is not valid. Got %i, pool->size: %i", constant_id, pool->size);
     return &pool->constants[constant_id - 1];
 }
 
@@ -133,7 +129,7 @@ rbs_constant_pool_id_to_constant(const rbs_constant_pool_t *pool, rbs_constant_i
  */
 rbs_constant_id_t
 rbs_constant_pool_find(const rbs_constant_pool_t *pool, const uint8_t *start, size_t length) {
-    rbs_assert(is_power_of_two(pool->capacity), "pool->capacity is not a power of two. Got %i", pool->capacity);
+    RBS_ASSERT(is_power_of_two(pool->capacity), "pool->capacity is not a power of two. Got %i", pool->capacity);
     const uint32_t mask = pool->capacity - 1;
 
     uint32_t hash = rbs_constant_pool_hash(start, length);
@@ -161,7 +157,7 @@ rbs_constant_pool_insert(rbs_constant_pool_t *pool, const uint8_t *start, size_t
         if (!rbs_constant_pool_resize(pool)) return RBS_CONSTANT_ID_UNSET;
     }
 
-    rbs_assert(is_power_of_two(pool->capacity), "pool->capacity is not a power of two. Got %i", pool->capacity);
+    RBS_ASSERT(is_power_of_two(pool->capacity), "pool->capacity is not a power of two. Got %i", pool->capacity);
     const uint32_t mask = pool->capacity - 1;
 
     uint32_t hash = rbs_constant_pool_hash(start, length);
@@ -202,7 +198,7 @@ rbs_constant_pool_insert(rbs_constant_pool_t *pool, const uint8_t *start, size_t
     // IDs are allocated starting at 1, since the value 0 denotes a non-existent
     // constant.
     uint32_t id = ++pool->size;
-    rbs_assert(pool->size < ((uint32_t) (1 << 30)), "pool->size is too large. Got %i", pool->size);
+    RBS_ASSERT(pool->size < ((uint32_t) (1 << 30)), "pool->size is too large. Got %i", pool->size);
 
     *bucket = (rbs_constant_pool_bucket_t) {
         .id = (unsigned int) (id & 0x3fffffff),

data/src/util/rbs_encoding.c

@@ -3,12 +3,6 @@
 
 #include <ctype.h>
 
-#if defined(__GNUC__)
-#define RBS_ATTRIBUTE_UNUSED __attribute__((unused))
-#else
-#define RBS_ATTRIBUTE_UNUSED
-#endif
-
 typedef uint32_t rbs_unicode_codepoint_t;
 
 #define UNICODE_ALPHA_CODEPOINTS_LENGTH 1450
@@ -4620,6 +4614,7 @@ rbs_unicode_codepoint_match(rbs_unicode_codepoint_t codepoint, const rbs_unicode
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
+// clang-format off
 static const uint8_t rbs_utf_8_dfa[] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1f
     0,
@@ -4991,6 +4986,7 @@ static const uint8_t rbs_utf_8_dfa[] = {
     1,
     1, // s7..s8
 };
+// clang-format on
 
 /**
  * Given a pointer to a string and the number of bytes remaining in the string,
@@ -4999,7 +4995,7 @@ static const uint8_t rbs_utf_8_dfa[] = {
  */
 static rbs_unicode_codepoint_t
 rbs_utf_8_codepoint(const uint8_t *b, ptrdiff_t n, size_t *width) {
-    rbs_assert(n >= 0, "n must be greater than or equal to 0. Got %ti", n);
+    RBS_ASSERT(n >= 0, "[rbs_unicode_codepoint_t] n must be greater than or equal to 0. Got %ti", n);
 
     size_t maximum = (n > 4) ? 4 : ((size_t) n);
     uint32_t codepoint;
@@ -5029,7 +5025,7 @@ rbs_utf_8_codepoint(const uint8_t *b, ptrdiff_t n, size_t *width) {
  */
 size_t
 rbs_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n) {
-    rbs_assert(n >= 0, "n must be greater than or equal to 0. Got %ti", n);
+    RBS_ASSERT(n >= 0, "[rbs_encoding_utf_8_char_width] n must be greater than or equal to 0. Got %ti", n);
 
     size_t maximum = (n > 4) ? 4 : ((size_t) n);
     uint32_t state = 0;

data/src/util/rbs_unescape.c

@@ -1,4 +1,5 @@
 #include "rbs/util/rbs_unescape.h"
+#include "rbs/util/rbs_encoding.h"
 #include <string.h>
 #include <stdlib.h>
 #include <ctype.h>
@@ -42,20 +43,44 @@ static int octal_to_int(const char *octal, int length) {
     return result;
 }
 
-int rbs_utf8_codelen(unsigned int c) {
-    if (c <= 0x7F) return 1;
-    if (c <= 0x7FF) return 2;
-    if (c <= 0xFFFF) return 3;
-    if (c <= 0x10FFFF) return 4;
-    return 1; // Invalid Unicode codepoint, treat as 1 byte
+// Fills buf starting at index 'start' with the UTF-8 encoding of 'codepoint'.
+// Returns the number of bytes written, or 0 when the output is not changed.
+//
+size_t rbs_utf8_fill_codepoint(char *buf, size_t start, size_t end, unsigned int codepoint) {
+    if (start + 4 > end) {
+        return 0;
+    }
+
+    if (codepoint <= 0x7F) {
+        buf[start] = codepoint & 0x7F;
+        return 1;
+    } else if (codepoint <= 0x7FF) {
+        buf[start + 0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+        buf[start + 1] = 0x80 | (codepoint & 0x3F);
+        return 2;
+    } else if (codepoint <= 0xFFFF) {
+        buf[start + 0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+        buf[start + 1] = 0x80 | ((codepoint >> 6) & 0x3F);
+        buf[start + 2] = 0x80 | (codepoint & 0x3F);
+        return 3;
+    } else if (codepoint <= 0x10FFFF) {
+        buf[start + 0] = 0xF0 | ((codepoint >> 18) & 0x07);
+        buf[start + 1] = 0x80 | ((codepoint >> 12) & 0x3F);
+        buf[start + 2] = 0x80 | ((codepoint >> 6) & 0x3F);
+        buf[start + 3] = 0x80 | (codepoint & 0x3F);
+        return 4;
+    } else {
+        return 0;
+    }
 }
 
-rbs_string_t unescape_string(rbs_allocator_t *allocator, const rbs_string_t string, bool is_double_quote) {
+rbs_string_t unescape_string(rbs_allocator_t *allocator, const rbs_string_t string, bool is_double_quote, bool is_unicode) {
     if (!string.start) return RBS_STRING_NULL;
 
     size_t len = string.end - string.start;
     const char *input = string.start;
 
+    // The output cannot be longer than the input even after unescaping.
     char *output = rbs_allocator_alloc_many(allocator, len + 1, char);
     if (!output) return RBS_STRING_NULL;
 
@@ -79,9 +104,21 @@ rbs_string_t unescape_string(rbs_allocator_t *allocator, const rbs_string_t stri
                     i += hex_len + 2;
                 } else if (input[i + 1] == 'u' && i + 5 < len) {
                     // Unicode escape
-                    int value = hex_to_int(input + i + 2, 4);
-                    output[j++] = (char) value;
-                    i += 6;
+
+                    if (is_unicode) {
+                        // The UTF-8 representation is at most 4 bytes, shorter than the input length.
+                        int value = hex_to_int(input + i + 2, 4);
+                        j += rbs_utf8_fill_codepoint(output, j, len + 1, value);
+                        i += 6;
+                    } else {
+                        // Copy the escape sequence as-is
+                        output[j++] = input[i++];
+                        output[j++] = input[i++];
+                        output[j++] = input[i++];
+                        output[j++] = input[i++];
+                        output[j++] = input[i++];
+                        output[j++] = input[i++];
+                    }
                 } else {
                     // Other escapes
                     int found = 0;
@@ -114,18 +151,17 @@ rbs_string_t unescape_string(rbs_allocator_t *allocator, const rbs_string_t stri
     return rbs_string_new(output, output + j);
 }
 
-rbs_string_t rbs_unquote_string(rbs_allocator_t *allocator, rbs_string_t input) {
-    unsigned int first_char = rbs_utf8_string_to_codepoint(input);
-    size_t byte_length = rbs_string_len(input);
+rbs_string_t rbs_unquote_string(rbs_allocator_t *allocator, rbs_string_t input, const rbs_encoding_t *encoding) {
+    unsigned int first_char = input.start[0];
+
+    const char *new_start = input.start;
+    const char *new_end = input.end;
 
-    ptrdiff_t start_offset = 0;
     if (first_char == '"' || first_char == '\'' || first_char == '`') {
-        int bs = rbs_utf8_codelen(first_char);
-        start_offset += bs;
-        byte_length -= 2 * bs;
+        new_start += 1;
+        new_end -= 1;
     }
 
-    const char *new_start = input.start + start_offset;
-    rbs_string_t string = rbs_string_new(new_start, new_start + byte_length);
-    return unescape_string(allocator, string, first_char == '"');
+    rbs_string_t string = rbs_string_new(new_start, new_end);
+    return unescape_string(allocator, string, first_char == '"', encoding == RBS_ENCODING_UTF_8_ENTRY);
 }