herb 0.8.10-arm-linux-gnu → 0.9.0-arm-linux-gnu

data/src/include/util/hb_allocator.h

@@ -0,0 +1,78 @@
+#ifndef HERB_ALLOCATOR_H
+#define HERB_ALLOCATOR_H
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#include "hb_arena.h"
+
+#ifndef HB_ALLOCATOR_DEFAULT_ARENA_SIZE
+#  define HB_ALLOCATOR_DEFAULT_ARENA_SIZE (1024 * 16)
+#endif
+
+typedef enum {
+  HB_ALLOCATOR_MALLOC,
+  HB_ALLOCATOR_ARENA,
+  HB_ALLOCATOR_TRACKING,
+} hb_allocator_type_T;
+
+typedef struct {
+  void* pointer;
+  size_t size;
+} hb_allocator_tracking_entry_T;
+
+typedef struct {
+  size_t allocation_count;
+  size_t deallocation_count;
+  size_t untracked_deallocation_count;
+  size_t bytes_allocated;
+  size_t bytes_deallocated;
+  hb_allocator_tracking_entry_T* buckets;
+  size_t buckets_capacity;
+  size_t buckets_used;
+  void** untracked_pointers;
+  size_t untracked_pointers_size;
+  size_t untracked_pointers_capacity;
+} hb_allocator_tracking_stats_T;
+
+typedef struct hb_allocator {
+  void* (*alloc)(struct hb_allocator* self, size_t size);
+  void* (*realloc)(struct hb_allocator* self, void* pointer, size_t old_size, size_t new_size);
+  void (*dealloc)(struct hb_allocator* self, void* pointer);
+  char* (*strdup)(struct hb_allocator* self, const char* string);
+  char* (*strndup)(struct hb_allocator* self, const char* string, size_t length);
+  void (*destroy)(struct hb_allocator* self);
+  void* context;
+} hb_allocator_T;
+
+bool hb_allocator_init(hb_allocator_T* allocator, hb_allocator_type_T type);
+bool hb_allocator_init_with_size(hb_allocator_T* allocator, hb_allocator_type_T type, size_t initial_size);
+void hb_allocator_destroy(hb_allocator_T* allocator);
+
+hb_allocator_T hb_allocator_with_malloc(void);
+hb_allocator_T hb_allocator_with_arena(hb_arena_T* arena);
+hb_allocator_T hb_allocator_with_tracking(void);
+
+hb_allocator_tracking_stats_T* hb_allocator_tracking_stats(hb_allocator_T* allocator);
+
+static inline void* hb_allocator_alloc(hb_allocator_T* allocator, size_t size) {
+  return allocator->alloc(allocator, size);
+}
+
+static inline void* hb_allocator_realloc(hb_allocator_T* allocator, void* pointer, size_t old_size, size_t new_size) {
+  return allocator->realloc(allocator, pointer, old_size, new_size);
+}
+
+static inline void hb_allocator_dealloc(hb_allocator_T* allocator, void* pointer) {
+  allocator->dealloc(allocator, pointer);
+}
+
+static inline char* hb_allocator_strdup(hb_allocator_T* allocator, const char* string) {
+  return allocator->strdup(allocator, string);
+}
+
+static inline char* hb_allocator_strndup(hb_allocator_T* allocator, const char* string, size_t length) {
+  return allocator->strndup(allocator, string, length);
+}
+
+#endif

data/src/include/util/hb_arena.h

@@ -20,12 +20,17 @@ typedef struct HB_ARENA_STRUCT {
   size_t allocation_count;
 } hb_arena_T;
 
+#define hb_arena_for_each_page(arena) for (hb_arena_page_T* page = (arena)->head; page != NULL; page = page->next)
+
+#define hb_arena_for_each_page_const(arena)                                                                            \
+  for (const hb_arena_page_T* page = (arena)->head; page != NULL; page = page->next)
+
 bool hb_arena_init(hb_arena_T* allocator, size_t initial_size);
 void* hb_arena_alloc(hb_arena_T* allocator, size_t size);
 size_t hb_arena_position(hb_arena_T* allocator);
 size_t hb_arena_capacity(hb_arena_T* allocator);
 void hb_arena_reset(hb_arena_T* allocator);
-void hb_arena_reset_to(hb_arena_T* allocator, size_t new_position);
+void hb_arena_reset_to(hb_arena_T* allocator, size_t target_position);
 void hb_arena_free(hb_arena_T* allocator);
 
 #endif

data/src/include/util/hb_arena_debug.h

@@ -3,6 +3,17 @@
 
 #include "hb_arena.h"
 
-void hb_arena_print_stats(const hb_arena_T* allocator);
+typedef struct {
+  size_t pages;
+  size_t total_capacity;
+  size_t total_used;
+  size_t total_available;
+  size_t allocations;
+  size_t fragmentation;
+  size_t default_page_size;
+} hb_arena_stats_T;
+
+hb_arena_stats_T hb_arena_get_stats(const hb_arena_T* arena);
+void hb_arena_print_stats(const hb_arena_T* arena);
 
 #endif

data/src/include/util/hb_array.h

@@ -1,21 +1,25 @@
 #ifndef HERB_ARRAY_H
 #define HERB_ARRAY_H
 
+#include <stdbool.h>
 #include <stdlib.h>
 
+struct hb_allocator;
+
 typedef struct HB_ARRAY_STRUCT {
   void** items;
   size_t size;
   size_t capacity;
+  struct hb_allocator* allocator;
 } hb_array_T;
 
-hb_array_T* hb_array_init(size_t capacity);
+hb_array_T* hb_array_init(size_t capacity, struct hb_allocator* allocator);
 
 void* hb_array_get(const hb_array_T* array, size_t index);
 void* hb_array_first(hb_array_T* array);
 void* hb_array_last(hb_array_T* array);
 
-void hb_array_append(hb_array_T* array, void* item);
+bool hb_array_append(hb_array_T* array, void* item);
 void hb_array_set(const hb_array_T* array, size_t index, void* item);
 void hb_array_free(hb_array_T** array);
 void hb_array_remove(hb_array_T* array, size_t index);
@@ -23,7 +27,7 @@ void hb_array_remove(hb_array_T* array, size_t index);
 size_t hb_array_index_of(hb_array_T* array, void* item);
 void hb_array_remove_item(hb_array_T* array, void* item);
 
-void hb_array_push(hb_array_T* array, void* item);
+bool hb_array_push(hb_array_T* array, void* item);
 void* hb_array_pop(hb_array_T* array);
 
 size_t hb_array_capacity(const hb_array_T* array);

data/src/include/util/hb_buffer.h

@@ -1,21 +1,21 @@
 #ifndef HERB_BUFFER_H
 #define HERB_BUFFER_H
 
-#include "hb_arena.h"
 #include "hb_string.h"
 
 #include <stdbool.h>
 #include <stdlib.h>
 
+struct hb_allocator;
+
 typedef struct HB_BUFFER_STRUCT {
-  hb_arena_T* allocator;
+  struct hb_allocator* allocator;
   char* value;
   size_t length;
   size_t capacity;
 } hb_buffer_T;
 
-bool hb_buffer_init(hb_buffer_T* buffer, size_t capacity);
-bool hb_buffer_init_arena(hb_buffer_T* buffer, hb_arena_T* allocator, size_t capacity);
+bool hb_buffer_init(hb_buffer_T* buffer, size_t capacity, struct hb_allocator* allocator);
 
 void hb_buffer_append(hb_buffer_T* buffer, const char* text);
 void hb_buffer_append_with_length(hb_buffer_T* buffer, const char* text, size_t length);
@@ -28,9 +28,11 @@ void hb_buffer_concat(hb_buffer_T* destination, hb_buffer_T* source);
 char* hb_buffer_value(const hb_buffer_T* buffer);
 
 size_t hb_buffer_length(const hb_buffer_T* buffer);
+bool hb_buffer_is_empty(const hb_buffer_T* buffer);
 size_t hb_buffer_capacity(const hb_buffer_T* buffer);
 size_t hb_buffer_sizeof(void);
 
+void hb_buffer_free(hb_buffer_T* buffer);
 void hb_buffer_clear(hb_buffer_T* buffer);
 
 #endif

data/src/include/util/hb_foreach.h

@@ -0,0 +1,79 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See /home/runner/work/herb/herb/templates/src/include/util/hb_foreach.h.erb
+
+#ifndef HB_FOREACH_H
+#define HB_FOREACH_H
+
+#define HB_ARG64(_1,_2,_3,_4,_5,_6,_7,_8,_9,_10,_11,_12,_13,_14,_15,_16,_17,_18,_19,_20,_21,_22,_23,_24,_25,_26,_27,_28,_29,_30,_31,_32,_33,_34,_35,_36,_37,_38,_39,_40,_41,_42,_43,_44,_45,_46,_47,_48,_49,_50,_51,_52,_53,_54,_55,_56,_57,_58,_59,_60,_61,_62,_63,_64,count,...) count
+#define HB_NARGS(...) HB_ARG64(__VA_ARGS__,64,63,62,61,60,59,58,57,56,55,54,53,52,51,50,49,48,47,46,45,44,43,42,41,40,39,38,37,36,35,34,33,32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1)
+
+#define HB_FOR_EACH_1(macro, head) macro(head)
+#define HB_FOR_EACH_2(macro, head, ...) macro(head), HB_FOR_EACH_1(macro, __VA_ARGS__)
+#define HB_FOR_EACH_3(macro, head, ...) macro(head), HB_FOR_EACH_2(macro, __VA_ARGS__)
+#define HB_FOR_EACH_4(macro, head, ...) macro(head), HB_FOR_EACH_3(macro, __VA_ARGS__)
+#define HB_FOR_EACH_5(macro, head, ...) macro(head), HB_FOR_EACH_4(macro, __VA_ARGS__)
+#define HB_FOR_EACH_6(macro, head, ...) macro(head), HB_FOR_EACH_5(macro, __VA_ARGS__)
+#define HB_FOR_EACH_7(macro, head, ...) macro(head), HB_FOR_EACH_6(macro, __VA_ARGS__)
+#define HB_FOR_EACH_8(macro, head, ...) macro(head), HB_FOR_EACH_7(macro, __VA_ARGS__)
+#define HB_FOR_EACH_9(macro, head, ...) macro(head), HB_FOR_EACH_8(macro, __VA_ARGS__)
+#define HB_FOR_EACH_10(macro, head, ...) macro(head), HB_FOR_EACH_9(macro, __VA_ARGS__)
+#define HB_FOR_EACH_11(macro, head, ...) macro(head), HB_FOR_EACH_10(macro, __VA_ARGS__)
+#define HB_FOR_EACH_12(macro, head, ...) macro(head), HB_FOR_EACH_11(macro, __VA_ARGS__)
+#define HB_FOR_EACH_13(macro, head, ...) macro(head), HB_FOR_EACH_12(macro, __VA_ARGS__)
+#define HB_FOR_EACH_14(macro, head, ...) macro(head), HB_FOR_EACH_13(macro, __VA_ARGS__)
+#define HB_FOR_EACH_15(macro, head, ...) macro(head), HB_FOR_EACH_14(macro, __VA_ARGS__)
+#define HB_FOR_EACH_16(macro, head, ...) macro(head), HB_FOR_EACH_15(macro, __VA_ARGS__)
+#define HB_FOR_EACH_17(macro, head, ...) macro(head), HB_FOR_EACH_16(macro, __VA_ARGS__)
+#define HB_FOR_EACH_18(macro, head, ...) macro(head), HB_FOR_EACH_17(macro, __VA_ARGS__)
+#define HB_FOR_EACH_19(macro, head, ...) macro(head), HB_FOR_EACH_18(macro, __VA_ARGS__)
+#define HB_FOR_EACH_20(macro, head, ...) macro(head), HB_FOR_EACH_19(macro, __VA_ARGS__)
+#define HB_FOR_EACH_21(macro, head, ...) macro(head), HB_FOR_EACH_20(macro, __VA_ARGS__)
+#define HB_FOR_EACH_22(macro, head, ...) macro(head), HB_FOR_EACH_21(macro, __VA_ARGS__)
+#define HB_FOR_EACH_23(macro, head, ...) macro(head), HB_FOR_EACH_22(macro, __VA_ARGS__)
+#define HB_FOR_EACH_24(macro, head, ...) macro(head), HB_FOR_EACH_23(macro, __VA_ARGS__)
+#define HB_FOR_EACH_25(macro, head, ...) macro(head), HB_FOR_EACH_24(macro, __VA_ARGS__)
+#define HB_FOR_EACH_26(macro, head, ...) macro(head), HB_FOR_EACH_25(macro, __VA_ARGS__)
+#define HB_FOR_EACH_27(macro, head, ...) macro(head), HB_FOR_EACH_26(macro, __VA_ARGS__)
+#define HB_FOR_EACH_28(macro, head, ...) macro(head), HB_FOR_EACH_27(macro, __VA_ARGS__)
+#define HB_FOR_EACH_29(macro, head, ...) macro(head), HB_FOR_EACH_28(macro, __VA_ARGS__)
+#define HB_FOR_EACH_30(macro, head, ...) macro(head), HB_FOR_EACH_29(macro, __VA_ARGS__)
+#define HB_FOR_EACH_31(macro, head, ...) macro(head), HB_FOR_EACH_30(macro, __VA_ARGS__)
+#define HB_FOR_EACH_32(macro, head, ...) macro(head), HB_FOR_EACH_31(macro, __VA_ARGS__)
+#define HB_FOR_EACH_33(macro, head, ...) macro(head), HB_FOR_EACH_32(macro, __VA_ARGS__)
+#define HB_FOR_EACH_34(macro, head, ...) macro(head), HB_FOR_EACH_33(macro, __VA_ARGS__)
+#define HB_FOR_EACH_35(macro, head, ...) macro(head), HB_FOR_EACH_34(macro, __VA_ARGS__)
+#define HB_FOR_EACH_36(macro, head, ...) macro(head), HB_FOR_EACH_35(macro, __VA_ARGS__)
+#define HB_FOR_EACH_37(macro, head, ...) macro(head), HB_FOR_EACH_36(macro, __VA_ARGS__)
+#define HB_FOR_EACH_38(macro, head, ...) macro(head), HB_FOR_EACH_37(macro, __VA_ARGS__)
+#define HB_FOR_EACH_39(macro, head, ...) macro(head), HB_FOR_EACH_38(macro, __VA_ARGS__)
+#define HB_FOR_EACH_40(macro, head, ...) macro(head), HB_FOR_EACH_39(macro, __VA_ARGS__)
+#define HB_FOR_EACH_41(macro, head, ...) macro(head), HB_FOR_EACH_40(macro, __VA_ARGS__)
+#define HB_FOR_EACH_42(macro, head, ...) macro(head), HB_FOR_EACH_41(macro, __VA_ARGS__)
+#define HB_FOR_EACH_43(macro, head, ...) macro(head), HB_FOR_EACH_42(macro, __VA_ARGS__)
+#define HB_FOR_EACH_44(macro, head, ...) macro(head), HB_FOR_EACH_43(macro, __VA_ARGS__)
+#define HB_FOR_EACH_45(macro, head, ...) macro(head), HB_FOR_EACH_44(macro, __VA_ARGS__)
+#define HB_FOR_EACH_46(macro, head, ...) macro(head), HB_FOR_EACH_45(macro, __VA_ARGS__)
+#define HB_FOR_EACH_47(macro, head, ...) macro(head), HB_FOR_EACH_46(macro, __VA_ARGS__)
+#define HB_FOR_EACH_48(macro, head, ...) macro(head), HB_FOR_EACH_47(macro, __VA_ARGS__)
+#define HB_FOR_EACH_49(macro, head, ...) macro(head), HB_FOR_EACH_48(macro, __VA_ARGS__)
+#define HB_FOR_EACH_50(macro, head, ...) macro(head), HB_FOR_EACH_49(macro, __VA_ARGS__)
+#define HB_FOR_EACH_51(macro, head, ...) macro(head), HB_FOR_EACH_50(macro, __VA_ARGS__)
+#define HB_FOR_EACH_52(macro, head, ...) macro(head), HB_FOR_EACH_51(macro, __VA_ARGS__)
+#define HB_FOR_EACH_53(macro, head, ...) macro(head), HB_FOR_EACH_52(macro, __VA_ARGS__)
+#define HB_FOR_EACH_54(macro, head, ...) macro(head), HB_FOR_EACH_53(macro, __VA_ARGS__)
+#define HB_FOR_EACH_55(macro, head, ...) macro(head), HB_FOR_EACH_54(macro, __VA_ARGS__)
+#define HB_FOR_EACH_56(macro, head, ...) macro(head), HB_FOR_EACH_55(macro, __VA_ARGS__)
+#define HB_FOR_EACH_57(macro, head, ...) macro(head), HB_FOR_EACH_56(macro, __VA_ARGS__)
+#define HB_FOR_EACH_58(macro, head, ...) macro(head), HB_FOR_EACH_57(macro, __VA_ARGS__)
+#define HB_FOR_EACH_59(macro, head, ...) macro(head), HB_FOR_EACH_58(macro, __VA_ARGS__)
+#define HB_FOR_EACH_60(macro, head, ...) macro(head), HB_FOR_EACH_59(macro, __VA_ARGS__)
+#define HB_FOR_EACH_61(macro, head, ...) macro(head), HB_FOR_EACH_60(macro, __VA_ARGS__)
+#define HB_FOR_EACH_62(macro, head, ...) macro(head), HB_FOR_EACH_61(macro, __VA_ARGS__)
+#define HB_FOR_EACH_63(macro, head, ...) macro(head), HB_FOR_EACH_62(macro, __VA_ARGS__)
+#define HB_FOR_EACH_64(macro, head, ...) macro(head), HB_FOR_EACH_63(macro, __VA_ARGS__)
+
+#define HB_CONCAT(left, right) left ## right
+#define HB_FOR_EACH_N(count) HB_CONCAT(HB_FOR_EACH_, count)
+#define HB_FOR_EACH(macro, ...) HB_FOR_EACH_N(HB_NARGS(__VA_ARGS__))(macro, __VA_ARGS__)
+
+#endif

data/src/include/util/hb_narray.h

@@ -1,26 +1,30 @@
 #ifndef HERB_NARRAY_H
 #define HERB_NARRAY_H
 
+#include <stdbool.h>
 #include <stdint.h>
 #include <stdlib.h>
-#include <stdbool.h>
+
+#include "hb_allocator.h"
 
 typedef struct HB_NARRAY_STRUCT {
+  hb_allocator_T* allocator;
   uint8_t* items;
   size_t item_size;
   size_t size;
   size_t capacity;
 } hb_narray_T;
 
-void hb_narray_init(hb_narray_T* array, size_t item_size, size_t initial_capacity);
-#define hb_narray_pointer_init(array, initial_capacity) (hb_narray_init(array, sizeof(void*), initial_capacity))
+bool hb_narray_init(hb_narray_T* array, size_t item_size, size_t initial_capacity, hb_allocator_T* allocator);
+#define hb_narray_pointer_init(array, initial_capacity, allocator)                                                     \
+  (hb_narray_init(array, sizeof(void*), initial_capacity, allocator))
 
 void* hb_narray_get(const hb_narray_T* array, size_t index);
 void* hb_narray_first(hb_narray_T* array);
 void* hb_narray_last(hb_narray_T* array);
 size_t hb_narray_size(const hb_narray_T* array);
 
-void hb_narray_append(hb_narray_T* array, void* item);
+bool hb_narray_append(hb_narray_T* array, void* item);
 void hb_narray_remove(hb_narray_T* array, size_t index);
 void hb_narray_deinit(hb_narray_T* array);
 

data/src/include/util/hb_string.h

@@ -4,26 +4,73 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
+#include <strings.h>
 
-#include "hb_arena.h"
+#include "hb_allocator.h"
+#include "hb_foreach.h"
 
 typedef struct HB_STRING_STRUCT {
   char* data;
   uint32_t length;
 } hb_string_T;
 
-hb_string_T hb_string(const char* null_terminated_c_string);
-hb_string_T hb_string_slice(hb_string_T string, uint32_t offset);
-bool hb_string_equals(hb_string_T a, hb_string_T b);
-bool hb_string_equals_case_insensitive(hb_string_T a, hb_string_T b);
-bool hb_string_starts_with(hb_string_T string, hb_string_T expected_prefix);
-bool hb_string_is_empty(hb_string_T string);
-hb_string_T hb_string_truncate(hb_string_T string, uint32_t max_length);
+#define HB_STRING_EMPTY ((hb_string_T) { .data = "", .length = 0 })
+#define HB_STRING_NULL ((hb_string_T) { .data = NULL, .length = 0 })
+
+#define HB_STRING_LITERAL(string) { .data = (char*) (string), .length = (uint32_t) (sizeof(string) - 1) }
+
+#define HB_STRING_LIST(...) { HB_FOR_EACH(HB_STRING_LITERAL, __VA_ARGS__) }
+
+#define hb_string(string)                                                                                              \
+  (__builtin_constant_p(string)                                                                                        \
+     ? ((hb_string_T) { .data = (char*) (string), .length = (uint32_t) __builtin_strlen(string) })                     \
+     : hb_string_from_c_string(string))
+
+hb_string_T hb_string_from_c_string(const char* null_terminated_c_string);
+
+static inline bool hb_string_is_null(hb_string_T string) {
+  return string.data == NULL;
+}
+
+static inline bool hb_string_is_empty(hb_string_T string) {
+  return string.data == NULL || string.length == 0;
+}
+
+static inline hb_string_T hb_string_slice(hb_string_T string, uint32_t offset) {
+  if (string.length < offset) { return HB_STRING_NULL; }
+
+  return (hb_string_T) { .data = string.data + offset, .length = string.length - offset };
+}
 
+static inline bool hb_string_equals(hb_string_T a, hb_string_T b) {
+  if (a.length != b.length) { return false; }
+
+  return strncmp(a.data, b.data, a.length) == 0;
+}
+
+static inline bool hb_string_equals_case_insensitive(hb_string_T a, hb_string_T b) {
+  if (a.length != b.length) { return false; }
+
+  return strncasecmp(a.data, b.data, a.length) == 0;
+}
+
+static inline bool hb_string_starts_with(hb_string_T string, hb_string_T expected_prefix) {
+  if (hb_string_is_empty(string) || hb_string_is_empty(expected_prefix)) { return false; }
+  if (string.length < expected_prefix.length) { return false; }
+
+  return strncmp(string.data, expected_prefix.data, expected_prefix.length) == 0;
+}
+
+hb_string_T hb_string_truncate(hb_string_T string, uint32_t max_length);
 hb_string_T hb_string_range(hb_string_T string, uint32_t from, uint32_t to);
+hb_string_T hb_string_trim_start(hb_string_T string);
+hb_string_T hb_string_trim_end(hb_string_T string);
+hb_string_T hb_string_trim(hb_string_T string);
+bool hb_string_is_blank(hb_string_T string);
+hb_string_T hb_string_copy(hb_string_T string, hb_allocator_T* allocator);
 
 char* hb_string_to_c_string_using_malloc(hb_string_T string);
-
 char* hb_string_to_c_string(hb_arena_T* allocator, hb_string_T string);
 
 #endif

data/src/include/util.h

@@ -5,10 +5,13 @@
 #include <stdbool.h>
 #include <stdlib.h>
 
+struct hb_allocator;
+
 int is_newline(int character);
+int is_whitespace(int character);
+hb_string_T escape_newlines(struct hb_allocator* allocator, hb_string_T input);
+hb_string_T quoted_string(struct hb_allocator* allocator, hb_string_T input);
 
-hb_string_T escape_newlines(hb_string_T input);
-hb_string_T quoted_string(hb_string_T input);
-char* herb_strdup(const char* s);
+char* convert_underscores_to_dashes(const char* input);
 
 #endif

data/src/include/version.h

@@ -1,6 +1,6 @@
 #ifndef HERB_VERSION_H
 #define HERB_VERSION_H
 
-#define HERB_VERSION "0.8.10"
+#define HERB_VERSION "0.9.0"
 
 #endif

data/src/io.c

@@ -1,4 +1,5 @@
 #include "include/io.h"
+#include "include/util/hb_allocator.h"
 #include "include/util/hb_buffer.h"
 
 #include <errno.h>
@@ -7,7 +8,7 @@
 
 #define FILE_READ_CHUNK 4096
 
-char* herb_read_file(const char* filename) {
+char* herb_read_file(const char* filename, struct hb_allocator* allocator) {
   if (!filename) { return NULL; }
 
   FILE* fp = fopen(filename, "rb");
@@ -18,7 +19,7 @@ char* herb_read_file(const char* filename) {
   }
 
   hb_buffer_T buffer;
-  hb_buffer_init(&buffer, 4096);
+  hb_buffer_init(&buffer, 4096, allocator);
 
   char chunk[FILE_READ_CHUNK];
   size_t bytes_read;

data/src/lexer.c

@@ -1,15 +1,18 @@
 #include "include/lexer_peek_helpers.h"
+#include "include/macros.h"
 #include "include/token.h"
 #include "include/utf8.h"
 #include "include/util.h"
-#include "include/util/hb_buffer.h"
 #include "include/util/hb_string.h"
 
 #include <ctype.h>
+#include <stdint.h>
 #include <string.h>
 
 #define LEXER_STALL_LIMIT 5
 
+static hb_string_T erb_open_patterns[] = HB_STRING_LIST("<%==", "<%%=", "<%graphql", "<%=", "<%#", "<%-", "<%%", "<%");
+
 static bool lexer_eof(const lexer_T* lexer) {
   return lexer->current_character == '\0' || lexer->stalled;
 }
@@ -31,11 +34,13 @@ static bool lexer_stalled(lexer_T* lexer) {
   return lexer->stalled;
 }
 
-void lexer_init(lexer_T* lexer, const char* source) {
+void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator) {
+  lexer->allocator = allocator;
+
   if (source != NULL) {
     lexer->source = hb_string(source);
   } else {
-    lexer->source = hb_string("");
+    lexer->source = HB_STRING_EMPTY;
   }
 
   lexer->current_character = lexer->source.data[0];
@@ -55,11 +60,11 @@ void lexer_init(lexer_T* lexer, const char* source) {
 }
 
 token_T* lexer_error(lexer_T* lexer, const char* message) {
-  char error_message[128];
+  char buffer[128];
 
   snprintf(
-    error_message,
-    sizeof(error_message),
+    buffer,
+    sizeof(buffer),
     "[Lexer] Error: %s (character '%c', line %u, col %u)\n",
     message,
     lexer->current_character,
@@ -67,7 +72,10 @@ token_T* lexer_error(lexer_T* lexer, const char* message) {
     lexer->current_column
   );
 
-  return token_init(hb_string(error_message), TOKEN_ERROR, lexer);
+  size_t length = strlen(buffer);
+  char* error_message = hb_allocator_strndup(lexer->allocator, buffer, length);
+
+  return token_init((hb_string_T) { .data = error_message, .length = (uint32_t) length }, TOKEN_ERROR, lexer);
 }
 
 static void lexer_advance(lexer_T* lexer) {
@@ -79,8 +87,8 @@ static void lexer_advance(lexer_T* lexer) {
   }
 }
 
-static void lexer_advance_utf8_bytes(lexer_T* lexer, int byte_count) {
-  if (byte_count <= 0) { return; }
+static void lexer_advance_utf8_bytes(lexer_T* lexer, uint32_t byte_count) {
+  if (byte_count == 0) { return; }
 
   if (lexer_has_more_characters(lexer) && !lexer_eof(lexer)) {
     if (!is_newline(lexer->current_character)) { lexer->current_column++; }
@@ -120,19 +128,17 @@ static token_T* lexer_advance_with_next(lexer_T* lexer, size_t count, token_type
 }
 
 static token_T* lexer_advance_current(lexer_T* lexer, const token_type_T type) {
-  char buffer[2];
-  buffer[0] = lexer->current_character;
-  buffer[1] = '\0';
-
-  return lexer_advance_with(lexer, hb_string(buffer), type);
+  return lexer_advance_with_next(lexer, 1, type);
 }
 
 static token_T* lexer_advance_utf8_character(lexer_T* lexer, const token_type_T type) {
-  int char_byte_length = utf8_sequence_length(lexer->source.data, lexer->current_position, lexer->source.length);
+  uint32_t char_byte_length = utf8_sequence_length(hb_string_slice(lexer->source, lexer->current_position));
+
   if (char_byte_length <= 1) { return lexer_advance_current(lexer, type); }
+
   uint32_t start_position = lexer->current_position;
 
-  for (int i = 0; i < char_byte_length; i++) {
+  for (uint32_t i = 0; i < char_byte_length; i++) {
     if (lexer->current_position + i >= lexer->source.length) { return lexer_advance_current(lexer, type); }
   }
 
@@ -171,7 +177,8 @@ static token_T* lexer_parse_identifier(lexer_T* lexer) {
 
   while ((isalnum(lexer->current_character) || lexer->current_character == '-' || lexer->current_character == '_'
           || lexer->current_character == ':')
-         && !lexer_peek_for_html_comment_end(lexer, 0) && !lexer_eof(lexer)) {
+         && !lexer_peek_for_html_comment_end(lexer, 0) && !lexer_peek_for_html_comment_invalid_end(lexer, 0)
+         && !lexer_eof(lexer)) {
 
     lexer_advance(lexer);
   }
@@ -185,13 +192,9 @@ static token_T* lexer_parse_identifier(lexer_T* lexer) {
 // ===== ERB Parsing
 
 static token_T* lexer_parse_erb_open(lexer_T* lexer) {
-  hb_string_T erb_patterns[] = { hb_string("<%=="), hb_string("<%%="), hb_string("<%="),       hb_string("<%#"),
-                                 hb_string("<%-"),  hb_string("<%%"),  hb_string("<%graphql"), hb_string("<%") };
-
   lexer->state = STATE_ERB_CONTENT;
-
-  for (size_t i = 0; i < sizeof(erb_patterns) / sizeof(erb_patterns[0]); i++) {
-    token_T* match = lexer_match_and_advance(lexer, erb_patterns[i], TOKEN_ERB_START);
+  for (size_t i = 0; i < sizeof(erb_open_patterns) / sizeof(erb_open_patterns[0]); i++) {
+    token_T* match = lexer_match_and_advance(lexer, erb_open_patterns[i], TOKEN_ERB_START);
     if (match) { return match; }
   }
 
@@ -203,11 +206,17 @@ static token_T* lexer_parse_erb_content(lexer_T* lexer) {
 
   while (!lexer_peek_erb_end(lexer, 0)) {
     if (lexer_eof(lexer)) {
-      token_T* token = token_init(
-        hb_string_range(lexer->source, start_position, lexer->current_position),
-        TOKEN_ERROR,
-        lexer
-      ); // Handle unexpected EOF
+      token_T* token =
+        token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_ERB_CONTENT, lexer);
+
+      return token;
+    }
+
+    if (lexer_peek_erb_start(lexer, 0)) {
+      lexer->state = STATE_DATA;
+
+      token_T* token =
+        token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_ERB_CONTENT, lexer);
 
       return token;
     }
@@ -244,7 +253,7 @@ static token_T* lexer_parse_erb_close(lexer_T* lexer) {
 // ===== Tokenizing Function
 
 token_T* lexer_next_token(lexer_T* lexer) {
-  if (lexer_eof(lexer)) { return token_init(hb_string(""), TOKEN_EOF, lexer); }
+  if (lexer_eof(lexer)) { return token_init(HB_STRING_EMPTY, TOKEN_EOF, lexer); }
   if (lexer_stalled(lexer)) { return lexer_error(lexer, "Lexer stalled after 5 iterations"); }
 
   if (lexer->state == STATE_ERB_CONTENT) { return lexer_parse_erb_content(lexer); }
@@ -302,7 +311,10 @@ token_T* lexer_next_token(lexer_T* lexer) {
     }
 
     case '-': {
-      token_T* token = lexer_match_and_advance(lexer, hb_string("-->"), TOKEN_HTML_COMMENT_END);
+      token_T* token = lexer_match_and_advance(lexer, hb_string("--!>"), TOKEN_HTML_COMMENT_INVALID_END);
+      if (token) { return token; }
+
+      token = lexer_match_and_advance(lexer, hb_string("-->"), TOKEN_HTML_COMMENT_END);
       return token ? token : lexer_advance_current(lexer, TOKEN_DASH);
     }