@herb-tools/node 0.7.4 → 0.8.0

package/extension/libherb/token.c

@@ -1,37 +1,35 @@
 #include "include/token.h"
-#include "include/json.h"
 #include "include/lexer.h"
 #include "include/position.h"
+#include "include/range.h"
 #include "include/token_struct.h"
 #include "include/util.h"
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-size_t token_sizeof(void) {
-  return sizeof(struct TOKEN_STRUCT);
-}
-
-token_T* token_init(const char* value, const token_type_T type, lexer_T* lexer) {
-  token_T* token = calloc(1, token_sizeof());
+token_T* token_init(hb_string_T value, const token_type_T type, lexer_T* lexer) {
+  token_T* token = calloc(1, sizeof(token_T));
 
   if (type == TOKEN_NEWLINE) {
     lexer->current_line++;
     lexer->current_column = 0;
   }
 
-  if (value) {
-    token->value = herb_strdup(value);
-  } else {
-    token->value = NULL;
-  }
+  token->value = hb_string_to_c_string_using_malloc(value);
 
   token->type = type;
-  token->range = range_init(lexer->previous_position, lexer->current_position);
-
-  token->location =
-    location_from(lexer->previous_line, lexer->previous_column, lexer->current_line, lexer->current_column);
+  token->range = (range_T) { .from = lexer->previous_position, .to = lexer->current_position };
+
+  location_from(
+    &token->location,
+    lexer->previous_line,
+    lexer->previous_column,
+    lexer->current_line,
+    lexer->current_column
+  );
 
   lexer->previous_line = lexer->current_line;
   lexer->previous_column = lexer->current_column;
@@ -82,93 +80,42 @@ const char* token_type_to_string(const token_type_T type) {
   return "Unknown token_type_T";
 }
 
-char* token_to_string(const token_T* token) {
+hb_string_T token_to_string(const token_T* token) {
   const char* type_string = token_type_to_string(token->type);
-  const char* template = "#<Herb::Token type=\"%s\" value=\"%s\" range=[%d, %d] start=(%d:%d) end=(%d:%d)>";
+  const char* template = "#<Herb::Token type=\"%s\" value=\"%.*s\" range=[%u, %u] start=(%u:%u) end=(%u:%u)>";
 
   char* string = calloc(strlen(type_string) + strlen(template) + strlen(token->value) + 16, sizeof(char));
-  char* escaped;
+  hb_string_T escaped;
 
   if (token->type == TOKEN_EOF) {
-    escaped = herb_strdup("<EOF>");
+    escaped = hb_string(herb_strdup("<EOF>"));
   } else {
-    escaped = escape_newlines(token->value);
+    escaped = escape_newlines(hb_string(token->value));
   }
 
   sprintf(
     string,
     template,
     type_string,
-    escaped,
-    token->range->from,
-    token->range->to,
-    token->location->start->line,
-    token->location->start->column,
-    token->location->end->line,
-    token->location->end->column
+    escaped.length,
+    escaped.data,
+    token->range.from,
+    token->range.to,
+    token->location.start.line,
+    token->location.start.column,
+    token->location.end.line,
+    token->location.end.column
   );
 
-  free(escaped);
-
-  return string;
-}
-
-char* token_to_json(const token_T* token) {
-  buffer_T json = buffer_new();
-
-  json_start_root_object(&json);
-  json_add_string(&json, "type", token_type_to_string(token->type));
-  json_add_string(&json, "value", token->value);
-
-  buffer_T range = buffer_new();
-  json_start_array(&json, "range");
-  json_add_size_t(&range, NULL, token->range->from);
-  json_add_size_t(&range, NULL, token->range->to);
-  buffer_concat(&json, &range);
-  buffer_free(&range);
-  json_end_array(&json);
-
-  buffer_T start = buffer_new();
-  json_start_object(&json, "start");
-  json_add_size_t(&start, "line", token->location->start->line);
-  json_add_size_t(&start, "column", token->location->start->column);
-  buffer_concat(&json, &start);
-  buffer_free(&start);
-  json_end_object(&json);
-
-  buffer_T end = buffer_new();
-  json_start_object(&json, "end");
-  json_add_size_t(&end, "line", token->location->end->line);
-  json_add_size_t(&end, "column", token->location->end->column);
-  buffer_concat(&json, &end);
-  buffer_free(&end);
-  json_end_object(&json);
-
-  json_end_object(&json);
-
-  return buffer_value(&json);
-}
-
-char* token_value(const token_T* token) {
-  return token->value;
-}
-
-int token_type(const token_T* token) {
-  return token->type;
-}
+  free(escaped.data);
 
-position_T* token_start_position(token_T* token) {
-  return token->location->start;
-}
-
-position_T* token_end_position(token_T* token) {
-  return token->location->end;
+  return hb_string(string);
 }
 
 token_T* token_copy(token_T* token) {
   if (!token) { return NULL; }
 
-  token_T* new_token = calloc(1, token_sizeof());
+  token_T* new_token = calloc(1, sizeof(token_T));
 
   if (!new_token) { return NULL; }
 
@@ -184,18 +131,20 @@ token_T* token_copy(token_T* token) {
   }
 
   new_token->type = token->type;
-  new_token->range = range_copy(token->range);
-  new_token->location = location_copy(token->location);
+  new_token->range = token->range;
+  new_token->location = token->location;
 
   return new_token;
 }
 
+bool token_value_empty(const token_T* token) {
+  return token == NULL || token->value == NULL || token->value[0] == '\0';
+}
+
 void token_free(token_T* token) {
   if (!token) { return; }
 
   if (token->value != NULL) { free(token->value); }
-  if (token->range != NULL) { range_free(token->range); }
-  if (token->location != NULL) { location_free(token->location); }
 
   free(token);
 }

package/extension/libherb/token.h

@@ -4,22 +4,16 @@
 #include "lexer_struct.h"
 #include "position.h"
 #include "token_struct.h"
+#include "util/hb_string.h"
 
-token_T* token_init(const char* value, token_type_T type, lexer_T* lexer);
-char* token_to_string(const token_T* token);
-char* token_to_json(const token_T* token);
+token_T* token_init(hb_string_T value, token_type_T type, lexer_T* lexer);
+hb_string_T token_to_string(const token_T* token);
 const char* token_type_to_string(token_type_T type);
 
-char* token_value(const token_T* token);
-int token_type(const token_T* token);
-
-position_T* token_start_position(token_T* token);
-position_T* token_end_position(token_T* token);
-
-size_t token_sizeof(void);
-
 token_T* token_copy(token_T* token);
 
 void token_free(token_T* token);
 
+bool token_value_empty(const token_T* token);
+
 #endif

package/extension/libherb/token_struct.h

@@ -50,8 +50,8 @@ typedef enum {
 
 typedef struct TOKEN_STRUCT {
   char* value;
-  range_T* range;
-  location_T* location;
+  range_T range;
+  location_T location;
   token_type_T type;
 } token_T;
 

package/extension/libherb/utf8.c

@@ -5,7 +5,7 @@
 //   110xxxxx = 2 bytes
 //   1110xxxx = 3 bytes
 //   11110xxx = 4 bytes
-int utf8_char_byte_length(unsigned char first_byte) {
+uint32_t utf8_char_byte_length(unsigned char first_byte) {
   if ((first_byte & 0x80) == 0) {
     return 1;
   } else if ((first_byte & 0xE0) == 0xC0) {
@@ -24,18 +24,18 @@ bool utf8_is_valid_continuation_byte(unsigned char byte) {
   return (byte & 0xC0) == 0x80;
 }
 
-int utf8_sequence_length(const char* str, size_t position, size_t max_length) {
+uint32_t utf8_sequence_length(const char* str, size_t position, size_t max_length) {
   if (position >= max_length) { return 0; }
 
   unsigned char first_byte = (unsigned char) str[position];
-  int expected_length = utf8_char_byte_length(first_byte);
+  uint32_t expected_length = utf8_char_byte_length(first_byte);
 
   if (position + expected_length > max_length) {
     return 1; // Not enough bytes, treat as single byte
   }
 
   if (expected_length > 1) {
-    for (int i = 1; i < expected_length; i++) {
+    for (uint32_t i = 1; i < expected_length; i++) {
       if (!utf8_is_valid_continuation_byte((unsigned char) str[position + i])) {
         return 1; // Invalid continuation byte, treat first byte as single byte
       }

package/extension/libherb/utf8.h

@@ -2,10 +2,11 @@
 #define HERB_UTF8_H
 
 #include <stdbool.h>
+#include <stdint.h>
 #include <stdlib.h>
 
-int utf8_char_byte_length(unsigned char first_byte);
-int utf8_sequence_length(const char* str, size_t position, size_t max_length);
+uint32_t utf8_char_byte_length(unsigned char first_byte);
+uint32_t utf8_sequence_length(const char* str, size_t position, size_t max_length);
 bool utf8_is_valid_continuation_byte(unsigned char byte);
 
 #endif

package/extension/libherb/util/hb_arena.c

@@ -0,0 +1,179 @@
+#include "../include/util/hb_arena.h"
+#include "../include/macros.h"
+#include "../include/util/hb_system.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+
+#define hb_arena_for_each_page(allocator, page)                                                                        \
+  for (hb_arena_page_T* page = (allocator)->head; page != NULL; page = page->next)
+
+static inline size_t hb_arena_align_size(size_t size, size_t alignment) {
+  assert(size <= SIZE_MAX - (alignment - 1));
+
+  return ((size + (alignment - 1)) / alignment) * alignment;
+}
+
+static inline bool hb_arena_page_has_capacity(hb_arena_page_T* page, size_t required_size) {
+  assert(page->position <= page->capacity);
+
+  return page->position + required_size <= page->capacity;
+}
+
+static inline void* hb_arena_page_alloc_from(hb_arena_page_T* page, size_t size) {
+  assert(size > 0);
+  assert(page->position + size <= page->capacity);
+
+  void* result = &page->memory[page->position];
+  page->position += size;
+
+  return result;
+}
+
+static inline void hb_arena_page_reset(hb_arena_page_T* page) {
+  page->position = 0;
+}
+
+static inline void hb_arena_reset_pages_after(hb_arena_page_T* start_page) {
+  for (hb_arena_page_T* page = start_page; page != NULL; page = page->next) {
+    hb_arena_page_reset(page);
+  }
+}
+
+static bool hb_arena_append_page(hb_arena_T* allocator, size_t minimum_size) {
+  assert(minimum_size > 0);
+
+  size_t page_size = MAX(allocator->default_page_size, minimum_size);
+
+  assert(page_size <= SIZE_MAX - sizeof(hb_arena_page_T));
+  size_t total_size = page_size + sizeof(hb_arena_page_T);
+
+  hb_arena_page_T* page = hb_system_allocate_memory(total_size);
+  if (page == NULL) { return false; }
+
+  *page = (hb_arena_page_T) { .next = NULL, .capacity = page_size, .position = 0 };
+
+  if (allocator->head == NULL) {
+    allocator->head = page;
+    allocator->tail = page;
+  } else {
+    hb_arena_page_T* last = allocator->head;
+
+    while (last->next != NULL) {
+      last = last->next;
+    }
+
+    last->next = page;
+    allocator->tail = page;
+  }
+
+  return true;
+}
+
+bool hb_arena_init(hb_arena_T* allocator, size_t initial_size) {
+  assert(initial_size > 0);
+
+  allocator->head = NULL;
+  allocator->tail = NULL;
+  allocator->default_page_size = initial_size;
+  allocator->allocation_count = 0;
+
+  return hb_arena_append_page(allocator, initial_size);
+}
+
+void* hb_arena_alloc(hb_arena_T* allocator, size_t size) {
+  assert(allocator->tail != NULL);
+  assert(size > 0);
+
+  size_t required_size = hb_arena_align_size(size, 8);
+
+  allocator->allocation_count++;
+
+  if (hb_arena_page_has_capacity(allocator->tail, required_size)) {
+    return hb_arena_page_alloc_from(allocator->tail, required_size);
+  }
+
+  for (hb_arena_page_T* page = allocator->tail->next; page != NULL; page = page->next) {
+    if (hb_arena_page_has_capacity(page, required_size)) {
+      allocator->tail = page;
+      return hb_arena_page_alloc_from(allocator->tail, required_size);
+    }
+  }
+
+  bool allocated = hb_arena_append_page(allocator, required_size);
+
+  if (!allocated) { return NULL; }
+
+  return hb_arena_page_alloc_from(allocator->tail, required_size);
+}
+
+size_t hb_arena_position(hb_arena_T* allocator) {
+  size_t total = 0;
+
+  hb_arena_for_each_page(allocator, page) {
+    total += page->position;
+  }
+
+  return total;
+}
+
+size_t hb_arena_capacity(hb_arena_T* allocator) {
+  size_t total = 0;
+
+  hb_arena_for_each_page(allocator, page) {
+    total += page->capacity;
+  }
+
+  return total;
+}
+
+void hb_arena_reset(hb_arena_T* allocator) {
+  hb_arena_for_each_page(allocator, page) {
+    hb_arena_page_reset(page);
+  }
+
+  allocator->tail = allocator->head;
+  allocator->allocation_count = 0;
+}
+
+void hb_arena_reset_to(hb_arena_T* allocator, size_t target_position) {
+  if (target_position == 0) {
+    hb_arena_reset(allocator);
+
+    return;
+  }
+
+  size_t accumulated = 0;
+
+  hb_arena_for_each_page(allocator, page) {
+    if (accumulated + page->capacity >= target_position) {
+      page->position = target_position - accumulated;
+      allocator->tail = page;
+
+      hb_arena_reset_pages_after(page->next);
+
+      return;
+    }
+
+    accumulated += page->capacity;
+    page->position = page->capacity;
+  }
+}
+
+void hb_arena_free(hb_arena_T* allocator) {
+  if (allocator->head == NULL) { return; }
+
+  for (hb_arena_page_T* current = allocator->head; current != NULL;) {
+    hb_arena_page_T* next = current->next;
+    size_t total_size = sizeof(hb_arena_page_T) + current->capacity;
+    hb_system_free_memory(current, total_size);
+
+    current = next;
+  }
+
+  allocator->head = NULL;
+  allocator->tail = NULL;
+  allocator->default_page_size = 0;
+}

package/extension/libherb/util/hb_arena.h

@@ -0,0 +1,31 @@
+#ifndef HERB_ARENA_H
+#define HERB_ARENA_H
+
+#include <stdbool.h>
+#include <stddef.h>
+
+typedef struct HB_ARENA_PAGE_STRUCT hb_arena_page_T;
+
+struct HB_ARENA_PAGE_STRUCT {
+  hb_arena_page_T* next;
+  size_t capacity;
+  size_t position;
+  char memory[];
+};
+
+typedef struct HB_ARENA_STRUCT {
+  hb_arena_page_T* head;
+  hb_arena_page_T* tail;
+  size_t default_page_size;
+  size_t allocation_count;
+} hb_arena_T;
+
+bool hb_arena_init(hb_arena_T* allocator, size_t initial_size);
+void* hb_arena_alloc(hb_arena_T* allocator, size_t size);
+size_t hb_arena_position(hb_arena_T* allocator);
+size_t hb_arena_capacity(hb_arena_T* allocator);
+void hb_arena_reset(hb_arena_T* allocator);
+void hb_arena_reset_to(hb_arena_T* allocator, size_t new_position);
+void hb_arena_free(hb_arena_T* allocator);
+
+#endif