herb 0.7.5 → 0.8.0

data/src/pretty_print.c

@@ -3,146 +3,149 @@
 #include "include/ast_node.h"
 #include "include/ast_nodes.h"
 #include "include/ast_pretty_print.h"
-#include "include/buffer.h"
 #include "include/errors.h"
 #include "include/token_struct.h"
 #include "include/util.h"
+#include "include/util/hb_buffer.h"
+#include "include/util/hb_string.h"
 
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 
-void pretty_print_indent(buffer_T* buffer, const size_t indent) {
+void pretty_print_indent(hb_buffer_T* buffer, const size_t indent) {
   for (size_t i = 0; i < indent; i++) {
-    buffer_append(buffer, "    ");
+    hb_buffer_append(buffer, "    ");
   }
 }
 
-void pretty_print_newline(const size_t indent, const size_t relative_indent, buffer_T* buffer) {
+void pretty_print_newline(const size_t indent, const size_t relative_indent, hb_buffer_T* buffer) {
   pretty_print_indent(buffer, indent);
   pretty_print_indent(buffer, relative_indent);
-  buffer_append(buffer, "\n");
+  hb_buffer_append(buffer, "\n");
 }
 
 void pretty_print_label(
-  const char* name,
+  hb_string_T name,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
   pretty_print_indent(buffer, indent);
   pretty_print_indent(buffer, relative_indent);
 
   if (last_property) {
-    buffer_append(buffer, "└── ");
+    hb_buffer_append(buffer, "└── ");
   } else {
-    buffer_append(buffer, "├── ");
+    hb_buffer_append(buffer, "├── ");
   }
 
-  buffer_append(buffer, name);
-  buffer_append(buffer, ": ");
+  hb_buffer_append_string(buffer, name);
+  hb_buffer_append(buffer, ": ");
 }
 
 void pretty_print_quoted_property(
-  const char* name,
-  const char* value,
+  hb_string_T name,
+  hb_string_T value,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
-  char* quoted = quoted_string(value);
+  hb_string_T quoted = quoted_string(value);
   pretty_print_property(name, quoted, indent, relative_indent, last_property, buffer);
-  free(quoted);
+  free(quoted.data);
 }
 
 void pretty_print_boolean_property(
-  const char* name,
+  hb_string_T name,
   bool value,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
-  pretty_print_property(name, value ? "true" : "false", indent, relative_indent, last_property, buffer);
+  pretty_print_property(name, hb_string(value ? "true" : "false"), indent, relative_indent, last_property, buffer);
 }
 
 void pretty_print_property(
-  const char* name,
-  const char* value,
+  hb_string_T name,
+  hb_string_T value,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
   pretty_print_label(name, indent, relative_indent, last_property, buffer);
-  buffer_append(buffer, value);
-  buffer_append(buffer, "\n");
+  hb_buffer_append_string(buffer, value);
+  hb_buffer_append(buffer, "\n");
 }
 
 void pretty_print_size_t_property(
   size_t value,
-  const char* name,
+  hb_string_T name,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
   pretty_print_label(name, indent, relative_indent, last_property, buffer);
-  char* string = size_t_to_string(value);
-  buffer_append(buffer, string);
-  buffer_append(buffer, "\n");
-  free(string);
+
+  char size_string[21];
+  snprintf(size_string, 21, "%zu", value);
+
+  hb_buffer_append(buffer, size_string);
+  hb_buffer_append(buffer, "\n");
 }
 
 void pretty_print_array(
-  const char* name,
-  array_T* array,
+  hb_string_T name,
+  hb_array_T* array,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
   if (array == NULL) {
-    pretty_print_property(name, "∅", indent, relative_indent, last_property, buffer);
+    pretty_print_property(name, hb_string("∅"), indent, relative_indent, last_property, buffer);
 
     return;
   }
 
-  if (array_size(array) == 0) {
-    pretty_print_property(name, "[]", indent, relative_indent, last_property, buffer);
+  if (hb_array_size(array) == 0) {
+    pretty_print_property(name, hb_string("[]"), indent, relative_indent, last_property, buffer);
 
     return;
   }
 
   pretty_print_label(name, indent, relative_indent, last_property, buffer);
 
-  buffer_append(buffer, "(");
+  hb_buffer_append(buffer, "(");
 
   char count[16];
-  sprintf(count, "%zu", array_size(array));
-  buffer_append(buffer, count);
-  buffer_append(buffer, ")\n");
+  sprintf(count, "%zu", hb_array_size(array));
+  hb_buffer_append(buffer, count);
+  hb_buffer_append(buffer, ")\n");
 
   if (indent < 20) {
-    for (size_t i = 0; i < array_size(array); i++) {
-      AST_NODE_T* child = array_get(array, i);
+    for (size_t i = 0; i < hb_array_size(array); i++) {
+      AST_NODE_T* child = hb_array_get(array, i);
       pretty_print_indent(buffer, indent);
       pretty_print_indent(buffer, relative_indent + 1);
 
-      if (i == array_size(array) - 1) {
-        buffer_append(buffer, "└── ");
+      if (i == hb_array_size(array) - 1) {
+        hb_buffer_append(buffer, "└── ");
       } else {
-        buffer_append(buffer, "├── ");
+        hb_buffer_append(buffer, "├── ");
       }
 
       ast_pretty_print_node(child, indent + 1, relative_indent + 1, buffer);
 
-      if (i != array_size(array) - 1) { pretty_print_newline(indent + 1, relative_indent, buffer); }
+      if (i != hb_array_size(array) - 1) { pretty_print_newline(indent + 1, relative_indent, buffer); }
     }
   }
-  buffer_append(buffer, "\n");
+  hb_buffer_append(buffer, "\n");
 }
 
 void pretty_print_errors(
@@ -150,16 +153,16 @@ void pretty_print_errors(
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
-  if (node->errors != NULL && array_size(node->errors) > 0) {
+  if (node->errors != NULL && hb_array_size(node->errors) > 0) {
     error_pretty_print_array("errors", node->errors, indent, relative_indent, last_property, buffer);
-    buffer_append(buffer, "\n");
+    hb_buffer_append(buffer, "\n");
   }
 }
 
-void pretty_print_location(location_T location, buffer_T* buffer) {
-  buffer_append(buffer, "(location: (");
+void pretty_print_location(location_T location, hb_buffer_T* buffer) {
+  hb_buffer_append(buffer, "(location: (");
   char location_string[128];
   sprintf(
     location_string,
@@ -169,73 +172,73 @@ void pretty_print_location(location_T location, buffer_T* buffer) {
     location.end.line,
     location.end.column
   );
-  buffer_append(buffer, location_string);
-  buffer_append(buffer, "))");
+  hb_buffer_append(buffer, location_string);
+  hb_buffer_append(buffer, "))");
 }
 
 void pretty_print_position_property(
   position_T* position,
-  const char* name,
+  hb_string_T name,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
   pretty_print_label(name, indent, relative_indent, last_property, buffer);
 
   if (position != NULL) {
-    buffer_append(buffer, "(");
+    hb_buffer_append(buffer, "(");
 
     char position_string[128];
 
     sprintf(position_string, "%u:%u", (position->line) ? position->line : 0, (position->column) ? position->column : 0);
 
-    buffer_append(buffer, position_string);
-    buffer_append(buffer, ")");
+    hb_buffer_append(buffer, position_string);
+    hb_buffer_append(buffer, ")");
   } else {
-    buffer_append(buffer, "∅");
+    hb_buffer_append(buffer, "∅");
   }
 
-  buffer_append(buffer, "\n");
+  hb_buffer_append(buffer, "\n");
 }
 
 void pretty_print_token_property(
   token_T* token,
-  const char* name,
+  hb_string_T name,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
   pretty_print_label(name, indent, relative_indent, last_property, buffer);
 
   if (token != NULL && token->value != NULL) {
-    char* quoted = quoted_string(token->value);
-    buffer_append(buffer, quoted);
-    free(quoted);
+    hb_string_T quoted = quoted_string(hb_string(token->value));
+    hb_buffer_append_string(buffer, quoted);
+    free(quoted.data);
 
-    buffer_append(buffer, " ");
+    hb_buffer_append(buffer, " ");
     pretty_print_location(token->location, buffer);
   } else {
-    buffer_append(buffer, "∅");
+    hb_buffer_append(buffer, "∅");
   }
 
-  buffer_append(buffer, "\n");
+  hb_buffer_append(buffer, "\n");
 }
 
 void pretty_print_string_property(
-  const char* string,
-  const char* name,
+  hb_string_T string,
+  hb_string_T name,
   const size_t indent,
   const size_t relative_indent,
   const bool last_property,
-  buffer_T* buffer
+  hb_buffer_T* buffer
 ) {
-  const char* value = "∅";
-  char* escaped = NULL;
-  char* quoted = NULL;
+  hb_string_T value = hb_string("∅");
+  hb_string_T escaped = { .data = NULL, .length = 0 };
+  hb_string_T quoted;
 
-  if (string != NULL) {
+  if (!hb_string_is_empty(string)) {
     escaped = escape_newlines(string);
     quoted = quoted_string(escaped);
     value = quoted;
@@ -243,35 +246,8 @@ void pretty_print_string_property(
 
   pretty_print_property(name, value, indent, relative_indent, last_property, buffer);
 
-  if (string != NULL) {
-    if (escaped != NULL) { free(escaped); }
-    if (quoted != NULL) { free(quoted); }
+  if (!hb_string_is_empty(string)) {
+    if (!hb_string_is_empty(escaped)) { free(escaped.data); }
+    if (!hb_string_is_empty(quoted)) { free(quoted.data); }
   }
 }
-
-void pretty_print_analyzed_ruby(analyzed_ruby_T* analyzed, const char* source) {
-  printf(
-    "------------------------\nanalyzed (%p)\n------------------------\n%s\n------------------------\n  if:     %i\n "
-    " elsif:  %i\n  else:   %i\n  end:    %i\n  block:  %i\n  block_closing: %i\n  case:   %i\n  when:   %i\n  for:    "
-    "%i\n  while:  %i\n "
-    " until:  %i\n  begin:  %i\n  "
-    "rescue: %i\n  ensure: %i\n  unless: %i\n==================\n\n",
-    (void*) analyzed,
-    source,
-    analyzed->has_if_node,
-    analyzed->has_elsif_node,
-    analyzed->has_else_node,
-    analyzed->has_end,
-    analyzed->has_block_node,
-    analyzed->has_block_closing,
-    analyzed->has_case_node,
-    analyzed->has_when_node,
-    analyzed->has_for_node,
-    analyzed->has_while_node,
-    analyzed->has_until_node,
-    analyzed->has_begin_node,
-    analyzed->has_rescue_node,
-    analyzed->has_ensure_node,
-    analyzed->has_unless_node
-  );
-}

data/src/token.c

@@ -1,32 +1,24 @@
 #include "include/token.h"
-#include "include/json.h"
 #include "include/lexer.h"
 #include "include/position.h"
 #include "include/range.h"
 #include "include/token_struct.h"
 #include "include/util.h"
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-size_t token_sizeof(void) {
-  return sizeof(struct TOKEN_STRUCT);
-}
-
-token_T* token_init(const char* value, const token_type_T type, lexer_T* lexer) {
-  token_T* token = calloc(1, token_sizeof());
+token_T* token_init(hb_string_T value, const token_type_T type, lexer_T* lexer) {
+  token_T* token = calloc(1, sizeof(token_T));
 
   if (type == TOKEN_NEWLINE) {
     lexer->current_line++;
     lexer->current_column = 0;
   }
 
-  if (value) {
-    token->value = herb_strdup(value);
-  } else {
-    token->value = NULL;
-  }
+  token->value = hb_string_to_c_string_using_malloc(value);
 
   token->type = type;
   token->range = (range_T) { .from = lexer->previous_position, .to = lexer->current_position };
@@ -88,24 +80,25 @@ const char* token_type_to_string(const token_type_T type) {
   return "Unknown token_type_T";
 }
 
-char* token_to_string(const token_T* token) {
+hb_string_T token_to_string(const token_T* token) {
   const char* type_string = token_type_to_string(token->type);
-  const char* template = "#<Herb::Token type=\"%s\" value=\"%s\" range=[%u, %u] start=(%u:%u) end=(%u:%u)>";
+  const char* template = "#<Herb::Token type=\"%s\" value=\"%.*s\" range=[%u, %u] start=(%u:%u) end=(%u:%u)>";
 
   char* string = calloc(strlen(type_string) + strlen(template) + strlen(token->value) + 16, sizeof(char));
-  char* escaped;
+  hb_string_T escaped;
 
   if (token->type == TOKEN_EOF) {
-    escaped = herb_strdup("<EOF>");
+    escaped = hb_string(herb_strdup("<EOF>"));
   } else {
-    escaped = escape_newlines(token->value);
+    escaped = escape_newlines(hb_string(token->value));
   }
 
   sprintf(
     string,
     template,
     type_string,
-    escaped,
+    escaped.length,
+    escaped.data,
     token->range.from,
     token->range.to,
     token->location.start.line,
@@ -114,59 +107,15 @@ char* token_to_string(const token_T* token) {
     token->location.end.column
   );
 
-  free(escaped);
-
-  return string;
-}
-
-char* token_to_json(const token_T* token) {
-  buffer_T json = buffer_new();
-
-  json_start_root_object(&json);
-  json_add_string(&json, "type", token_type_to_string(token->type));
-  json_add_string(&json, "value", token->value);
-
-  buffer_T range = buffer_new();
-  json_start_array(&json, "range");
-  json_add_size_t(&range, NULL, token->range.from);
-  json_add_size_t(&range, NULL, token->range.to);
-  buffer_concat(&json, &range);
-  buffer_free(&range);
-  json_end_array(&json);
-
-  buffer_T start = buffer_new();
-  json_start_object(&json, "start");
-  json_add_size_t(&start, "line", token->location.start.line);
-  json_add_size_t(&start, "column", token->location.start.column);
-  buffer_concat(&json, &start);
-  buffer_free(&start);
-  json_end_object(&json);
-
-  buffer_T end = buffer_new();
-  json_start_object(&json, "end");
-  json_add_size_t(&end, "line", token->location.end.line);
-  json_add_size_t(&end, "column", token->location.end.column);
-  buffer_concat(&json, &end);
-  buffer_free(&end);
-  json_end_object(&json);
-
-  json_end_object(&json);
-
-  return buffer_value(&json);
-}
-
-char* token_value(const token_T* token) {
-  return token->value;
-}
+  free(escaped.data);
 
-int token_type(const token_T* token) {
-  return token->type;
+  return hb_string(string);
 }
 
 token_T* token_copy(token_T* token) {
   if (!token) { return NULL; }
 
-  token_T* new_token = calloc(1, token_sizeof());
+  token_T* new_token = calloc(1, sizeof(token_T));
 
   if (!new_token) { return NULL; }
 
@@ -188,6 +137,10 @@ token_T* token_copy(token_T* token) {
   return new_token;
 }
 
+bool token_value_empty(const token_T* token) {
+  return token == NULL || token->value == NULL || token->value[0] == '\0';
+}
+
 void token_free(token_T* token) {
   if (!token) { return; }
 

data/src/utf8.c

@@ -5,7 +5,7 @@
 //   110xxxxx = 2 bytes
 //   1110xxxx = 3 bytes
 //   11110xxx = 4 bytes
-int utf8_char_byte_length(unsigned char first_byte) {
+uint32_t utf8_char_byte_length(unsigned char first_byte) {
   if ((first_byte & 0x80) == 0) {
     return 1;
   } else if ((first_byte & 0xE0) == 0xC0) {
@@ -24,18 +24,18 @@ bool utf8_is_valid_continuation_byte(unsigned char byte) {
   return (byte & 0xC0) == 0x80;
 }
 
-int utf8_sequence_length(const char* str, size_t position, size_t max_length) {
+uint32_t utf8_sequence_length(const char* str, size_t position, size_t max_length) {
   if (position >= max_length) { return 0; }
 
   unsigned char first_byte = (unsigned char) str[position];
-  int expected_length = utf8_char_byte_length(first_byte);
+  uint32_t expected_length = utf8_char_byte_length(first_byte);
 
   if (position + expected_length > max_length) {
     return 1; // Not enough bytes, treat as single byte
   }
 
   if (expected_length > 1) {
-    for (int i = 1; i < expected_length; i++) {
+    for (uint32_t i = 1; i < expected_length; i++) {
       if (!utf8_is_valid_continuation_byte((unsigned char) str[position + i])) {
         return 1; // Invalid continuation byte, treat first byte as single byte
       }