herb 0.8.10-arm-linux-gnu → 0.9.0-arm-linux-gnu

data/src/lexer_peek_helpers.c

@@ -1,22 +1,10 @@
 #include "include/lexer_peek_helpers.h"
 #include "include/lexer.h"
-#include "include/lexer_struct.h"
-#include "include/macros.h"
 #include "include/token.h"
-#include "include/util/hb_string.h"
 
 #include <ctype.h>
-#include <stdbool.h>
 
-char lexer_backtrack(const lexer_T* lexer, uint32_t offset) {
-  return lexer->source.data[MAX(lexer->current_position - offset, 0)];
-}
-
-char lexer_peek(const lexer_T* lexer, uint32_t offset) {
-  return lexer->source.data[MIN(lexer->current_position + offset, lexer->source.length)];
-}
-
-bool lexer_peek_for(const lexer_T* lexer, uint32_t offset, hb_string_T pattern, const bool case_insensitive) {
+static bool lexer_peek_for(const lexer_T* lexer, uint32_t offset, hb_string_T pattern, bool case_insensitive) {
   hb_string_T remaining_source = hb_string_slice(lexer->source, lexer->current_position + offset);
   remaining_source.length = MIN(pattern.length, remaining_source.length);
 
@@ -47,31 +35,19 @@ bool lexer_peek_for_html_comment_start(const lexer_T* lexer, uint32_t offset) {
   return lexer_peek_for(lexer, offset, hb_string("<!--"), false);
 }
 
-bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("-->"), false);
-}
-
-bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("%>"), false);
-}
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset) {
+  if (lexer_peek(lexer, offset) != '<' || lexer_peek(lexer, offset + 1) != '/') { return false; }
 
-bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("-%>"), false);
-}
+  uint32_t position = offset + 2;
 
-bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("%%>"), false);
-}
+  while (lexer_peek(lexer, position) == ' ' || lexer_peek(lexer, position) == '\t'
+         || lexer_peek(lexer, position) == '\n' || lexer_peek(lexer, position) == '\r') {
+    position++;
+  }
 
-bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("=%>"), false);
-}
+  char character = lexer_peek(lexer, position);
 
-bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset) {
-  return (
-    lexer_peek_erb_close_tag(lexer, offset) || lexer_peek_erb_dash_close_tag(lexer, offset)
-    || lexer_peek_erb_percent_close_tag(lexer, offset) || lexer_peek_erb_equals_close_tag(lexer, offset)
-  );
+  return isalpha(character) || character == '_';
 }
 
 bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type) {
@@ -84,13 +60,13 @@ bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T tok
   token_T* token = lexer_next_token(lexer);
 
   while (token && (token->type == TOKEN_WHITESPACE || token->type == TOKEN_NEWLINE)) {
-    token_free(token);
+    token_free(token, lexer->allocator);
     token = lexer_next_token(lexer);
   }
 
   bool result = (token && token->type == token_type);
 
-  if (token) { token_free(token); }
+  if (token) { token_free(token, lexer->allocator); }
 
   lexer->current_position = saved_position;
   lexer->current_line = saved_line;
@@ -100,41 +76,3 @@ bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T tok
 
   return result;
 }
-
-bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset) {
-  if (lexer_peek(lexer, offset) != '<' || lexer_peek(lexer, offset + 1) != '/') { return false; }
-
-  uint32_t pos = offset + 2;
-
-  while (lexer_peek(lexer, pos) == ' ' || lexer_peek(lexer, pos) == '\t' || lexer_peek(lexer, pos) == '\n'
-         || lexer_peek(lexer, pos) == '\r') {
-    pos++;
-  }
-
-  char c = lexer_peek(lexer, pos);
-
-  return isalpha(c) || c == '_';
-}
-
-lexer_state_snapshot_T lexer_save_state(lexer_T* lexer) {
-  lexer_state_snapshot_T snapshot = { .position = lexer->current_position,
-                                      .line = lexer->current_line,
-                                      .column = lexer->current_column,
-                                      .previous_position = lexer->previous_position,
-                                      .previous_line = lexer->previous_line,
-                                      .previous_column = lexer->previous_column,
-                                      .current_character = lexer->current_character,
-                                      .state = lexer->state };
-  return snapshot;
-}
-
-void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot) {
-  lexer->current_position = snapshot.position;
-  lexer->current_line = snapshot.line;
-  lexer->current_column = snapshot.column;
-  lexer->previous_position = snapshot.previous_position;
-  lexer->previous_line = snapshot.previous_line;
-  lexer->previous_column = snapshot.previous_column;
-  lexer->current_character = snapshot.current_character;
-  lexer->state = snapshot.state;
-}

data/src/location.c

@@ -17,8 +17,8 @@ void location_from_positions(location_T* location, position_T start, position_T
   location->end = end;
 }
 
-location_T* location_create(position_T start, position_T end) {
-  location_T* location = malloc(sizeof(location_T));
+location_T* location_create(position_T start, position_T end, hb_allocator_T* allocator) {
+  location_T* location = hb_allocator_alloc(allocator, sizeof(location_T));
 
   if (location != NULL) {
     location->start = start;

data/src/main.c

@@ -1,13 +1,21 @@
 #define _POSIX_C_SOURCE 199309L // Enables `clock_gettime()`
 
-#include "include/analyze.h"
 #include "include/ast_node.h"
 #include "include/ast_nodes.h"
-#include "include/ast_pretty_print.h"
+
+#ifndef HERB_EXCLUDE_PRETTYPRINT
+#  include "include/ast_pretty_print.h"
+#endif
+
 #include "include/extract.h"
 #include "include/herb.h"
 #include "include/io.h"
+#include "include/lex_helpers.h"
+#include "include/macros.h"
 #include "include/ruby_parser.h"
+#include "include/util/hb_allocator.h"
+#include "include/util/hb_arena.h"
+#include "include/util/hb_arena_debug.h"
 #include "include/util/hb_buffer.h"
 #include "include/util/string.h"
 
@@ -38,7 +46,7 @@ int main(const int argc, char* argv[]) {
   if (argc < 2) {
     puts("./herb [command] [options]\n");
 
-    puts("Herb 🌿 Powerful and seamless HTML-aware ERB parsing and tooling.\n");
+    puts("Herb 🌿 Powerful and seamless HTML-aware ERB toolchain.\n");
 
     puts("./herb lex [file]      -  Lex a file");
     puts("./herb parse [file]    -  Parse a file");
@@ -54,74 +62,90 @@ int main(const int argc, char* argv[]) {
     return EXIT_FAILURE;
   }
 
-  hb_buffer_T output;
+  hb_allocator_T malloc_allocator = hb_allocator_with_malloc();
+  char* source = herb_read_file(argv[2], &malloc_allocator);
 
-  if (!hb_buffer_init(&output, 4096)) { return 1; }
+  hb_allocator_T allocator;
+  if (!hb_allocator_init(&allocator, HB_ALLOCATOR_ARENA)) {
+    fprintf(stderr, "Failed to initialize allocator\n");
+    hb_allocator_dealloc(&malloc_allocator, source);
+    return EXIT_FAILURE;
+  }
 
-  char* source = herb_read_file(argv[2]);
+  hb_buffer_T output;
+  if (!hb_buffer_init(&output, 4096, &allocator)) { return 1; }
 
   struct timespec start, end;
   clock_gettime(CLOCK_MONOTONIC, &start);
 
+  int silent = 0;
+  if (argc > 3 && string_equals(argv[3], "--silent")) { silent = 1; }
+
   if (string_equals(argv[1], "lex")) {
-    herb_lex_to_buffer(source, &output);
+    herb_lex_to_buffer(source, &output, &allocator);
     clock_gettime(CLOCK_MONOTONIC, &end);
 
+    if (!silent) { hb_arena_print_stats((hb_arena_T*) allocator.context); }
+
     puts(output.value);
     print_time_diff(start, end, "lexing");
 
-    free(output.value);
-    free(source);
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+    hb_allocator_dealloc(&malloc_allocator, source);
 
     return EXIT_SUCCESS;
   }
 
   if (string_equals(argv[1], "parse")) {
-    AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL);
-
-    herb_analyze_parse_tree(root, source);
+    AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL, &allocator);
 
     clock_gettime(CLOCK_MONOTONIC, &end);
 
-    int silent = 0;
-    if (argc > 3 && string_equals(argv[3], "--silent")) { silent = 1; }
-
     if (!silent) {
+      hb_arena_print_stats((hb_arena_T*) allocator.context);
+
+#ifndef HERB_EXCLUDE_PRETTYPRINT
       ast_pretty_print_node((AST_NODE_T*) root, 0, 0, &output);
       puts(output.value);
+#endif
 
       print_time_diff(start, end, "parsing");
     }
 
-    ast_node_free((AST_NODE_T*) root);
-    free(output.value);
-    free(source);
+    ast_node_free((AST_NODE_T*) root, &allocator);
+
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+    hb_allocator_dealloc(&malloc_allocator, source);
 
     return EXIT_SUCCESS;
   }
 
   if (string_equals(argv[1], "ruby")) {
-    herb_extract_ruby_to_buffer(source, &output);
+    herb_extract_ruby_to_buffer(source, &output, &allocator);
     clock_gettime(CLOCK_MONOTONIC, &end);
 
     puts(output.value);
     print_time_diff(start, end, "extracting Ruby");
 
-    free(output.value);
-    free(source);
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+    hb_allocator_dealloc(&malloc_allocator, source);
 
     return EXIT_SUCCESS;
   }
 
   if (string_equals(argv[1], "html")) {
-    herb_extract_html_to_buffer(source, &output);
+    herb_extract_html_to_buffer(source, &output, &allocator);
     clock_gettime(CLOCK_MONOTONIC, &end);
 
     puts(output.value);
     print_time_diff(start, end, "extracting HTML");
 
-    free(output.value);
-    free(source);
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+    hb_allocator_dealloc(&malloc_allocator, source);
 
     return EXIT_SUCCESS;
   }
@@ -129,14 +153,15 @@ int main(const int argc, char* argv[]) {
   if (string_equals(argv[1], "prism")) {
     printf("HTML+ERB File: \n%s\n", source);
 
-    char* ruby_source = herb_extract(source, HERB_EXTRACT_LANGUAGE_RUBY);
+    char* ruby_source = herb_extract(source, HERB_EXTRACT_LANGUAGE_RUBY, &allocator);
     printf("Extracted Ruby: \n%s\n", ruby_source);
 
     herb_parse_ruby_to_stdout(ruby_source);
 
-    free(ruby_source);
-    free(output.value);
-    free(source);
+    hb_allocator_dealloc(&allocator, ruby_source);
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+    hb_allocator_dealloc(&malloc_allocator, source);
 
     return EXIT_SUCCESS;
   }