@herb-tools/node 0.7.4 → 0.8.0

package/extension/libherb/lexer.h

@@ -4,10 +4,8 @@
 #include "lexer_struct.h"
 #include "token_struct.h"
 
-lexer_T* lexer_init(const char* source);
+void lexer_init(lexer_T* lexer, const char* source);
 token_T* lexer_next_token(lexer_T* lexer);
 token_T* lexer_error(lexer_T* lexer, const char* message);
 
-void lexer_free(lexer_T* lexer);
-
 #endif

package/extension/libherb/lexer_peek_helpers.c

@@ -3,80 +3,83 @@
 #include "include/lexer_struct.h"
 #include "include/macros.h"
 #include "include/token.h"
+#include "include/util/hb_string.h"
 
 #include <ctype.h>
 #include <stdbool.h>
 
-char lexer_backtrack(const lexer_T* lexer, const int offset) {
-  return lexer->source[MAX(lexer->current_position - offset, 0)];
+char lexer_backtrack(const lexer_T* lexer, uint32_t offset) {
+  return lexer->source.data[MAX(lexer->current_position - offset, 0)];
 }
 
-char lexer_peek(const lexer_T* lexer, const int offset) {
-  return lexer->source[MIN(lexer->current_position + offset, lexer->source_length)];
+char lexer_peek(const lexer_T* lexer, uint32_t offset) {
+  return lexer->source.data[MIN(lexer->current_position + offset, lexer->source.length)];
 }
 
-bool lexer_peek_for(const lexer_T* lexer, const int offset, const char* pattern, const bool case_insensitive) {
-  for (int index = 0; pattern[index]; index++) {
-    const char character = lexer_peek(lexer, offset + index);
+bool lexer_peek_for(const lexer_T* lexer, uint32_t offset, hb_string_T pattern, const bool case_insensitive) {
+  hb_string_T remaining_source = hb_string_slice(lexer->source, lexer->current_position + offset);
+  remaining_source.length = MIN(pattern.length, remaining_source.length);
 
-    if (case_insensitive) {
-      if (tolower(character) != tolower(pattern[index])) { return false; }
-    } else {
-      if (character != pattern[index]) { return false; }
-    }
+  if (case_insensitive) {
+    return hb_string_equals_case_insensitive(remaining_source, pattern);
+  } else {
+    return hb_string_equals(remaining_source, pattern);
   }
+}
 
-  return true;
+bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("<!DOCTYPE"), true);
 }
 
-bool lexer_peek_for_doctype(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "<!DOCTYPE", true);
+bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("<?xml"), true);
 }
 
-bool lexer_peek_for_xml_declaration(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "<?xml", true);
+bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("<![CDATA["), false);
 }
 
-bool lexer_peek_for_cdata_start(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "<![CDATA[", false);
+bool lexer_peek_for_cdata_end(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("]]>"), false);
 }
 
-bool lexer_peek_for_cdata_end(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "]]>", false);
+bool lexer_peek_for_html_comment_start(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("<!--"), false);
 }
 
-bool lexer_peek_for_html_comment_start(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "<!--", false);
+bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("-->"), false);
 }
 
-bool lexer_peek_for_html_comment_end(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "-->", false);
+bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("%>"), false);
 }
 
-bool lexer_peek_erb_close_tag(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "%>", false);
+bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("-%>"), false);
 }
 
-bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "-%>", false);
+bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("%%>"), false);
 }
 
-bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, const int offset) {
-  return lexer_peek_for(lexer, offset, "%%>", false);
+bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_for(lexer, offset, hb_string("=%>"), false);
 }
 
-bool lexer_peek_erb_end(const lexer_T* lexer, const int offset) {
+bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset) {
   return (
     lexer_peek_erb_close_tag(lexer, offset) || lexer_peek_erb_dash_close_tag(lexer, offset)
-    || lexer_peek_erb_percent_close_tag(lexer, offset)
+    || lexer_peek_erb_percent_close_tag(lexer, offset) || lexer_peek_erb_equals_close_tag(lexer, offset)
   );
 }
 
 bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type) {
-  size_t saved_position = lexer->current_position;
-  size_t saved_line = lexer->current_line;
-  size_t saved_column = lexer->current_column;
+  uint32_t saved_position = lexer->current_position;
+  uint32_t saved_line = lexer->current_line;
+  uint32_t saved_column = lexer->current_column;
   char saved_character = lexer->current_character;
+  lexer_state_T saved_state = lexer->state;
 
   token_T* token = lexer_next_token(lexer);
 
@@ -93,14 +96,15 @@ bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T tok
   lexer->current_line = saved_line;
   lexer->current_column = saved_column;
   lexer->current_character = saved_character;
+  lexer->state = saved_state;
 
   return result;
 }
 
-bool lexer_peek_for_close_tag_start(const lexer_T* lexer, const int offset) {
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset) {
   if (lexer_peek(lexer, offset) != '<' || lexer_peek(lexer, offset + 1) != '/') { return false; }
 
-  int pos = offset + 2;
+  uint32_t pos = offset + 2;
 
   while (lexer_peek(lexer, pos) == ' ' || lexer_peek(lexer, pos) == '\t' || lexer_peek(lexer, pos) == '\n'
          || lexer_peek(lexer, pos) == '\r') {

package/extension/libherb/lexer_peek_helpers.h

@@ -5,38 +5,40 @@
 #include "token_struct.h"
 
 #include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 
 typedef struct {
-  size_t position;
-  size_t line;
-  size_t column;
-  size_t previous_position;
-  size_t previous_line;
-  size_t previous_column;
+  uint32_t position;
+  uint32_t line;
+  uint32_t column;
+  uint32_t previous_position;
+  uint32_t previous_line;
+  uint32_t previous_column;
   char current_character;
   lexer_state_T state;
 } lexer_state_snapshot_T;
 
-char lexer_peek(const lexer_T* lexer, int offset);
-bool lexer_peek_for_doctype(const lexer_T* lexer, int offset);
-bool lexer_peek_for_xml_declaration(const lexer_T* lexer, int offset);
-bool lexer_peek_for_cdata_start(const lexer_T* lexer, int offset);
-bool lexer_peek_for_cdata_end(const lexer_T* lexer, int offset);
+char lexer_peek(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_cdata_end(const lexer_T* lexer, uint32_t offset);
 
-bool lexer_peek_for_html_comment_start(const lexer_T* lexer, int offset);
-bool lexer_peek_for_html_comment_end(const lexer_T* lexer, int offset);
+bool lexer_peek_for_html_comment_start(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset);
 
-bool lexer_peek_erb_close_tag(const lexer_T* lexer, int offset);
-bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, int offset);
-bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, int offset);
-bool lexer_peek_erb_end(const lexer_T* lexer, int offset);
+bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset);
 
-char lexer_backtrack(const lexer_T* lexer, int offset);
+char lexer_backtrack(const lexer_T* lexer, uint32_t offset);
 
 bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type);
-bool lexer_peek_for_close_tag_start(const lexer_T* lexer, int offset);
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset);
 
 lexer_state_snapshot_T lexer_save_state(lexer_T* lexer);
 void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot);

package/extension/libherb/lexer_struct.h

@@ -1,7 +1,10 @@
 #ifndef HERB_LEXER_STRUCT_H
 #define HERB_LEXER_STRUCT_H
 
+#include "util/hb_string.h"
+
 #include <stdbool.h>
+#include <stdint.h>
 #include <stdlib.h>
 
 typedef enum {
@@ -11,21 +14,20 @@ typedef enum {
 } lexer_state_T;
 
 typedef struct LEXER_STRUCT {
-  const char* source;
-  size_t source_length;
+  hb_string_T source;
 
-  size_t current_line;
-  size_t current_column;
-  size_t current_position;
+  uint32_t current_line;
+  uint32_t current_column;
+  uint32_t current_position;
 
-  size_t previous_line;
-  size_t previous_column;
-  size_t previous_position;
+  uint32_t previous_line;
+  uint32_t previous_column;
+  uint32_t previous_position;
 
   char current_character;
   lexer_state_T state;
-  size_t stall_counter;
-  size_t last_position;
+  uint32_t stall_counter;
+  uint32_t last_position;
   bool stalled;
 } lexer_T;
 

package/extension/libherb/location.c

@@ -1,41 +1,13 @@
 #include "include/location.h"
-#include "include/memory.h"
 #include "include/position.h"
 
-size_t location_sizeof(void) {
-  return sizeof(location_T);
-}
-
-location_T* location_init(position_T* start, position_T* end) {
-  location_T* location = safe_malloc(location_sizeof());
-
-  location->start = start;
-  location->end = end;
-
-  return location;
-}
-
-location_T* location_from(size_t start_line, size_t start_column, size_t end_line, size_t end_column) {
-  return location_init(position_init(start_line, start_column), position_init(end_line, end_column));
-}
-
-position_T* location_start(location_T* location) {
-  return location->start;
-}
-
-position_T* location_end(location_T* location) {
-  return location->end;
-}
-
-location_T* location_copy(location_T* location) {
-  if (location == NULL) { return NULL; }
-
-  return location_init(position_copy(location->start), position_copy(location->end));
-}
-
-void location_free(location_T* location) {
-  if (location->start != NULL) { position_free(location->start); }
-  if (location->end != NULL) { position_free(location->end); }
-
-  free(location);
+void location_from(
+  location_T* location,
+  uint32_t start_line,
+  uint32_t start_column,
+  uint32_t end_line,
+  uint32_t end_column
+) {
+  location->start = (position_T) { .line = start_line, .column = start_column };
+  location->end = (position_T) { .line = end_line, .column = end_column };
 }

package/extension/libherb/location.h

@@ -1,25 +1,22 @@
 #ifndef HERB_LOCATION_H
 #define HERB_LOCATION_H
 
+#include <stdint.h>
 #include <stdlib.h>
 
 #include "position.h"
 
 typedef struct LOCATION_STRUCT {
-  position_T* start;
-  position_T* end;
+  position_T start;
+  position_T end;
 } location_T;
 
-location_T* location_init(position_T* start, position_T* end);
-location_T* location_from(size_t start_line, size_t start_column, size_t end_line, size_t end_column);
-
-position_T* location_start(location_T* location);
-position_T* location_end_(location_T* location);
-
-size_t location_sizeof(void);
-
-location_T* location_copy(location_T* location);
-
-void location_free(location_T* location);
+void location_from(
+  location_T* location,
+  uint32_t start_line,
+  uint32_t start_column,
+  uint32_t end_line,
+  uint32_t end_column
+);
 
 #endif

package/extension/libherb/macros.h

@@ -5,6 +5,10 @@
 
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 
+#define KB(kb) (1024 * (kb))
+
+#define MB(mb) (1024 * KB(mb))
+
 #define unlikely(x) __builtin_expect(!!(x), 0)
 
 #endif

package/extension/libherb/main.c

@@ -4,11 +4,11 @@
 #include "include/ast_node.h"
 #include "include/ast_nodes.h"
 #include "include/ast_pretty_print.h"
-#include "include/buffer.h"
 #include "include/extract.h"
 #include "include/herb.h"
 #include "include/io.h"
 #include "include/ruby_parser.h"
+#include "include/util/hb_buffer.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -39,7 +39,6 @@ int main(const int argc, char* argv[]) {
     printf("Herb 🌿 Powerful and seamless HTML-aware ERB parsing and tooling.\n\n");
 
     printf("./herb lex [file]      -  Lex a file\n");
-    printf("./herb lex_json [file] -  Lex a file and return the result as json.\n");
     printf("./herb parse [file]    -  Parse a file\n");
     printf("./herb ruby [file]     -  Extract Ruby from a file\n");
     printf("./herb html [file]     -  Extract HTML from a file\n");
@@ -53,9 +52,9 @@ int main(const int argc, char* argv[]) {
     return 1;
   }
 
-  buffer_T output;
+  hb_buffer_T output;
 
-  if (!buffer_init(&output)) { return 1; }
+  if (!hb_buffer_init(&output, 4096)) { return 1; }
 
   char* source = herb_read_file(argv[2]);
 
@@ -74,7 +73,7 @@ int main(const int argc, char* argv[]) {
     print_time_diff(start, end, "visiting");
 
     ast_node_free((AST_NODE_T*) root);
-    buffer_free(&output);
+    free(output.value);
     free(source);
 
     return 0;
@@ -87,18 +86,7 @@ int main(const int argc, char* argv[]) {
     printf("%s\n", output.value);
     print_time_diff(start, end, "lexing");
 
-    buffer_free(&output);
-    free(source);
-
-    return 0;
-  }
-
-  if (strcmp(argv[1], "lex_json") == 0) {
-    herb_lex_json_to_buffer(source, &output);
-
-    printf("%s\n", output.value);
-
-    buffer_free(&output);
+    free(output.value);
     free(source);
 
     return 0;
@@ -106,15 +94,23 @@ int main(const int argc, char* argv[]) {
 
   if (strcmp(argv[1], "parse") == 0) {
     AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL);
+
+    herb_analyze_parse_tree(root, source);
+
     clock_gettime(CLOCK_MONOTONIC, &end);
 
-    ast_pretty_print_node((AST_NODE_T*) root, 0, 0, &output);
-    printf("%s\n", output.value);
+    int silent = 0;
+    if (argc > 3 && strcmp(argv[3], "--silent") == 0) { silent = 1; }
+
+    if (!silent) {
+      ast_pretty_print_node((AST_NODE_T*) root, 0, 0, &output);
+      printf("%s\n", output.value);
 
-    print_time_diff(start, end, "parsing");
+      print_time_diff(start, end, "parsing");
+    }
 
     ast_node_free((AST_NODE_T*) root);
-    buffer_free(&output);
+    free(output.value);
     free(source);
 
     return 0;
@@ -127,7 +123,7 @@ int main(const int argc, char* argv[]) {
     printf("%s\n", output.value);
     print_time_diff(start, end, "extracting Ruby");
 
-    buffer_free(&output);
+    free(output.value);
     free(source);
 
     return 0;
@@ -140,7 +136,7 @@ int main(const int argc, char* argv[]) {
     printf("%s\n", output.value);
     print_time_diff(start, end, "extracting HTML");
 
-    buffer_free(&output);
+    free(output.value);
     free(source);
 
     return 0;