herb 0.8.9-arm-linux-gnu → 0.9.0-arm-linux-gnu

data/src/extract.c

@@ -1,40 +1,96 @@
 #include "include/herb.h"
-#include "include/io.h"
-#include "include/lexer.h"
+#include "include/util/hb_allocator.h"
 #include "include/util/hb_array.h"
 #include "include/util/hb_buffer.h"
+#include "include/util/hb_string.h"
+#include "include/util/string.h"
 
+#include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 
-void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
-  hb_array_T* tokens = herb_lex(source);
+const herb_extract_ruby_options_T HERB_EXTRACT_RUBY_DEFAULT_OPTIONS = { .semicolons = true,
+                                                                        .comments = false,
+                                                                        .preserve_positions = true };
+
+void herb_extract_ruby_to_buffer_with_options(
+  const char* source,
+  hb_buffer_T* output,
+  const herb_extract_ruby_options_T* options,
+  hb_allocator_T* allocator
+) {
+  herb_extract_ruby_options_T extract_options = options ? *options : HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+
+  hb_array_T* tokens = herb_lex(source, allocator);
   bool skip_erb_content = false;
   bool is_comment_tag = false;
+  bool is_erb_comment_tag = false;
+  bool need_newline = false;
 
   for (size_t i = 0; i < hb_array_size(tokens); i++) {
     const token_T* token = hb_array_get(tokens, i);
 
     switch (token->type) {
       case TOKEN_NEWLINE: {
-        hb_buffer_append(output, token->value);
+        hb_buffer_append_string(output, token->value);
+        need_newline = false;
         break;
       }
 
       case TOKEN_ERB_START: {
-        if (strcmp(token->value, "<%#") == 0) {
-          skip_erb_content = true;
-          is_comment_tag = true;
-        } else if (strcmp(token->value, "<%%") == 0 || strcmp(token->value, "<%%=") == 0
-                   || strcmp(token->value, "<%graphql") == 0) {
+        is_erb_comment_tag = hb_string_equals(token->value, hb_string("<%#"));
+
+        if (is_erb_comment_tag) {
+          if (extract_options.comments) {
+            skip_erb_content = false;
+            is_comment_tag = false;
+
+            if (extract_options.preserve_positions) {
+              bool is_multiline = false;
+
+              if (i + 1 < hb_array_size(tokens)) {
+                const token_T* next = hb_array_get(tokens, i + 1);
+
+                if (next->type == TOKEN_ERB_CONTENT && !hb_string_is_null(next->value)
+                    && memchr(next->value.data, '\n', next->value.length) != NULL) {
+                  is_multiline = true;
+                }
+              }
+
+              if (is_multiline) {
+                hb_buffer_append_char(output, '#');
+                hb_buffer_append_whitespace(output, 2);
+              } else {
+                hb_buffer_append_whitespace(output, 2);
+                hb_buffer_append_char(output, '#');
+              }
+            } else {
+              if (need_newline) { hb_buffer_append_char(output, '\n'); }
+              hb_buffer_append_char(output, '#');
+              need_newline = true;
+            }
+          } else {
+            skip_erb_content = true;
+            is_comment_tag = true;
+            if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
+          }
+        } else if (hb_string_equals(token->value, hb_string("<%%")) || hb_string_equals(token->value, hb_string("<%%="))
+                   || hb_string_equals(token->value, hb_string("<%graphql"))) {
           skip_erb_content = true;
           is_comment_tag = false;
+          if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
         } else {
           skip_erb_content = false;
           is_comment_tag = false;
+
+          if (extract_options.preserve_positions) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          } else if (need_newline) {
+            hb_buffer_append_char(output, '\n');
+            need_newline = false;
+          }
         }
 
-        hb_buffer_append_whitespace(output, range_length(token->range));
         break;
       }
 
@@ -42,26 +98,65 @@ void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
         if (skip_erb_content == false) {
           bool is_inline_comment = false;
 
-          if (!is_comment_tag && token->value != NULL) {
-            const char* content = token->value;
+          if (!extract_options.comments && !is_comment_tag && !hb_string_is_empty(token->value)) {
+            hb_string_T trimmed = hb_string_trim_start(token->value);
 
-            while (*content == ' ' || *content == '\t') {
-              content++;
-            }
-
-            if (*content == '#' && token->location.start.line == token->location.end.line) {
+            if (!hb_string_is_empty(trimmed) && trimmed.data[0] == '#'
+                && token->location.start.line == token->location.end.line) {
               is_comment_tag = true;
               is_inline_comment = true;
             }
           }
 
           if (is_inline_comment) {
-            hb_buffer_append_whitespace(output, range_length(token->range));
+            if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
+          } else if (is_erb_comment_tag && !hb_string_is_null(token->value)) {
+            const char* content = token->value.data;
+            size_t content_remaining = token->value.length;
+
+            while (content_remaining > 0) {
+              if (*content == '\n') {
+                hb_buffer_append_char(output, '\n');
+                content++;
+                content_remaining--;
+
+                if (content_remaining > 0 && extract_options.preserve_positions && *content == ' ') {
+                  content++;
+                  content_remaining--;
+                }
+
+                hb_buffer_append_char(output, '#');
+              } else {
+                hb_buffer_append_char(output, *content);
+                content++;
+                content_remaining--;
+              }
+            }
+
+            if (!extract_options.preserve_positions) { need_newline = true; }
           } else {
-            hb_buffer_append(output, token->value);
+            hb_buffer_append_string(output, token->value);
+
+            if (!extract_options.preserve_positions) { need_newline = true; }
           }
         } else {
-          hb_buffer_append_whitespace(output, range_length(token->range));
+          if (is_erb_comment_tag && extract_options.preserve_positions && !hb_string_is_null(token->value)) {
+            const char* content = token->value.data;
+            size_t content_remaining = token->value.length;
+
+            while (content_remaining > 0) {
+              if (*content == '\n') {
+                hb_buffer_append_char(output, '\n');
+              } else {
+                hb_buffer_append_char(output, ' ');
+              }
+
+              content++;
+              content_remaining--;
+            }
+          } else if (extract_options.preserve_positions) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          }
         }
 
         break;
@@ -69,31 +164,43 @@ void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
 
       case TOKEN_ERB_END: {
         bool was_comment = is_comment_tag;
+        bool was_erb_comment = is_erb_comment_tag;
         skip_erb_content = false;
         is_comment_tag = false;
+        is_erb_comment_tag = false;
 
-        if (was_comment) {
-          hb_buffer_append_whitespace(output, range_length(token->range));
-          break;
+        if (extract_options.preserve_positions) {
+          if (was_comment) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          } else if (was_erb_comment && extract_options.comments) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          } else if (extract_options.semicolons) {
+            hb_buffer_append_char(output, ' ');
+            hb_buffer_append_char(output, ';');
+            hb_buffer_append_whitespace(output, range_length(token->range) - 2);
+          } else {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          }
         }
 
-        hb_buffer_append_char(output, ' ');
-        hb_buffer_append_char(output, ';');
-        hb_buffer_append_whitespace(output, range_length(token->range) - 2);
         break;
       }
 
       default: {
-        hb_buffer_append_whitespace(output, range_length(token->range));
+        if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
       }
     }
   }
 
-  herb_free_tokens(&tokens);
+  herb_free_tokens(&tokens, allocator);
+}
+
+void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {
+  herb_extract_ruby_to_buffer_with_options(source, output, NULL, allocator);
 }
 
-void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output) {
-  hb_array_T* tokens = herb_lex(source);
+void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {
+  hb_array_T* tokens = herb_lex(source, allocator);
 
   for (size_t i = 0; i < hb_array_size(tokens); i++) {
     const token_T* token = hb_array_get(tokens, i);
@@ -102,43 +209,35 @@ void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output) {
       case TOKEN_ERB_START:
       case TOKEN_ERB_CONTENT:
       case TOKEN_ERB_END: hb_buffer_append_whitespace(output, range_length(token->range)); break;
-      default: hb_buffer_append(output, token->value);
+      default: hb_buffer_append_string(output, token->value);
     }
   }
 
-  herb_free_tokens(&tokens);
+  herb_free_tokens(&tokens, allocator);
 }
 
-char* herb_extract_ruby_with_semicolons(const char* source) {
+char* herb_extract_ruby_with_semicolons(const char* source, hb_allocator_T* allocator) {
   if (!source) { return NULL; }
 
   hb_buffer_T output;
-  hb_buffer_init(&output, strlen(source));
+  hb_buffer_init(&output, strlen(source), allocator);
 
-  herb_extract_ruby_to_buffer(source, &output);
+  herb_extract_ruby_to_buffer(source, &output, allocator);
 
   return output.value;
 }
 
-char* herb_extract(const char* source, const herb_extract_language_T language) {
+char* herb_extract(const char* source, const herb_extract_language_T language, hb_allocator_T* allocator) {
   if (!source) { return NULL; }
 
   hb_buffer_T output;
-  hb_buffer_init(&output, strlen(source));
+  hb_buffer_init(&output, strlen(source), allocator);
 
   switch (language) {
-    case HERB_EXTRACT_LANGUAGE_RUBY: herb_extract_ruby_to_buffer(source, &output); break;
-    case HERB_EXTRACT_LANGUAGE_HTML: herb_extract_html_to_buffer(source, &output); break;
+    case HERB_EXTRACT_LANGUAGE_RUBY: herb_extract_ruby_to_buffer(source, &output, allocator); break;
+    case HERB_EXTRACT_LANGUAGE_HTML: herb_extract_html_to_buffer(source, &output, allocator); break;
+    default: assert(0 && "invalid extract language");
   }
 
   return output.value;
 }
-
-char* herb_extract_from_file(const char* path, const herb_extract_language_T language) {
-  char* source = herb_read_file(path);
-  char* output = herb_extract(source, language);
-
-  free(source);
-
-  return output;
-}

data/src/herb.c

@@ -1,21 +1,25 @@
 #include "include/herb.h"
-#include "include/io.h"
+#include "include/analyze/analyze.h"
+#include "include/analyze/prism_annotate.h"
 #include "include/lexer.h"
 #include "include/parser.h"
 #include "include/token.h"
+#include "include/util/hb_allocator.h"
 #include "include/util/hb_array.h"
-#include "include/util/hb_buffer.h"
 #include "include/version.h"
 
 #include <prism.h>
 #include <stdlib.h>
+#include <string.h>
+
+HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source, hb_allocator_T* allocator) {
+  if (!source) { source = ""; }
 
-HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source) {
   lexer_T lexer = { 0 };
-  lexer_init(&lexer, source);
+  lexer_init(&lexer, source, allocator);
 
   token_T* token = NULL;
-  hb_array_T* tokens = hb_array_init(128);
+  hb_array_T* tokens = hb_array_init(128, allocator);
 
   while ((token = lexer_next_token(&lexer))->type != TOKEN_EOF) {
     hb_array_append(tokens, token);
@@ -26,15 +30,18 @@ HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source) {
   return tokens;
 }
 
-HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(const char* source, parser_options_T* options) {
+HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(
+  const char* source,
+  const parser_options_T* options,
+  hb_allocator_T* allocator
+) {
   if (!source) { source = ""; }
 
   lexer_T lexer = { 0 };
-  lexer_init(&lexer, source);
+  lexer_init(&lexer, source, allocator);
   parser_T parser = { 0 };
 
   parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
-
   if (options != NULL) { parser_options = *options; }
 
   herb_parser_init(&parser, &lexer, parser_options);
@@ -43,40 +50,28 @@ HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(const char* source, parse
 
   herb_parser_deinit(&parser);
 
-  return document;
-}
-
-HERB_EXPORTED_FUNCTION hb_array_T* herb_lex_file(const char* path) {
-  char* source = herb_read_file(path);
-  hb_array_T* tokens = herb_lex(source);
-
-  free(source);
-
-  return tokens;
-}
-
-HERB_EXPORTED_FUNCTION void herb_lex_to_buffer(const char* source, hb_buffer_T* output) {
-  hb_array_T* tokens = herb_lex(source);
-
-  for (size_t i = 0; i < hb_array_size(tokens); i++) {
-    token_T* token = hb_array_get(tokens, i);
-
-    hb_string_T type = token_to_string(token);
-    hb_buffer_append_string(output, type);
-    free(type.data);
-
-    hb_buffer_append(output, "\n");
+  if (parser_options.analyze) { herb_analyze_parse_tree(document, source, &parser_options, allocator); }
+
+  if (parser_options.prism_nodes || parser_options.prism_program) {
+    herb_annotate_prism_nodes(
+      document,
+      source,
+      parser_options.prism_nodes,
+      parser_options.prism_nodes_deep,
+      parser_options.prism_program,
+      allocator
+    );
   }
 
-  herb_free_tokens(&tokens);
+  return document;
 }
 
-HERB_EXPORTED_FUNCTION void herb_free_tokens(hb_array_T** tokens) {
+HERB_EXPORTED_FUNCTION void herb_free_tokens(hb_array_T** tokens, hb_allocator_T* allocator) {
   if (!tokens || !*tokens) { return; }
 
   for (size_t i = 0; i < hb_array_size(*tokens); i++) {
     token_T* token = hb_array_get(*tokens, i);
-    if (token) { token_free(token); }
+    if (token) { token_free(token, allocator); }
   }
 
   hb_array_free(tokens);
@@ -89,3 +84,26 @@ HERB_EXPORTED_FUNCTION const char* herb_version(void) {
 HERB_EXPORTED_FUNCTION const char* herb_prism_version(void) {
   return PRISM_VERSION;
 }
+
+HERB_EXPORTED_FUNCTION herb_ruby_parse_result_T* herb_parse_ruby(const char* source, size_t length) {
+  if (!source) { return NULL; }
+
+  herb_ruby_parse_result_T* result = malloc(sizeof(herb_ruby_parse_result_T));
+  if (!result) { return NULL; }
+
+  memset(&result->options, 0, sizeof(pm_options_t));
+  pm_parser_init(&result->parser, (const uint8_t*) source, length, &result->options);
+  result->root = pm_parse(&result->parser);
+
+  return result;
+}
+
+HERB_EXPORTED_FUNCTION void herb_free_ruby_parse_result(herb_ruby_parse_result_T* result) {
+  if (!result) { return; }
+
+  if (result->root) { pm_node_destroy(&result->parser, result->root); }
+
+  pm_parser_free(&result->parser);
+  pm_options_free(&result->options);
+  free(result);
+}