@herb-tools/node 0.8.9 → 0.9.0

package/extension/libherb/errors.h

@@ -1,5 +1,5 @@
 // NOTE: This file is generated by the templates/template.rb script and should not
-// be modified manually. See /Users/marcoroth/Development/herb-release-0.8.9/templates/src/include/errors.h.erb
+// be modified manually. See /Users/marcoroth/Development/herb-release-0.9.0/templates/src/include/errors.h.erb
 
 #ifndef HERB_ERRORS_H
 #define HERB_ERRORS_H
@@ -8,8 +8,10 @@
 #include "location.h"
 #include "position.h"
 #include "token.h"
+#include "util/hb_allocator.h"
 #include "util/hb_array.h"
 #include "util/hb_buffer.h"
+#include "util/hb_string.h"
 
 typedef enum {
   UNEXPECTED_ERROR,
@@ -17,27 +19,38 @@ typedef enum {
   MISSING_OPENING_TAG_ERROR,
   MISSING_CLOSING_TAG_ERROR,
   TAG_NAMES_MISMATCH_ERROR,
-  QUOTES_MISMATCH_ERROR,
   VOID_ELEMENT_CLOSING_TAG_ERROR,
   UNCLOSED_ELEMENT_ERROR,
   RUBY_PARSE_ERROR,
   ERB_CONTROL_FLOW_SCOPE_ERROR,
-  MISSINGERB_END_TAG_ERROR,
+  MISSING_ERB_END_TAG_ERROR,
   ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR,
+  ERB_CASE_WITH_CONDITIONS_ERROR,
+  CONDITIONAL_ELEMENT_MULTIPLE_TAGS_ERROR,
+  CONDITIONAL_ELEMENT_CONDITION_MISMATCH_ERROR,
+  INVALID_COMMENT_CLOSING_TAG_ERROR,
+  OMITTED_CLOSING_TAG_ERROR,
+  UNCLOSED_OPEN_TAG_ERROR,
+  UNCLOSED_CLOSE_TAG_ERROR,
+  UNCLOSED_QUOTE_ERROR,
+  MISSING_ATTRIBUTE_VALUE_ERROR,
+  UNCLOSED_ERB_TAG_ERROR,
+  STRAY_ERB_CLOSING_TAG_ERROR,
+  NESTED_ERB_TAG_ERROR,
 } error_type_T;
 
 typedef struct ERROR_STRUCT {
   error_type_T type;
   location_T location;
-  char* message;
+  hb_string_T message;
 } ERROR_T;
 
 
 typedef struct {
   ERROR_T base;
-  const char* description;
-  const char* expected;
-  const char* found;
+  hb_string_T description;
+  hb_string_T expected;
+  hb_string_T found;
 } UNEXPECTED_ERROR_T;
 
 typedef struct {
@@ -62,17 +75,11 @@ typedef struct {
   token_T* closing_tag;
 } TAG_NAMES_MISMATCH_ERROR_T;
 
-typedef struct {
-  ERROR_T base;
-  token_T* opening_quote;
-  token_T* closing_quote;
-} QUOTES_MISMATCH_ERROR_T;
-
 typedef struct {
   ERROR_T base;
   token_T* tag_name;
-  const char* expected;
-  const char* found;
+  hb_string_T expected;
+  hb_string_T found;
 } VOID_ELEMENT_CLOSING_TAG_ERROR_T;
 
 typedef struct {
@@ -82,68 +89,162 @@ typedef struct {
 
 typedef struct {
   ERROR_T base;
-  const char* error_message;
-  const char* diagnostic_id;
-  const char* level;
+  hb_string_T error_message;
+  hb_string_T diagnostic_id;
+  hb_string_T level;
 } RUBY_PARSE_ERROR_T;
 
 typedef struct {
   ERROR_T base;
-  const char* keyword;
+  hb_string_T keyword;
 } ERB_CONTROL_FLOW_SCOPE_ERROR_T;
 
 typedef struct {
   ERROR_T base;
-  const char* keyword;
-} MISSINGERB_END_TAG_ERROR_T;
+  hb_string_T keyword;
+} MISSING_ERB_END_TAG_ERROR_T;
 
 typedef struct {
   ERROR_T base;
   /* no additional fields */
 } ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR_T;
 
-UNEXPECTED_ERROR_T* unexpected_error_init(const char* description, const char* expected, const char* found, position_T start, position_T end);
-void append_unexpected_error(const char* description, const char* expected, const char* found, position_T start, position_T end, hb_array_T* errors);
-UNEXPECTED_TOKEN_ERROR_T* unexpected_token_error_init(token_type_T expected_type, token_T* found, position_T start, position_T end);
-void append_unexpected_token_error(token_type_T expected_type, token_T* found, position_T start, position_T end, hb_array_T* errors);
-MISSING_OPENING_TAG_ERROR_T* missing_opening_tag_error_init(token_T* closing_tag, position_T start, position_T end);
-void append_missing_opening_tag_error(token_T* closing_tag, position_T start, position_T end, hb_array_T* errors);
-MISSING_CLOSING_TAG_ERROR_T* missing_closing_tag_error_init(token_T* opening_tag, position_T start, position_T end);
-void append_missing_closing_tag_error(token_T* opening_tag, position_T start, position_T end, hb_array_T* errors);
-TAG_NAMES_MISMATCH_ERROR_T* tag_names_mismatch_error_init(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end);
-void append_tag_names_mismatch_error(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end, hb_array_T* errors);
-QUOTES_MISMATCH_ERROR_T* quotes_mismatch_error_init(token_T* opening_quote, token_T* closing_quote, position_T start, position_T end);
-void append_quotes_mismatch_error(token_T* opening_quote, token_T* closing_quote, position_T start, position_T end, hb_array_T* errors);
-VOID_ELEMENT_CLOSING_TAG_ERROR_T* void_element_closing_tag_error_init(token_T* tag_name, const char* expected, const char* found, position_T start, position_T end);
-void append_void_element_closing_tag_error(token_T* tag_name, const char* expected, const char* found, position_T start, position_T end, hb_array_T* errors);
-UNCLOSED_ELEMENT_ERROR_T* unclosed_element_error_init(token_T* opening_tag, position_T start, position_T end);
-void append_unclosed_element_error(token_T* opening_tag, position_T start, position_T end, hb_array_T* errors);
-RUBY_PARSE_ERROR_T* ruby_parse_error_init(const char* error_message, const char* diagnostic_id, const char* level, position_T start, position_T end);
-void append_ruby_parse_error(const char* error_message, const char* diagnostic_id, const char* level, position_T start, position_T end, hb_array_T* errors);
-ERB_CONTROL_FLOW_SCOPE_ERROR_T* erb_control_flow_scope_error_init(const char* keyword, position_T start, position_T end);
-void append_erb_control_flow_scope_error(const char* keyword, position_T start, position_T end, hb_array_T* errors);
-MISSINGERB_END_TAG_ERROR_T* missingerb_end_tag_error_init(const char* keyword, position_T start, position_T end);
-void append_missingerb_end_tag_error(const char* keyword, position_T start, position_T end, hb_array_T* errors);
-ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR_T* erb_multiple_blocks_in_tag_error_init(position_T start, position_T end);
-void append_erb_multiple_blocks_in_tag_error(position_T start, position_T end, hb_array_T* errors);
+typedef struct {
+  ERROR_T base;
+  /* no additional fields */
+} ERB_CASE_WITH_CONDITIONS_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  size_t line;
+  size_t column;
+} CONDITIONAL_ELEMENT_MULTIPLE_TAGS_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  hb_string_T tag_name;
+  hb_string_T open_condition;
+  size_t open_line;
+  size_t open_column;
+  hb_string_T close_condition;
+  size_t close_line;
+  size_t close_column;
+} CONDITIONAL_ELEMENT_CONDITION_MISMATCH_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* closing_tag;
+} INVALID_COMMENT_CLOSING_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_tag;
+  position_T insertion_point;
+} OMITTED_CLOSING_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* tag_name;
+} UNCLOSED_OPEN_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* tag_name;
+} UNCLOSED_CLOSE_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_quote;
+} UNCLOSED_QUOTE_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  hb_string_T attribute_name;
+} MISSING_ATTRIBUTE_VALUE_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_tag;
+} UNCLOSED_ERB_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  /* no additional fields */
+} STRAY_ERB_CLOSING_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_tag;
+  size_t nested_tag_line;
+  size_t nested_tag_column;
+} NESTED_ERB_TAG_ERROR_T;
+
+UNEXPECTED_ERROR_T* unexpected_error_init(hb_string_T description, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unexpected_error(hb_string_T description, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNEXPECTED_TOKEN_ERROR_T* unexpected_token_error_init(token_type_T expected_type, token_T* found, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unexpected_token_error(token_type_T expected_type, token_T* found, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_OPENING_TAG_ERROR_T* missing_opening_tag_error_init(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_opening_tag_error(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_CLOSING_TAG_ERROR_T* missing_closing_tag_error_init(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_closing_tag_error(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+TAG_NAMES_MISMATCH_ERROR_T* tag_names_mismatch_error_init(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_tag_names_mismatch_error(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+VOID_ELEMENT_CLOSING_TAG_ERROR_T* void_element_closing_tag_error_init(token_T* tag_name, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator);
+void append_void_element_closing_tag_error(token_T* tag_name, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_ELEMENT_ERROR_T* unclosed_element_error_init(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_element_error(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+RUBY_PARSE_ERROR_T* ruby_parse_error_init(hb_string_T error_message, hb_string_T diagnostic_id, hb_string_T level, position_T start, position_T end, hb_allocator_T* allocator);
+void append_ruby_parse_error(hb_string_T error_message, hb_string_T diagnostic_id, hb_string_T level, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+ERB_CONTROL_FLOW_SCOPE_ERROR_T* erb_control_flow_scope_error_init(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator);
+void append_erb_control_flow_scope_error(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_ERB_END_TAG_ERROR_T* missing_erb_end_tag_error_init(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_erb_end_tag_error(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR_T* erb_multiple_blocks_in_tag_error_init(position_T start, position_T end, hb_allocator_T* allocator);
+void append_erb_multiple_blocks_in_tag_error(position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+ERB_CASE_WITH_CONDITIONS_ERROR_T* erb_case_with_conditions_error_init(position_T start, position_T end, hb_allocator_T* allocator);
+void append_erb_case_with_conditions_error(position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+CONDITIONAL_ELEMENT_MULTIPLE_TAGS_ERROR_T* conditional_element_multiple_tags_error_init(size_t line, size_t column, position_T start, position_T end, hb_allocator_T* allocator);
+void append_conditional_element_multiple_tags_error(size_t line, size_t column, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+CONDITIONAL_ELEMENT_CONDITION_MISMATCH_ERROR_T* conditional_element_condition_mismatch_error_init(hb_string_T tag_name, hb_string_T open_condition, size_t open_line, size_t open_column, hb_string_T close_condition, size_t close_line, size_t close_column, position_T start, position_T end, hb_allocator_T* allocator);
+void append_conditional_element_condition_mismatch_error(hb_string_T tag_name, hb_string_T open_condition, size_t open_line, size_t open_column, hb_string_T close_condition, size_t close_line, size_t close_column, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+INVALID_COMMENT_CLOSING_TAG_ERROR_T* invalid_comment_closing_tag_error_init(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_invalid_comment_closing_tag_error(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+OMITTED_CLOSING_TAG_ERROR_T* omitted_closing_tag_error_init(token_T* opening_tag, position_T insertion_point, position_T start, position_T end, hb_allocator_T* allocator);
+void append_omitted_closing_tag_error(token_T* opening_tag, position_T insertion_point, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_OPEN_TAG_ERROR_T* unclosed_open_tag_error_init(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_open_tag_error(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_CLOSE_TAG_ERROR_T* unclosed_close_tag_error_init(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_close_tag_error(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_QUOTE_ERROR_T* unclosed_quote_error_init(token_T* opening_quote, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_quote_error(token_T* opening_quote, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_ATTRIBUTE_VALUE_ERROR_T* missing_attribute_value_error_init(hb_string_T attribute_name, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_attribute_value_error(hb_string_T attribute_name, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_ERB_TAG_ERROR_T* unclosed_erb_tag_error_init(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_erb_tag_error(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+STRAY_ERB_CLOSING_TAG_ERROR_T* stray_erb_closing_tag_error_init(position_T start, position_T end, hb_allocator_T* allocator);
+void append_stray_erb_closing_tag_error(position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+NESTED_ERB_TAG_ERROR_T* nested_erb_tag_error_init(token_T* opening_tag, size_t nested_tag_line, size_t nested_tag_column, position_T start, position_T end, hb_allocator_T* allocator);
+void append_nested_erb_tag_error(token_T* opening_tag, size_t nested_tag_line, size_t nested_tag_column, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
 
 void error_init(ERROR_T* error, error_type_T type, position_T start, position_T end);
 
 size_t error_sizeof(void);
 error_type_T error_type(ERROR_T* error);
 
-char* error_message(ERROR_T* error);
+hb_string_T error_message(ERROR_T* error);
 
-const char* error_type_to_string(ERROR_T* error);
-const char* error_human_type(ERROR_T* error);
+hb_string_T error_type_to_string(ERROR_T* error);
+hb_string_T error_human_type(ERROR_T* error);
 
-void error_free(ERROR_T* error);
+void error_free(ERROR_T* error, hb_allocator_T* allocator);
 
-void error_pretty_print(ERROR_T* error, size_t indent, size_t relative_indent, hb_buffer_T* buffer);
+#ifndef HERB_EXCLUDE_PRETTYPRINT
+  void error_pretty_print(ERROR_T* error, size_t indent, size_t relative_indent, hb_buffer_T* buffer);
 
-void error_pretty_print_array(
-  const char* name, hb_array_T* array, size_t indent, size_t relative_indent, bool last_property,
-  hb_buffer_T* buffer
-);
+  void error_pretty_print_array(
+    const char* name, hb_array_T* array, size_t indent, size_t relative_indent, bool last_property,
+    hb_buffer_T* buffer
+  );
+#endif
 
 #endif

package/extension/libherb/extract.c

@@ -1,40 +1,96 @@
 #include "include/herb.h"
-#include "include/io.h"
-#include "include/lexer.h"
+#include "include/util/hb_allocator.h"
 #include "include/util/hb_array.h"
 #include "include/util/hb_buffer.h"
+#include "include/util/hb_string.h"
+#include "include/util/string.h"
 
+#include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 
-void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
-  hb_array_T* tokens = herb_lex(source);
+const herb_extract_ruby_options_T HERB_EXTRACT_RUBY_DEFAULT_OPTIONS = { .semicolons = true,
+                                                                        .comments = false,
+                                                                        .preserve_positions = true };
+
+void herb_extract_ruby_to_buffer_with_options(
+  const char* source,
+  hb_buffer_T* output,
+  const herb_extract_ruby_options_T* options,
+  hb_allocator_T* allocator
+) {
+  herb_extract_ruby_options_T extract_options = options ? *options : HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+
+  hb_array_T* tokens = herb_lex(source, allocator);
   bool skip_erb_content = false;
   bool is_comment_tag = false;
+  bool is_erb_comment_tag = false;
+  bool need_newline = false;
 
   for (size_t i = 0; i < hb_array_size(tokens); i++) {
     const token_T* token = hb_array_get(tokens, i);
 
     switch (token->type) {
       case TOKEN_NEWLINE: {
-        hb_buffer_append(output, token->value);
+        hb_buffer_append_string(output, token->value);
+        need_newline = false;
         break;
       }
 
       case TOKEN_ERB_START: {
-        if (strcmp(token->value, "<%#") == 0) {
-          skip_erb_content = true;
-          is_comment_tag = true;
-        } else if (strcmp(token->value, "<%%") == 0 || strcmp(token->value, "<%%=") == 0
-                   || strcmp(token->value, "<%graphql") == 0) {
+        is_erb_comment_tag = hb_string_equals(token->value, hb_string("<%#"));
+
+        if (is_erb_comment_tag) {
+          if (extract_options.comments) {
+            skip_erb_content = false;
+            is_comment_tag = false;
+
+            if (extract_options.preserve_positions) {
+              bool is_multiline = false;
+
+              if (i + 1 < hb_array_size(tokens)) {
+                const token_T* next = hb_array_get(tokens, i + 1);
+
+                if (next->type == TOKEN_ERB_CONTENT && !hb_string_is_null(next->value)
+                    && memchr(next->value.data, '\n', next->value.length) != NULL) {
+                  is_multiline = true;
+                }
+              }
+
+              if (is_multiline) {
+                hb_buffer_append_char(output, '#');
+                hb_buffer_append_whitespace(output, 2);
+              } else {
+                hb_buffer_append_whitespace(output, 2);
+                hb_buffer_append_char(output, '#');
+              }
+            } else {
+              if (need_newline) { hb_buffer_append_char(output, '\n'); }
+              hb_buffer_append_char(output, '#');
+              need_newline = true;
+            }
+          } else {
+            skip_erb_content = true;
+            is_comment_tag = true;
+            if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
+          }
+        } else if (hb_string_equals(token->value, hb_string("<%%")) || hb_string_equals(token->value, hb_string("<%%="))
+                   || hb_string_equals(token->value, hb_string("<%graphql"))) {
           skip_erb_content = true;
           is_comment_tag = false;
+          if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
         } else {
           skip_erb_content = false;
           is_comment_tag = false;
+
+          if (extract_options.preserve_positions) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          } else if (need_newline) {
+            hb_buffer_append_char(output, '\n');
+            need_newline = false;
+          }
         }
 
-        hb_buffer_append_whitespace(output, range_length(token->range));
         break;
       }
 
@@ -42,26 +98,65 @@ void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
         if (skip_erb_content == false) {
           bool is_inline_comment = false;
 
-          if (!is_comment_tag && token->value != NULL) {
-            const char* content = token->value;
+          if (!extract_options.comments && !is_comment_tag && !hb_string_is_empty(token->value)) {
+            hb_string_T trimmed = hb_string_trim_start(token->value);
 
-            while (*content == ' ' || *content == '\t') {
-              content++;
-            }
-
-            if (*content == '#' && token->location.start.line == token->location.end.line) {
+            if (!hb_string_is_empty(trimmed) && trimmed.data[0] == '#'
+                && token->location.start.line == token->location.end.line) {
               is_comment_tag = true;
               is_inline_comment = true;
             }
           }
 
           if (is_inline_comment) {
-            hb_buffer_append_whitespace(output, range_length(token->range));
+            if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
+          } else if (is_erb_comment_tag && !hb_string_is_null(token->value)) {
+            const char* content = token->value.data;
+            size_t content_remaining = token->value.length;
+
+            while (content_remaining > 0) {
+              if (*content == '\n') {
+                hb_buffer_append_char(output, '\n');
+                content++;
+                content_remaining--;
+
+                if (content_remaining > 0 && extract_options.preserve_positions && *content == ' ') {
+                  content++;
+                  content_remaining--;
+                }
+
+                hb_buffer_append_char(output, '#');
+              } else {
+                hb_buffer_append_char(output, *content);
+                content++;
+                content_remaining--;
+              }
+            }
+
+            if (!extract_options.preserve_positions) { need_newline = true; }
           } else {
-            hb_buffer_append(output, token->value);
+            hb_buffer_append_string(output, token->value);
+
+            if (!extract_options.preserve_positions) { need_newline = true; }
           }
         } else {
-          hb_buffer_append_whitespace(output, range_length(token->range));
+          if (is_erb_comment_tag && extract_options.preserve_positions && !hb_string_is_null(token->value)) {
+            const char* content = token->value.data;
+            size_t content_remaining = token->value.length;
+
+            while (content_remaining > 0) {
+              if (*content == '\n') {
+                hb_buffer_append_char(output, '\n');
+              } else {
+                hb_buffer_append_char(output, ' ');
+              }
+
+              content++;
+              content_remaining--;
+            }
+          } else if (extract_options.preserve_positions) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          }
         }
 
         break;
@@ -69,31 +164,43 @@ void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
 
       case TOKEN_ERB_END: {
         bool was_comment = is_comment_tag;
+        bool was_erb_comment = is_erb_comment_tag;
         skip_erb_content = false;
         is_comment_tag = false;
+        is_erb_comment_tag = false;
 
-        if (was_comment) {
-          hb_buffer_append_whitespace(output, range_length(token->range));
-          break;
+        if (extract_options.preserve_positions) {
+          if (was_comment) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          } else if (was_erb_comment && extract_options.comments) {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          } else if (extract_options.semicolons) {
+            hb_buffer_append_char(output, ' ');
+            hb_buffer_append_char(output, ';');
+            hb_buffer_append_whitespace(output, range_length(token->range) - 2);
+          } else {
+            hb_buffer_append_whitespace(output, range_length(token->range));
+          }
         }
 
-        hb_buffer_append_char(output, ' ');
-        hb_buffer_append_char(output, ';');
-        hb_buffer_append_whitespace(output, range_length(token->range) - 2);
         break;
       }
 
       default: {
-        hb_buffer_append_whitespace(output, range_length(token->range));
+        if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
       }
     }
   }
 
-  herb_free_tokens(&tokens);
+  herb_free_tokens(&tokens, allocator);
+}
+
+void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {
+  herb_extract_ruby_to_buffer_with_options(source, output, NULL, allocator);
 }
 
-void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output) {
-  hb_array_T* tokens = herb_lex(source);
+void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {
+  hb_array_T* tokens = herb_lex(source, allocator);
 
   for (size_t i = 0; i < hb_array_size(tokens); i++) {
     const token_T* token = hb_array_get(tokens, i);
@@ -102,43 +209,35 @@ void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output) {
       case TOKEN_ERB_START:
       case TOKEN_ERB_CONTENT:
       case TOKEN_ERB_END: hb_buffer_append_whitespace(output, range_length(token->range)); break;
-      default: hb_buffer_append(output, token->value);
+      default: hb_buffer_append_string(output, token->value);
     }
   }
 
-  herb_free_tokens(&tokens);
+  herb_free_tokens(&tokens, allocator);
 }
 
-char* herb_extract_ruby_with_semicolons(const char* source) {
+char* herb_extract_ruby_with_semicolons(const char* source, hb_allocator_T* allocator) {
   if (!source) { return NULL; }
 
   hb_buffer_T output;
-  hb_buffer_init(&output, strlen(source));
+  hb_buffer_init(&output, strlen(source), allocator);
 
-  herb_extract_ruby_to_buffer(source, &output);
+  herb_extract_ruby_to_buffer(source, &output, allocator);
 
   return output.value;
 }
 
-char* herb_extract(const char* source, const herb_extract_language_T language) {
+char* herb_extract(const char* source, const herb_extract_language_T language, hb_allocator_T* allocator) {
   if (!source) { return NULL; }
 
   hb_buffer_T output;
-  hb_buffer_init(&output, strlen(source));
+  hb_buffer_init(&output, strlen(source), allocator);
 
   switch (language) {
-    case HERB_EXTRACT_LANGUAGE_RUBY: herb_extract_ruby_to_buffer(source, &output); break;
-    case HERB_EXTRACT_LANGUAGE_HTML: herb_extract_html_to_buffer(source, &output); break;
+    case HERB_EXTRACT_LANGUAGE_RUBY: herb_extract_ruby_to_buffer(source, &output, allocator); break;
+    case HERB_EXTRACT_LANGUAGE_HTML: herb_extract_html_to_buffer(source, &output, allocator); break;
+    default: assert(0 && "invalid extract language");
   }
 
   return output.value;
 }
-
-char* herb_extract_from_file(const char* path, const herb_extract_language_T language) {
-  char* source = herb_read_file(path);
-  char* output = herb_extract(source, language);
-
-  free(source);
-
-  return output;
-}

package/extension/libherb/extract.h

@@ -1,19 +1,35 @@
 #ifndef HERB_EXTRACT_H
 #define HERB_EXTRACT_H
 
+#include "util/hb_allocator.h"
 #include "util/hb_buffer.h"
 
+#include <stdbool.h>
+
 typedef enum {
   HERB_EXTRACT_LANGUAGE_RUBY,
   HERB_EXTRACT_LANGUAGE_HTML,
 } herb_extract_language_T;
 
-void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output);
-void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output);
+typedef struct {
+  bool semicolons;
+  bool comments;
+  bool preserve_positions;
+} herb_extract_ruby_options_T;
+
+extern const herb_extract_ruby_options_T HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+
+void herb_extract_ruby_to_buffer_with_options(
+  const char* source,
+  hb_buffer_T* output,
+  const herb_extract_ruby_options_T* options,
+  hb_allocator_T* allocator
+);
+void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator);
+void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator);
 
-char* herb_extract_ruby_with_semicolons(const char* source);
+char* herb_extract_ruby_with_semicolons(const char* source, hb_allocator_T* allocator);
 
-char* herb_extract(const char* source, herb_extract_language_T language);
-char* herb_extract_from_file(const char* path, herb_extract_language_T language);
+char* herb_extract(const char* source, herb_extract_language_T language, hb_allocator_T* allocator);
 
 #endif