herb 0.11.0-aarch64-linux-gnu → 0.11.1-aarch64-linux-gnu

data/src/extract.c

@@ -1,9 +1,12 @@
 #include "include/herb.h"
+#include "include/lexer/lexer.h"
+#include "include/lexer/token.h"
 #include "include/lib/hb_allocator.h"
 #include "include/lib/hb_array.h"
 #include "include/lib/hb_buffer.h"
 #include "include/lib/hb_string.h"
 #include "include/lib/string.h"
+#include "include/util/util.h"
 
 #include <assert.h>
 #include <stdlib.h>
@@ -16,192 +19,250 @@ const herb_extract_ruby_options_T HERB_EXTRACT_RUBY_DEFAULT_OPTIONS = { .semicol
                                                                         .erb_openers = NULL,
                                                                         .erb_opener_count = 0 };
 
-void herb_extract_ruby_to_buffer_with_options(
-  const char* source,
-  hb_buffer_T* output,
-  const herb_extract_ruby_options_T* options,
-  hb_allocator_T* allocator
-) {
-  herb_extract_ruby_options_T extract_options = options ? *options : HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+typedef struct {
+  herb_extract_ruby_options_T options;
+  hb_buffer_T* output;
+  bool skip_erb_content;
+  bool is_comment_tag;
+  bool is_erb_comment_tag;
+  bool need_newline;
+} extract_ruby_state_T;
 
-  parser_options_T lex_options = HERB_DEFAULT_PARSER_OPTIONS;
-  lex_options.erb_openers = extract_options.erb_openers;
-  lex_options.erb_opener_count = extract_options.erb_opener_count;
+static void extract_ruby_data(extract_ruby_state_T* state, const char* data, uint32_t from, uint32_t to) {
+  uint32_t run_start = from;
 
-  hb_array_T* tokens = herb_lex_with_options(source, &lex_options, allocator);
-  bool skip_erb_content = false;
-  bool is_comment_tag = false;
-  bool is_erb_comment_tag = false;
-  bool need_newline = false;
+  for (uint32_t position = from; position < to; position++) {
+    if (!is_newline(data[position])) { continue; }
 
-  for (size_t i = 0; i < hb_array_size(tokens); i++) {
-    const token_T* token = hb_array_get(tokens, i);
+    if (state->options.preserve_positions) { hb_buffer_append_whitespace(state->output, position - run_start); }
 
-    switch (token->type) {
-      case TOKEN_NEWLINE: {
-        hb_buffer_append_string(output, token->value);
-        need_newline = false;
-        break;
-      }
+    hb_buffer_append_char(state->output, data[position]);
+    state->need_newline = false;
+    run_start = position + 1;
+  }
 
-      case TOKEN_ERB_START: {
-        is_erb_comment_tag = hb_string_equals(token->value, hb_string("<%#"));
+  if (state->options.preserve_positions) { hb_buffer_append_whitespace(state->output, to - run_start); }
+}
 
-        if (is_erb_comment_tag) {
-          if (extract_options.comments) {
-            skip_erb_content = false;
-            is_comment_tag = false;
+static void extract_ruby_token(extract_ruby_state_T* state, const token_T* token, const token_T* next) {
+  switch (token->type) {
+    case TOKEN_NEWLINE: {
+      hb_buffer_append_string(state->output, token->value);
+      state->need_newline = false;
+      break;
+    }
 
-            if (extract_options.preserve_positions) {
-              bool is_multiline = false;
+    case TOKEN_ERB_START: {
+      state->is_erb_comment_tag = hb_string_equals(token->value, hb_string("<%#"));
 
-              if (i + 1 < hb_array_size(tokens)) {
-                const token_T* next = hb_array_get(tokens, i + 1);
+      if (state->is_erb_comment_tag) {
+        if (state->options.comments) {
+          state->skip_erb_content = false;
+          state->is_comment_tag = false;
 
-                if (next->type == TOKEN_ERB_CONTENT && !hb_string_is_null(next->value)
-                    && memchr(next->value.data, '\n', next->value.length) != NULL) {
-                  is_multiline = true;
-                }
-              }
+          if (state->options.preserve_positions) {
+            bool is_multiline = false;
 
-              if (is_multiline) {
-                hb_buffer_append_char(output, '#');
-                hb_buffer_append_whitespace(output, 2);
-              } else {
-                hb_buffer_append_whitespace(output, 2);
-                hb_buffer_append_char(output, '#');
-              }
+            if (next && next->type == TOKEN_ERB_CONTENT && !hb_string_is_null(next->value)
+                && memchr(next->value.data, '\n', next->value.length) != NULL) {
+              is_multiline = true;
+            }
+
+            if (is_multiline) {
+              hb_buffer_append_char(state->output, '#');
+              hb_buffer_append_whitespace(state->output, 2);
             } else {
-              if (need_newline) { hb_buffer_append_char(output, '\n'); }
-              hb_buffer_append_char(output, '#');
-              need_newline = true;
+              hb_buffer_append_whitespace(state->output, 2);
+              hb_buffer_append_char(state->output, '#');
             }
           } else {
-            skip_erb_content = true;
-            is_comment_tag = true;
-            if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
+            if (state->need_newline) { hb_buffer_append_char(state->output, '\n'); }
+            hb_buffer_append_char(state->output, '#');
+            state->need_newline = true;
           }
-        } else if (hb_string_equals(token->value, hb_string("<%%")) || hb_string_equals(token->value, hb_string("<%%="))
-                   || (erb_opening_is_custom(token->value) && !extract_options.custom_tags)) {
-          skip_erb_content = true;
-          is_comment_tag = false;
-          if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
         } else {
-          skip_erb_content = false;
-          is_comment_tag = false;
-
-          if (extract_options.preserve_positions) {
-            hb_buffer_append_whitespace(output, range_length(token->range));
-          } else if (need_newline) {
-            hb_buffer_append_char(output, '\n');
-            need_newline = false;
+          state->skip_erb_content = true;
+          state->is_comment_tag = true;
+          if (state->options.preserve_positions) {
+            hb_buffer_append_whitespace(state->output, range_length(token->range));
           }
         }
-
-        break;
+      } else if (hb_string_equals(token->value, hb_string("<%%")) || hb_string_equals(token->value, hb_string("<%%="))
+                 || (erb_opening_is_custom(token->value) && !state->options.custom_tags)) {
+        state->skip_erb_content = true;
+        state->is_comment_tag = false;
+        if (state->options.preserve_positions) {
+          hb_buffer_append_whitespace(state->output, range_length(token->range));
+        }
+      } else {
+        state->skip_erb_content = false;
+        state->is_comment_tag = false;
+
+        if (state->options.preserve_positions) {
+          hb_buffer_append_whitespace(state->output, range_length(token->range));
+        } else if (state->need_newline) {
+          hb_buffer_append_char(state->output, '\n');
+          state->need_newline = false;
+        }
       }
 
-      case TOKEN_ERB_CONTENT: {
-        if (skip_erb_content == false) {
-          bool is_inline_comment = false;
+      break;
+    }
 
-          if (!extract_options.comments && !is_comment_tag && !hb_string_is_empty(token->value)) {
-            hb_string_T trimmed = hb_string_trim_start(token->value);
+    case TOKEN_ERB_CONTENT: {
+      if (state->skip_erb_content == false) {
+        bool is_inline_comment = false;
 
-            if (!hb_string_is_empty(trimmed) && trimmed.data[0] == '#'
-                && token->location.start.line == token->location.end.line) {
-              is_comment_tag = true;
-              is_inline_comment = true;
-            }
-          }
+        if (!state->options.comments && !state->is_comment_tag && !hb_string_is_empty(token->value)) {
+          hb_string_T trimmed = hb_string_trim_start(token->value);
 
-          if (is_inline_comment) {
-            if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
-          } else if (is_erb_comment_tag && !hb_string_is_null(token->value)) {
-            const char* content = token->value.data;
-            size_t content_remaining = token->value.length;
+          if (!hb_string_is_empty(trimmed) && trimmed.data[0] == '#'
+              && token->location.start.line == token->location.end.line) {
+            state->is_comment_tag = true;
+            is_inline_comment = true;
+          }
+        }
 
-            while (content_remaining > 0) {
-              if (*content == '\n') {
-                hb_buffer_append_char(output, '\n');
-                content++;
-                content_remaining--;
+        if (is_inline_comment) {
+          if (state->options.preserve_positions) {
+            hb_buffer_append_whitespace(state->output, range_length(token->range));
+          }
+        } else if (state->is_erb_comment_tag && !hb_string_is_null(token->value)) {
+          const char* content = token->value.data;
+          size_t content_remaining = token->value.length;
 
-                if (content_remaining > 0 && extract_options.preserve_positions && *content == ' ') {
-                  content++;
-                  content_remaining--;
-                }
+          while (content_remaining > 0) {
+            if (*content == '\n') {
+              hb_buffer_append_char(state->output, '\n');
+              content++;
+              content_remaining--;
 
-                hb_buffer_append_char(output, '#');
-              } else {
-                hb_buffer_append_char(output, *content);
+              if (content_remaining > 0 && state->options.preserve_positions && *content == ' ') {
                 content++;
                 content_remaining--;
               }
-            }
-
-            if (!extract_options.preserve_positions) { need_newline = true; }
-          } else {
-            hb_buffer_append_string(output, token->value);
-
-            if (!extract_options.preserve_positions) { need_newline = true; }
-          }
-        } else {
-          if (is_erb_comment_tag && extract_options.preserve_positions && !hb_string_is_null(token->value)) {
-            const char* content = token->value.data;
-            size_t content_remaining = token->value.length;
-
-            while (content_remaining > 0) {
-              if (*content == '\n') {
-                hb_buffer_append_char(output, '\n');
-              } else {
-                hb_buffer_append_char(output, ' ');
-              }
 
+              hb_buffer_append_char(state->output, '#');
+            } else {
+              hb_buffer_append_char(state->output, *content);
               content++;
               content_remaining--;
             }
-          } else if (extract_options.preserve_positions) {
-            hb_buffer_append_whitespace(output, range_length(token->range));
           }
-        }
 
-        break;
-      }
+          if (!state->options.preserve_positions) { state->need_newline = true; }
+        } else {
+          hb_buffer_append_string(state->output, token->value);
 
-      case TOKEN_ERB_END: {
-        bool was_comment = is_comment_tag;
-        bool was_erb_comment = is_erb_comment_tag;
-        skip_erb_content = false;
-        is_comment_tag = false;
-        is_erb_comment_tag = false;
-
-        if (extract_options.preserve_positions) {
-          if (was_comment) {
-            hb_buffer_append_whitespace(output, range_length(token->range));
-          } else if (was_erb_comment && extract_options.comments) {
-            hb_buffer_append_whitespace(output, range_length(token->range));
-          } else if (extract_options.semicolons) {
-            size_t length = range_length(token->range);
-
-            if (length >= 2) { hb_buffer_append_char(output, ' '); }
-            if (length >= 1) { hb_buffer_append_char(output, ';'); }
-            if (length >= 2) { hb_buffer_append_whitespace(output, length - 2); }
-          } else {
-            hb_buffer_append_whitespace(output, range_length(token->range));
-          }
+          if (!state->options.preserve_positions) { state->need_newline = true; }
         }
+      } else {
+        if (state->is_erb_comment_tag && state->options.preserve_positions && !hb_string_is_null(token->value)) {
+          const char* content = token->value.data;
+          size_t content_remaining = token->value.length;
+
+          while (content_remaining > 0) {
+            if (*content == '\n') {
+              hb_buffer_append_char(state->output, '\n');
+            } else {
+              hb_buffer_append_char(state->output, ' ');
+            }
 
-        break;
+            content++;
+            content_remaining--;
+          }
+        } else if (state->options.preserve_positions) {
+          hb_buffer_append_whitespace(state->output, range_length(token->range));
+        }
       }
 
-      default: {
-        if (extract_options.preserve_positions) { hb_buffer_append_whitespace(output, range_length(token->range)); }
+      break;
+    }
+
+    case TOKEN_ERB_END: {
+      bool was_comment = state->is_comment_tag;
+      bool was_erb_comment = state->is_erb_comment_tag;
+      state->skip_erb_content = false;
+      state->is_comment_tag = false;
+      state->is_erb_comment_tag = false;
+
+      if (state->options.preserve_positions) {
+        if (was_comment) {
+          hb_buffer_append_whitespace(state->output, range_length(token->range));
+        } else if (was_erb_comment && state->options.comments) {
+          hb_buffer_append_whitespace(state->output, range_length(token->range));
+        } else if (state->options.semicolons) {
+          size_t length = range_length(token->range);
+
+          if (length >= 2) { hb_buffer_append_char(state->output, ' '); }
+          if (length >= 1) { hb_buffer_append_char(state->output, ';'); }
+          if (length >= 2) { hb_buffer_append_whitespace(state->output, length - 2); }
+        } else {
+          hb_buffer_append_whitespace(state->output, range_length(token->range));
+        }
       }
+
+      break;
+    }
+
+    default: {
+      if (state->options.preserve_positions) { hb_buffer_append_whitespace(state->output, range_length(token->range)); }
     }
   }
+}
 
-  herb_free_tokens(&tokens, allocator);
+void herb_extract_ruby_to_buffer_with_options(
+  const char* source,
+  hb_buffer_T* output,
+  const herb_extract_ruby_options_T* options,
+  hb_allocator_T* allocator
+) {
+  extract_ruby_state_T state = {
+    .options = options ? *options : HERB_EXTRACT_RUBY_DEFAULT_OPTIONS,
+    .output = output,
+    .skip_erb_content = false,
+    .is_comment_tag = false,
+    .is_erb_comment_tag = false,
+    .need_newline = false,
+  };
+
+  parser_options_T lex_options = HERB_DEFAULT_PARSER_OPTIONS;
+  lex_options.erb_openers = state.options.erb_openers;
+  lex_options.erb_opener_count = state.options.erb_opener_count;
+
+  lexer_T lexer = { 0 };
+  lexer_init(&lexer, source ? source : "", allocator);
+  lexer_apply_erb_openers(&lexer, &lex_options);
+
+  const char* data = lexer.source.data;
+  uint32_t length = (uint32_t) lexer.source.length;
+  token_T* pending = NULL;
+
+  while (true) {
+    if (!pending && lexer.state == STATE_DATA) {
+      uint32_t position = lexer.current_position;
+
+      while (position < length && !(data[position] == '<' && data[position + 1] == '%')) {
+        position++;
+      }
+
+      extract_ruby_data(&state, data, lexer.current_position, position);
+      lexer_skip_data_to(&lexer, position);
+    }
+
+    token_T* token = pending ? pending : lexer_next_token(&lexer);
+    pending = NULL;
+
+    if (token->type == TOKEN_EOF) {
+      token_free(token, allocator);
+      break;
+    }
+
+    if (token->type == TOKEN_ERB_START) { pending = lexer_next_token(&lexer); }
+
+    extract_ruby_token(&state, token, pending);
+    token_free(token, allocator);
+  }
 }
 
 void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {

data/src/include/analyze/analyze.h

@@ -26,28 +26,6 @@ typedef struct ANALYZE_RUBY_CONTEXT_STRUCT {
   const parser_options_T* options;
 } analyze_ruby_context_T;
 
-typedef enum {
-  CONTROL_TYPE_IF,
-  CONTROL_TYPE_ELSIF,
-  CONTROL_TYPE_ELSE,
-  CONTROL_TYPE_END,
-  CONTROL_TYPE_CASE,
-  CONTROL_TYPE_CASE_MATCH,
-  CONTROL_TYPE_WHEN,
-  CONTROL_TYPE_IN,
-  CONTROL_TYPE_BEGIN,
-  CONTROL_TYPE_RESCUE,
-  CONTROL_TYPE_ENSURE,
-  CONTROL_TYPE_UNLESS,
-  CONTROL_TYPE_WHILE,
-  CONTROL_TYPE_UNTIL,
-  CONTROL_TYPE_FOR,
-  CONTROL_TYPE_BLOCK,
-  CONTROL_TYPE_BLOCK_CLOSE,
-  CONTROL_TYPE_YIELD,
-  CONTROL_TYPE_UNKNOWN
-} control_type_t;
-
 typedef struct {
   int loop_depth;
   int rescue_depth;

data/src/include/analyze/analyzed_ruby.h

@@ -5,6 +5,29 @@
 #include "../lib/hb_string.h"
 
 #include <prism.h>
+#include <stdbool.h>
+
+typedef enum {
+  CONTROL_TYPE_IF,
+  CONTROL_TYPE_ELSIF,
+  CONTROL_TYPE_ELSE,
+  CONTROL_TYPE_END,
+  CONTROL_TYPE_CASE,
+  CONTROL_TYPE_CASE_MATCH,
+  CONTROL_TYPE_WHEN,
+  CONTROL_TYPE_IN,
+  CONTROL_TYPE_BEGIN,
+  CONTROL_TYPE_RESCUE,
+  CONTROL_TYPE_ENSURE,
+  CONTROL_TYPE_UNLESS,
+  CONTROL_TYPE_WHILE,
+  CONTROL_TYPE_UNTIL,
+  CONTROL_TYPE_FOR,
+  CONTROL_TYPE_BLOCK,
+  CONTROL_TYPE_BLOCK_CLOSE,
+  CONTROL_TYPE_YIELD,
+  CONTROL_TYPE_UNKNOWN
+} control_type_t;
 
 typedef struct ANALYZED_RUBY_STRUCT {
   pm_parser_t parser;
@@ -31,6 +54,8 @@ typedef struct ANALYZED_RUBY_STRUCT {
   int yield_node_count;
   int then_keyword_count;
   int unclosed_control_flow_count;
+  control_type_t control_type;
+  bool control_type_detected;
 } analyzed_ruby_T;
 
 analyzed_ruby_T* init_analyzed_ruby(hb_string_T source);

data/src/include/analyze/helpers.h

@@ -42,29 +42,9 @@ bool is_brace_block(pm_location_t opening_location);
 bool is_closing_brace(pm_location_t location);
 bool has_valid_block_closing(pm_location_t opening_loc, pm_location_t closing_loc);
 
-bool search_begin_nodes(const pm_node_t* node, void* data);
-bool search_block_nodes(const pm_node_t* node, void* data);
-bool search_case_match_nodes(const pm_node_t* node, void* data);
-bool search_case_nodes(const pm_node_t* node, void* data);
-bool search_for_nodes(const pm_node_t* node, void* data);
-bool search_if_nodes(const pm_node_t* node, void* data);
-bool search_in_nodes(const pm_node_t* node, void* data);
-bool search_then_keywords(const pm_node_t* node, void* data);
+bool search_control_flow_nodes(const pm_node_t* node, void* data);
 bool search_unclosed_control_flows(const pm_node_t* node, void* data);
-bool search_unless_nodes(const pm_node_t* node, void* data);
-bool search_until_nodes(const pm_node_t* node, void* data);
-bool search_when_nodes(const pm_node_t* node, void* data);
-bool search_while_nodes(const pm_node_t* node, void* data);
-bool search_yield_nodes(const pm_node_t* node, void* data);
-
-bool search_unexpected_block_closing_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_else_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_elsif_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_end_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_ensure_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_in_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_rescue_nodes(analyzed_ruby_T* analyzed);
-bool search_unexpected_when_nodes(analyzed_ruby_T* analyzed);
+void search_unexpected_keyword_errors(analyzed_ruby_T* analyzed);
 
 void check_erb_node_for_missing_end(const AST_NODE_T* node, hb_allocator_T* allocator, const parser_options_T* options);
 

data/src/include/ast/ast_nodes.h

@@ -478,6 +478,7 @@ typedef struct AST_ERB_YIELD_NODE_STRUCT {
   token_T* tag_opening;
   token_T* content;
   token_T* tag_closing;
+  herb_prism_node_T prism_node;
 } AST_ERB_YIELD_NODE_T;
 
 typedef struct AST_ERB_IN_NODE_STRUCT {
@@ -565,7 +566,7 @@ AST_RUBY_RENDER_KEYWORDS_NODE_T* ast_ruby_render_keywords_node_init(token_T* par
 AST_ERB_RENDER_NODE_T* ast_erb_render_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, analyzed_ruby_T* analyzed_ruby, herb_prism_node_T prism_node, struct AST_RUBY_RENDER_KEYWORDS_NODE_STRUCT* keywords, hb_array_T* body, hb_array_T* block_arguments, struct AST_ERB_RESCUE_NODE_STRUCT* rescue_clause, struct AST_ERB_ELSE_NODE_STRUCT* else_clause, struct AST_ERB_ENSURE_NODE_STRUCT* ensure_clause, struct AST_ERB_END_NODE_STRUCT* end_node, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_RUBY_PARAMETER_NODE_T* ast_ruby_parameter_node_init(token_T* name, struct AST_RUBY_LITERAL_NODE_STRUCT* default_value, hb_string_T kind, bool required, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_ERB_STRICT_LOCALS_NODE_T* ast_erb_strict_locals_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, analyzed_ruby_T* analyzed_ruby, herb_prism_node_T prism_node, hb_array_T* locals, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
-AST_ERB_YIELD_NODE_T* ast_erb_yield_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
+AST_ERB_YIELD_NODE_T* ast_erb_yield_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, herb_prism_node_T prism_node, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_ERB_IN_NODE_T* ast_erb_in_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, location_T* then_keyword, hb_array_T* statements, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_HERB_DIRECTIVE_NODE_T* ast_herb_directive_node_init(token_T* tag_opening, token_T* content, token_T* tag_closing, token_T* key, token_T* arguments, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);
 AST_HERB_STATE_DECLARATION_NODE_T* ast_herb_state_declaration_node_init(token_T* name, struct AST_RUBY_LITERAL_NODE_STRUCT* default_value, hb_string_T kind, position_T start_position, position_T end_position, hb_array_T* errors, hb_allocator_T* allocator);

data/src/include/lexer/lexer.h

@@ -7,5 +7,6 @@
 void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator);
 token_T* lexer_next_token(lexer_T* lexer);
 token_T* lexer_error(lexer_T* lexer, const char* message);
+void lexer_skip_data_to(lexer_T* lexer, uint32_t position);
 
 #endif

data/src/include/prism/herb_prism_node.h

@@ -2,12 +2,17 @@
 #define HERB_PRISM_NODE_H
 
 #include <prism.h>
+#include <stdint.h>
 
 typedef struct {
   pm_node_t* node;
   pm_parser_t* parser;
+  uint32_t source_start;
+  uint32_t source_end;
 } herb_prism_node_T;
 
-#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL })
+#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL, .source_start = 0, .source_end = 0 })
+
+void herb_prism_node_serialize(herb_prism_node_T prism_node, pm_buffer_t* buffer);
 
 #endif

data/src/include/version.h

@@ -1,6 +1,6 @@
 #ifndef HERB_VERSION_H
 #define HERB_VERSION_H
 
-#define HERB_VERSION "0.11.0"
+#define HERB_VERSION "0.11.1"
 
 #endif

data/src/lexer.c

@@ -430,6 +430,38 @@ static token_T* lexer_parse_erb_close(lexer_T* lexer) {
   return lexer_advance_with(lexer, hb_string("%>"), TOKEN_ERB_END);
 }
 
+void lexer_skip_data_to(lexer_T* lexer, uint32_t position) {
+  if (position > lexer->source.length) { position = lexer->source.length; }
+
+  while (lexer->current_position < position && !lexer_eof(lexer)) {
+    char character = lexer->current_character;
+
+    if (character == '\r' && lexer_peek(lexer, 1) == '\n') {
+      lexer->current_position += 2;
+      lexer->current_line++;
+      lexer->current_column = 0;
+    } else if (is_newline(character)) {
+      lexer->current_position++;
+      lexer->current_line++;
+      lexer->current_column = 0;
+    } else if ((unsigned char) character < 0x80) {
+      lexer->current_position++;
+      lexer->current_column++;
+    } else {
+      uint32_t length = utf8_sequence_length(hb_string_slice(lexer->source, lexer->current_position));
+
+      lexer->current_position += length > 0 ? length : 1;
+      lexer->current_column++;
+    }
+
+    lexer->current_character = lexer->source.data[lexer->current_position];
+  }
+
+  lexer->previous_line = lexer->current_line;
+  lexer->previous_column = lexer->current_column;
+  lexer->previous_position = lexer->current_position;
+}
+
 // ===== Tokenizing Function
 
 token_T* lexer_next_token(lexer_T* lexer) {