@herb-tools/node 0.8.10 → 0.9.0

package/extension/libherb/include/errors.h

@@ -1,5 +1,5 @@
 // NOTE: This file is generated by the templates/template.rb script and should not
-// be modified manually. See /Users/marcoroth/Development/herb-release-0.8.10/templates/src/include/errors.h.erb
+// be modified manually. See /Users/marcoroth/Development/herb-release-0.9.0/templates/src/include/errors.h.erb
 
 #ifndef HERB_ERRORS_H
 #define HERB_ERRORS_H
@@ -8,8 +8,10 @@
 #include "location.h"
 #include "position.h"
 #include "token.h"
+#include "util/hb_allocator.h"
 #include "util/hb_array.h"
 #include "util/hb_buffer.h"
+#include "util/hb_string.h"
 
 typedef enum {
   UNEXPECTED_ERROR,
@@ -17,28 +19,38 @@ typedef enum {
   MISSING_OPENING_TAG_ERROR,
   MISSING_CLOSING_TAG_ERROR,
   TAG_NAMES_MISMATCH_ERROR,
-  QUOTES_MISMATCH_ERROR,
   VOID_ELEMENT_CLOSING_TAG_ERROR,
   UNCLOSED_ELEMENT_ERROR,
   RUBY_PARSE_ERROR,
   ERB_CONTROL_FLOW_SCOPE_ERROR,
-  MISSINGERB_END_TAG_ERROR,
+  MISSING_ERB_END_TAG_ERROR,
   ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR,
   ERB_CASE_WITH_CONDITIONS_ERROR,
+  CONDITIONAL_ELEMENT_MULTIPLE_TAGS_ERROR,
+  CONDITIONAL_ELEMENT_CONDITION_MISMATCH_ERROR,
+  INVALID_COMMENT_CLOSING_TAG_ERROR,
+  OMITTED_CLOSING_TAG_ERROR,
+  UNCLOSED_OPEN_TAG_ERROR,
+  UNCLOSED_CLOSE_TAG_ERROR,
+  UNCLOSED_QUOTE_ERROR,
+  MISSING_ATTRIBUTE_VALUE_ERROR,
+  UNCLOSED_ERB_TAG_ERROR,
+  STRAY_ERB_CLOSING_TAG_ERROR,
+  NESTED_ERB_TAG_ERROR,
 } error_type_T;
 
 typedef struct ERROR_STRUCT {
   error_type_T type;
   location_T location;
-  char* message;
+  hb_string_T message;
 } ERROR_T;
 
 
 typedef struct {
   ERROR_T base;
-  const char* description;
-  const char* expected;
-  const char* found;
+  hb_string_T description;
+  hb_string_T expected;
+  hb_string_T found;
 } UNEXPECTED_ERROR_T;
 
 typedef struct {
@@ -63,17 +75,11 @@ typedef struct {
   token_T* closing_tag;
 } TAG_NAMES_MISMATCH_ERROR_T;
 
-typedef struct {
-  ERROR_T base;
-  token_T* opening_quote;
-  token_T* closing_quote;
-} QUOTES_MISMATCH_ERROR_T;
-
 typedef struct {
   ERROR_T base;
   token_T* tag_name;
-  const char* expected;
-  const char* found;
+  hb_string_T expected;
+  hb_string_T found;
 } VOID_ELEMENT_CLOSING_TAG_ERROR_T;
 
 typedef struct {
@@ -83,20 +89,20 @@ typedef struct {
 
 typedef struct {
   ERROR_T base;
-  const char* error_message;
-  const char* diagnostic_id;
-  const char* level;
+  hb_string_T error_message;
+  hb_string_T diagnostic_id;
+  hb_string_T level;
 } RUBY_PARSE_ERROR_T;
 
 typedef struct {
   ERROR_T base;
-  const char* keyword;
+  hb_string_T keyword;
 } ERB_CONTROL_FLOW_SCOPE_ERROR_T;
 
 typedef struct {
   ERROR_T base;
-  const char* keyword;
-} MISSINGERB_END_TAG_ERROR_T;
+  hb_string_T keyword;
+} MISSING_ERB_END_TAG_ERROR_T;
 
 typedef struct {
   ERROR_T base;
@@ -108,50 +114,137 @@ typedef struct {
   /* no additional fields */
 } ERB_CASE_WITH_CONDITIONS_ERROR_T;
 
-UNEXPECTED_ERROR_T* unexpected_error_init(const char* description, const char* expected, const char* found, position_T start, position_T end);
-void append_unexpected_error(const char* description, const char* expected, const char* found, position_T start, position_T end, hb_array_T* errors);
-UNEXPECTED_TOKEN_ERROR_T* unexpected_token_error_init(token_type_T expected_type, token_T* found, position_T start, position_T end);
-void append_unexpected_token_error(token_type_T expected_type, token_T* found, position_T start, position_T end, hb_array_T* errors);
-MISSING_OPENING_TAG_ERROR_T* missing_opening_tag_error_init(token_T* closing_tag, position_T start, position_T end);
-void append_missing_opening_tag_error(token_T* closing_tag, position_T start, position_T end, hb_array_T* errors);
-MISSING_CLOSING_TAG_ERROR_T* missing_closing_tag_error_init(token_T* opening_tag, position_T start, position_T end);
-void append_missing_closing_tag_error(token_T* opening_tag, position_T start, position_T end, hb_array_T* errors);
-TAG_NAMES_MISMATCH_ERROR_T* tag_names_mismatch_error_init(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end);
-void append_tag_names_mismatch_error(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end, hb_array_T* errors);
-QUOTES_MISMATCH_ERROR_T* quotes_mismatch_error_init(token_T* opening_quote, token_T* closing_quote, position_T start, position_T end);
-void append_quotes_mismatch_error(token_T* opening_quote, token_T* closing_quote, position_T start, position_T end, hb_array_T* errors);
-VOID_ELEMENT_CLOSING_TAG_ERROR_T* void_element_closing_tag_error_init(token_T* tag_name, const char* expected, const char* found, position_T start, position_T end);
-void append_void_element_closing_tag_error(token_T* tag_name, const char* expected, const char* found, position_T start, position_T end, hb_array_T* errors);
-UNCLOSED_ELEMENT_ERROR_T* unclosed_element_error_init(token_T* opening_tag, position_T start, position_T end);
-void append_unclosed_element_error(token_T* opening_tag, position_T start, position_T end, hb_array_T* errors);
-RUBY_PARSE_ERROR_T* ruby_parse_error_init(const char* error_message, const char* diagnostic_id, const char* level, position_T start, position_T end);
-void append_ruby_parse_error(const char* error_message, const char* diagnostic_id, const char* level, position_T start, position_T end, hb_array_T* errors);
-ERB_CONTROL_FLOW_SCOPE_ERROR_T* erb_control_flow_scope_error_init(const char* keyword, position_T start, position_T end);
-void append_erb_control_flow_scope_error(const char* keyword, position_T start, position_T end, hb_array_T* errors);
-MISSINGERB_END_TAG_ERROR_T* missingerb_end_tag_error_init(const char* keyword, position_T start, position_T end);
-void append_missingerb_end_tag_error(const char* keyword, position_T start, position_T end, hb_array_T* errors);
-ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR_T* erb_multiple_blocks_in_tag_error_init(position_T start, position_T end);
-void append_erb_multiple_blocks_in_tag_error(position_T start, position_T end, hb_array_T* errors);
-ERB_CASE_WITH_CONDITIONS_ERROR_T* erb_case_with_conditions_error_init(position_T start, position_T end);
-void append_erb_case_with_conditions_error(position_T start, position_T end, hb_array_T* errors);
+typedef struct {
+  ERROR_T base;
+  size_t line;
+  size_t column;
+} CONDITIONAL_ELEMENT_MULTIPLE_TAGS_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  hb_string_T tag_name;
+  hb_string_T open_condition;
+  size_t open_line;
+  size_t open_column;
+  hb_string_T close_condition;
+  size_t close_line;
+  size_t close_column;
+} CONDITIONAL_ELEMENT_CONDITION_MISMATCH_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* closing_tag;
+} INVALID_COMMENT_CLOSING_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_tag;
+  position_T insertion_point;
+} OMITTED_CLOSING_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* tag_name;
+} UNCLOSED_OPEN_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* tag_name;
+} UNCLOSED_CLOSE_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_quote;
+} UNCLOSED_QUOTE_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  hb_string_T attribute_name;
+} MISSING_ATTRIBUTE_VALUE_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_tag;
+} UNCLOSED_ERB_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  /* no additional fields */
+} STRAY_ERB_CLOSING_TAG_ERROR_T;
+
+typedef struct {
+  ERROR_T base;
+  token_T* opening_tag;
+  size_t nested_tag_line;
+  size_t nested_tag_column;
+} NESTED_ERB_TAG_ERROR_T;
+
+UNEXPECTED_ERROR_T* unexpected_error_init(hb_string_T description, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unexpected_error(hb_string_T description, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNEXPECTED_TOKEN_ERROR_T* unexpected_token_error_init(token_type_T expected_type, token_T* found, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unexpected_token_error(token_type_T expected_type, token_T* found, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_OPENING_TAG_ERROR_T* missing_opening_tag_error_init(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_opening_tag_error(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_CLOSING_TAG_ERROR_T* missing_closing_tag_error_init(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_closing_tag_error(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+TAG_NAMES_MISMATCH_ERROR_T* tag_names_mismatch_error_init(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_tag_names_mismatch_error(token_T* opening_tag, token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+VOID_ELEMENT_CLOSING_TAG_ERROR_T* void_element_closing_tag_error_init(token_T* tag_name, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator);
+void append_void_element_closing_tag_error(token_T* tag_name, hb_string_T expected, hb_string_T found, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_ELEMENT_ERROR_T* unclosed_element_error_init(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_element_error(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+RUBY_PARSE_ERROR_T* ruby_parse_error_init(hb_string_T error_message, hb_string_T diagnostic_id, hb_string_T level, position_T start, position_T end, hb_allocator_T* allocator);
+void append_ruby_parse_error(hb_string_T error_message, hb_string_T diagnostic_id, hb_string_T level, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+ERB_CONTROL_FLOW_SCOPE_ERROR_T* erb_control_flow_scope_error_init(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator);
+void append_erb_control_flow_scope_error(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_ERB_END_TAG_ERROR_T* missing_erb_end_tag_error_init(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_erb_end_tag_error(hb_string_T keyword, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+ERB_MULTIPLE_BLOCKS_IN_TAG_ERROR_T* erb_multiple_blocks_in_tag_error_init(position_T start, position_T end, hb_allocator_T* allocator);
+void append_erb_multiple_blocks_in_tag_error(position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+ERB_CASE_WITH_CONDITIONS_ERROR_T* erb_case_with_conditions_error_init(position_T start, position_T end, hb_allocator_T* allocator);
+void append_erb_case_with_conditions_error(position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+CONDITIONAL_ELEMENT_MULTIPLE_TAGS_ERROR_T* conditional_element_multiple_tags_error_init(size_t line, size_t column, position_T start, position_T end, hb_allocator_T* allocator);
+void append_conditional_element_multiple_tags_error(size_t line, size_t column, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+CONDITIONAL_ELEMENT_CONDITION_MISMATCH_ERROR_T* conditional_element_condition_mismatch_error_init(hb_string_T tag_name, hb_string_T open_condition, size_t open_line, size_t open_column, hb_string_T close_condition, size_t close_line, size_t close_column, position_T start, position_T end, hb_allocator_T* allocator);
+void append_conditional_element_condition_mismatch_error(hb_string_T tag_name, hb_string_T open_condition, size_t open_line, size_t open_column, hb_string_T close_condition, size_t close_line, size_t close_column, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+INVALID_COMMENT_CLOSING_TAG_ERROR_T* invalid_comment_closing_tag_error_init(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_invalid_comment_closing_tag_error(token_T* closing_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+OMITTED_CLOSING_TAG_ERROR_T* omitted_closing_tag_error_init(token_T* opening_tag, position_T insertion_point, position_T start, position_T end, hb_allocator_T* allocator);
+void append_omitted_closing_tag_error(token_T* opening_tag, position_T insertion_point, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_OPEN_TAG_ERROR_T* unclosed_open_tag_error_init(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_open_tag_error(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_CLOSE_TAG_ERROR_T* unclosed_close_tag_error_init(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_close_tag_error(token_T* tag_name, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_QUOTE_ERROR_T* unclosed_quote_error_init(token_T* opening_quote, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_quote_error(token_T* opening_quote, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+MISSING_ATTRIBUTE_VALUE_ERROR_T* missing_attribute_value_error_init(hb_string_T attribute_name, position_T start, position_T end, hb_allocator_T* allocator);
+void append_missing_attribute_value_error(hb_string_T attribute_name, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+UNCLOSED_ERB_TAG_ERROR_T* unclosed_erb_tag_error_init(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator);
+void append_unclosed_erb_tag_error(token_T* opening_tag, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+STRAY_ERB_CLOSING_TAG_ERROR_T* stray_erb_closing_tag_error_init(position_T start, position_T end, hb_allocator_T* allocator);
+void append_stray_erb_closing_tag_error(position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
+NESTED_ERB_TAG_ERROR_T* nested_erb_tag_error_init(token_T* opening_tag, size_t nested_tag_line, size_t nested_tag_column, position_T start, position_T end, hb_allocator_T* allocator);
+void append_nested_erb_tag_error(token_T* opening_tag, size_t nested_tag_line, size_t nested_tag_column, position_T start, position_T end, hb_allocator_T* allocator, hb_array_T* errors);
 
 void error_init(ERROR_T* error, error_type_T type, position_T start, position_T end);
 
 size_t error_sizeof(void);
 error_type_T error_type(ERROR_T* error);
 
-char* error_message(ERROR_T* error);
+hb_string_T error_message(ERROR_T* error);
 
-const char* error_type_to_string(ERROR_T* error);
-const char* error_human_type(ERROR_T* error);
+hb_string_T error_type_to_string(ERROR_T* error);
+hb_string_T error_human_type(ERROR_T* error);
 
-void error_free(ERROR_T* error);
+void error_free(ERROR_T* error, hb_allocator_T* allocator);
 
-void error_pretty_print(ERROR_T* error, size_t indent, size_t relative_indent, hb_buffer_T* buffer);
+#ifndef HERB_EXCLUDE_PRETTYPRINT
+  void error_pretty_print(ERROR_T* error, size_t indent, size_t relative_indent, hb_buffer_T* buffer);
 
-void error_pretty_print_array(
-  const char* name, hb_array_T* array, size_t indent, size_t relative_indent, bool last_property,
-  hb_buffer_T* buffer
-);
+  void error_pretty_print_array(
+    const char* name, hb_array_T* array, size_t indent, size_t relative_indent, bool last_property,
+    hb_buffer_T* buffer
+  );
+#endif
 
 #endif

package/extension/libherb/include/extract.h

@@ -1,19 +1,35 @@
 #ifndef HERB_EXTRACT_H
 #define HERB_EXTRACT_H
 
+#include "util/hb_allocator.h"
 #include "util/hb_buffer.h"
 
+#include <stdbool.h>
+
 typedef enum {
   HERB_EXTRACT_LANGUAGE_RUBY,
   HERB_EXTRACT_LANGUAGE_HTML,
 } herb_extract_language_T;
 
-void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output);
-void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output);
+typedef struct {
+  bool semicolons;
+  bool comments;
+  bool preserve_positions;
+} herb_extract_ruby_options_T;
+
+extern const herb_extract_ruby_options_T HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+
+void herb_extract_ruby_to_buffer_with_options(
+  const char* source,
+  hb_buffer_T* output,
+  const herb_extract_ruby_options_T* options,
+  hb_allocator_T* allocator
+);
+void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator);
+void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator);
 
-char* herb_extract_ruby_with_semicolons(const char* source);
+char* herb_extract_ruby_with_semicolons(const char* source, hb_allocator_T* allocator);
 
-char* herb_extract(const char* source, herb_extract_language_T language);
-char* herb_extract_from_file(const char* path, herb_extract_language_T language);
+char* herb_extract(const char* source, herb_extract_language_T language, hb_allocator_T* allocator);
 
 #endif

package/extension/libherb/include/herb.h

@@ -5,26 +5,38 @@
 #include "extract.h"
 #include "macros.h"
 #include "parser.h"
+#include "util/hb_allocator.h"
 #include "util/hb_array.h"
 #include "util/hb_buffer.h"
 
+#include <prism.h>
+#include <stdbool.h>
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-HERB_EXPORTED_FUNCTION void herb_lex_to_buffer(const char* source, hb_buffer_T* output);
+HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source, hb_allocator_T* allocator);
 
-HERB_EXPORTED_FUNCTION hb_array_T* herb_lex(const char* source);
-HERB_EXPORTED_FUNCTION hb_array_T* herb_lex_file(const char* path);
-
-HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(const char* source, parser_options_T* options);
+HERB_EXPORTED_FUNCTION AST_DOCUMENT_NODE_T* herb_parse(
+  const char* source,
+  const parser_options_T* options,
+  hb_allocator_T* allocator
+);
 
 HERB_EXPORTED_FUNCTION const char* herb_version(void);
 HERB_EXPORTED_FUNCTION const char* herb_prism_version(void);
 
-HERB_EXPORTED_FUNCTION void herb_free_tokens(hb_array_T** tokens);
+typedef struct {
+  pm_parser_t parser;
+  pm_node_t* root;
+  pm_options_t options;
+} herb_ruby_parse_result_T;
+
+HERB_EXPORTED_FUNCTION herb_ruby_parse_result_T* herb_parse_ruby(const char* source, size_t length);
+HERB_EXPORTED_FUNCTION void herb_free_ruby_parse_result(herb_ruby_parse_result_T* result);
+HERB_EXPORTED_FUNCTION void herb_free_tokens(hb_array_T** tokens, hb_allocator_T* allocator);
 
 #ifdef __cplusplus
 }

package/extension/libherb/include/herb_prism_node.h

@@ -0,0 +1,13 @@
+#ifndef HERB_PRISM_NODE_H
+#define HERB_PRISM_NODE_H
+
+#include <prism.h>
+
+typedef struct {
+  pm_node_t* node;
+  pm_parser_t* parser;
+} herb_prism_node_T;
+
+#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL })
+
+#endif

package/extension/libherb/include/html_util.h

@@ -4,9 +4,14 @@
 #include "util/hb_string.h"
 #include <stdbool.h>
 
+struct hb_allocator;
+
 bool is_void_element(hb_string_T tag_name);
+bool has_optional_end_tag(hb_string_T tag_name);
+bool should_implicitly_close(hb_string_T open_tag_name, hb_string_T next_tag_name);
+bool parent_closes_element(hb_string_T open_tag_name, hb_string_T parent_close_tag_name);
 
-hb_string_T html_closing_tag_string(hb_string_T tag_name);
-hb_string_T html_self_closing_tag_string(hb_string_T tag_name);
+hb_string_T html_closing_tag_string(hb_string_T tag_name, struct hb_allocator* allocator);
+hb_string_T html_self_closing_tag_string(hb_string_T tag_name, struct hb_allocator* allocator);
 
 #endif

package/extension/libherb/include/io.h

@@ -4,6 +4,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-char* herb_read_file(const char* filename);
+struct hb_allocator;
+
+char* herb_read_file(const char* filename, struct hb_allocator* allocator);
 
 #endif

package/extension/libherb/include/lex_helpers.h

@@ -0,0 +1,29 @@
+#ifndef HERB_LEX_HELPERS_H
+#define HERB_LEX_HELPERS_H
+
+#include "herb.h"
+#include "token.h"
+#include "util/hb_allocator.h"
+#include "util/hb_array.h"
+#include "util/hb_buffer.h"
+#include "util/hb_string.h"
+
+#include <stdlib.h>
+
+static inline void herb_lex_to_buffer(const char* source, hb_buffer_T* output, hb_allocator_T* allocator) {
+  hb_array_T* tokens = herb_lex(source, allocator);
+
+  for (size_t i = 0; i < hb_array_size(tokens); i++) {
+    token_T* token = hb_array_get(tokens, i);
+
+    hb_string_T type = token_to_string(allocator, token);
+    hb_buffer_append_string(output, type);
+    hb_allocator_dealloc(allocator, type.data);
+
+    hb_buffer_append(output, "\n");
+  }
+
+  herb_free_tokens(&tokens, allocator);
+}
+
+#endif

package/extension/libherb/include/lexer.h

@@ -4,7 +4,7 @@
 #include "lexer_struct.h"
 #include "token_struct.h"
 
-void lexer_init(lexer_T* lexer, const char* source);
+void lexer_init(lexer_T* lexer, const char* source, hb_allocator_T* allocator);
 token_T* lexer_next_token(lexer_T* lexer);
 token_T* lexer_error(lexer_T* lexer, const char* message);
 

package/extension/libherb/include/lexer_peek_helpers.h

@@ -2,8 +2,11 @@
 #define HERB_LEXER_PEEK_HELPERS_H
 
 #include "lexer_struct.h"
+#include "macros.h"
 #include "token_struct.h"
+#include "util/hb_string.h"
 
+#include <ctype.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -20,27 +23,98 @@ typedef struct {
   lexer_state_T state;
 } lexer_state_snapshot_T;
 
-char lexer_peek(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset);
 bool lexer_peek_for_cdata_end(const lexer_T* lexer, uint32_t offset);
-
 bool lexer_peek_for_html_comment_start(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset);
+bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type);
+bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset);
 
-bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset);
-bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset);
+static inline char lexer_peek(const lexer_T* lexer, uint32_t offset) {
+  return lexer->source.data[MIN(lexer->current_position + offset, lexer->source.length)];
+}
 
-char lexer_backtrack(const lexer_T* lexer, uint32_t offset);
+static inline char lexer_backtrack(const lexer_T* lexer, uint32_t offset) {
+  return lexer->source.data[MAX(lexer->current_position - offset, 0)];
+}
 
-bool lexer_peek_for_token_type_after_whitespace(lexer_T* lexer, token_type_T token_type);
-bool lexer_peek_for_close_tag_start(const lexer_T* lexer, uint32_t offset);
+static inline bool lexer_peek_for_html_comment_end(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '-'
+      && lexer->source.data[position + 1] == '-' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_for_html_comment_invalid_end(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 3 < lexer->source.length && lexer->source.data[position] == '-'
+      && lexer->source.data[position + 1] == '-' && lexer->source.data[position + 2] == '!'
+      && lexer->source.data[position + 3] == '>';
+}
+
+static inline bool lexer_peek_erb_start(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 1 < lexer->source.length && lexer->source.data[position] == '<'
+      && lexer->source.data[position + 1] == '%';
+}
+
+static inline bool lexer_peek_erb_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 1 < lexer->source.length && lexer->source.data[position] == '%'
+      && lexer->source.data[position + 1] == '>';
+}
+
+static inline bool lexer_peek_erb_dash_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '-'
+      && lexer->source.data[position + 1] == '%' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_erb_percent_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '%'
+      && lexer->source.data[position + 1] == '%' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_erb_equals_close_tag(const lexer_T* lexer, uint32_t offset) {
+  uint32_t position = lexer->current_position + offset;
+
+  return position + 2 < lexer->source.length && lexer->source.data[position] == '='
+      && lexer->source.data[position + 1] == '%' && lexer->source.data[position + 2] == '>';
+}
+
+static inline bool lexer_peek_erb_end(const lexer_T* lexer, uint32_t offset) {
+  return lexer_peek_erb_close_tag(lexer, offset) || lexer_peek_erb_dash_close_tag(lexer, offset)
+      || lexer_peek_erb_percent_close_tag(lexer, offset) || lexer_peek_erb_equals_close_tag(lexer, offset);
+}
+
+static inline lexer_state_snapshot_T lexer_save_state(lexer_T* lexer) {
+  lexer_state_snapshot_T snapshot = { .position = lexer->current_position,
+                                      .line = lexer->current_line,
+                                      .column = lexer->current_column,
+                                      .previous_position = lexer->previous_position,
+                                      .previous_line = lexer->previous_line,
+                                      .previous_column = lexer->previous_column,
+                                      .current_character = lexer->current_character,
+                                      .state = lexer->state };
+  return snapshot;
+}
 
-lexer_state_snapshot_T lexer_save_state(lexer_T* lexer);
-void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot);
+static inline void lexer_restore_state(lexer_T* lexer, lexer_state_snapshot_T snapshot) {
+  lexer->current_position = snapshot.position;
+  lexer->current_line = snapshot.line;
+  lexer->current_column = snapshot.column;
+  lexer->previous_position = snapshot.previous_position;
+  lexer->previous_line = snapshot.previous_line;
+  lexer->previous_column = snapshot.previous_column;
+  lexer->current_character = snapshot.current_character;
+  lexer->state = snapshot.state;
+}
 
 #endif

package/extension/libherb/include/lexer_struct.h

@@ -1,6 +1,7 @@
 #ifndef HERB_LEXER_STRUCT_H
 #define HERB_LEXER_STRUCT_H
 
+#include "util/hb_allocator.h"
 #include "util/hb_string.h"
 
 #include <stdbool.h>
@@ -14,6 +15,7 @@ typedef enum {
 } lexer_state_T;
 
 typedef struct LEXER_STRUCT {
+  hb_allocator_T* allocator;
   hb_string_T source;
 
   uint32_t current_line;

package/extension/libherb/include/location.h

@@ -5,6 +5,7 @@
 #include <stdlib.h>
 
 #include "position.h"
+#include "util/hb_allocator.h"
 
 typedef struct LOCATION_STRUCT {
   position_T start;
@@ -21,6 +22,6 @@ void location_from(
 
 void location_from_positions(location_T* location, position_T start, position_T end);
 
-location_T* location_create(position_T start, position_T end);
+location_T* location_create(position_T start, position_T end, hb_allocator_T* allocator);
 
 #endif