herb 0.10.4 → 0.11.1

data/src/include/parser/parser.h

@@ -6,47 +6,29 @@
 #include "../lib/hb_allocator.h"
 #include "../lib/hb_array.h"
 #include "../lib/hb_clock.h"
+#include "parser_options.h"
 
-#include <stdint.h>
+#define HERB_MAX_FOREIGN_CONTENT_ELEMENTS 16
 
 typedef enum {
-  FOREIGN_CONTENT_UNKNOWN = 0,
-  FOREIGN_CONTENT_SCRIPT,
-  FOREIGN_CONTENT_STYLE,
-  // FOREIGN_CONTENT_RUBY,
-  // FOREIGN_CONTENT_TEMPLATE
-} foreign_content_type_T;
+  FOREIGN_CONTENT_NONE = 0,
+  FOREIGN_CONTENT_RAW_TEXT,
+  FOREIGN_CONTENT_RCDATA,
+} foreign_content_kind_T;
 
 typedef enum { PARSER_STATE_DATA, PARSER_STATE_FOREIGN_CONTENT } parser_state_T;
 
-typedef struct PARSER_OPTIONS_STRUCT {
-  bool track_whitespace;
-  bool analyze;
-  bool strict;
-  bool action_view_helpers;
-  bool transform_conditionals;
-  bool render_nodes;
-  bool strict_locals;
-  bool prism_program;
-  bool prism_nodes;
-  bool prism_nodes_deep;
-  bool dot_notation_tags;
-  bool html;
-  uint32_t start_line;
-  uint32_t start_column;
-  uint32_t timeout_ms;
-  uint32_t max_errors;
-  uint32_t* error_count;
-  uint64_t deadline_ms;
-} parser_options_T;
-
 typedef struct MATCH_TAGS_CONTEXT_STRUCT {
   hb_array_T* errors;
   const parser_options_T* options;
   hb_allocator_T* allocator;
 } match_tags_context_T;
 
-extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
+static inline void lexer_apply_erb_openers(lexer_T* lexer, const parser_options_T* options) {
+  if (options == NULL || options->erb_openers == NULL) { return; }
+
+  lexer->erb_openers = (erb_openers_T) { .items = options->erb_openers, .count = options->erb_opener_count };
+}
 
 static inline bool parser_options_past_deadline(const parser_options_T* options) {
   if (options == NULL || options->timeout_ms == 0) { return false; }
@@ -77,7 +59,11 @@ typedef struct PARSER_STRUCT {
   token_T* current_token;
   hb_array_T* open_tags_stack;
   parser_state_T state;
-  foreign_content_type_T foreign_content_type;
+  foreign_content_kind_T foreign_content_kind;
+  hb_string_T foreign_content_tag_name;
+  size_t svg_depth;
+  bool xml_document;
+  uint32_t foreign_content_absent_from[HERB_MAX_FOREIGN_CONTENT_ELEMENTS];
   parser_options_T options;
   size_t consecutive_error_count;
   bool in_recovery_mode;

data/src/include/parser/parser_helpers.h

@@ -15,7 +15,7 @@ token_T* parser_pop_open_tag(const parser_T* parser);
 
 void parser_append_unexpected_error_impl(
   parser_T* parser,
-  hb_array_T* errors,
+  hb_array_T** errors,
   const char* description,
   token_type_T first_token,
   ...
@@ -25,11 +25,11 @@ void parser_append_unexpected_error_impl(
 
 void parser_append_unexpected_error_string(
   parser_T* parser,
-  hb_array_T* errors,
+  hb_array_T** errors,
   const char* description,
   const char* expected
 );
-void parser_append_unexpected_token_error(parser_T* parser, token_type_T expected_type, hb_array_T* errors);
+void parser_append_unexpected_token_error(parser_T* parser, token_type_T expected_type, hb_array_T** errors);
 
 void parser_append_literal_node_from_buffer(
   const parser_T* parser,
@@ -40,32 +40,34 @@ void parser_append_literal_node_from_buffer(
 
 bool parser_in_svg_context(const parser_T* parser);
 
-foreign_content_type_T parser_get_foreign_content_type(hb_string_T tag_name);
+foreign_content_kind_T parser_get_foreign_content_kind(hb_string_T tag_name);
 bool parser_is_foreign_content_tag(hb_string_T tag_name);
-hb_string_T parser_get_foreign_content_closing_tag(foreign_content_type_T type);
+bool parser_foreign_content_has_end_tag(hb_string_T tag_name);
+bool parser_foreign_content_is_html_only(hb_string_T tag_name);
+int parser_foreign_content_element_index(hb_string_T tag_name);
 
-void parser_enter_foreign_content(parser_T* parser, foreign_content_type_T type);
+void parser_enter_foreign_content(parser_T* parser, foreign_content_kind_T kind, hb_string_T tag_name);
 void parser_exit_foreign_content(parser_T* parser);
 
-bool parser_is_expected_closing_tag_name(hb_string_T tag_name, foreign_content_type_T expected_type);
+bool parser_is_foreign_content_closing_tag_name(const parser_T* parser, hb_string_T tag_name);
 
 token_T* parser_advance(parser_T* parser);
 token_T* parser_consume_if_present(parser_T* parser, token_type_T type);
-token_T* parser_consume_expected(parser_T* parser, token_type_T type, hb_array_T* array);
+token_T* parser_consume_expected(parser_T* parser, token_type_T type, hb_array_T** array);
 
 AST_HTML_ELEMENT_NODE_T* parser_handle_missing_close_tag(
   parser_T* parser,
   AST_HTML_OPEN_TAG_NODE_T* open_tag,
   hb_array_T* body,
-  hb_array_T* errors
+  hb_array_T** errors
 );
 void parser_handle_mismatched_tags(
   const parser_T* parser,
   const AST_HTML_CLOSE_TAG_NODE_T* close_tag,
-  hb_array_T* errors
+  hb_array_T** errors
 );
 
-void parser_synchronize(parser_T* parser, hb_array_T* errors);
+void parser_synchronize(parser_T* parser, hb_array_T** errors);
 
 bool parser_can_close_ancestor(const parser_T* parser, hb_string_T tag_name);
 size_t parser_find_ancestor_depth(const parser_T* parser, hb_string_T tag_name);

data/src/include/parser/parser_options.h

@@ -0,0 +1,41 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See templates/src/include/parser/parser_options.h.erb
+
+#ifndef HERB_PARSER_OPTIONS_H
+#define HERB_PARSER_OPTIONS_H
+
+#include "../lib/hb_string.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct PARSER_OPTIONS_STRUCT {
+  bool action_view_helpers;
+  bool analyze;
+  bool dot_notation_tags;
+  const hb_string_T* erb_openers;
+  size_t erb_opener_count;
+  bool herb_directives;
+  bool html;
+  bool iteration_nodes;
+  uint32_t max_errors;
+  bool prism_nodes;
+  bool prism_nodes_deep;
+  bool prism_program;
+  bool render_nodes;
+  bool strict;
+  bool strict_locals;
+  uint32_t timeout_ms;
+  bool track_locations;
+  bool track_whitespace;
+  bool transform_conditionals;
+  uint64_t deadline_ms;
+  uint32_t* error_count;
+  uint32_t start_column;
+  uint32_t start_line;
+} parser_options_T;
+
+extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
+
+#endif

data/src/include/prism/herb_prism_node.h

@@ -2,12 +2,17 @@
 #define HERB_PRISM_NODE_H
 
 #include <prism.h>
+#include <stdint.h>
 
 typedef struct {
   pm_node_t* node;
   pm_parser_t* parser;
+  uint32_t source_start;
+  uint32_t source_end;
 } herb_prism_node_T;
 
-#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL })
+#define HERB_PRISM_NODE_EMPTY ((herb_prism_node_T) { .node = NULL, .parser = NULL, .source_start = 0, .source_end = 0 })
+
+void herb_prism_node_serialize(herb_prism_node_T prism_node, pm_buffer_t* buffer);
 
 #endif

data/src/include/prism/ruby_parser.h

@@ -1,6 +1,12 @@
 #ifndef HERB_RUBY_PARSER_H
 #define HERB_RUBY_PARSER_H
 
+#include "../lib/hb_string.h"
+
+#include <stdbool.h>
+
 void herb_parse_ruby_to_stdout(char* source);
 
+bool herb_ruby_fragment_is_parseable(hb_string_T source);
+
 #endif

data/src/include/util/html_elements.h

@@ -0,0 +1,85 @@
+// NOTE: This file is generated by the templates/template.rb script and should not
+// be modified manually. See templates/src/include/util/html_elements.h.erb
+
+#ifndef HERB_HTML_ELEMENTS_H
+#define HERB_HTML_ELEMENTS_H
+
+#include "../lib/hb_string.h"
+
+// https://developer.mozilla.org/en-US/docs/Glossary/Void_element
+static hb_string_T void_tags[] = HB_STRING_LIST(
+  "area",
+  "base",
+  "basefont",
+  "bgsound",
+  "br",
+  "col",
+  "embed",
+  "frame",
+  "hr",
+  "img",
+  "input",
+  "keygen",
+  "link",
+  "meta",
+  "param",
+  "source",
+  "track",
+  "wbr"
+);
+
+// https://html.spec.whatwg.org/multipage/common-microsyntaxes.html#boolean-attributes
+static hb_string_T boolean_attributes[] = HB_STRING_LIST(
+  "allowfullscreen",
+  "async",
+  "autofocus",
+  "autoplay",
+  "checked",
+  "compact",
+  "controls",
+  "declare",
+  "default",
+  "defer",
+  "disabled",
+  "formnovalidate",
+  "hidden",
+  "inert",
+  "ismap",
+  "itemscope",
+  "loop",
+  "multiple",
+  "muted",
+  "nomodule",
+  "nohref",
+  "noresize",
+  "noshade",
+  "novalidate",
+  "nowrap",
+  "open",
+  "playsinline",
+  "readonly",
+  "required",
+  "reversed",
+  "scoped",
+  "seamless",
+  "selected",
+  "sortable",
+  "truespeed",
+  "typemustmatch"
+);
+
+// https://html.spec.whatwg.org/multipage/rendering.html#the-page
+static hb_string_T whitespace_preserving_tags[] = HB_STRING_LIST(
+  "iframe",
+  "listing",
+  "noembed",
+  "noframes",
+  "plaintext",
+  "pre",
+  "script",
+  "style",
+  "textarea",
+  "xmp"
+);
+
+#endif

data/src/include/util/html_util.h

@@ -7,6 +7,7 @@
 struct hb_allocator;
 
 bool is_void_element(hb_string_T tag_name);
+bool is_whitespace_preserving_element(hb_string_T tag_name);
 bool is_boolean_attribute(hb_string_T attribute_name);
 bool has_optional_end_tag(hb_string_T tag_name);
 bool should_implicitly_close(hb_string_T open_tag_name, hb_string_T next_tag_name);

data/src/include/version.h

@@ -1,6 +1,6 @@
 #ifndef HERB_VERSION_H
 #define HERB_VERSION_H
 
-#define HERB_VERSION "0.10.4"
+#define HERB_VERSION "0.11.1"
 
 #endif

data/src/lexer/lexer_peek_helpers.c

@@ -19,8 +19,22 @@ bool lexer_peek_for_doctype(const lexer_T* lexer, uint32_t offset) {
   return lexer_peek_for(lexer, offset, hb_string("<!DOCTYPE"), true);
 }
 
+static bool is_xml_name_character(char character) {
+  return isalnum(character) || character == '-' || character == '_' || character == ':' || character == '.';
+}
+
 bool lexer_peek_for_xml_declaration(const lexer_T* lexer, uint32_t offset) {
-  return lexer_peek_for(lexer, offset, hb_string("<?xml"), true);
+  hb_string_T pattern = hb_string("<?xml");
+
+  if (!lexer_peek_for(lexer, offset, pattern, true)) { return false; }
+
+  return !is_xml_name_character(lexer_peek(lexer, offset + pattern.length));
+}
+
+bool lexer_peek_for_xml_processing_instruction(const lexer_T* lexer, uint32_t offset) {
+  if (lexer_peek(lexer, offset) != '<' || lexer_peek(lexer, offset + 1) != '?') { return false; }
+
+  return isalpha(lexer_peek(lexer, offset + 2));
 }
 
 bool lexer_peek_for_cdata_start(const lexer_T* lexer, uint32_t offset) {

data/src/lexer/token.c

@@ -5,6 +5,7 @@
 #include "../include/lib/hb_string.h"
 #include "../include/location/position.h"
 #include "../include/location/range.h"
+#include "../include/util/utf8.h"
 #include "../include/util/util.h"
 
 #include <stdarg.h>
@@ -25,6 +26,7 @@ token_T* token_init(hb_string_T value, const token_type_T type, lexer_T* lexer)
   }
 
   token->value = value;
+  token->owns_value = false;
 
   token->type = type;
   token->range = (range_T) { .from = lexer->previous_position, .to = lexer->current_position };
@@ -53,6 +55,7 @@ hb_string_T token_type_to_string(const token_type_T type) {
     case TOKEN_HTML_DOCTYPE: return hb_string("TOKEN_HTML_DOCTYPE");
     case TOKEN_XML_DECLARATION: return hb_string("TOKEN_XML_DECLARATION");
     case TOKEN_XML_DECLARATION_END: return hb_string("TOKEN_XML_DECLARATION_END");
+    case TOKEN_XML_PROCESSING_INSTRUCTION_START: return hb_string("TOKEN_XML_PROCESSING_INSTRUCTION_START");
     case TOKEN_CDATA_START: return hb_string("TOKEN_CDATA_START");
     case TOKEN_CDATA_END: return hb_string("TOKEN_CDATA_END");
     case TOKEN_HTML_TAG_START: return hb_string("TOKEN_HTML_TAG_START");
@@ -94,6 +97,7 @@ hb_string_T token_type_to_friendly_string(const token_type_T type) {
     case TOKEN_HTML_DOCTYPE: return hb_string("`<!DOCTYPE`");
     case TOKEN_XML_DECLARATION: return hb_string("`<?xml`");
     case TOKEN_XML_DECLARATION_END: return hb_string("`?>`");
+    case TOKEN_XML_PROCESSING_INSTRUCTION_START: return hb_string("`<?`");
     case TOKEN_CDATA_START: return hb_string("`<![CDATA[`");
     case TOKEN_CDATA_END: return hb_string("`]]>`");
     case TOKEN_HTML_TAG_START: return hb_string("`<`");
@@ -215,7 +219,8 @@ token_T* token_copy(token_T* token, hb_allocator_T* allocator) {
 
   if (!new_token) { return NULL; }
 
-  new_token->value = token->value;
+  new_token->value = token->owns_value ? hb_string_copy(token->value, allocator) : token->value;
+  new_token->owns_value = token->owns_value;
 
   new_token->type = token->type;
   new_token->range = token->range;
@@ -224,6 +229,81 @@ token_T* token_copy(token_T* token, hb_allocator_T* allocator) {
   return new_token;
 }
 
+static position_T token_position_after(position_T position, hb_string_T value, uint32_t offset) {
+  uint32_t index = 0;
+
+  while (index < offset && index < value.length) {
+    if (is_newline(value.data[index])) {
+      position.line++;
+      position.column = 0;
+      index++;
+
+      continue;
+    }
+
+    position.column++;
+    index += utf8_sequence_length(hb_string_slice(value, index));
+  }
+
+  return position;
+}
+
+static token_T* token_from_slice(
+  const token_T* token,
+  hb_string_T value,
+  uint32_t offset,
+  position_T start,
+  position_T end,
+  hb_allocator_T* allocator
+) {
+  token_T* slice = hb_allocator_alloc(allocator, sizeof(token_T));
+
+  if (!slice) { return NULL; }
+
+  slice->value = token->owns_value ? hb_string_copy(value, allocator) : value;
+  slice->owns_value = token->owns_value;
+
+  slice->type = token->type;
+  slice->range = (range_T) { .from = token->range.from + offset, .to = token->range.from + offset + value.length };
+
+  location_from_positions(&slice->location, start, end);
+
+  return slice;
+}
+
+bool token_split(
+  const token_T* token,
+  const uint32_t offset,
+  hb_allocator_T* allocator,
+  token_T** head,
+  token_T** tail
+) {
+  if (!token || !head || !tail) { return false; }
+  if (offset == 0 || offset >= token->value.length) { return false; }
+  if (utf8_is_valid_continuation_byte((unsigned char) token->value.data[offset])) { return false; }
+
+  const position_T split = token_position_after(token->location.start, token->value, offset);
+
+  token_T* head_token =
+    token_from_slice(token, hb_string_range(token->value, 0, offset), 0, token->location.start, split, allocator);
+
+  if (!head_token) { return false; }
+
+  token_T* tail_token =
+    token_from_slice(token, hb_string_slice(token->value, offset), offset, split, token->location.end, allocator);
+
+  if (!tail_token) {
+    token_free(head_token, allocator);
+
+    return false;
+  }
+
+  *head = head_token;
+  *tail = tail_token;
+
+  return true;
+}
+
 bool token_value_empty(const token_T* token) {
   return token == NULL || hb_string_is_empty(token->value);
 }
@@ -237,5 +317,7 @@ bool token_is_escaped_erb_tag_opening(const token_T* token) {
 void token_free(token_T* token, hb_allocator_T* allocator) {
   if (!token) { return; }
 
+  if (token->owns_value) { hb_allocator_dealloc(allocator, (void*) token->value.data); }
+
   hb_allocator_dealloc(allocator, token);
 }