herb 0.8.10-arm-linux-gnu → 0.9.0-arm-linux-gnu

data/templates/src/include/errors.h.erb

@@ -5,8 +5,10 @@
 #include "location.h"
 #include "position.h"
 #include "token.h"
+#include "util/hb_allocator.h"
 #include "util/hb_array.h"
 #include "util/hb_buffer.h"
+#include "util/hb_string.h"
 
 typedef enum {
 <%- errors.each do |error| -%>
@@ -17,7 +19,7 @@ typedef enum {
 typedef struct ERROR_STRUCT {
   error_type_T type;
   location_T location;
-  char* message;
+  hb_string_T message;
 } ERROR_T;
 
 <%- errors.each do |error| -%>
@@ -31,7 +33,7 @@ typedef struct {
 
 <%- errors.each do |error| -%>
 <%- error_arguments = error.fields.any? ? error.fields.map { |field| [field.c_type, " ", field.name].join } : [] -%>
-<%- arguments = error_arguments + ["position_T start", "position_T end"] -%>
+<%- arguments = error_arguments + ["position_T start", "position_T end", "hb_allocator_T* allocator"] -%>
 <%= error.struct_type %>* <%= error.human %>_init(<%= arguments.join(", ") %>);
 void append_<%= error.human %>(<%= (arguments << "hb_array_T* errors").join(", ") %>);
 <%- end -%>
@@ -41,18 +43,20 @@ void error_init(ERROR_T* error, error_type_T type, position_T start, position_T
 size_t error_sizeof(void);
 error_type_T error_type(ERROR_T* error);
 
-char* error_message(ERROR_T* error);
+hb_string_T error_message(ERROR_T* error);
 
-const char* error_type_to_string(ERROR_T* error);
-const char* error_human_type(ERROR_T* error);
+hb_string_T error_type_to_string(ERROR_T* error);
+hb_string_T error_human_type(ERROR_T* error);
 
-void error_free(ERROR_T* error);
+void error_free(ERROR_T* error, hb_allocator_T* allocator);
 
-void error_pretty_print(ERROR_T* error, size_t indent, size_t relative_indent, hb_buffer_T* buffer);
+#ifndef HERB_EXCLUDE_PRETTYPRINT
+  void error_pretty_print(ERROR_T* error, size_t indent, size_t relative_indent, hb_buffer_T* buffer);
 
-void error_pretty_print_array(
-  const char* name, hb_array_T* array, size_t indent, size_t relative_indent, bool last_property,
-  hb_buffer_T* buffer
-);
+  void error_pretty_print_array(
+    const char* name, hb_array_T* array, size_t indent, size_t relative_indent, bool last_property,
+    hb_buffer_T* buffer
+  );
+#endif
 
 #endif

data/templates/src/include/util/hb_foreach.h.erb

@@ -0,0 +1,20 @@
+<% max = 64 -%>
+#ifndef HB_FOREACH_H
+#define HB_FOREACH_H
+
+#define HB_ARG<%= max %>(<%= (1..max).map { |i| "_#{i}" }.join(",") %>,count,...) count
+#define HB_NARGS(...) HB_ARG<%= max %>(__VA_ARGS__,<%= max.downto(1).to_a.join(",") %>)
+
+<% (1..max).each do |i| -%>
+<% if i == 1 -%>
+#define HB_FOR_EACH_1(macro, head) macro(head)
+<% else -%>
+#define HB_FOR_EACH_<%= i %>(macro, head, ...) macro(head), HB_FOR_EACH_<%= i - 1 %>(macro, __VA_ARGS__)
+<% end -%>
+<% end -%>
+
+#define HB_CONCAT(left, right) left ## right
+#define HB_FOR_EACH_N(count) HB_CONCAT(HB_FOR_EACH_, count)
+#define HB_FOR_EACH(macro, ...) HB_FOR_EACH_N(HB_NARGS(__VA_ARGS__))(macro, __VA_ARGS__)
+
+#endif

data/templates/src/parser_match_tags.c.erb

@@ -4,14 +4,20 @@
 #include "include/visitor.h"
 
 bool match_tags_visitor(const AST_NODE_T* node, void* data) {
-  hb_array_T* errors = (hb_array_T*) data;
+  match_tags_context_T* context = (match_tags_context_T*) data;
 
   if (node == NULL) { return false; }
 
   switch (node->type) {
     <%- nodes.each do |node| -%>
     <%- array_fields = node.fields.select { |f| f.is_a?(Herb::Template::ArrayField) && f.name != "errors" } -%>
-    <%- single_node_fields = node.fields.select { |f| f.is_a?(Herb::Template::NodeField) } -%>
+
+    <%- skip_fields = case node.name
+      when "HTMLConditionalElementNode" then ["open_tag", "close_tag", "open_conditional", "close_conditional"]
+      when "HTMLConditionalOpenTagNode" then ["conditional"]
+      else []
+    end -%>
+    <%- single_node_fields = node.fields.select { |f| f.is_a?(Herb::Template::NodeField) && !skip_fields.include?(f.name) } -%>
 
     <%- if array_fields.any? || single_node_fields.any? -%>
     case <%= node.type %>: {
@@ -19,12 +25,12 @@ bool match_tags_visitor(const AST_NODE_T* node, void* data) {
 
       <%- array_fields.each do |field| -%>
       if (<%= node.human %>-><%= field.name %> != NULL) {
-        match_tags_in_node_array(<%= node.human %>-><%= field.name %>, errors);
+        match_tags_in_node_array(<%= node.human %>-><%= field.name %>, context->errors, context->options, context->allocator);
       }
       <%- end -%>
       <%- single_node_fields.each do |field| -%>
       if (<%= node.human %>-><%= field.name %> != NULL) {
-        herb_visit_node((AST_NODE_T*) <%= node.human %>-><%= field.name %>, match_tags_visitor, errors);
+        herb_visit_node((AST_NODE_T*) <%= node.human %>-><%= field.name %>, match_tags_visitor, context);
       }
       <%- end -%>
     } break;

data/templates/src/visitor.c.erb

@@ -18,13 +18,13 @@ void herb_visit_child_nodes(const AST_NODE_T *node, bool (*visitor)(const AST_NO
 
   switch (node->type) {
     <%- nodes.each do |node| -%>
-    <%- if node.fields.count { |field| [Herb::Template::NodeField, Herb::Template::ArrayField].include?(field.class) }.positive? -%>
+    <%- if node.fields.count { |field| [Herb::Template::NodeField, Herb::Template::BorrowedNodeField, Herb::Template::ArrayField].include?(field.class) }.positive? -%>
     case <%= node.type %>: {
       const <%= node.struct_type %>* <%= node.human %> = ((const <%= node.struct_type %> *) node);
 
       <%- node.fields.each do |field| -%>
       <%- case field -%>
-      <%- when Herb::Template::NodeField -%>
+      <%- when Herb::Template::NodeField, Herb::Template::BorrowedNodeField -%>
       if (<%= node.human %>-><%= field.name %> != NULL) {
         herb_visit_node((AST_NODE_T *) <%= node.human %>-><%= field.name %>, visitor, data);
       }

data/templates/template.rb

@@ -7,6 +7,16 @@ require "yaml"
 
 module Herb
   module Template
+    def self.underscore(name)
+      name
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+    end
+
+    ALWAYS_INVISIBLE_FIELD_CLASSES = ["PrismContextField", "AnalyzedRubyField"].freeze
+    CONDITIONALLY_INVISIBLE_FIELD_CLASSES = ["PrismSerializedField", "PrismNodeField"].freeze
+    ALL_INVISIBLE_FIELD_CLASSES = (ALWAYS_INVISIBLE_FIELD_CLASSES + CONDITIONALLY_INVISIBLE_FIELD_CLASSES).freeze
+
     class Field
       attr_reader :name, :options
 
@@ -14,6 +24,45 @@ module Herb
         @name = name
         @options = options
       end
+
+      def always_invisible?
+        ALWAYS_INVISIBLE_FIELD_CLASSES.include?(self.class.name.split("::").last)
+      end
+
+      def conditionally_invisible?
+        CONDITIONALLY_INVISIBLE_FIELD_CLASSES.include?(self.class.name.split("::").last)
+      end
+
+      def invisible?
+        ALL_INVISIBLE_FIELD_CLASSES.include?(self.class.name.split("::").last)
+      end
+    end
+
+    class FieldVisibility
+      attr_reader :field, :prism_field_name
+
+      def initialize(field:, static_last:, dynamic_last:, prism_field_name:)
+        @field = field
+        @static_last = static_last
+        @dynamic_last = dynamic_last
+        @prism_field_name = prism_field_name
+      end
+
+      def static_last?
+        @static_last
+      end
+
+      def dynamic_last?
+        @dynamic_last
+      end
+
+      def symbol
+        @static_last ? "└── " : "├── "
+      end
+
+      def prefix
+        @static_last ? "    " : "│   "
+      end
     end
 
     class ArrayField < Field
@@ -23,10 +72,17 @@ module Herb
       end
 
       def ruby_type
-        return "Array" unless specific_kind
-        return "Array[Herb::AST::#{specific_kind}]" if specific_kind.end_with?("Node")
-
-        "Array[#{specific_kind}]"
+        if specific_kind
+          if specific_kind.end_with?("Node")
+            "Array[Herb::AST::#{specific_kind}]"
+          else
+            "Array[#{specific_kind}]"
+          end
+        elsif union_kind
+          "Array[(#{union_kind.map { |k| "Herb::AST::#{k}" }.join(" | ")})]"
+        else
+          "Array"
+        end
       end
 
       def c_type
@@ -35,7 +91,7 @@ module Herb
 
       def c_item_type
         if specific_kind
-          "AST_#{specific_kind.gsub(/(?<=[a-zA-Z])(?=[A-Z][a-z])/, "_").upcase}_T*"
+          "AST_#{Template.underscore(specific_kind).upcase}_T*"
         else
           "void*"
         end
@@ -58,14 +114,20 @@ module Herb
 
       def c_type
         if specific_kind
-          "struct AST_#{specific_kind.gsub(/(?<=[a-zA-Z])(?=[A-Z][a-z])/, "_").upcase}_STRUCT*"
+          "struct AST_#{Template.underscore(specific_kind).upcase}_STRUCT*"
         else
           "AST_NODE_T*"
         end
       end
 
       def ruby_type
-        "Herb::AST::#{specific_kind || "Node"}"
+        if specific_kind
+          "Herb::AST::#{specific_kind}"
+        elsif union_kind
+          "(#{union_kind.map { |k| "Herb::AST::#{k}" }.join(" | ")})"
+        else
+          "Herb::AST::Node"
+        end
       end
 
       def specific_kind
@@ -75,6 +137,29 @@ module Herb
       def union_kind
         @kind if @kind.is_a?(Array)
       end
+
+      def union_type_name
+        return nil unless union_kind
+
+        union_kind.sort.join("Or")
+      end
+
+      def rust_type
+        if specific_kind && specific_kind != "Node"
+          "Option<Box<#{specific_kind}>>"
+        elsif union_kind
+          "Option<#{union_type_name}>"
+        else
+          "Option<Box<AnyNode>>"
+        end
+      end
+
+      def java_type
+        specific_kind || "Node" # Java uses base type for union_kind, could use sealed interface in future
+      end
+    end
+
+    class BorrowedNodeField < NodeField
     end
 
     class TokenField < Field
@@ -103,7 +188,7 @@ module Herb
       end
 
       def c_type
-        "const char*"
+        "hb_string_T"
       end
     end
 
@@ -113,7 +198,7 @@ module Herb
       end
 
       def c_type
-        "position_T*"
+        "position_T"
       end
     end
 
@@ -159,11 +244,33 @@ module Herb
 
     class PrismNodeField < Field
       def ruby_type
-        "Prism::Node"
+        "String"
+      end
+
+      def c_type
+        "herb_prism_node_T"
+      end
+
+      def rust_type
+        "Option<Vec<u8>>"
+      end
+
+      def java_type
+        "byte[]"
+      end
+
+      def js_type
+        "Uint8Array | null"
+      end
+    end
+
+    class PrismContextField < Field
+      def ruby_type
+        "nil"
       end
 
       def c_type
-        "pm_node_t*"
+        "herb_prism_context_T*"
       end
     end
 
@@ -187,13 +294,35 @@ module Herb
       end
     end
 
+    class PrismSerializedField < Field
+      def ruby_type
+        "String"
+      end
+
+      def c_type
+        "prism_serialized_T"
+      end
+
+      def java_type
+        "byte[]"
+      end
+
+      def rust_type
+        "Option<Vec<u8>>"
+      end
+
+      def js_type
+        "Uint8Array | null"
+      end
+    end
+
     class ElementSourceField < Field
       def ruby_type
         "String"
       end
 
       def c_type
-        "element_source_t"
+        "hb_string_T"
       end
     end
 
@@ -214,19 +343,22 @@ module Herb
 
       def field_type_for(name)
         case name
-        when "array"      then ArrayField
-        when "node"       then NodeField
-        when "token"      then TokenField
-        when "token_type" then TokenTypeField
-        when "string"     then StringField
-        when "position"   then PositionField
-        when "location"   then LocationField
-        when "size_t"     then SizeTField
-        when "boolean"    then BooleanField
-        when "prism_node" then PrismNodeField
-        when "analyzed_ruby" then AnalyzedRubyField
-        when "element_source" then ElementSourceField
-        when "void*" then VoidPointerField
+        when "array"            then ArrayField
+        when "node"             then NodeField
+        when "borrowed_node"    then BorrowedNodeField
+        when "token"            then TokenField
+        when "token_type"       then TokenTypeField
+        when "string"           then StringField
+        when "position"         then PositionField
+        when "location"         then LocationField
+        when "size_t"           then SizeTField
+        when "boolean"          then BooleanField
+        when "prism_node"       then PrismNodeField
+        when "prism_context"    then PrismContextField
+        when "analyzed_ruby"    then AnalyzedRubyField
+        when "prism_serialized" then PrismSerializedField
+        when "element_source"   then ElementSourceField
+        when "void*"            then VoidPointerField
         else raise("Unknown field type: #{name.inspect}")
         end
       end
@@ -242,7 +374,7 @@ module Herb
         @message_template = config.dig("message", "template")
         @message_arguments = config.dig("message", "arguments")
 
-        camelized = @name.gsub(/(?<=[a-zA-Z])(?=[A-Z][a-z])/, "_")
+        camelized = Template.underscore(@name)
         @type = camelized.upcase
         @struct_type = "#{camelized.upcase}_T"
         @struct_name = "#{camelized.upcase}_STRUCT"
@@ -269,7 +401,7 @@ module Herb
 
       def initialize(config)
         @name = config.fetch("name")
-        camelized = @name.gsub(/(?<=[a-zA-Z])(?=[A-Z][a-z])/, "_")
+        camelized = Template.underscore(@name)
         @type = "AST_#{camelized.upcase}"
         @struct_type = "AST_#{camelized.upcase}_T"
         @struct_name = "AST_#{camelized.upcase}_STRUCT"
@@ -287,6 +419,34 @@ module Herb
       def c_type
         @struct_type
       end
+
+      def field_visibilities
+        @field_visibilities ||= begin
+          prism_fields = @fields.select(&:conditionally_invisible?)
+          last_prism_field_name = prism_fields.last&.name
+
+          @fields.each_with_index.map do |field, index|
+            remaining = @fields[(index + 1)..] || []
+            all_remaining_always_invisible = remaining.all?(&:always_invisible?)
+            all_remaining_invisible = remaining.all?(&:invisible?)
+            any_remaining_conditionally_invisible = remaining.any?(&:conditionally_invisible?)
+
+            static_last = all_remaining_always_invisible
+            dynamic_last = !static_last && all_remaining_invisible && any_remaining_conditionally_invisible
+
+            FieldVisibility.new(
+              field: field,
+              static_last: static_last,
+              dynamic_last: dynamic_last,
+              prism_field_name: dynamic_last ? last_prism_field_name : nil
+            )
+          end
+        end
+      end
+
+      def field_visibility(index)
+        field_visibilities[index]
+      end
     end
 
     class PrintfMessageTemplate
@@ -308,7 +468,7 @@ module Herb
         base_length = template.length
         total_size = base_length
 
-        format_specifiers = template.scan(/%[sdulfz]/)
+        format_specifiers = template.scan(/%(?:zu|llu|lf|ld|[sdulf])/)
 
         format_specifiers.each_with_index do |specifier, _i|
           estimated_size = ESTIMATED_SIZES[specifier] || 16 # Default extra buffer
@@ -381,7 +541,7 @@ module Herb
                         )
                       end
 
-      rendered_template = read_template(template_path.to_s).result_with_hash({ nodes: nodes, errors: errors })
+      rendered_template = read_template(template_path.to_s).result_with_hash({ nodes: nodes, errors: errors, union_kinds: union_kinds })
       content = heading_for(name, template_file) + rendered_template
 
       check_gitignore(name)
@@ -425,6 +585,21 @@ module Herb
       (config.dig("nodes", "types") || []).map { |node| NodeType.new(node) }
     end
 
+    # Collect all unique union kinds from node fields
+    def self.union_kinds
+      union_kinds_set = Set.new
+
+      nodes.each do |node|
+        node.fields.each do |field|
+          if field.respond_to?(:union_kind) && field.union_kind
+            union_kinds_set.add(field.union_kind.sort)
+          end
+        end
+      end
+
+      union_kinds_set.to_a.sort
+    end
+
     def self.errors
       (config.dig("errors", "types") || []).map { |node| ErrorType.new(node) }
     end

data/templates/wasm/error_helpers.cpp.erb

@@ -18,7 +18,7 @@ extern "C" {
 
 using namespace emscripten;
 
-val CreateLocation(location_T* location);
+val CreateLocation(location_T location);
 val CreateToken(token_T* token);
 val NodeFromCStruct(AST_NODE_T* node);
 val NodesArrayFromCArray(hb_array_T* array);
@@ -34,21 +34,25 @@ val <%= error.name %>FromCStruct(<%= error.struct_type %>* <%= error.human %>) {
   val Object = val::global("Object");
   val result = Object.new_();
 
-  result.set("type", CreateString(error_type_to_string(&<%= error.human %>->base)));
-  result.set("message", CreateString(<%= error.human %>->base.message));
+  result.set("type", CreateStringFromHbString(error_type_to_string(&<%= error.human %>->base)));
+  result.set("message", CreateStringFromHbString(<%= error.human %>->base.message));
   result.set("location", CreateLocation(<%= error.human %>->base.location));
   <%- error.fields.each do |field| -%>
   <%- case field -%>
   <%- when Herb::Template::StringField -%>
-  result.set("<%= field.name %>", CreateString(<%= error.human %>-><%= field.name %>));
+  result.set("<%= field.name %>", CreateStringFromHbString(<%= error.human %>-><%= field.name %>));
   <%- when Herb::Template::NodeField -%>
   result.set("<%= field.name %>", NodeFromCStruct((AST_NODE_T*)<%= error.human %>-><%= field.name %>));
   <%- when Herb::Template::TokenField -%>
   result.set("<%= field.name %>", CreateToken(<%= error.human %>-><%= field.name %>));
   <%- when Herb::Template::TokenTypeField -%>
-  result.set("<%= field.name %>", CreateString(token_type_to_string(<%= error.human %>-><%= field.name %>)));
+  result.set("<%= field.name %>", CreateStringFromHbString(token_type_to_string(<%= error.human %>-><%= field.name %>)));
   <%- when Herb::Template::BooleanField -%>
   result.set("<%= field.name %>", <%= error.human %>-><%= field.name %>);
+  <%- when Herb::Template::PositionField -%>
+  result.set("<%= field.name %>", CreatePosition(<%= error.human %>-><%= field.name %>));
+  <%- when Herb::Template::SizeTField -%>
+  result.set("<%= field.name %>", val((double)<%= error.human %>-><%= field.name %>));
   <%- when Herb::Template::ArrayField -%>
   result.set("<%= field.name %>", NodesArrayFromCArray(<%= error.human %>-><%= field.name %>));
   <%- else -%>

data/templates/wasm/nodes.cpp.erb

@@ -28,8 +28,8 @@ val <%= node.name %>FromCStruct(<%= node.struct_type %>* <%= node.human %>) {
   <%- node.fields.each do |field| -%>
   <%- case field -%>
   <%- when Herb::Template::StringField -%>
-  result.set("<%= field.name %>", CreateString(<%= node.human %>-><%= field.name %>));
-  <%- when Herb::Template::NodeField -%>
+  result.set("<%= field.name %>", CreateStringFromHbString(<%= node.human %>-><%= field.name %>));
+  <%- when Herb::Template::NodeField, Herb::Template::BorrowedNodeField -%>
   result.set("<%= field.name %>", NodeFromCStruct((AST_NODE_T*) <%= node.human %>-><%= field.name %>));
   <%- when Herb::Template::TokenField -%>
   result.set("<%= field.name %>", CreateToken(<%= node.human %>-><%= field.name %>));
@@ -38,14 +38,51 @@ val <%= node.name %>FromCStruct(<%= node.struct_type %>* <%= node.human %>) {
   <%- when Herb::Template::ArrayField -%>
   result.set("<%= field.name %>", NodesArrayFromCArray(<%= node.human %>-><%= field.name %>));
   <%- when Herb::Template::ElementSourceField -%>
-  result.set("<%= field.name %>", CreateStringFromHbString(element_source_to_string(<%= node.human %>-><%= field.name %>)));
+  result.set("<%= field.name %>", CreateStringFromHbString(<%= node.human %>-><%= field.name %>));
   <%- when Herb::Template::LocationField -%>
   if (<%= node.human %>-><%= field.name %>) {
     result.set("<%= field.name %>", CreateLocation(*<%= node.human %>-><%= field.name %>));
   } else {
     result.set("<%= field.name %>", val::null());
   }
+  <%- when Herb::Template::PrismSerializedField -%>
+  if (<%= node.human %>-><%= field.name %>.data != NULL && <%= node.human %>-><%= field.name %>.length > 0) {
+    val jsArray = val::array();
+
+    for (size_t i = 0; i < <%= node.human %>-><%= field.name %>.length; i++) {
+      jsArray.call<void>("push", (int)<%= node.human %>-><%= field.name %>.data[i]);
+    }
+
+    result.set("<%= field.name %>", jsArray);
+  } else {
+    result.set("<%= field.name %>", val::null());
+  }
+  <%- when Herb::Template::PrismNodeField -%>
+  if (<%= node.human %>-><%= field.name %>.node != NULL && <%= node.human %>-><%= field.name %>.parser != NULL) {
+    pm_buffer_t pm_buffer = { 0 };
+    pm_serialize(<%= node.human %>-><%= field.name %>.parser, <%= node.human %>-><%= field.name %>.node, &pm_buffer);
+
+    if (pm_buffer.length > 0) {
+      val jsArray = val::array();
+
+      for (size_t i = 0; i < pm_buffer.length; i++) {
+        jsArray.call<void>("push", (int)((uint8_t*)pm_buffer.value)[i]);
+      }
+
+      result.set("<%= field.name %>", jsArray);
+    } else {
+      result.set("<%= field.name %>", val::null());
+    }
+
+    pm_buffer_free(&pm_buffer);
+  } else {
+    result.set("<%= field.name %>", val::null());
+  }
+  <%- when Herb::Template::AnalyzedRubyField, Herb::Template::PrismContextField, Herb::Template::VoidPointerField -%>
+  /* <%= field.name %> is internal parser state, not exposed to JavaScript */
+  result.set("<%= field.name %>", val::null());
   <%- else -%>
+  /* Unhandled field type: <%= field.class.name %> */
   result.set("<%= field.name %>", val::null());
   <%- end -%>
   <%- end -%>
@@ -77,7 +114,7 @@ val NodesArrayFromCArray(hb_array_T* array) {
     AST_NODE_T* child_node = (AST_NODE_T*) hb_array_get(array, i);
 
     if (child_node) {
-      jsArray.set(i, NodeFromCStruct(child_node));
+      jsArray.call<void>("push", NodeFromCStruct(child_node));
     }
   }