herb 0.11.1 → 0.11.2

data/templates/src/analyze/action_view/generated_handlers.c.erb

@@ -16,9 +16,9 @@
 bool detect_<%= helper.safe_name %>(pm_call_node_t* call_node, pm_parser_t* parser) {
   if (!call_node || !call_node->name) { return false; }
 
-  pm_constant_t* constant = pm_constant_pool_id_to_constant(&parser->constant_pool, call_node->name);
+  const pm_constant_t* constant = pm_parser_constant(parser, call_node->name);
 
-  return constant && constant->length == <%= helper.name.length %> && strncmp((const char*) constant->start, "<%= helper.name %>", <%= helper.name.length %>) == 0;
+  return constant && pm_constant_length(constant) == <%= helper.name.length %> && strncmp((const char*) pm_constant_start(constant), "<%= helper.name %>", <%= helper.name.length %>) == 0;
 }
 <%- elsif helper.receiver_call_detect? -%>
 
@@ -29,9 +29,9 @@ bool detect_<%= helper.safe_name %>(pm_call_node_t* call_node, pm_parser_t* pars
   pm_call_node_t* receiver_node = (pm_call_node_t*) call_node->receiver;
   if (!receiver_node->name) { return false; }
 
-  pm_constant_t* constant = pm_constant_pool_id_to_constant(&parser->constant_pool, receiver_node->name);
+  const pm_constant_t* constant = pm_parser_constant(parser, receiver_node->name);
 
-  return constant && constant->length == 3 && strncmp((const char*) constant->start, "tag", 3) == 0;
+  return constant && pm_constant_length(constant) == 3 && strncmp((const char*) pm_constant_start(constant), "tag", 3) == 0;
 }
 <%- end -%>
 <%- end -%>
@@ -79,14 +79,17 @@ extern bool is_ruby_introspection_method(hb_string_T method_name);
 static char* extract_<%= helper.safe_name %>_tag_name(pm_call_node_t* call_node, pm_parser_t* parser, hb_allocator_T* allocator) {
   if (!call_node || !call_node->name) { return NULL; }
 
-  pm_constant_t* constant = pm_constant_pool_id_to_constant(&parser->constant_pool, call_node->name);
+  const pm_constant_t* constant = pm_parser_constant(parser, call_node->name);
   if (!constant) { return NULL; }
 
-  if (is_ruby_introspection_method(hb_string_from_data((const char*) constant->start, constant->length))) { return NULL; }
+  const char* constant_start = (const char*) pm_constant_start(constant);
+  size_t constant_length = pm_constant_length(constant);
 
-  char* name = hb_allocator_strndup(allocator, (const char*) constant->start, constant->length);
+  if (is_ruby_introspection_method(hb_string_from_data(constant_start, constant_length))) { return NULL; }
 
-  for (size_t i = 0; i < constant->length && name[i] != '\0'; i++) {
+  char* name = hb_allocator_strndup(allocator, constant_start, constant_length);
+
+  for (size_t i = 0; i < constant_length && name[i] != '\0'; i++) {
     if (name[i] == '_') { name[i] = '-'; }
   }
 
@@ -111,9 +114,7 @@ static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* _call_node,
 }
 <%- elsif content.first_arg? -%>
 
-static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node, pm_parser_t* _parser, hb_allocator_T* allocator) {
-  (void) _parser;
-
+static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node, pm_parser_t* parser, hb_allocator_T* allocator) {
   if (!call_node || !call_node->arguments) { return NULL; }
 
   pm_arguments_node_t* arguments = call_node->arguments;
@@ -131,27 +132,26 @@ static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node,
     return hb_allocator_strndup(allocator, (const char*) pm_string_source(&string_node->unescaped), length);
   }
 
-  size_t source_length = first_argument->location.end - first_argument->location.start;
-  return hb_allocator_strndup(allocator, (const char*) first_argument->location.start, source_length);
+  const uint8_t* source = pm_parser_start(parser);
+  size_t source_length = first_argument->location.length;
+  return hb_allocator_strndup(allocator, (const char*) (source + first_argument->location.start), source_length);
 }
 <%- elsif content.block_or_arg? -%>
 
-static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node, pm_parser_t* <%= helper.receiver_call_detect? ? "parser" : "_parser" %>, hb_allocator_T* allocator) {
-  (void) <%= helper.receiver_call_detect? ? "parser" : "_parser" %>;
-
+static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node, pm_parser_t* parser, hb_allocator_T* allocator) {
   if (!call_node) { return NULL; }
 
   <%- if helper.receiver_call_detect? -%>
   if (call_node->name) {
-    pm_constant_t* constant = pm_constant_pool_id_to_constant(&parser->constant_pool, call_node->name);
+    const pm_constant_t* constant = pm_parser_constant(parser, call_node->name);
 
-    if (constant && is_ruby_introspection_method(hb_string_from_data((const char*) constant->start, constant->length))) {
+    if (constant && is_ruby_introspection_method(hb_string_from_data((const char*) pm_constant_start(constant), pm_constant_length(constant)))) {
       return NULL;
     }
   }
   <%- end -%>
 
-  char* block_content = extract_inline_block_content(call_node, allocator);
+  char* block_content = extract_inline_block_content(call_node, parser, allocator);
   if (block_content) { return block_content; }
 
   <%- arg_index = (content.arg_position || 1) - 1 -%>
@@ -173,7 +173,8 @@ static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node,
           return hb_allocator_strndup(allocator, (const char*) pm_string_source(&string_node->unescaped), length);
         }
 
-        size_t source_length = argument->location.end - argument->location.start;
+        const uint8_t* source = pm_parser_start(parser);
+        size_t source_length = argument->location.length;
 
         <%- if content.to_s_suffix_when_single -%>
         if (arguments->arguments.size == 1 && argument->type != PM_NIL_NODE) {
@@ -182,7 +183,7 @@ static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node,
           size_t total_length = source_length + suffix_length;
           char* ruby_expression = hb_allocator_alloc(allocator, total_length + 1);
 
-          memcpy(ruby_expression, (const char*) argument->location.start, source_length);
+          memcpy(ruby_expression, (const char*) (source + argument->location.start), source_length);
           memcpy(ruby_expression + source_length, suffix, suffix_length);
           ruby_expression[total_length] = '\0';
 
@@ -190,7 +191,7 @@ static char* extract_<%= helper.safe_name %>_content(pm_call_node_t* call_node,
         }
         <%- end -%>
 
-        return hb_allocator_strndup(allocator, (const char*) argument->location.start, source_length);
+        return hb_allocator_strndup(allocator, (const char*) (source + argument->location.start), source_length);
 
       <%- if content.skip_if_hash -%>
       }

data/templates/src/include/parser/parser_options.h.erb

@@ -7,9 +7,15 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#define HERB_PARSER_OPTIONS_STRING_CAPACITY 16
+
 typedef struct PARSER_OPTIONS_STRUCT {
 <%- parser_options.all_c_fields.each do |field| -%>
+<%- if field.string? -%>
+  char <%= field.c_name %>[HERB_PARSER_OPTIONS_STRING_CAPACITY];
+<%- else -%>
   <%= field.c_type %> <%= field.c_name %>;
+<%- end -%>
 <%- if field.string_array? -%>
   size_t <%= field.count_name %>;
 <%- end -%>
@@ -17,5 +23,9 @@ typedef struct PARSER_OPTIONS_STRUCT {
 } parser_options_T;
 
 extern const parser_options_T HERB_DEFAULT_PARSER_OPTIONS;
+<%- parser_options.all_c_fields.select(&:validator).each do |field| -%>
+
+bool <%= field.validator %>(const char* value, size_t length);
+<%- end -%>
 
 #endif

data/templates/src/include/prism_version.h.erb

@@ -0,0 +1,17 @@
+<%-
+  lockfile = File.expand_path("../../../Gemfile.lock", __dir__)
+  prism_git = File.exist?(lockfile) ? File.read(lockfile).split(/\n\n+/).find { |block| block.start_with?("GIT") && block.match?(/^\s+prism \(/) } : nil
+  prism_revision = prism_git && !prism_git.match?(/^\s+tag:/) ? prism_git[/^\s+revision: (\h{7})/, 1] : nil
+-%>
+#ifndef HERB_PRISM_VERSION_H
+#define HERB_PRISM_VERSION_H
+
+#include <prism/version.h>
+
+<%- if prism_revision -%>
+#define HERB_PRISM_VERSION PRISM_VERSION "+<%= prism_revision %>"
+<%- else -%>
+#define HERB_PRISM_VERSION PRISM_VERSION
+<%- end -%>
+
+#endif

data/templates/src/parser_options_defaults.c.erb

@@ -2,7 +2,7 @@
 
 #include <stddef.h>
 
-<%- entries = parser_options.all_c_fields.flat_map { |field| field.string_array? ? [[field.c_name, "NULL"], [field.count_name, "0"]] : [[field.c_name, field.pointer? ? "NULL" : field.c_default.to_s]] } -%>
+<%- entries = parser_options.all_c_fields.flat_map { |field| field.string_array? ? [[field.c_name, "NULL"], [field.count_name, "0"]] : field.string? ? [[field.c_name, field.default ? field.default.inspect : "{ 0 }"]] : [[field.c_name, field.pointer? ? "NULL" : field.c_default.to_s]] } -%>
 <%- prefix = "const parser_options_T HERB_DEFAULT_PARSER_OPTIONS = { " -%>
 <%- entries.each_with_index do |(name, value), index| -%>
 <%= index.zero? ? prefix : " " * prefix.length %>.<%= name %> = <%= value %><%= index < entries.length - 1 ? "," : " };" %>

data/templates/template.rb

@@ -1176,7 +1176,8 @@ module Herb
     end
 
     class ParserOptionField
-      attr_reader :name, :type, :default, :description, :c_name, :nullable, :null_sentinel, :ruby_unit, :default_ms, :count_name
+      attr_reader :name, :type, :default, :description, :c_name, :nullable, :null_sentinel, :ruby_unit, :default_ms, :count_name,
+                  :validator
 
       def initialize(config)
         @name = config.fetch("name")
@@ -1193,6 +1194,7 @@ module Herb
         @only = config["only"]
         @skip = config.fetch("skip", [])
         @lex = config.fetch("lex", false)
+        @validator = config["validator"]
       end
 
       def for?(language)
@@ -1217,6 +1219,10 @@ module Herb
         @type == "string_array"
       end
 
+      def string?
+        @type == "string"
+      end
+
       def uint64?
         @type == "uint64"
       end
@@ -1253,17 +1259,19 @@ module Herb
 
       def java_jni_signature
         return "()Z" if boolean?
+        return "()Ljava/lang/String;" if string?
         return "()Ljava/lang/Integer;" if nullable?
 
         "()I"
       end
 
       def ruby_constant?
-        !string_array?
+        !string_array? && !string?
       end
 
       def ruby_default
         return "[]" if string_array?
+        return @default.nil? ? "nil" : @default.inspect if string?
         return "nil" if nullable? && @default.nil?
 
         if @ruby_unit == "seconds" && @default_ms
@@ -1290,12 +1298,14 @@ module Herb
 
       def js_default
         return "[]" if string_array?
+        return @default.nil? ? "null" : @default.inspect if string?
         return "null" if nullable? && @default.nil?
 
         @default_ms || @default
       end
 
       def java_default
+        return @default.nil? ? "null" : @default.inspect if string?
         return "null" if nullable? && @default.nil?
 
         @default_ms || @default
@@ -1303,6 +1313,7 @@ module Herb
 
       def rust_default
         return "None" if string_array?
+        return @default.nil? ? "None" : "Some(#{@default.inspect}.to_string())" if string?
         return "None" if nullable? && @default.nil?
 
         value = @default_ms || @default
@@ -1322,6 +1333,7 @@ module Herb
       def ruby_type
         return "bool" if boolean?
         return "Array[String]" if string_array?
+        return "String?" if string?
         return "Numeric" if @ruby_unit == "seconds"
 
         nullable? ? "Integer?" : "Integer"
@@ -1330,12 +1342,14 @@ module Herb
       def typescript_type
         return "boolean" if boolean?
         return "string[]" if string_array?
+        return "string | null" if string?
 
         nullable? ? "number | null" : "number"
       end
 
       def java_type
         return "boolean" if boolean?
+        return "String" if string?
 
         nullable? ? "Integer" : "int"
       end
@@ -1343,6 +1357,7 @@ module Herb
       def rust_type
         return "bool" if boolean?
         return "Option<Vec<String>>" if string_array?
+        return "Option<String>" if string?
 
         nullable? ? "Option<u32>" : "u32"
       end

data/templates/wasm/nodes.cpp.erb

@@ -54,17 +54,17 @@ val <%= node.name %>FromCStruct(<%= node.struct_type %>* <%= node.human %>, cons
   }
   <%- when Herb::Template::PrismNodeField -%>
   if (<%= node.human %>-><%= field.name %>.node != NULL && <%= node.human %>-><%= field.name %>.parser != NULL) {
-    pm_buffer_t pm_buffer = { 0 };
-    herb_prism_node_serialize(<%= node.human %>-><%= field.name %>, &pm_buffer);
+    pm_buffer_t* pm_buffer = pm_buffer_new();
+    herb_prism_node_serialize(<%= node.human %>-><%= field.name %>, pm_buffer);
 
-    if (pm_buffer.length > 0) {
-      val view = val(typed_memory_view(pm_buffer.length, (const uint8_t*) pm_buffer.value));
+    if (pm_buffer_length(pm_buffer) > 0) {
+      val view = val(typed_memory_view(pm_buffer_length(pm_buffer), (const uint8_t*) pm_buffer_value(pm_buffer)));
       result.set("<%= field.name %>", val::global("Uint8Array").new_(view));
     } else {
       result.set("<%= field.name %>", val::null());
     }
 
-    pm_buffer_free(&pm_buffer);
+    pm_buffer_free(pm_buffer);
   } else {
     result.set("<%= field.name %>", val::null());
   }

data/templates/wasm/parser_options_helpers.cpp.erb

@@ -1,5 +1,8 @@
 #include "parser_options_helpers.h"
 
+#include <cstring>
+#include <string>
+
 void herb_extract_parser_options(emscripten::val options, parser_options_T* parser_options) {
   if (options.isUndefined() || options.isNull() || options.typeOf().as<std::string>() != "object") { return; }
 
@@ -9,6 +12,27 @@ void herb_extract_parser_options(emscripten::val options, parser_options_T* pars
     parser_options-><%= field.c_name %> = options["<%= field.snake_case %>"].as<bool>();
   }
 
+<%- elsif field.string? -%>
+  if (options.hasOwnProperty("<%= field.snake_case %>")) {
+    emscripten::val value = options["<%= field.snake_case %>"];
+
+    if (value.isString()) {
+      std::string string = value.as<std::string>();
+
+<%- if field.validator -%>
+      if (!<%= field.validator %>(string.c_str(), string.length())) {
+        emscripten::val::global("Error").new_("<%= field.snake_case %> \"" + string + "\" is not supported").throw_();
+      }
+<%- else -%>
+      if (string.length() >= HERB_PARSER_OPTIONS_STRING_CAPACITY) {
+        emscripten::val::global("Error").new_(std::string("<%= field.snake_case %> is too long")).throw_();
+      }
+<%- end -%>
+
+      memcpy(parser_options-><%= field.c_name %>, string.c_str(), string.length() + 1);
+    }
+  }
+
 <%- elsif field.nullable? -%>
   if (options.hasOwnProperty("<%= field.snake_case %>")) {
     emscripten::val value = options["<%= field.snake_case %>"];

data/vendor/prism/Rakefile

@@ -3,6 +3,15 @@
 require "bundler/gem_tasks"
 require "rake/clean"
 
+begin
+  require "rake_compiler_dock"
+rescue LoadError
+  # rake_compiler_dock is not available in versioned Gemfiles (gemfiles/2.7/, etc.)
+  # that don't include the gem. The native gem tasks will not be defined.
+end
+
+PRISM_SPEC = Bundler.load_gemspec("prism.gemspec")
+
 task default: %i[compile test]
 
 require_relative "templates/template"
@@ -28,14 +37,55 @@ task build: [:check_manifest, :templates]
 # the C extension
 task "compile:prism" => ["templates"] # must be before the ExtensionTask is created
 
+RCD_CROSS_PLATFORMS = %w[
+  aarch64-linux-gnu
+  aarch64-linux-musl
+  aarch64-mingw-ucrt
+  arm-linux-gnu
+  arm-linux-musl
+  arm64-darwin
+  x64-mingw-ucrt
+  x86_64-darwin
+  x86_64-linux-gnu
+  x86_64-linux-musl
+]
+
+if defined?(RakeCompilerDock)
+  RakeCompilerDock.set_ruby_cc_version(">= 3.3")
+
+  namespace "gem" do
+    RCD_CROSS_PLATFORMS.each do |platform|
+      desc "build native gem for #{platform}"
+      task platform do
+        RakeCompilerDock.sh(<<~EOF, platform: platform, verbose: true)
+          gem install bundler --no-document &&
+          bundle &&
+          bundle exec rake gem:#{platform}:build
+        EOF
+      end
+
+      namespace platform do
+        # this runs in the rake-compiler-dock docker container
+        task "build" => ["templates"] do
+          Rake::Task["native:#{platform}"].invoke
+          Rake::Task["pkg/#{PRISM_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke
+        end
+      end
+    end
+
+    desc "build native gem for all platforms"
+    task "all" => [RCD_CROSS_PLATFORMS, "build"].flatten
+  end
+end
+
 if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"]
   require "rake/extensiontask"
 
-  Rake::ExtensionTask.new(:compile) do |ext|
-    ext.name = "prism"
+  Rake::ExtensionTask.new("prism", PRISM_SPEC) do |ext|
     ext.ext_dir = "ext/prism"
     ext.lib_dir = "lib/prism"
-    ext.gem_spec = Gem::Specification.load("prism.gemspec")
+    ext.cross_compile = true
+    ext.cross_platform = RCD_CROSS_PLATFORMS
   end
 elsif RUBY_ENGINE == "jruby"
   require "rake/javaextensiontask"
@@ -43,11 +93,10 @@ elsif RUBY_ENGINE == "jruby"
   # This compiles java to make sure any templating changes produces valid code.
   Rake::JavaExtensionTask.new(:compile) do |ext|
     ext.name = "prism"
-    ext.ext_dir = "java"
+    ext.ext_dir = "java/api"
     ext.lib_dir = "tmp"
-    ext.source_version = "1.8"
-    ext.target_version = "1.8"
-    ext.gem_spec = Gem::Specification.load("prism.gemspec")
+    ext.release = "21"
+    ext.gem_spec = PRISM_SPEC
   end
 end
 
@@ -55,6 +104,7 @@ end
 CLOBBER.concat(Prism::Template::TEMPLATES)
 CLOBBER.concat(["build"])
 CLOBBER << "lib/prism/prism.#{RbConfig::CONFIG["DLEXT"]}"
+CLOBBER << "java/wasm/src/main/resources/prism.wasm"
 
 Prism::Template::TEMPLATES.each do |filepath|
   desc "Generate #{filepath}"
@@ -76,3 +126,22 @@ namespace :build do
   desc "Build a development version of the gem"
   task dev: ["build:dev_version_set", "build", "build:dev_version_clear"]
 end
+
+task :build_in_docker => :templates do
+  # Versions from https://github.com/ruby/ruby/blob/master/.github/workflows/compilers.yml
+  versions = (7..15).to_a
+  versions.each do |version|
+    dockerfile = <<~DOCKERFILE
+    FROM docker.io/library/gcc:#{version}
+    COPY Makefile /prism/
+    COPY src /prism/src
+    COPY include /prism/include
+    WORKDIR /prism
+    RUN gcc --version
+    RUN make SOEXT=so
+    DOCKERFILE
+    File.write 'Dockerfile', dockerfile
+    sh "docker build ."
+    rm 'Dockerfile'
+  end
+end