expressir 1.3.0 → 1.3.1

data/lib/expressir/express/visitor.rb

@@ -40,7 +40,7 @@ module Expressir
 
       private_constant :REMARK_CHANNEL
 
-      # @param [Array<::ExpressParser::Token>] tokens
+      # @param [::ExpressParser::TokenVector] Rice-wrapped std::vector<TokenProxy>
       # @param [Boolean] include_source attach original source code to model elements
       def initialize(tokens, include_source: nil)
         @tokens = tokens
@@ -101,7 +101,11 @@ module Expressir
           [ctx.start.token_index, ctx.stop.token_index]
         end
 
-        @tokens[start_index..stop_index]
+        selected_tokens = []
+        (start_index..stop_index).each do |i|
+          selected_tokens << @tokens[i]
+        end
+        selected_tokens
       end
 
       def attach_source(ctx, node)
@@ -150,15 +154,14 @@ module Expressir
       end
 
       def attach_remarks(ctx, node)
-        @remark_tokens = get_tokens(ctx)
-        @remark_tokens = @remark_tokens.select{ |x| x.channel == REMARK_CHANNEL
-        }
+        remark_tokens = get_tokens(ctx)
+        remark_tokens = remark_tokens.select{ |x| x.channel == REMARK_CHANNEL }
 
         # skip already attached remarks
-        @remark_tokens = @remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
+        remark_tokens = remark_tokens.select{|x| !@attached_remark_tokens.include?(x.token_index)}
 
         # parse remarks, find remark targets
-        tagged_remark_tokens = @remark_tokens.map do |remark_token|
+        tagged_remark_tokens = remark_tokens.map do |remark_token|
           _, remark_tag, remark_text = if remark_token.text.start_with?('--')
             remark_token.text.match(/^--"([^"]*)"(.*)$/).to_a
           else
@@ -181,7 +184,7 @@ module Expressir
           remark_target.remarks << remark_text
 
           # mark remark as attached, so that it is not attached again at higher nesting level
-          @attached_remark_tokens << remark_token
+          @attached_remark_tokens << remark_token.token_index
         end
       end
 

data/lib/expressir/model/declarations/schema.rb

@@ -90,13 +90,13 @@ module Expressir
           return [] unless parent
 
           interfaces.flat_map do |interface|
-            schema = parent.children_by_id[interface.schema.id.downcase]
+            schema = parent.children_by_id[interface.schema.id.safe_downcase]
             if schema
               schema_safe_children = schema.safe_children
-              schema_safe_children_by_id = schema_safe_children.select{|x| x.id}.map{|x| [x.id.downcase, x]}.to_h
+              schema_safe_children_by_id = schema_safe_children.select{|x| x.id}.map{|x| [x.id.safe_downcase, x]}.to_h
               if interface.items.length > 0
                 interface.items.map do |interface_item|
-                  base_item = schema_safe_children_by_id[interface_item.ref.id.downcase]
+                  base_item = schema_safe_children_by_id[interface_item.ref.id.safe_downcase]
                   if base_item
                     id = interface_item.id || base_item.id
                     create_interfaced_item(id, base_item)

data/lib/expressir/model/model_element.rb

@@ -46,7 +46,7 @@ module Expressir
       def find(path)
         return self if path.empty?
 
-        path_parts = path.downcase.split(/\./).map do |current_path|
+        path_parts = path.safe_downcase.split(/\./).map do |current_path|
           _, _, current_path = current_path.rpartition(":") # ignore prefix
           current_path
         end
@@ -82,7 +82,7 @@ module Expressir
 
       # @return [Hash<String, Declaration>]
       def children_by_id
-        @children_by_id ||= children.select{|x| x.id}.map{|x| [x.id.downcase, x]}.to_h
+        @children_by_id ||= children.select{|x| x.id}.map{|x| [x.id.safe_downcase, x]}.to_h
       end
 
       # @return [nil]
@@ -90,7 +90,7 @@ module Expressir
         @children_by_id = nil
         nil
       end
-      
+
       # @param [String] root_path
       # @param [Express::Formatter] formatter
       # @param [Boolean] include_empty
@@ -131,7 +131,7 @@ module Expressir
 
         hash
       end
-      
+
       # @param [Hash] hash
       # @param [String] root_path
       # @return [ModelElement]

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "1.3.0".freeze
+  VERSION = "1.3.1".freeze
 end

data/lib/expressir.rb

@@ -7,6 +7,26 @@ Dir[File.join(__dir__, "expressir", "express", "*.rb")].sort.each do |fea|
   require fea
 end
 
+# ..........................................................
+# https://bugs.ruby-lang.org/issues/19319
+# The issue is that this bug is fixed for 3.1 and above,
+# but not for 3.0 or 2.7, so we need a "safe" function
+# ..........................................................
+
+if RUBY_VERSION < "3.1"
+  class String
+    def safe_downcase
+      each_char.map(&:downcase).join
+    end
+  end
+else
+  class String
+    def safe_downcase
+      downcase
+    end
+  end
+end
+
 module Expressir
   class Error < StandardError; end
 

data/rakelib/antlr4-native.rake

@@ -72,12 +72,12 @@ def create_pp_class_definition(parser_source_lines)
           return detail::To_Ruby<SyntaxContextProxy>().convert(proxy);
         }
 
-        Array getTokens() {
-          Array a;
+        Object getTokens() {
+          std::vector<TokenProxy> tk;
           for (auto token : tokens -> getTokens()) {
-            a.push(new TokenProxy(token));
+            tk.push_back(TokenProxy(token));
           }
-          return a;
+          return detail::To_Ruby<std::vector<TokenProxy>>().convert(tk);
         }
 
         Object visit(VisitorProxy* visitor) {
@@ -88,8 +88,8 @@ def create_pp_class_definition(parser_source_lines)
 
           try {
             return std::any_cast<Object>(result);
-          } catch(std::bad_cast) {
-           return Qnil;
+          } catch(std::bad_any_cast) {
+            return Qnil;
           }
         }
 
@@ -105,7 +105,7 @@ def create_pp_class_definition(parser_source_lines)
 end
 
 def create_class_api(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit, Return().keepAlive());" }
+  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit);" }
   parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
 
 
@@ -116,17 +116,29 @@ def create_class_api(parser_source_lines)
 
     rb_cParserExt = define_class_under<ParserProxyExt>(rb_mExpressParser, "ParserExt")
       .define_constructor(Constructor<ParserProxyExt, string>())
-      .define_method("syntax", &ParserProxyExt::syntax, Return().keepAlive())
-      .define_method("tokens", &ParserProxyExt::getTokens)
-      .define_method("visit", &ParserProxyExt::visit, Return().keepAlive());
+      .define_method("syntax", &ParserProxyExt::syntax)
+      .define_method("tokens", &ParserProxyExt::getTokens, Return().keepAlive())
+      .define_method("visit", &ParserProxyExt::visit);
+
+    define_vector<std::vector<TokenProxy>>("TokenVector");
+
+  CPP
+end
+
+def create_vector_definition(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit);" }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
 
   CPP
 end
 
 def generate_extended_parser
   # Generate extended parser that provide Ruby access to token stream
-  parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
-  parser_source_lines = File.read(parser_source_file).split("\n")
+  parser_source_file = File.join("ext", "express_parser", "express_parser.cpp")
+  parser_source_lines = File.read(parser_source_file)
+    .gsub!("bad_cast", "bad_any_cast")
+    .gsub!("return detail::To_Ruby<Token*>().convert(token)", "return detail::To_Ruby<TokenProxy>().convert(TokenProxy(token))")
+    .split("\n")
   create_class_declarations(parser_source_lines)
   create_tp_class_definition(parser_source_lines)
   create_pp_class_definition(parser_source_lines)

data/rakelib/cross-ruby.rake

@@ -370,7 +370,7 @@ end
 require "rake/extensiontask"
 
 Rake::ExtensionTask.new("express_parser", GEMSPEC) do |ext|
-  ext.ext_dir = "ext/express-parser"
+  ext.ext_dir = "ext/express_parser"
   ext.lib_dir = File.join(*["lib", "expressir", "express", ENV.fetch("FAT_DIR", nil)].compact)
   ext.config_options << ENV.fetch("EXTOPTS", nil)
   ext.cross_compile  = true

data/spec/acceptance/version_spec.rb

@@ -3,6 +3,9 @@ require "spec_helper"
 RSpec.describe "Expressir" do
   describe "version" do
     it "has a version number" do |example|
+      GC.stress = ENV["GC_STRESS"] == "true"
+      puts "Running tests in GC stress mode. It may take a couple of hours ..." if GC.stress
+
       print "\n[#{example.description}] "
       expect(Expressir::VERSION).not_to be nil
 

data/spec/expressir/express/cache_spec.rb

@@ -40,7 +40,7 @@ RSpec.describe Expressir::Express::Cache do
       end.to raise_error(Errno::ENOENT)
     end
 
-    it "parses a file" do |example|
+    it "loads a cache file" do |example|
       print "\n[#{example.description}] "
       temp_file = Tempfile.new
 
@@ -63,7 +63,7 @@ RSpec.describe Expressir::Express::Cache do
       GC.verify_internal_consistency
     end
 
-    it "fails parsing a file from a different Expressir version" do |example|
+    it "fails parsing a cache from a different Expressir version" do |example|
       print "\n[#{example.description}] "
       temp_file = Tempfile.new
 

data/spec/expressir/express/formatter_spec.rb

@@ -11,9 +11,7 @@ RSpec.describe Expressir::Express::Formatter do
       print "\n[#{example.description}] "
       exp_file = Expressir.root_path.join("spec", "syntax", "single.exp")
       formatted_exp_file = Expressir.root_path.join("spec", "syntax", "single_formatted.exp")
-
       repo = Expressir::Express::Parser.from_file(exp_file)
-
       result = Expressir::Express::Formatter.format(repo)
       # File.write(formatted_exp_file, result)
       expected_result = File.read(formatted_exp_file)