expressir 1.2.5-arm64-darwin → 1.2.6-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcc517f98f669640a476b91c1bf0f8b172be09cace048f2ab3e3e30144c558cf
-  data.tar.gz: 5b149c5bd39d81408c20d5715717cdff98921ce710ec71867ccc6ead40b8e31c
+  metadata.gz: 5a60eb7fe4b2ddc2c04fc256cdbd92ec236dae6b7fc9cfec69466186522dc4ec
+  data.tar.gz: f43f2ea6fe33146931f8339818db8e47f6040942da6a436b331e7a6de591a328
 SHA512:
-  metadata.gz: 1b810959667eb300c59cf535912f00af8e0b75a35cc8e265a9cdce680d86e69e7623721f64f3a1560b477aa856f2b943a3c7c4a789dc70f4ea02ed05e9c3e431
-  data.tar.gz: 597ce31a42daed544ffed4c02f6698f7414602c81f3a6a5adf43063e78612895f8342b3307479bc9d7fbddf012c04138458abf6cb9f28132e04ee3c7eadc4cff
+  metadata.gz: 322ea0cf458cbeafdae98719596fb6cb804e3c8b725ca0b874c460bf6d0606f2b3af9a157d2105887a6e15974c459bc21cebc6cf51a4412c378183b99409f570
+  data.tar.gz: 3a3817309088c49dcff0ef6852c78328d1a049311fc0e9d1a862155f76685a3a0c63fdca47a8ec62e0b6d1e4ec91a5ee24355513c18af4573fde4efcf61fb643

data/.github/workflows/rake.yml

@@ -29,23 +29,11 @@ jobs:
   rake:
     name: test on ruby-${{ matrix.ruby }} ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
         ruby: [ '3.1', '3.0', '2.7' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
-        experimental: [ false ]
-        include:
-          - ruby: 'head'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: 'head'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: 'head'
-            os: 'macos-latest'
-            experimental: true
 
     steps:
       - name: Checkout

data/README.adoc

@@ -75,6 +75,14 @@ Run the test suite
 bin/rspec
 ----
 
+=== Grammar updates
+EXPRESS grammar is lined as git submodule to ext/express-grammar
+Shoudl you update it, run ```rake generate```. This command will generate source code for updated native extension using antlr4-native gem.
+Please note that we create several classes on top of antlr4-native output so using embedded rake task is a real requirement.
+
+When new extension is gnerated and tested plase check in updated C++ files to git (```rake generate``` is NOT a CI step, 
+extension source files are pulled from the repo).
+
 == Installation
 
 Add this line to your application's `Gemfile`:

data/lib/expressir/express/parser.rb

@@ -31,13 +31,12 @@ module Expressir
       # @param [Boolean] include_source attach original source code to model elements
       # @return [Model::Repository]
       def self.from_file(file, skip_references: nil, include_source: nil)
-        @input = File.read(file)
 
       # An important note re memory management
-      # parse, syntax, visitor methods return complex tree structures created in netive (C++) extension
+      # parse, syntax, visitor methods return complex tree structures created in native (C++) extension
       # visit method references nodes and leaves of this structures but it is totally untransparent for Ruby GarbageCllector
-      # so in this class we keep those C++ structure marked for GC so theu are not freed 
-        @parser = ::ExpressParser::Parser.parse(@input)
+      # so in this class we keep those C++ structure marked for GC so they are not freed 
+        @parser = ::ExpressParser::ParserExt.new(file.to_s)
         @parse_tree = @parser.syntax()
 
         @visitor = Visitor.new(@parser.tokens, include_source: include_source)

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "1.2.5".freeze
+  VERSION = "1.2.6".freeze
 end

data/rakelib/antlr4-native.rake

@@ -2,34 +2,138 @@ require "fileutils"
 require "antlr4-native"
 require "rake"
 
-def create_tokens_api(parser_source_lines)
-  # - add ParserProxy tokens method, simple compensation for missing exposed BufferedTokenStream
-  i = parser_source_lines.index { |x| x == "  Object syntax() {" }
-  parser_source_lines[i + 6] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
+def create_class_declarations(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "Class rb_cContextProxy;" }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
 
-    Array getTokens() {
-      Array a;
+    Class rb_cParserExt;
+    Class rb_cTokenExt;
 
-      std::vector<Token*> tokens = this -> tokens -> getTokens();
+  CPP
+end
+
+def create_tp_class_definition(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "class ContextProxy {" }
+  parser_source_lines[i - 2] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
+
+
+    class TokenProxy : public Object {
+      public:
+        TokenProxy(Token* orig) {
+          this -> orig = orig;
+        }
+
+        std::string getText() {
+          return orig->getText();
+        }
+
+        size_t getChannel() {
+          return orig->getChannel();
+        }
+
+        size_t getTokenIndex() {
+          return orig->getTokenIndex();
+        }
+
+      private:
+        Token * orig = nullptr;
+    };
+
+  CPP
+end
+
+def create_pp_class_definition(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "extern \"C\"" }
+  parser_source_lines[i - 2] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
+
+    class ParserProxyExt : public Object {
+      public:
+        ParserProxyExt(Object self, string file) {
+          ifstream stream;
+          stream.open(file);
+          input = new ANTLRInputStream(stream);
+          lexer = new ExpressLexer(input);
+          tokens = new CommonTokenStream(lexer);
+          parser = new ExpressParser(tokens);
+          stream.close();
+        };
+
+        ~ParserProxyExt() {
+          delete parser;
+          delete tokens;
+          delete lexer;
+          delete input;
+        }
+
+        Object syntax() {
+          auto ctx = parser -> syntax();
+
+          SyntaxContextProxy proxy((ExpressParser::SyntaxContext*) ctx);
+          return detail::To_Ruby<SyntaxContextProxy>().convert(proxy);
+        }
 
-      for (auto &token : tokens) {
-        a.push(token);
-      }
+        Array getTokens() {
+          Array a;
+          for (auto token : tokens -> getTokens()) {
+            a.push(new TokenProxy(token));
+          }
+          return a;
+        }
+
+        Object visit(VisitorProxy* visitor) {
+          auto result = visitor -> visit(parser -> syntax());
+
+          lexer -> reset();
+          parser -> reset();
+
+          try {
+            return std::any_cast<Object>(result);
+          } catch(std::bad_cast) {
+           return Qnil;
+          }
+        }
+
+      private:
+        ANTLRInputStream* input;
+        ExpressLexer* lexer;
+        CommonTokenStream* tokens;
+        ExpressParser* parser;
+    };
 
-      return a;
-    }
 
   CPP
 end
 
-def create_tokens_method(parser_source_lines)
-  i = parser_source_lines.index { |x| x == '    .define_method("syntax", &ParserProxy::syntax, Return().keepAlive())' }
-  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "    #{x}" }.join("\n")
+def create_class_api(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit, Return().keepAlive());" }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
+
+
+    rb_cTokenExt = define_class_under<TokenProxy>(rb_mExpressParser, "TokenExt")
+      .define_method("text", &TokenProxy::getText)
+      .define_method("channel", &TokenProxy::getChannel)
+      .define_method("token_index", &TokenProxy::getTokenIndex);
+
+    rb_cParserExt = define_class_under<ParserProxyExt>(rb_mExpressParser, "ParserExt")
+      .define_constructor(Constructor<ParserProxyExt, Object, string>())
+      .define_method("syntax", &ParserProxyExt::syntax, Return().keepAlive())
+      .define_method("tokens", &ParserProxyExt::getTokens)
+      .define_method("visit", &ParserProxyExt::visit, Return().keepAlive());
 
-    .define_method("tokens", &ParserProxy::getTokens)
   CPP
 end
 
+def generate_extended_parser
+  # Generate extended parser that provide Ruby access to token stream
+  parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
+  parser_source_lines = File.read(parser_source_file).split("\n")
+  create_class_declarations(parser_source_lines)
+  create_tp_class_definition(parser_source_lines)
+  create_pp_class_definition(parser_source_lines)
+  create_class_api(parser_source_lines)
+  File.write(parser_source_file, "#{parser_source_lines.join("\n")}\n")
+end
+
 desc "Generate parser  (Usage: 'rake generate <grammar_file>')"
 task "generate" do
   grammar_file = ARGV[1]
@@ -38,7 +142,6 @@ task "generate" do
   end
 
   puts "Generating parser from '#{grammar_file}'"
-
   # ANTLR does weird things if the grammar file isn't in the current working directory
   temp_grammar_file = File.join(FileUtils.pwd, File.basename(grammar_file))
   FileUtils.cp(grammar_file, temp_grammar_file)
@@ -51,13 +154,8 @@ task "generate" do
   )
   generator.generate
 
-  # fix issues with generated parser
-  parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
-  parser_source_lines = File.read(parser_source_file).split("\n")
-  create_tokens_api(parser_source_lines)
-  create_tokens_method(parser_source_lines)
-  File.write(parser_source_file, "#{parser_source_lines.join("\n")}\n")
-
+  puts "Generating extended parser"
+  generate_extended_parser
   # cleanup
   FileUtils.rm(temp_grammar_file)
 end

data/spec/acceptance/version_spec.rb

@@ -5,6 +5,11 @@ RSpec.describe "Expressir" do
     it "has a version number" do |example|
       print "\n[#{example.description}] "
       expect(Expressir::VERSION).not_to be nil
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "displays the current verison" do |example|
@@ -12,6 +17,11 @@ RSpec.describe "Expressir" do
       command = %w(version)
       output = capture_stdout { Expressir::Cli.start(command) }
       expect(output).to include("Version #{Expressir::VERSION}")
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 end

data/spec/expressir/express/cache_spec.rb

@@ -24,6 +24,11 @@ RSpec.describe Expressir::Express::Cache do
         temp_file.close
         temp_file.unlink
       end
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 
@@ -44,6 +49,11 @@ RSpec.describe Expressir::Express::Cache do
         temp_file.close
         temp_file.unlink
       end
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "fails parsing a file from a different Expressir version" do |example|
@@ -62,6 +72,11 @@ RSpec.describe Expressir::Express::Cache do
         temp_file.close
         temp_file.unlink
       end
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 end

data/spec/expressir/express/formatter_spec.rb

@@ -19,6 +19,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object (multiple.exp)" do |example|
@@ -33,6 +38,9 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object (remark.exp)" do |example|
@@ -47,6 +55,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object (syntax.exp)" do |example|
@@ -61,6 +74,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object with schema head formatter (syntax.exp)" do |example|
@@ -78,6 +96,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object with hyperlink formatter (syntax.exp)" do |example|
@@ -95,6 +118,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object with hyperlink formatter (multiple.exp)" do |example|
@@ -112,6 +140,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "exports an object with schema head and hyperlink formatter (multiple.exp)" do |example|
@@ -130,6 +163,11 @@ RSpec.describe Expressir::Express::Formatter do
       expected_result = File.read(formatted_exp_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 end

data/spec/expressir/express/parser_spec.rb

@@ -15,6 +15,11 @@ RSpec.describe Expressir::Express::Parser do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses a file (multiple.exp)" do |example|
@@ -28,6 +33,11 @@ RSpec.describe Expressir::Express::Parser do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses a file (syntax.exp)" do |example|
@@ -41,6 +51,11 @@ RSpec.describe Expressir::Express::Parser do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses a file (remark.exp)" do |example|
@@ -54,6 +69,11 @@ RSpec.describe Expressir::Express::Parser do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses a file including original source (multiple.exp)" do |example|
@@ -74,6 +94,11 @@ RSpec.describe Expressir::Express::Parser do
       stop_index = input.index("END_ENTITY;") + "END_ENTITY;".length - 1
       expected_result = input[start_index..stop_index]
       expect(entity.source).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 
@@ -99,6 +124,11 @@ RSpec.describe Expressir::Express::Parser do
       expect(schemas[3].id).to eq("multiple_schema3")
       expect(schemas[4].file).to eq(exp_files[1].to_s)
       expect(schemas[4].id).to eq("multiple_schema4")
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 end

data/spec/expressir/model/model_element_spec.rb

@@ -17,6 +17,11 @@ RSpec.describe Expressir::Model::ModelElement do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 
@@ -32,6 +37,11 @@ RSpec.describe Expressir::Model::ModelElement do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses an object (multiple.yaml)" do |example|
@@ -45,6 +55,11 @@ RSpec.describe Expressir::Model::ModelElement do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses an object (syntax.yaml)" do |example|
@@ -58,6 +73,10 @@ RSpec.describe Expressir::Model::ModelElement do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "parses an object (remark.yaml)" do |example|
@@ -71,6 +90,11 @@ RSpec.describe Expressir::Model::ModelElement do
       expected_result = File.read(yaml_file)
 
       expect(result).to eq(expected_result)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 
@@ -88,6 +112,11 @@ RSpec.describe Expressir::Model::ModelElement do
       # schema scope
       schema = repo.schemas.first
       expect(schema.find("empty_entity")).to be_instance_of(Expressir::Model::Declarations::Entity)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "finds an object (multiple.exp)" do |example|
@@ -111,6 +140,11 @@ RSpec.describe Expressir::Model::ModelElement do
       expect(schema.find("attribute_entity2")).to be_instance_of(Expressir::Model::Declarations::Entity)
       expect(schema.find("attribute_entity3")).to be_instance_of(Expressir::Model::Declarations::Entity)
       expect(schema.find("attribute_entity4")).to be_instance_of(Expressir::Model::Declarations::Entity)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "finds an object (syntax.exp)" do |example|
@@ -126,6 +160,11 @@ RSpec.describe Expressir::Model::ModelElement do
       # schema scope
       schema = repo.schemas.first
       expect(schema.find("empty_entity")).to be_instance_of(Expressir::Model::Declarations::Entity)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
 
     it "finds an object (remark.exp)" do |example|
@@ -269,6 +308,11 @@ RSpec.describe Expressir::Model::ModelElement do
       expect(entity.find("remark_type.wr:WR1")).to be_instance_of(Expressir::Model::Declarations::WhereRule)
       expect(entity.find("remark_type.IP1")).to be_instance_of(Expressir::Model::Declarations::RemarkItem)
       expect(entity.find("remark_type.ip:IP1")).to be_instance_of(Expressir::Model::Declarations::RemarkItem)
+
+      # Validate Object Space
+      GC.start
+      GC.verify_compaction_references
+      GC.verify_internal_consistency
     end
   end
 end

data/spec/spec_helper.rb

@@ -14,4 +14,11 @@ RSpec.configure do |config|
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
+
+  config.around(:example) do |ex|
+    ex.run
+  rescue SystemExit => e
+    puts "Got SystemExit: #{e.inspect}."
+    raise
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: expressir
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.2.6
 platform: arm64-darwin
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-17 00:00:00.000000000 Z
+date: 2022-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor