expressir 1.2.3-x64-mingw32 → 1.2.6-x64-mingw32

data/lib/expressir/express/parser.rb

@@ -31,25 +31,27 @@ module Expressir
       # @param [Boolean] include_source attach original source code to model elements
       # @return [Model::Repository]
       def self.from_file(file, skip_references: nil, include_source: nil)
-        input = File.read(file)
 
-        parser = ::ExpressParser::Parser.parse(input)
+      # An important note re memory management
+      # parse, syntax, visitor methods return complex tree structures created in native (C++) extension
+      # visit method references nodes and leaves of this structures but it is totally untransparent for Ruby GarbageCllector
+      # so in this class we keep those C++ structure marked for GC so they are not freed 
+        @parser = ::ExpressParser::ParserExt.new(file.to_s)
+        @parse_tree = @parser.syntax()
 
-        parse_tree = parser.syntax()
+        @visitor = Visitor.new(@parser.tokens, include_source: include_source)
+        @repository = @visitor.visit(@parse_tree)
 
-        visitor = Visitor.new(parser.tokens, include_source: include_source)
-        repository = visitor.visit(parse_tree)
-
-        repository.schemas.each do |schema|
+        @repository.schemas.each do |schema|
           schema.file = file.to_s
         end
 
         unless skip_references
-          resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-          resolve_references_model_visitor.visit(repository)
+          @resolve_references_model_visitor = ResolveReferencesModelVisitor.new
+          @resolve_references_model_visitor.visit(@repository)
         end
 
-        repository
+        @repository
       end
 
       # Parses Express files into an Express model
@@ -65,16 +67,16 @@ module Expressir
           repository.schemas
         end.flatten
 
-        repository = Model::Repository.new(
+        @repository = Model::Repository.new(
           schemas: schemas
         )
 
-        unless skip_references
-          resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-          resolve_references_model_visitor.visit(repository)
+        unless @skip_references
+          @resolve_references_model_visitor = ResolveReferencesModelVisitor.new
+          @resolve_references_model_visitor.visit(@repository)
         end
 
-        repository
+        @repository
       end
     end
   end

data/lib/expressir/express/visitor.rb

@@ -31,6 +31,8 @@ require "set"
 # - prevents segfault in ANTLR4 C++ runtime, not sure why they are caused
 # - e.g. see visit_schema_decl
 
+require 'objspace'
+
 module Expressir
   module Express
     class Visitor < ::ExpressParser::Visitor
@@ -138,13 +140,15 @@ module Expressir
       end
 
       def attach_remarks(ctx, node)
-        remark_tokens = get_tokens(ctx).select{|x| x.channel == REMARK_CHANNEL}
+        @remark_tokens = get_tokens(ctx)
+        @remark_tokens = @remark_tokens.select{ |x| x.channel == REMARK_CHANNEL
+        }
 
         # skip already attached remarks
-        remark_tokens = remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
+        @remark_tokens = @remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
 
         # parse remarks, find remark targets
-        tagged_remark_tokens = remark_tokens.map do |remark_token|
+        tagged_remark_tokens = @remark_tokens.map do |remark_token|
           _, remark_tag, remark_text = if remark_token.text.start_with?('--')
             remark_token.text.match(/^--"([^"]*)"(.*)$/).to_a
           else

data/lib/expressir/model.rb

@@ -1,79 +1,79 @@
-require 'expressir/model/model_element'
+require "expressir/model/model_element"
 
-require 'expressir/model/cache'
-require 'expressir/model/data_type'
-require 'expressir/model/declaration'
-require 'expressir/model/expression'
-require 'expressir/model/identifier'
-require 'expressir/model/literal'
-require 'expressir/model/reference'
-require 'expressir/model/repository'
-require 'expressir/model/statement'
-require 'expressir/model/supertype_expression'
-require 'expressir/model/data_types/aggregate'
-require 'expressir/model/data_types/array'
-require 'expressir/model/data_types/bag'
-require 'expressir/model/data_types/binary'
-require 'expressir/model/data_types/boolean'
-require 'expressir/model/data_types/enumeration'
-require 'expressir/model/data_types/enumeration_item'
-require 'expressir/model/data_types/generic_entity'
-require 'expressir/model/data_types/generic'
-require 'expressir/model/data_types/integer'
-require 'expressir/model/data_types/list'
-require 'expressir/model/data_types/logical'
-require 'expressir/model/data_types/number'
-require 'expressir/model/data_types/real'
-require 'expressir/model/data_types/set'
-require 'expressir/model/data_types/select'
-require 'expressir/model/data_types/string'
-require 'expressir/model/declarations/attribute'
-require 'expressir/model/declarations/constant'
-require 'expressir/model/declarations/entity'
-require 'expressir/model/declarations/function'
-require 'expressir/model/declarations/interface'
-require 'expressir/model/declarations/interface_item'
-require 'expressir/model/declarations/interfaced_item'
-require 'expressir/model/declarations/parameter'
-require 'expressir/model/declarations/procedure'
-require 'expressir/model/declarations/remark_item'
-require 'expressir/model/declarations/rule'
-require 'expressir/model/declarations/schema'
-require 'expressir/model/declarations/schema_version'
-require 'expressir/model/declarations/schema_version_item'
-require 'expressir/model/declarations/subtype_constraint'
-require 'expressir/model/declarations/type'
-require 'expressir/model/declarations/unique_rule'
-require 'expressir/model/declarations/variable'
-require 'expressir/model/declarations/where_rule'
-require 'expressir/model/expressions/aggregate_initializer'
-require 'expressir/model/expressions/aggregate_initializer_item'
-require 'expressir/model/expressions/binary_expression'
-require 'expressir/model/expressions/entity_constructor'
-require 'expressir/model/expressions/function_call'
-require 'expressir/model/expressions/interval'
-require 'expressir/model/expressions/query_expression'
-require 'expressir/model/expressions/unary_expression'
-require 'expressir/model/literals/binary'
-require 'expressir/model/literals/integer'
-require 'expressir/model/literals/logical'
-require 'expressir/model/literals/real'
-require 'expressir/model/literals/string'
-require 'expressir/model/references/attribute_reference'
-require 'expressir/model/references/group_reference'
-require 'expressir/model/references/index_reference'
-require 'expressir/model/references/simple_reference'
-require 'expressir/model/statements/alias'
-require 'expressir/model/statements/assignment'
-require 'expressir/model/statements/case'
-require 'expressir/model/statements/case_action'
-require 'expressir/model/statements/compound'
-require 'expressir/model/statements/escape'
-require 'expressir/model/statements/if'
-require 'expressir/model/statements/null'
-require 'expressir/model/statements/procedure_call'
-require 'expressir/model/statements/repeat'
-require 'expressir/model/statements/return'
-require 'expressir/model/statements/skip'
-require 'expressir/model/supertype_expressions/binary_supertype_expression'
-require 'expressir/model/supertype_expressions/oneof_supertype_expression'
+require "expressir/model/cache"
+require "expressir/model/data_type"
+require "expressir/model/declaration"
+require "expressir/model/expression"
+require "expressir/model/identifier"
+require "expressir/model/literal"
+require "expressir/model/reference"
+require "expressir/model/repository"
+require "expressir/model/statement"
+require "expressir/model/supertype_expression"
+require "expressir/model/data_types/aggregate"
+require "expressir/model/data_types/array"
+require "expressir/model/data_types/bag"
+require "expressir/model/data_types/binary"
+require "expressir/model/data_types/boolean"
+require "expressir/model/data_types/enumeration"
+require "expressir/model/data_types/enumeration_item"
+require "expressir/model/data_types/generic_entity"
+require "expressir/model/data_types/generic"
+require "expressir/model/data_types/integer"
+require "expressir/model/data_types/list"
+require "expressir/model/data_types/logical"
+require "expressir/model/data_types/number"
+require "expressir/model/data_types/real"
+require "expressir/model/data_types/set"
+require "expressir/model/data_types/select"
+require "expressir/model/data_types/string"
+require "expressir/model/declarations/attribute"
+require "expressir/model/declarations/constant"
+require "expressir/model/declarations/entity"
+require "expressir/model/declarations/function"
+require "expressir/model/declarations/interface"
+require "expressir/model/declarations/interface_item"
+require "expressir/model/declarations/interfaced_item"
+require "expressir/model/declarations/parameter"
+require "expressir/model/declarations/procedure"
+require "expressir/model/declarations/remark_item"
+require "expressir/model/declarations/rule"
+require "expressir/model/declarations/schema"
+require "expressir/model/declarations/schema_version"
+require "expressir/model/declarations/schema_version_item"
+require "expressir/model/declarations/subtype_constraint"
+require "expressir/model/declarations/type"
+require "expressir/model/declarations/unique_rule"
+require "expressir/model/declarations/variable"
+require "expressir/model/declarations/where_rule"
+require "expressir/model/expressions/aggregate_initializer"
+require "expressir/model/expressions/aggregate_initializer_item"
+require "expressir/model/expressions/binary_expression"
+require "expressir/model/expressions/entity_constructor"
+require "expressir/model/expressions/function_call"
+require "expressir/model/expressions/interval"
+require "expressir/model/expressions/query_expression"
+require "expressir/model/expressions/unary_expression"
+require "expressir/model/literals/binary"
+require "expressir/model/literals/integer"
+require "expressir/model/literals/logical"
+require "expressir/model/literals/real"
+require "expressir/model/literals/string"
+require "expressir/model/references/attribute_reference"
+require "expressir/model/references/group_reference"
+require "expressir/model/references/index_reference"
+require "expressir/model/references/simple_reference"
+require "expressir/model/statements/alias"
+require "expressir/model/statements/assignment"
+require "expressir/model/statements/case"
+require "expressir/model/statements/case_action"
+require "expressir/model/statements/compound"
+require "expressir/model/statements/escape"
+require "expressir/model/statements/if"
+require "expressir/model/statements/null"
+require "expressir/model/statements/procedure_call"
+require "expressir/model/statements/repeat"
+require "expressir/model/statements/return"
+require "expressir/model/statements/skip"
+require "expressir/model/supertype_expressions/binary_supertype_expression"
+require "expressir/model/supertype_expressions/oneof_supertype_expression"

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "1.2.3".freeze
+  VERSION = "1.2.6".freeze
 end

data/rakelib/antlr4-native.rake

@@ -0,0 +1,161 @@
+require "fileutils"
+require "antlr4-native"
+require "rake"
+
+def create_class_declarations(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "Class rb_cContextProxy;" }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
+
+    Class rb_cParserExt;
+    Class rb_cTokenExt;
+
+  CPP
+end
+
+def create_tp_class_definition(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "class ContextProxy {" }
+  parser_source_lines[i - 2] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
+
+
+    class TokenProxy : public Object {
+      public:
+        TokenProxy(Token* orig) {
+          this -> orig = orig;
+        }
+
+        std::string getText() {
+          return orig->getText();
+        }
+
+        size_t getChannel() {
+          return orig->getChannel();
+        }
+
+        size_t getTokenIndex() {
+          return orig->getTokenIndex();
+        }
+
+      private:
+        Token * orig = nullptr;
+    };
+
+  CPP
+end
+
+def create_pp_class_definition(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "extern \"C\"" }
+  parser_source_lines[i - 2] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
+
+    class ParserProxyExt : public Object {
+      public:
+        ParserProxyExt(Object self, string file) {
+          ifstream stream;
+          stream.open(file);
+          input = new ANTLRInputStream(stream);
+          lexer = new ExpressLexer(input);
+          tokens = new CommonTokenStream(lexer);
+          parser = new ExpressParser(tokens);
+          stream.close();
+        };
+
+        ~ParserProxyExt() {
+          delete parser;
+          delete tokens;
+          delete lexer;
+          delete input;
+        }
+
+        Object syntax() {
+          auto ctx = parser -> syntax();
+
+          SyntaxContextProxy proxy((ExpressParser::SyntaxContext*) ctx);
+          return detail::To_Ruby<SyntaxContextProxy>().convert(proxy);
+        }
+
+        Array getTokens() {
+          Array a;
+          for (auto token : tokens -> getTokens()) {
+            a.push(new TokenProxy(token));
+          }
+          return a;
+        }
+
+        Object visit(VisitorProxy* visitor) {
+          auto result = visitor -> visit(parser -> syntax());
+
+          lexer -> reset();
+          parser -> reset();
+
+          try {
+            return std::any_cast<Object>(result);
+          } catch(std::bad_cast) {
+           return Qnil;
+          }
+        }
+
+      private:
+        ANTLRInputStream* input;
+        ExpressLexer* lexer;
+        CommonTokenStream* tokens;
+        ExpressParser* parser;
+    };
+
+
+  CPP
+end
+
+def create_class_api(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit, Return().keepAlive());" }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
+
+
+    rb_cTokenExt = define_class_under<TokenProxy>(rb_mExpressParser, "TokenExt")
+      .define_method("text", &TokenProxy::getText)
+      .define_method("channel", &TokenProxy::getChannel)
+      .define_method("token_index", &TokenProxy::getTokenIndex);
+
+    rb_cParserExt = define_class_under<ParserProxyExt>(rb_mExpressParser, "ParserExt")
+      .define_constructor(Constructor<ParserProxyExt, Object, string>())
+      .define_method("syntax", &ParserProxyExt::syntax, Return().keepAlive())
+      .define_method("tokens", &ParserProxyExt::getTokens)
+      .define_method("visit", &ParserProxyExt::visit, Return().keepAlive());
+
+  CPP
+end
+
+def generate_extended_parser
+  # Generate extended parser that provide Ruby access to token stream
+  parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
+  parser_source_lines = File.read(parser_source_file).split("\n")
+  create_class_declarations(parser_source_lines)
+  create_tp_class_definition(parser_source_lines)
+  create_pp_class_definition(parser_source_lines)
+  create_class_api(parser_source_lines)
+  File.write(parser_source_file, "#{parser_source_lines.join("\n")}\n")
+end
+
+desc "Generate parser  (Usage: 'rake generate <grammar_file>')"
+task "generate" do
+  grammar_file = ARGV[1]
+  if grammar_file.nil?
+    grammar_file = File.expand_path(File.join("..", "ext", "express-grammar", "Express.g4"), __dir__)
+  end
+
+  puts "Generating parser from '#{grammar_file}'"
+  # ANTLR does weird things if the grammar file isn't in the current working directory
+  temp_grammar_file = File.join(FileUtils.pwd, File.basename(grammar_file))
+  FileUtils.cp(grammar_file, temp_grammar_file)
+
+  # generate parser
+  generator = Antlr4Native::Generator.new(
+    grammar_files: [File.basename(temp_grammar_file)],
+    output_dir: "ext",
+    parser_root_method: "syntax",
+  )
+  generator.generate
+
+  puts "Generating extended parser"
+  generate_extended_parser
+  # cleanup
+  FileUtils.rm(temp_grammar_file)
+end