expressir 1.2.1-x86_64-linux → 1.2.5-x86_64-linux

data/lib/expressir/express/parser.rb

@@ -31,25 +31,28 @@ module Expressir
       # @param [Boolean] include_source attach original source code to model elements
       # @return [Model::Repository]
       def self.from_file(file, skip_references: nil, include_source: nil)
-        input = File.read(file)
+        @input = File.read(file)
 
-        parser = ::ExpressParser::Parser.parse(input)
+      # An important note re memory management
+      # parse, syntax, visitor methods return complex tree structures created in netive (C++) extension
+      # visit method references nodes and leaves of this structures but it is totally untransparent for Ruby GarbageCllector
+      # so in this class we keep those C++ structure marked for GC so theu are not freed 
+        @parser = ::ExpressParser::Parser.parse(@input)
+        @parse_tree = @parser.syntax()
 
-        parse_tree = parser.syntax()
+        @visitor = Visitor.new(@parser.tokens, include_source: include_source)
+        @repository = @visitor.visit(@parse_tree)
 
-        visitor = Visitor.new(parser.tokens, include_source: include_source)
-        repository = visitor.visit(parse_tree)
-
-        repository.schemas.each do |schema|
+        @repository.schemas.each do |schema|
           schema.file = file.to_s
         end
 
         unless skip_references
-          resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-          resolve_references_model_visitor.visit(repository)
+          @resolve_references_model_visitor = ResolveReferencesModelVisitor.new
+          @resolve_references_model_visitor.visit(@repository)
         end
 
-        repository
+        @repository
       end
 
       # Parses Express files into an Express model
@@ -65,16 +68,16 @@ module Expressir
           repository.schemas
         end.flatten
 
-        repository = Model::Repository.new(
+        @repository = Model::Repository.new(
           schemas: schemas
         )
 
-        unless skip_references
-          resolve_references_model_visitor = ResolveReferencesModelVisitor.new
-          resolve_references_model_visitor.visit(repository)
+        unless @skip_references
+          @resolve_references_model_visitor = ResolveReferencesModelVisitor.new
+          @resolve_references_model_visitor.visit(@repository)
         end
 
-        repository
+        @repository
       end
     end
   end

data/lib/expressir/express/visitor.rb

@@ -31,6 +31,8 @@ require "set"
 # - prevents segfault in ANTLR4 C++ runtime, not sure why they are caused
 # - e.g. see visit_schema_decl
 
+require 'objspace'
+
 module Expressir
   module Express
     class Visitor < ::ExpressParser::Visitor
@@ -138,13 +140,15 @@ module Expressir
       end
 
       def attach_remarks(ctx, node)
-        remark_tokens = get_tokens(ctx).select{|x| x.channel == REMARK_CHANNEL}
+        @remark_tokens = get_tokens(ctx)
+        @remark_tokens = @remark_tokens.select{ |x| x.channel == REMARK_CHANNEL
+        }
 
         # skip already attached remarks
-        remark_tokens = remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
+        @remark_tokens = @remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
 
         # parse remarks, find remark targets
-        tagged_remark_tokens = remark_tokens.map do |remark_token|
+        tagged_remark_tokens = @remark_tokens.map do |remark_token|
           _, remark_tag, remark_text = if remark_token.text.start_with?('--')
             remark_token.text.match(/^--"([^"]*)"(.*)$/).to_a
           else

data/lib/expressir/model.rb

@@ -1,79 +1,79 @@
-require 'expressir/model/model_element'
+require "expressir/model/model_element"
 
-require 'expressir/model/cache'
-require 'expressir/model/data_type'
-require 'expressir/model/declaration'
-require 'expressir/model/expression'
-require 'expressir/model/identifier'
-require 'expressir/model/literal'
-require 'expressir/model/reference'
-require 'expressir/model/repository'
-require 'expressir/model/statement'
-require 'expressir/model/supertype_expression'
-require 'expressir/model/data_types/aggregate'
-require 'expressir/model/data_types/array'
-require 'expressir/model/data_types/bag'
-require 'expressir/model/data_types/binary'
-require 'expressir/model/data_types/boolean'
-require 'expressir/model/data_types/enumeration'
-require 'expressir/model/data_types/enumeration_item'
-require 'expressir/model/data_types/generic_entity'
-require 'expressir/model/data_types/generic'
-require 'expressir/model/data_types/integer'
-require 'expressir/model/data_types/list'
-require 'expressir/model/data_types/logical'
-require 'expressir/model/data_types/number'
-require 'expressir/model/data_types/real'
-require 'expressir/model/data_types/set'
-require 'expressir/model/data_types/select'
-require 'expressir/model/data_types/string'
-require 'expressir/model/declarations/attribute'
-require 'expressir/model/declarations/constant'
-require 'expressir/model/declarations/entity'
-require 'expressir/model/declarations/function'
-require 'expressir/model/declarations/interface'
-require 'expressir/model/declarations/interface_item'
-require 'expressir/model/declarations/interfaced_item'
-require 'expressir/model/declarations/parameter'
-require 'expressir/model/declarations/procedure'
-require 'expressir/model/declarations/remark_item'
-require 'expressir/model/declarations/rule'
-require 'expressir/model/declarations/schema'
-require 'expressir/model/declarations/schema_version'
-require 'expressir/model/declarations/schema_version_item'
-require 'expressir/model/declarations/subtype_constraint'
-require 'expressir/model/declarations/type'
-require 'expressir/model/declarations/unique_rule'
-require 'expressir/model/declarations/variable'
-require 'expressir/model/declarations/where_rule'
-require 'expressir/model/expressions/aggregate_initializer'
-require 'expressir/model/expressions/aggregate_initializer_item'
-require 'expressir/model/expressions/binary_expression'
-require 'expressir/model/expressions/entity_constructor'
-require 'expressir/model/expressions/function_call'
-require 'expressir/model/expressions/interval'
-require 'expressir/model/expressions/query_expression'
-require 'expressir/model/expressions/unary_expression'
-require 'expressir/model/literals/binary'
-require 'expressir/model/literals/integer'
-require 'expressir/model/literals/logical'
-require 'expressir/model/literals/real'
-require 'expressir/model/literals/string'
-require 'expressir/model/references/attribute_reference'
-require 'expressir/model/references/group_reference'
-require 'expressir/model/references/index_reference'
-require 'expressir/model/references/simple_reference'
-require 'expressir/model/statements/alias'
-require 'expressir/model/statements/assignment'
-require 'expressir/model/statements/case'
-require 'expressir/model/statements/case_action'
-require 'expressir/model/statements/compound'
-require 'expressir/model/statements/escape'
-require 'expressir/model/statements/if'
-require 'expressir/model/statements/null'
-require 'expressir/model/statements/procedure_call'
-require 'expressir/model/statements/repeat'
-require 'expressir/model/statements/return'
-require 'expressir/model/statements/skip'
-require 'expressir/model/supertype_expressions/binary_supertype_expression'
-require 'expressir/model/supertype_expressions/oneof_supertype_expression'
+require "expressir/model/cache"
+require "expressir/model/data_type"
+require "expressir/model/declaration"
+require "expressir/model/expression"
+require "expressir/model/identifier"
+require "expressir/model/literal"
+require "expressir/model/reference"
+require "expressir/model/repository"
+require "expressir/model/statement"
+require "expressir/model/supertype_expression"
+require "expressir/model/data_types/aggregate"
+require "expressir/model/data_types/array"
+require "expressir/model/data_types/bag"
+require "expressir/model/data_types/binary"
+require "expressir/model/data_types/boolean"
+require "expressir/model/data_types/enumeration"
+require "expressir/model/data_types/enumeration_item"
+require "expressir/model/data_types/generic_entity"
+require "expressir/model/data_types/generic"
+require "expressir/model/data_types/integer"
+require "expressir/model/data_types/list"
+require "expressir/model/data_types/logical"
+require "expressir/model/data_types/number"
+require "expressir/model/data_types/real"
+require "expressir/model/data_types/set"
+require "expressir/model/data_types/select"
+require "expressir/model/data_types/string"
+require "expressir/model/declarations/attribute"
+require "expressir/model/declarations/constant"
+require "expressir/model/declarations/entity"
+require "expressir/model/declarations/function"
+require "expressir/model/declarations/interface"
+require "expressir/model/declarations/interface_item"
+require "expressir/model/declarations/interfaced_item"
+require "expressir/model/declarations/parameter"
+require "expressir/model/declarations/procedure"
+require "expressir/model/declarations/remark_item"
+require "expressir/model/declarations/rule"
+require "expressir/model/declarations/schema"
+require "expressir/model/declarations/schema_version"
+require "expressir/model/declarations/schema_version_item"
+require "expressir/model/declarations/subtype_constraint"
+require "expressir/model/declarations/type"
+require "expressir/model/declarations/unique_rule"
+require "expressir/model/declarations/variable"
+require "expressir/model/declarations/where_rule"
+require "expressir/model/expressions/aggregate_initializer"
+require "expressir/model/expressions/aggregate_initializer_item"
+require "expressir/model/expressions/binary_expression"
+require "expressir/model/expressions/entity_constructor"
+require "expressir/model/expressions/function_call"
+require "expressir/model/expressions/interval"
+require "expressir/model/expressions/query_expression"
+require "expressir/model/expressions/unary_expression"
+require "expressir/model/literals/binary"
+require "expressir/model/literals/integer"
+require "expressir/model/literals/logical"
+require "expressir/model/literals/real"
+require "expressir/model/literals/string"
+require "expressir/model/references/attribute_reference"
+require "expressir/model/references/group_reference"
+require "expressir/model/references/index_reference"
+require "expressir/model/references/simple_reference"
+require "expressir/model/statements/alias"
+require "expressir/model/statements/assignment"
+require "expressir/model/statements/case"
+require "expressir/model/statements/case_action"
+require "expressir/model/statements/compound"
+require "expressir/model/statements/escape"
+require "expressir/model/statements/if"
+require "expressir/model/statements/null"
+require "expressir/model/statements/procedure_call"
+require "expressir/model/statements/repeat"
+require "expressir/model/statements/return"
+require "expressir/model/statements/skip"
+require "expressir/model/supertype_expressions/binary_supertype_expression"
+require "expressir/model/supertype_expressions/oneof_supertype_expression"

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "1.2.1".freeze
+  VERSION = "1.2.5".freeze
 end

data/rakelib/antlr4-native.rake

@@ -0,0 +1,63 @@
+require "fileutils"
+require "antlr4-native"
+require "rake"
+
+def create_tokens_api(parser_source_lines)
+  # - add ParserProxy tokens method, simple compensation for missing exposed BufferedTokenStream
+  i = parser_source_lines.index { |x| x == "  Object syntax() {" }
+  parser_source_lines[i + 6] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
+
+    Array getTokens() {
+      Array a;
+
+      std::vector<Token*> tokens = this -> tokens -> getTokens();
+
+      for (auto &token : tokens) {
+        a.push(token);
+      }
+
+      return a;
+    }
+
+  CPP
+end
+
+def create_tokens_method(parser_source_lines)
+  i = parser_source_lines.index { |x| x == '    .define_method("syntax", &ParserProxy::syntax, Return().keepAlive())' }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "    #{x}" }.join("\n")
+
+    .define_method("tokens", &ParserProxy::getTokens)
+  CPP
+end
+
+desc "Generate parser  (Usage: 'rake generate <grammar_file>')"
+task "generate" do
+  grammar_file = ARGV[1]
+  if grammar_file.nil?
+    grammar_file = File.expand_path(File.join("..", "ext", "express-grammar", "Express.g4"), __dir__)
+  end
+
+  puts "Generating parser from '#{grammar_file}'"
+
+  # ANTLR does weird things if the grammar file isn't in the current working directory
+  temp_grammar_file = File.join(FileUtils.pwd, File.basename(grammar_file))
+  FileUtils.cp(grammar_file, temp_grammar_file)
+
+  # generate parser
+  generator = Antlr4Native::Generator.new(
+    grammar_files: [File.basename(temp_grammar_file)],
+    output_dir: "ext",
+    parser_root_method: "syntax",
+  )
+  generator.generate
+
+  # fix issues with generated parser
+  parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
+  parser_source_lines = File.read(parser_source_file).split("\n")
+  create_tokens_api(parser_source_lines)
+  create_tokens_method(parser_source_lines)
+  File.write(parser_source_file, "#{parser_source_lines.join("\n")}\n")
+
+  # cleanup
+  FileUtils.rm(temp_grammar_file)
+end