expressir 0.2.5 → 0.2.10

data/lib/expressir/express_exp/parser.rb

@@ -1,10 +1,15 @@
-require 'express_parser'
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "#{$1}/express_parser"
+rescue LoadError
+  require_relative "express_parser"
+end
 require 'expressir/express_exp/visitor'
 
 module Expressir
   module ExpressExp
     class Parser
-      def self.from_exp(file)
+      def self.from_file(file)
         input = File.read(file)
 
 =begin
@@ -31,6 +36,19 @@ module Expressir
 
         repo
       end
+
+      def self.from_files(files)
+        schemas = files.map{|file| self.from_file(file).schemas}.flatten
+
+        Model::Repository.new({
+          schemas: schemas
+        })
+      end
+
+      # deprecated
+      def self.from_exp(file)
+        self.from_file(file)
+      end
     end
   end
 end

data/lib/expressir/express_exp/visitor.rb

@@ -1,9 +1,12 @@
-require 'express_parser'
-require 'expressir/model'
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "#{$1}/express_parser"
+rescue LoadError
+  require_relative "express_parser"
+end
+require "expressir/model"
+require "set"
 
-# static shorthands are unwrapped
-# - entity attributes, function/procedure parameters, local variables
-#
 # reference type is not recognized
 # see note in A.1.5 Interpreted identifiers
 # > It is expected that identifiers matching these syntax rules are known to an implementation.
@@ -20,6 +23,13 @@ require 'expressir/model'
 # > A syntactic construct such as ARRAY[1:3] OF REAL satisfies two syntactic productions —
 # > aggregation_type and general_aggregation_type. It is considered to be instantiable no matter which
 # > production it is required to satisfy in the syntax.
+#
+# static shorthands are unwrapped
+# - entity attributes, function/procedure parameters, local variables
+#
+# all access to ctx members must happen before calling other visitor code
+# - prevents segfault in ANTLR4 C++ runtime, not sure why they are caused
+# - e.g. see visit_schema_decl
 
 module Expressir
   module ExpressExp
@@ -28,91 +38,148 @@ module Expressir
 
       def initialize(tokens)
         @tokens = tokens
-        @attached_remarks = Set.new
+        @attached_remark_tokens = ::Set.new
 
         super()
       end
 
       def visit(ctx)
         result = super(ctx)
+        attach_source(ctx, result)
         attach_parent(ctx, result)
         attach_remarks(ctx, result)
         result
       end
 
-      def visit_if(ctx)
-        visit(ctx) if ctx
+      def visit_if(ctx, default = nil)
+        if ctx
+          visit(ctx)
+        else
+          default
+        end
       end
     
       def visit_if_map(ctx)
-        ctx.map{|ctx2| visit(ctx2)} if ctx
+        if ctx
+          ctx.map{|ctx2| visit(ctx2)}
+        else
+          []
+        end
       end
     
       def visit_if_map_flatten(ctx)
-        ctx.map{|ctx2| visit(ctx2)}.flatten if ctx
+        if ctx
+          ctx.map{|ctx2| visit(ctx2)}.flatten
+        else
+          []
+        end
+      end
+
+      def get_tokens_source(tokens)
+        if tokens.last.text == '<EOF>'
+          tokens.pop
+        end
+
+        tokens.map{|x| x.text}.join('').force_encoding('UTF-8')
+      end
+
+      def get_tokens(ctx)
+        start_index, stop_index = if ctx.instance_of? ::ExpressParser::SyntaxContext
+          [0, @tokens.size - 1]
+        else
+          [ctx.start.token_index, ctx.stop.token_index]
+        end
+
+        @tokens[start_index..stop_index]
+      end
+
+      def get_head_tokens(ctx)
+        start_index, stop_index = if ctx.instance_of? ::ExpressParser::SchemaDeclContext
+          start_index = ctx.start.token_index
+          stop_index = if ctx.schema_body.interface_specification.length > 0
+            ctx.schema_body.interface_specification.last.stop.token_index
+          elsif ctx.schema_version_id
+            ctx.schema_version_id.stop.token_index + 1
+          else
+            ctx.schema_id.stop.token_index + 1
+          end
+
+          [start_index, stop_index]
+        end
+
+        if start_index and stop_index
+          @tokens[start_index..stop_index]
+        end
+      end
+
+      def attach_source(ctx, node)
+        if node.class.method_defined? :source
+          tokens = get_tokens(ctx)
+          node.source = get_tokens_source(tokens)
+        end
+
+        if node.class.method_defined? :head_source
+          tokens = get_head_tokens(ctx)
+          node.head_source = get_tokens_source(tokens)
+        end
       end
 
       def attach_parent(ctx, node)
         if node.class.method_defined? :children
           node.children.each do |child_node|
-            if child_node.class.method_defined? :parent
+            if child_node.class.method_defined? :parent and !child_node.parent
               child_node.parent = node
             end
           end
         end
       end
 
-      def attach_remarks(ctx, node)
-        # get remark tokens
-        start_index, stop_index = if ctx.instance_of? ::ExpressParser::SyntaxContext
-          [0, @tokens.size - 1]
-        else
-          [ctx.start.token_index, ctx.stop.token_index]
+      def find_remark_target(node, path)
+        current_node = node
+        target_node = nil
+
+        if current_node.class.method_defined? :find_or_create
+          target_node = current_node.find_or_create(path)
+        end
+        while !target_node and current_node.class.method_defined? :parent and current_node.parent.class.method_defined? :find_or_create
+          current_node = current_node.parent
+          target_node = current_node.find_or_create(path)
         end
-        # puts [start_index, stop_index, ctx.class].inspect
-
-        remark_tokens = @tokens[start_index..stop_index].select{|x| x.channel == REMARK_CHANNEL}
-        if remark_tokens
-          remark_tokens.each do |remark_token|
-            remark_text = remark_token.text
-
-            # check if it is tagged remark
-            match = if remark_text.start_with?('--')
-              remark_text[2..-1].match(/^"([^"]*)"(.*)$/)
-            elsif remark_text.start_with?('(*') and remark_text.end_with?('*)')
-              remark_text[2..-3].match(/^"([^"]*)"(.*)$/m)
-            end
-            if !match
-              next
-            end
 
-            # don't attach already attached tagged remark
-            if @attached_remarks.include? remark_token
-              next
-            end
+        target_node
+      end
 
-            # attach tagged remark
-            remark_tag = match[1]
-            remark_content = match[2].strip
+      def attach_remarks(ctx, node)
+        remark_tokens = get_tokens(ctx).select{|x| x.channel == REMARK_CHANNEL}
 
-            target_node = nil
-            current_node = node
-            if current_node.class.method_defined? :find
-              target_node = current_node.find(remark_tag)
-            end
-            while !target_node and current_node.class.method_defined? :parent and current_node.parent.class.method_defined? :find
-              current_node = current_node.parent
-              target_node = current_node.find(remark_tag)
-            end
+        # skip already attached remarks
+        remark_tokens = remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
 
-            if target_node
-              target_node.remarks ||= []
-              target_node.remarks << remark_content
+        # parse remarks, find remark targets
+        tagged_remark_tokens = remark_tokens.map do |remark_token|
+          _, remark_tag, remark_text = if remark_token.text.start_with?('--')
+            remark_token.text.match(/^--"([^"]*)"(.*)$/).to_a
+          else
+            remark_token.text.match(/^\(\*"([^"]*)"(.*)\*\)$/m).to_a
+          end
 
-              # mark remark as attached, so that it is not attached again at higher nesting level
-              @attached_remarks << remark_token
-            end
+          if remark_tag
+            remark_target = find_remark_target(node, remark_tag)
+          end
+          if remark_text
+            remark_text = remark_text.strip.force_encoding('UTF-8')
           end
+
+          [remark_token, remark_target, remark_text]
+        end.select{|x| x[1]}
+
+        tagged_remark_tokens.each do |remark_token, remark_target, remark_text|
+          # attach remark
+          remark_target.remarks ||= []
+          remark_target.remarks << remark_text
+
+          # mark remark as attached, so that it is not attached again at higher nesting level
+          @attached_remark_tokens << remark_token
         end
       end
 
@@ -365,8 +432,8 @@ module Expressir
 
         bound1 = visit_if(ctx__bound_spec__bound1)
         bound2 = visit_if(ctx__bound_spec__bound2)
-        optional = !!ctx__OPTIONAL
-        unique = !!ctx__UNIQUE
+        optional = ctx__OPTIONAL && true
+        unique = ctx__UNIQUE && true
         base_type = visit_if(ctx__instantiable_type)
         
         Model::Types::Array.new({
@@ -450,7 +517,7 @@ module Expressir
         ctx__width_spec__FIXED = ctx__width_spec&.FIXED
 
         width = visit_if(ctx__width_spec__width)
-        fixed = !!ctx__width_spec__FIXED
+        fixed = ctx__width_spec__FIXED && true
 
         Model::Types::Binary.new({
           width: width,
@@ -702,7 +769,7 @@ module Expressir
         ctx__entity_body__where_clause = ctx__entity_body&.where_clause
 
         id = visit_if(ctx__entity_head__entity_id)
-        abstract = !!(ctx__entity_head__subsuper__supertype_constraint__abstract_entity_declaration || ctx__entity_head__subsuper__supertype_constraint__abstract_supertype_declaration)
+        abstract = (ctx__entity_head__subsuper__supertype_constraint__abstract_entity_declaration || ctx__entity_head__subsuper__supertype_constraint__abstract_supertype_declaration) && true
         supertype_expression = visit_if(ctx__entity_head__subsuper__supertype_constraint__abstract_supertype_declaration || ctx__entity_head__subsuper__supertype_constraint__supertype_rule)
         subtype_of = visit_if(ctx__entity_head__subsuper__subtype_declaration)
         attributes = [
@@ -710,8 +777,8 @@ module Expressir
           *visit_if(ctx__entity_body__derive_clause),
           *visit_if(ctx__entity_body__inverse_clause)
         ]
-        unique = visit_if(ctx__entity_body__unique_clause)
-        where = visit_if(ctx__entity_body__where_clause)
+        unique = visit_if(ctx__entity_body__unique_clause, [])
+        where = visit_if(ctx__entity_body__where_clause, [])
 
         Model::Entity.new({
           id: id,
@@ -784,10 +851,10 @@ module Expressir
         ctx__enumeration_extension__type_ref = ctx__enumeration_extension&.type_ref
         ctx__enumeration_extension__enumeration_items = ctx__enumeration_extension&.enumeration_items
 
-        extensible = !!ctx__EXTENSIBLE
-        items = visit_if(ctx__enumeration_items)
+        extensible = ctx__EXTENSIBLE && true
+        items = visit_if(ctx__enumeration_items, [])
         extension_type = visit_if(ctx__enumeration_extension__type_ref)
-        extension_items = visit_if(ctx__enumeration_extension__enumeration_items)
+        extension_items = visit_if(ctx__enumeration_extension__enumeration_items, [])
 
         Model::Types::Enumeration.new({
           extensible: extensible,
@@ -807,7 +874,7 @@ module Expressir
         ctx__parameter_type = ctx.parameter_type
 
         attributes = visit_if_map(ctx__attribute_decl)
-        optional = !!ctx__OPTIONAL
+        optional = ctx__OPTIONAL && true
         type = visit_if(ctx__parameter_type)
 
         attributes.map do |attribute|
@@ -881,7 +948,7 @@ module Expressir
         ctx__actual_parameter_list = ctx.actual_parameter_list
 
         ref = visit_if(ctx__built_in_function || ctx__function_ref)
-        parameters = visit_if(ctx__actual_parameter_list)
+        parameters = visit_if(ctx__actual_parameter_list, [])
 
         Model::Expressions::Call.new({
           ref: ref,
@@ -904,15 +971,24 @@ module Expressir
         parameters = visit_if_map_flatten(ctx__function_head__formal_parameter)
         return_type = visit_if(ctx__function_head__parameter_type)
         declarations = visit_if_map(ctx__algorithm_head__declaration)
-        constants = visit_if(ctx__algorithm_head__constant_decl)
-        variables = visit_if(ctx__algorithm_head__local_decl)
+        types = declarations.select{|x| x.instance_of? Model::Type}
+        entities = declarations.select{|x| x.instance_of? Model::Entity}
+        subtype_constraints = declarations.select{|x| x.instance_of? Model::SubtypeConstraint}
+        functions = declarations.select{|x| x.instance_of? Model::Function}
+        procedures = declarations.select{|x| x.instance_of? Model::Procedure}
+        constants = visit_if(ctx__algorithm_head__constant_decl, [])
+        variables = visit_if(ctx__algorithm_head__local_decl, [])
         statements = visit_if_map(ctx__stmt)
 
         Model::Function.new({
           id: id,
           parameters: parameters,
           return_type: return_type,
-          declarations: declarations,
+          types: types,
+          entities: entities,
+          subtype_constraints: subtype_constraints,
+          functions: functions,
+          procedures: procedures,
           constants: constants,
           variables: variables,
           statements: statements
@@ -957,8 +1033,8 @@ module Expressir
 
         bound1 = visit_if(ctx__bound_spec__bound1)
         bound2 = visit_if(ctx__bound_spec__bound2)
-        optional = !!ctx__OPTIONAL
-        unique = !!ctx__UNIQUE
+        optional = ctx__OPTIONAL && true
+        unique = ctx__UNIQUE && true
         base_type = visit_if(ctx__parameter_type)
         
         Model::Types::Array.new({
@@ -996,7 +1072,7 @@ module Expressir
 
         bound1 = visit_if(ctx__bound_spec__bound1)
         bound2 = visit_if(ctx__bound_spec__bound2)
-        unique = !!ctx__UNIQUE
+        unique = ctx__UNIQUE && true
         base_type = visit_if(ctx__parameter_type)
         
         Model::Types::List.new({
@@ -1071,8 +1147,8 @@ module Expressir
         ctx__if_stmt_else_statements = ctx.if_stmt_else_statements
 
         expression = visit_if(ctx__logical_expression)
-        statements = visit_if(ctx__if_stmt_statements)
-        else_statements = visit_if(ctx__if_stmt_else_statements)
+        statements = visit_if(ctx__if_stmt_statements, [])
+        else_statements = visit_if(ctx__if_stmt_else_statements, [])
 
         Model::Statements::If.new({
           expression: expression,
@@ -1286,7 +1362,7 @@ module Expressir
 
         bound1 = visit_if(ctx__bound_spec__bound1)
         bound2 = visit_if(ctx__bound_spec__bound2)
-        unique = !!ctx__UNIQUE
+        unique = ctx__UNIQUE && true
         base_type = visit_if(ctx__instantiable_type)
         
         Model::Types::List.new({
@@ -1502,7 +1578,7 @@ module Expressir
         ctx__actual_parameter_list = ctx.actual_parameter_list
 
         ref = visit_if(ctx__built_in_procedure || ctx__procedure_ref)
-        parameters = visit_if(ctx__actual_parameter_list)
+        parameters = visit_if(ctx__actual_parameter_list, [])
 
         Model::Statements::Call.new({
           ref: ref,
@@ -1523,14 +1599,23 @@ module Expressir
         id = visit_if(ctx__procedure_head__procedure_id)
         parameters = visit_if_map_flatten(ctx__procedure_head__procedure_head_parameter)
         declarations = visit_if_map(ctx__algorithm_head__declaration)
-        constants = visit_if(ctx__algorithm_head__constant_decl)
-        variables = visit_if(ctx__algorithm_head__local_decl)
+        types = declarations.select{|x| x.instance_of? Model::Type}
+        entities = declarations.select{|x| x.instance_of? Model::Entity}
+        subtype_constraints = declarations.select{|x| x.instance_of? Model::SubtypeConstraint}
+        functions = declarations.select{|x| x.instance_of? Model::Function}
+        procedures = declarations.select{|x| x.instance_of? Model::Procedure}
+        constants = visit_if(ctx__algorithm_head__constant_decl, [])
+        variables = visit_if(ctx__algorithm_head__local_decl, [])
         statements = visit_if_map(ctx__stmt)
 
         Model::Procedure.new({
           id: id,
           parameters: parameters,
-          declarations: declarations,
+          types: types,
+          entities: entities,
+          subtype_constraints: subtype_constraints,
+          functions: functions,
+          procedures: procedures,
           constants: constants,
           variables: variables,
           statements: statements
@@ -1609,12 +1694,12 @@ module Expressir
         ctx__logical_expression = ctx.logical_expression
 
         id = visit_if(ctx__variable_id)
-        source = visit_if(ctx__aggregate_source)
+        aggregate_source = visit_if(ctx__aggregate_source)
         expression = visit_if(ctx__logical_expression)
 
         Model::Expressions::QueryExpression.new({
           id: id,
-          source: source,
+          aggregate_source: aggregate_source,
           expression: expression
         })
       end
@@ -1807,15 +1892,24 @@ module Expressir
         id = visit_if(ctx__rule_head__rule_id)
         applies_to = visit_if_map(ctx__rule_head__entity_ref)
         declarations = visit_if_map(ctx__algorithm_head__declaration)
-        constants = visit_if(ctx__algorithm_head__constant_decl)
-        variables = visit_if(ctx__algorithm_head__local_decl)
+        types = declarations.select{|x| x.instance_of? Model::Type}
+        entities = declarations.select{|x| x.instance_of? Model::Entity}
+        subtype_constraints = declarations.select{|x| x.instance_of? Model::SubtypeConstraint}
+        functions = declarations.select{|x| x.instance_of? Model::Function}
+        procedures = declarations.select{|x| x.instance_of? Model::Procedure}
+        constants = visit_if(ctx__algorithm_head__constant_decl, [])
+        variables = visit_if(ctx__algorithm_head__local_decl, [])
         statements = visit_if_map(ctx__stmt)
-        where = visit_if(ctx__where_clause)
+        where = visit_if(ctx__where_clause, [])
 
         Model::Rule.new({
           id: id,
           applies_to: applies_to,
-          declarations: declarations,
+          types: types,
+          entities: entities,
+          subtype_constraints: subtype_constraints,
+          functions: functions,
+          procedures: procedures,
           constants: constants,
           variables: variables,
           statements: statements,
@@ -1861,15 +1955,26 @@ module Expressir
         id = visit_if(ctx__schema_id)
         version = visit_if(ctx__schema_version_id)
         interfaces = visit_if_map(ctx__schema_body__interface_specification)
-        constants = visit_if(ctx__schema_body__constant_decl)
+        constants = visit_if(ctx__schema_body__constant_decl, [])
         declarations = visit_if_map(ctx__schema_body__schema_body_declaration)
+        types = declarations.select{|x| x.instance_of? Model::Type}
+        entities = declarations.select{|x| x.instance_of? Model::Entity}
+        subtype_constraints = declarations.select{|x| x.instance_of? Model::SubtypeConstraint}
+        functions = declarations.select{|x| x.instance_of? Model::Function}
+        procedures = declarations.select{|x| x.instance_of? Model::Procedure}
+        rules = declarations.select{|x| x.instance_of? Model::Rule}
 
         Model::Schema.new({
           id: id,
           version: version,
           interfaces: interfaces,
           constants: constants,
-          declarations: declarations
+          types: types,
+          entities: entities,
+          subtype_constraints: subtype_constraints,
+          functions: functions,
+          procedures: procedures,
+          rules: rules
         })
       end
 
@@ -1909,11 +2014,11 @@ module Expressir
         ctx__select_extension__type_ref = ctx.select_extension&.type_ref
         ctx__select_extension__select_list = ctx__select_extension&.select_list
 
-        extensible = !!ctx__EXTENSIBLE
-        generic_entity = !!ctx__GENERIC_ENTITY
-        items = visit_if(ctx__select_list)
+        extensible = ctx__EXTENSIBLE && true
+        generic_entity = ctx__GENERIC_ENTITY && true
+        items = visit_if(ctx__select_list, [])
         extension_type = visit_if(ctx__select_extension__type_ref)
-        extension_items = visit_if(ctx__select_extension__select_list)
+        extension_items = visit_if(ctx__select_extension__select_list, [])
 
         Model::Types::Select.new({
           extensible: extensible,
@@ -2046,7 +2151,7 @@ module Expressir
         ctx__width_spec__FIXED = ctx__width_spec&.FIXED
 
         width = visit_if(ctx__width_spec__width)
-        fixed = !!ctx__width_spec__FIXED
+        fixed = ctx__width_spec__FIXED && true
 
         Model::Types::String.new({
           width: width,
@@ -2079,7 +2184,7 @@ module Expressir
 
         id = visit_if(ctx__subtype_constraint_head__subtype_constraint_id)
         applies_to = visit_if(ctx__subtype_constraint_head__entity_ref)
-        abstract = !!ctx__subtype_constraint_body__abstract_supertype
+        abstract = ctx__subtype_constraint_body__abstract_supertype && true
         total_over = visit_if(ctx__subtype_constraint_body__total_over)
         supertype_expression = visit_if(ctx__subtype_constraint_body__supertype_expression)
 
@@ -2215,7 +2320,7 @@ module Expressir
 
         id = visit_if(ctx__type_id)
         type = visit_if(ctx__underlying_type)
-        where = visit_if(ctx__where_clause)
+        where = visit_if(ctx__where_clause, [])
 
         Model::Type.new({
           id: id,
@@ -2429,7 +2534,7 @@ module Expressir
       def handle_simple_string_literal(ctx)
         ctx__text = ctx.text
 
-        value = ctx__text[1..(ctx__text.length - 2)]
+        value = ctx__text[1..(ctx__text.length - 2)].force_encoding('UTF-8')
 
         Model::Literals::String.new({
           value: value
@@ -2439,7 +2544,7 @@ module Expressir
       def handle_encoded_string_literal(ctx)
         ctx__text = ctx.text
 
-        value = ctx__text[1..(ctx__text.length - 2)]
+        value = ctx__text[1..(ctx__text.length - 2)].force_encoding('UTF-8')
 
         Model::Literals::String.new({
           value: value,