expressir 0.2.4 → 0.2.9

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathRuleElement.h

@@ -0,0 +1,26 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "XPathElement.h"
+
+namespace antlr4 {
+namespace tree {
+namespace xpath {
+
+  class ANTLR4CPP_PUBLIC XPathRuleElement : public XPathElement {
+  public:
+    XPathRuleElement(const std::string &ruleName, size_t ruleIndex);
+
+    virtual std::vector<ParseTree *> evaluate(ParseTree *t) override;
+
+  protected:
+    size_t _ruleIndex = 0;
+  };
+
+} // namespace xpath
+} // namespace tree
+} // namespace antlr4

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathTokenAnywhereElement.cpp

@@ -0,0 +1,20 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "tree/ParseTree.h"
+#include "tree/Trees.h"
+
+#include "XPathTokenAnywhereElement.h"
+
+using namespace antlr4::tree;
+using namespace antlr4::tree::xpath;
+
+XPathTokenAnywhereElement::XPathTokenAnywhereElement(const std::string &tokenName, int tokenType) : XPathElement(tokenName) {
+  this->tokenType = tokenType;
+}
+
+std::vector<ParseTree *> XPathTokenAnywhereElement::evaluate(ParseTree *t) {
+  return Trees::findAllTokenNodes(t, tokenType);
+}

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathTokenAnywhereElement.h

@@ -0,0 +1,25 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "XPathElement.h"
+
+namespace antlr4 {
+namespace tree {
+namespace xpath {
+
+  class ANTLR4CPP_PUBLIC XPathTokenAnywhereElement : public XPathElement {
+  protected:
+    int tokenType = 0;
+  public:
+    XPathTokenAnywhereElement(const std::string &tokenName, int tokenType);
+
+    virtual std::vector<ParseTree *> evaluate(ParseTree *t) override;
+  };
+
+} // namespace xpath
+} // namespace tree
+} // namespace antlr4

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathTokenElement.cpp

@@ -0,0 +1,33 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "tree/ParseTree.h"
+#include "tree/Trees.h"
+#include "support/CPPUtils.h"
+#include "Token.h"
+
+#include "XPathTokenElement.h"
+
+using namespace antlr4;
+using namespace antlr4::tree;
+using namespace antlr4::tree::xpath;
+
+XPathTokenElement::XPathTokenElement(const std::string &tokenName, size_t tokenType) : XPathElement(tokenName) {
+  _tokenType = tokenType;
+}
+
+std::vector<ParseTree *> XPathTokenElement::evaluate(ParseTree *t) {
+  // return all children of t that match nodeName
+  std::vector<ParseTree *> nodes;
+  for (auto c : t->children) {
+    if (antlrcpp::is<TerminalNode *>(c)) {
+      TerminalNode *tnode = dynamic_cast<TerminalNode *>(c);
+      if ((tnode->getSymbol()->getType() == _tokenType && !_invert) || (tnode->getSymbol()->getType() != _tokenType && _invert)) {
+        nodes.push_back(tnode);
+      }
+    }
+  }
+  return nodes;
+}

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathTokenElement.h

@@ -0,0 +1,26 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "XPathElement.h"
+
+namespace antlr4 {
+namespace tree {
+namespace xpath {
+
+  class ANTLR4CPP_PUBLIC XPathTokenElement : public XPathElement {
+  public:
+    XPathTokenElement(const std::string &tokenName, size_t tokenType);
+
+    virtual std::vector<ParseTree *> evaluate(ParseTree *t) override;
+
+  protected:
+    size_t _tokenType = 0;
+  };
+
+} // namespace xpath
+} // namespace tree
+} // namespace antlr4

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathWildcardAnywhereElement.cpp

@@ -0,0 +1,23 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "XPath.h"
+#include "tree/ParseTree.h"
+#include "tree/Trees.h"
+
+#include "XPathWildcardAnywhereElement.h"
+
+using namespace antlr4::tree;
+using namespace antlr4::tree::xpath;
+
+XPathWildcardAnywhereElement::XPathWildcardAnywhereElement() : XPathElement(XPath::WILDCARD) {
+}
+
+std::vector<ParseTree *> XPathWildcardAnywhereElement::evaluate(ParseTree *t) {
+  if (_invert) {
+    return {}; // !* is weird but valid (empty)
+  }
+  return Trees::getDescendants(t);
+}

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathWildcardAnywhereElement.h

@@ -0,0 +1,23 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "XPathElement.h"
+
+namespace antlr4 {
+namespace tree {
+namespace xpath {
+
+  class ANTLR4CPP_PUBLIC XPathWildcardAnywhereElement : public XPathElement {
+  public:
+    XPathWildcardAnywhereElement();
+
+    virtual std::vector<ParseTree *> evaluate(ParseTree *t) override;
+  };
+
+} // namespace xpath
+} // namespace tree
+} // namespace antlr4

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathWildcardElement.cpp

@@ -0,0 +1,24 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "XPath.h"
+#include "tree/ParseTree.h"
+#include "tree/Trees.h"
+
+#include "XPathWildcardElement.h"
+
+using namespace antlr4::tree;
+using namespace antlr4::tree::xpath;
+
+XPathWildcardElement::XPathWildcardElement() : XPathElement(XPath::WILDCARD) {
+}
+
+std::vector<ParseTree *> XPathWildcardElement::evaluate(ParseTree *t) {
+  if (_invert) {
+    return {}; // !* is weird but valid (empty)
+  }
+
+  return t->children;
+}

data/ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath/XPathWildcardElement.h

@@ -0,0 +1,23 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "XPathElement.h"
+
+namespace antlr4 {
+namespace tree {
+namespace xpath {
+
+  class ANTLR4CPP_PUBLIC XPathWildcardElement : public XPathElement {
+  public:
+    XPathWildcardElement();
+
+    virtual std::vector<ParseTree *> evaluate(ParseTree *t) override;
+  };
+
+} // namespace xpath
+} // namespace tree
+} // namespace antlr4

data/ext/express-parser/express_parser.cpp

@@ -1,24 +1,26 @@
 #include <iostream>
 
-#include "antlr4-runtime.h"
+#include <antlr4-runtime.h>
 
-#include "ExpressParser.h"
-#include "ExpressBaseVisitor.h"
-#include "ExpressLexer.h"
+#include "antlrgen/ExpressParser.h"
+#include "antlrgen/ExpressBaseVisitor.h"
+#include "antlrgen/ExpressLexer.h"
 
-#include "rice/Array.hpp"
-#include "rice/Class.hpp"
-#include "rice/Constructor.hpp"
-#include "rice/Director.hpp"
+#include <rice/Array.hpp>
+#include <rice/Class.hpp>
+#include <rice/Constructor.hpp>
+#include <rice/Director.hpp>
 
 #ifdef _WIN32
-#undef FALSE
-#undef TRUE
 #undef OPTIONAL
 #undef IN
 #undef OUT
 #endif
 
+// ruby-3.0
+#undef FALSE
+#undef TRUE
+
 using namespace std;
 using namespace Rice;
 using namespace antlr4;

data/ext/express-parser/extconf.rb

@@ -1,34 +1,57 @@
-require 'mkmf-rice'
+require 'rubygems'
+require 'mkmf'
 
 extension_name = 'express_parser'
-dir_config(extension_name)
-
-have_library('stdc++')
-
-$CFLAGS << ' -std=c++14 -DANTLR4CPP_STATIC'
-
-include_paths = [
-  '.',
-  'antlrgen',
-  'antlr4-upstream/runtime/Cpp/runtime/src',
-  'antlr4-upstream/runtime/Cpp/runtime/src/atn',
-  'antlr4-upstream/runtime/Cpp/runtime/src/dfa',
-  'antlr4-upstream/runtime/Cpp/runtime/src/misc',
-  'antlr4-upstream/runtime/Cpp/runtime/src/support',
-  'antlr4-upstream/runtime/Cpp/runtime/src/tree',
-  'antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern',
-  'antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath'
+cross_build = enable_config("cross-build")
+
+antlr4_src = 'antlr4-upstream/runtime/Cpp/runtime/src'
+src_paths = [
+  ".",
+  "antlrgen",
+  "#{antlr4_src}",
+  "#{antlr4_src}/atn",
+  "#{antlr4_src}/dfa",
+  "#{antlr4_src}/misc",
+  "#{antlr4_src}/support",
+  "#{antlr4_src}/tree",
+  "#{antlr4_src}/tree/pattern",
+  "#{antlr4_src}/tree/xpath",
 ]
 
-$srcs = []
+if cross_build
+  rice_subdir = "rice-embed"
+  rice_root = Gem.loaded_specs["rice"].full_gem_path
+  rice_src = File.join(rice_root, "rice")
+  rice_embed = File.join(__dir__, rice_subdir)
+  unless File.exists?(rice_embed)
+    FileUtils.mkdir_p(rice_embed)
+    FileUtils.cp_r(Dir.glob(rice_src), rice_embed)
+    File.delete(*Dir.glob("#{rice_embed}/**/*.o"))
+    File.delete(*Dir.glob("#{rice_embed}/**/Makefile"))
+  end
 
-include_paths.each do |include_path|
-  $INCFLAGS << " -I#{__dir__}/#{include_path}"
-  $VPATH << "#{__dir__}/#{include_path}"
+  src_paths.push("#{rice_subdir}/rice", "#{rice_subdir}/rice/detail")
 
-  Dir.glob("#{__dir__}/#{include_path}/*.cpp").each do |path|
-    $srcs << path
+  $INCFLAGS << " -I#{__dir__}/#{rice_subdir}"
+
+  if RbConfig::CONFIG['target_os'] =~ /mingw32|mswin/
+    # workaround for 'w64-mingw32-as: express_parser.o: too many sections'
+    $CXXFLAGS << " -O3 -Wa,-mbig-obj"
   end
+else
+  require 'mkmf-rice'
+
+  dir_config(extension_name)
+end
+
+$CPPFLAGS << " -std=c++14 -DANTLR4CPP_STATIC -DHAVE_CXX11"
+$INCFLAGS << " -I#{__dir__}/#{antlr4_src}"
+$srcs = []
+
+src_paths.each do |src_path|
+  abs_src_path = File.join(__dir__, src_path)
+  $VPATH << abs_src_path
+  $srcs.push(*Dir["#{abs_src_path}/*.cpp"])
 end
 
-create_makefile(extension_name)
+create_makefile "expressir/express_exp/#{extension_name}"

data/lib/expressir/express_exp/formatter.rb

@@ -395,7 +395,7 @@ module Expressir
               "\n",
               INDENT,
               '(',
-              node.items.map{|x| format(x)},
+              node.items.map{|x| format(x)}.join(', '),
               ')'
             ].join('')
           end,
@@ -832,7 +832,7 @@ module Expressir
           ' ',
           '<*',
           ' ',
-          format(node.source),
+          format(node.aggregate_source),
           ' ',
           '|',
           ' ',
@@ -1437,11 +1437,7 @@ module Expressir
         [
           *format_remarks(node),
           *if node.class.method_defined? :children
-            node.children.flat_map do |x|
-              if x != node
-                format_scope_remarks(x)
-              end
-            end
+            node.children.select{|x| x.parent == node}.flat_map{|x| format_scope_remarks(x)}
           end
         ]
       end

data/lib/expressir/express_exp/parser.rb

@@ -1,10 +1,15 @@
-require 'express_parser'
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "#{$1}/express_parser"
+rescue LoadError
+  require_relative "express_parser"
+end
 require 'expressir/express_exp/visitor'
 
 module Expressir
   module ExpressExp
     class Parser
-      def self.from_exp(file)
+      def self.from_file(file)
         input = File.read(file)
 
 =begin
@@ -31,6 +36,19 @@ module Expressir
 
         repo
       end
+
+      def self.from_files(files)
+        schemas = files.map{|file| self.from_file(file).schemas}.flatten
+
+        Model::Repository.new({
+          schemas: schemas
+        })
+      end
+
+      # deprecated
+      def self.from_exp(file)
+        self.from_file(file)
+      end
     end
   end
 end

data/lib/expressir/express_exp/visitor.rb

@@ -1,9 +1,12 @@
-require 'express_parser'
-require 'expressir/model'
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "#{$1}/express_parser"
+rescue LoadError
+  require_relative "express_parser"
+end
+require "expressir/model"
+require "set"
 
-# static shorthands are unwrapped
-# - entity attributes, function/procedure parameters, local variables
-#
 # reference type is not recognized
 # see note in A.1.5 Interpreted identifiers
 # > It is expected that identifiers matching these syntax rules are known to an implementation.
@@ -20,6 +23,9 @@ require 'expressir/model'
 # > A syntactic construct such as ARRAY[1:3] OF REAL satisfies two syntactic productions —
 # > aggregation_type and general_aggregation_type. It is considered to be instantiable no matter which
 # > production it is required to satisfy in the syntax.
+#
+# static shorthands are unwrapped
+# - entity attributes, function/procedure parameters, local variables
 
 module Expressir
   module ExpressExp
@@ -28,13 +34,14 @@ module Expressir
 
       def initialize(tokens)
         @tokens = tokens
-        @attached_remarks = Set.new
+        @attached_remark_tokens = ::Set.new
 
         super()
       end
 
       def visit(ctx)
         result = super(ctx)
+        attach_source(ctx, result)
         attach_parent(ctx, result)
         attach_remarks(ctx, result)
         result
@@ -52,67 +59,111 @@ module Expressir
         ctx.map{|ctx2| visit(ctx2)}.flatten if ctx
       end
 
+      def get_tokens_source(tokens)
+        if tokens.last.text == '<EOF>'
+          tokens.pop
+        end
+
+        tokens.map{|x| x.text}.join('').force_encoding('UTF-8')
+      end
+
+      def get_tokens(ctx)
+        start_index, stop_index = if ctx.instance_of? ::ExpressParser::SyntaxContext
+          [0, @tokens.size - 1]
+        else
+          [ctx.start.token_index, ctx.stop.token_index]
+        end
+
+        @tokens[start_index..stop_index]
+      end
+
+      def get_head_tokens(ctx)
+        start_index, stop_index = if ctx.instance_of? ::ExpressParser::SchemaDeclContext
+          start_index = ctx.start.token_index
+          stop_index = if ctx.schema_body.interface_specification.length > 0
+            ctx.schema_body.interface_specification.last.stop.token_index
+          elsif ctx.schema_version_id
+            ctx.schema_version_id.stop.token_index + 1
+          else
+            ctx.schema_id.stop.token_index + 1
+          end
+
+          [start_index, stop_index]
+        end
+
+        if start_index and stop_index
+          @tokens[start_index..stop_index]
+        end
+      end
+
+      def attach_source(ctx, node)
+        if node.class.method_defined? :source
+          tokens = get_tokens(ctx)
+          node.source = get_tokens_source(tokens)
+        end
+
+        if node.class.method_defined? :head_source
+          tokens = get_head_tokens(ctx)
+          node.head_source = get_tokens_source(tokens)
+        end
+      end
+
       def attach_parent(ctx, node)
         if node.class.method_defined? :children
           node.children.each do |child_node|
-            if child_node.class.method_defined? :parent
+            if child_node.class.method_defined? :parent and !child_node.parent
               child_node.parent = node
             end
           end
         end
       end
 
-      def attach_remarks(ctx, node)
-        # get remark tokens
-        start_index, stop_index = if ctx.instance_of? ::ExpressParser::SyntaxContext
-          [0, @tokens.size - 1]
-        else
-          [ctx.start.token_index, ctx.stop.token_index]
+      def find_remark_target(node, path)
+        current_node = node
+        target_node = nil
+
+        if current_node.class.method_defined? :find_or_create
+          target_node = current_node.find_or_create(path)
+        end
+        while !target_node and current_node.class.method_defined? :parent and current_node.parent.class.method_defined? :find_or_create
+          current_node = current_node.parent
+          target_node = current_node.find_or_create(path)
         end
-        # puts [start_index, stop_index, ctx.class].inspect
-
-        remark_tokens = @tokens[start_index..stop_index].select{|x| x.channel == REMARK_CHANNEL}
-        if remark_tokens
-          remark_tokens.each do |remark_token|
-            remark_text = remark_token.text
-
-            # check if it is tagged remark
-            match = if remark_text.start_with?('--')
-              remark_text[2..-1].match(/^"([^"]*)"(.*)$/)
-            elsif remark_text.start_with?('(*') and remark_text.end_with?('*)')
-              remark_text[2..-3].match(/^"([^"]*)"(.*)$/m)
-            end
-            if !match
-              next
-            end
 
-            # don't attach already attached tagged remark
-            if @attached_remarks.include? remark_token
-              next
-            end
+        target_node
+      end
 
-            # attach tagged remark
-            remark_tag = match[1]
-            remark_content = match[2].strip
+      def attach_remarks(ctx, node)
+        remark_tokens = get_tokens(ctx).select{|x| x.channel == REMARK_CHANNEL}
 
-            target_node = nil
-            current_node = node
-            if current_node.class.method_defined? :find
-              target_node = current_node.find(remark_tag)
-            end
-            while !target_node and current_node.class.method_defined? :parent and current_node.parent.class.method_defined? :find
-              current_node = current_node.parent
-              target_node = current_node.find(remark_tag)
-            end
+        # skip already attached remarks
+        remark_tokens = remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
 
-            if target_node
-              target_node.remarks ||= []
-              target_node.remarks << remark_content
+        # parse remarks, find remark targets
+        tagged_remark_tokens = remark_tokens.map do |remark_token|
+          _, remark_tag, remark_text = if remark_token.text.start_with?('--')
+            remark_token.text.match(/^--"([^"]*)"(.*)$/).to_a
+          else
+            remark_token.text.match(/^\(\*"([^"]*)"(.*)\*\)$/m).to_a
+          end
 
-              # mark remark as attached, so that it is not attached again at higher nesting level
-              @attached_remarks << remark_token
-            end
+          if remark_tag
+            remark_target = find_remark_target(node, remark_tag)
+          end
+          if remark_text
+            remark_text = remark_text.strip.force_encoding('UTF-8')
           end
+
+          [remark_token, remark_target, remark_text]
+        end.select{|x| x[1]}
+
+        tagged_remark_tokens.each do |remark_token, remark_target, remark_text|
+          # attach remark
+          remark_target.remarks ||= []
+          remark_target.remarks << remark_text
+
+          # mark remark as attached, so that it is not attached again at higher nesting level
+          @attached_remark_tokens << remark_token
         end
       end
 
@@ -1609,12 +1660,12 @@ module Expressir
         ctx__logical_expression = ctx.logical_expression
 
         id = visit_if(ctx__variable_id)
-        source = visit_if(ctx__aggregate_source)
+        aggregate_source = visit_if(ctx__aggregate_source)
         expression = visit_if(ctx__logical_expression)
 
         Model::Expressions::QueryExpression.new({
           id: id,
-          source: source,
+          aggregate_source: aggregate_source,
           expression: expression
         })
       end
@@ -2429,7 +2480,7 @@ module Expressir
       def handle_simple_string_literal(ctx)
         ctx__text = ctx.text
 
-        value = ctx__text[1..(ctx__text.length - 2)]
+        value = ctx__text[1..(ctx__text.length - 2)].force_encoding('UTF-8')
 
         Model::Literals::String.new({
           value: value
@@ -2439,7 +2490,7 @@ module Expressir
       def handle_encoded_string_literal(ctx)
         ctx__text = ctx.text
 
-        value = ctx__text[1..(ctx__text.length - 2)]
+        value = ctx__text[1..(ctx__text.length - 2)].force_encoding('UTF-8')
 
         Model::Literals::String.new({
           value: value,