vorax 0.1.0pre

data/lib/vorax/parser/grammars/statement.rl

@@ -0,0 +1,83 @@
+%%{
+
+machine statement;
+
+action parse_start {
+  eof = pe
+}
+
+action parse_error {
+}
+
+action mark_as_anonymous {
+  stmt_type = 'ANONYMOUS'
+}
+
+action mark_as_sqlplus_command {
+  stmt_type = 'SQLPLUS'
+}
+
+action mark_as_sql {
+  stmt_type = nil
+}
+
+action mark_type {
+  tail = data[(0...p)]
+  type = tail[/\w+\Z/]
+  stmt_type = type.upcase if type
+}
+
+action mark_body {
+  stmt_type << ' BODY'
+}
+
+include common "common.rl";
+
+# parsing rules baby
+anonymous_block = ((K_BEGIN | K_DECLARE) ws+) @mark_as_anonymous;
+simple_module = (K_TRIGGER | K_FUNCTION | K_PROCEDURE) %mark_type;
+package_module = (K_PACKAGE %mark_type) (ws+ K_BODY %mark_body)?;
+type_module = (K_TYPE %mark_type) (ws+ K_BODY %mark_body)?;
+java_module = ((K_AND ws+ (K_RESOLVE | K_COMPILE) ws+ K_NOFORCE ws+) |
+                (K_AND ws+ (K_RESOLVE | K_COMPILE) ws+) |
+                (K_NOFORCE ws+))? (K_JAVA %mark_type);
+plsql_module = K_CREATE ws+ (K_OR ws+ K_REPLACE ws+)? 
+               (simple_module | 
+                package_module | 
+                type_module | 
+                java_module) ws+;
+set_transaction = (K_SET ws+ K_TRANSACTION ws+) @ mark_as_sql;
+sqlplus_command = (((K_ACCEPT | K_ARCHIVE | K_ATTRIBUTE | K_BREAK | K_BTITLE | K_CLEAR | K_COLUMN |
+                  K_COMPUTE | K_CONNECT | K_COPY | K_DEFINE | K_DESCRIBE | K_DISCONNECT | K_EXECUTE |
+                  K_EXIT | K_HELP | K_HOST | K_PASSWORD | K_PAUSE | K_PRINT | K_PROMPT |
+                  K_RECOVER | K_REMARK | K_REPFOOTER | K_REPHEADER | K_RUN | K_SAVE | K_SET | K_SHOW | K_SHUTDOWN |
+                  K_SPOOL | K_START | K_STARTUP | K_STORE | K_TIMING | K_TITLE | K_UNDEFINE | K_VARIABLE | K_WHENEVER |
+                  K_XQUERY) ws+) | ('@@' | '@' | '/' | '!')) @ mark_as_sqlplus_command;
+ 
+main := ws* (anonymous_block | set_transaction | sqlplus_command | plsql_module) >parse_start $err(parse_error);
+
+}%%
+
+module Vorax
+
+  module Parser
+
+    # Gets the type of the provided statement.
+    # 
+    # @param data [String] the statement
+    # @return [String] 'SQLPLUS' for an sqlplus statement, 'FUNCTION|PROCEDURE|PACKAGE|TYPE...' for
+    #   a PL/SQL block, 'ANONYMOUS' for an anonymous plsql block
+    def self.statement_type(data)
+      stmt_type = nil
+      data << "\n"
+      %% write data;
+      %% write init;
+      %% write exec;
+      data.chop!
+      return stmt_type
+    end
+
+end
+
+end
+

data/lib/vorax/parser/parser.rb

@@ -0,0 +1,320 @@
+# encoding: utf-8
+
+module Vorax
+
+  # Provides parsing utilities.
+  module Parser
+
+    END_LINE = /\r\n?|\n/ unless defined?(END_LINE)
+    SQLPLUS_TERMINATOR = END_LINE unless defined?(SQLPLUS_TERMINATOR)
+    SEMI_COLON_TERMINATOR = /;/ unless defined?(SEMI_COLON_TERMINATOR)
+    SLASH_TERMINATOR = Regexp.new('(?:' + END_LINE.to_s + '\s*\/[ \t]*' + END_LINE.to_s + ')') unless defined?(SLASH_TERMINATOR)
+    PLSQL_SPEC = /(?:\bpackage\b|\btype\b)/i
+    SUBPROG = /(?:\bfunction\b|\bprocedure\b)/i
+    BEGIN_MODULE = /(?:\bbegin\b)/i
+    END_MODULE = /(?:\bend\b)/i
+    
+    # Given an expression with  parenthesis, it is walking it so that to
+    # keep track of the open/close paren, in a balanced way.
+    #
+    # @param text [String] the string to be walked
+    # @return [String] the paren expression
+    def self.walk_balanced_paren(text)
+      walker = PLSQLWalker.new(text)
+      level = 0
+      start_pos = 0
+      end_pos = 0
+      walker.register_spot(/[(]/) do |scanner|
+        start_pos = scanner.pos - 1 if level == 0
+        level += 1
+      end
+      walker.register_spot(/[)]/) do |scanner|
+        level -= 1
+        if level <= 0
+          end_pos = scanner.pos
+          scanner.terminate
+        end
+      end
+      walker.walk
+      text[start_pos, end_pos]
+    end
+
+    # Remove all comments from the provided statement. Pay attention that every
+    # comment is replaced by a blank in order to cover the case where a comment is
+    # used as a whitespace (e.g. select * from/*comment*/dual).
+    #
+    # @param statement [String] the statement to be cleaned up of comments
+    # @return [String] the statement without any comment
+    def self.remove_all_comments(statement)
+      comment_areas = []
+      result = statement
+      walker = PLSQLWalker.new(statement, false)
+      
+      callback = lambda do |scanner, end_pattern|
+        start_pos = scanner.pos - scanner.matched.length
+        text = scanner.scan_until(end_pattern)
+        if text
+          comment_areas << (start_pos..scanner.pos - 1)
+        else
+          scanner.terminate
+        end
+      end
+
+      walker.register_spot(PLSQLWalker::BEGIN_ML_COMMENT) do |scanner|
+        callback.call(scanner, PLSQLWalker::END_ML_COMMENT)
+      end
+
+      walker.register_spot(PLSQLWalker::BEGIN_SL_COMMENT) do |scanner|
+        callback.call(scanner, PLSQLWalker::END_SL_COMMENT)
+      end
+
+      walker.register_default_plsql_quoting_spot()
+      walker.register_default_double_quoting_spot()
+      walker.register_default_single_quoting_spot()
+      walker.walk
+      offset = 0
+      comment_areas.each do |interval| 
+        r = (interval.min - offset .. interval.max - offset)
+        result[r] = " "
+        offset += (interval.max - interval.min)
+      end
+      result
+    end
+
+    # Remove the trailing comments from the provided statement.
+    #
+    # @param statement [String] the statement to be cleaned up
+    # @return the statement without the trailing comments.
+    def self.remove_trailing_comments(statement)
+      stmt = statement
+      begin
+        stmt.gsub!(/(?:--[^\n]*\s*\z)|(?:\/\*.*?\*\/\s*\z)/m, '')
+      end while !$~.nil?
+      stmt
+    end
+
+    # Get the function/procedure to which the argument on the
+    # provided position belongs.
+    #
+    # @param statement [String] the statement to be parsed
+    # @param position [int] the position index where the 
+    #   argument should be given
+    def self.argument_belongs_to(statement, position = nil)
+      position = statement.length unless position
+      stmt = Parser.remove_all_comments(statement[(0...position)])
+      stmt.reverse!
+      level = 0
+      walker = PLSQLWalker.new(stmt, false)
+      arg_owner = ""
+
+      squote_fallback = lambda do |scanner|
+        scanner.skip_until(PLSQLWalker::BEGIN_SINGLE_QUOTING)
+        if scanner.matched == "'"
+          begin
+            scanner.skip_until(/\'+/)
+          end while (scanner.matched != "'" && !scanner.eos?)
+        end
+      end
+
+      extract_module = lambda do |scanner|
+        module_name = ""
+        while !scanner.eos?
+          # consume leading whitspaces
+          scanner.scan(/\s*/)
+          if scanner.check(/"/) == '"'
+            # we have a quoted identifier
+            module_name << scanner.scan(/"/)
+            module_name << scanner.scan_until(/"/)
+          else
+            # unquoted identifier
+            module_name << scanner.scan(/\S+/)
+          end
+          # consume trailing whitespaces
+          scanner.scan(/\s*/)
+
+          # might be a dblink
+          if scanner.check(/@/) == '@' 
+            module_name << scanner.scan(/@/)
+            next
+          end
+
+          # might be package or a schema
+          if scanner.check(/\./) == '.' 
+            module_name << scanner.scan(/\./)
+            next
+          end
+          scanner.terminate
+        end
+        module_name.reverse!
+      end
+      
+      walker.register_spot(/'[\]})>]/) do |scanner|
+        # pay attention, it's reveresed
+        if scanner.matched =~ /\'\]/
+          squote_fallback.call(scanner) unless scanner.skip_until(/\[\'q/) 
+        elsif scanner.matched =~ /\'[}]/
+          squote_fallback.call(scanner) unless scanner.skip_until(/[{]\'q/) 
+        elsif scanner.matched =~ /\'[)]/
+          squote_fallback.call(scanner) unless scanner.skip_until(/[(]\'q/) 
+        elsif scanner.matched =~ /\'[>]/
+          squote_fallback.call(scanner) unless scanner.skip_until(/[<]\'q/) 
+        end
+      end
+
+      walker.register_spot(/[)]/) do |scanner|
+        level += 1
+      end
+
+      walker.register_spot(/[(]/) do |scanner|
+        if level == 0
+          arg_owner = extract_module.call(scanner)
+        else
+          level -= 1
+          scanner.terminate if level < 0 #give up, it's an invalid statement
+        end
+      end
+
+      walker.walk
+      return arg_owner
+    end
+    
+    # Given the html output of a script, it extracts all tables into a nice
+    # ruby array. This method returns a hash with the following meaning:
+    #   :resultset => an array with resultsets from all queries which
+    #                 generated the <html> output. For example, if the
+    #                 html parameter contains the output of two valid
+    #                 queries, the :resultset will contain:
+    #
+    #                 [                      # an array with all result sets 
+    #                   [                    # the resultset of the first query
+    #                     [val11, val12],    
+    #                     [val21, val22],    
+    #                     ...
+    #                     [valn1, valn2]
+    #                   ],
+    #                   [                    # the result set of the second query
+    #                     [v11, v12, v13],
+    #                     [v21, v22, v23],
+    #                     ...
+    #                     [vn1, vn2, vn3]
+    #                   ]
+    #                 ]
+    # If errors are detected into the output they are extracted into the
+    # :errors attribute of the returining hash.
+    # 
+    # @param html [String] the html to be parsed
+    # @return a hash with the parsed content
+    def self.query_result(html)
+      nbsp = Nokogiri::HTML("&nbsp;").text
+      hash = {:resultset => [], :errors => []}
+      doc = Nokogiri::HTML(html)
+      hash[:errors] = doc.xpath('/html/body/text()').map{ |n| n.text }.grep(/\nORA-[0-9]+/)
+      doc.xpath('//table').each do |table|
+        resultset = []
+        table.xpath('tr').each do |tr|
+          row = []
+          # replace nbsp with a plain blank in order to not confuse 
+          # the ragel parser, in case it's used
+          tr.xpath('td').each { |td| row << td.text.strip.gsub(nbsp, " ") }
+          resultset << row unless row.empty?
+        end
+        hash[:resultset] << resultset
+      end
+      return hash
+    end
+
+    # Prepare the provided statement for sqlplus execution. The prepare phase consists in
+    # adding the right end separator according to the statement type.
+    #
+    # @param statement [String] the statement to be prepared
+    # @return [String] the statement with the proper end separator appended
+    def self.prepare_exec(statement)
+      stmt = Parser.remove_trailing_comments(statement)
+      type = Parser.statement_type(stmt)
+      if type == 'SQLPLUS'
+        # do nothing
+      elsif !type.nil?
+        # a plsql block. We need a trailing /
+        stmt = "#{stmt.strip}\n/\n" if stmt !~ /\n\s*\/\s*\z/
+      else
+        # normal statement. It should have a trailing ;
+        stmt = "#{stmt.strip};" if stmt !~ /;\s*\z/
+      end
+      return stmt
+    end
+
+    # Get the current statement for the provided position.
+    #
+    # @param script_content [String] the script within which the current statement must
+    #   be detected
+    # @param position [int] the absolute position within the script content for which
+    #   the current statement must be found out
+    # @param params [Hash] additional options. The following parameters may be
+    #   provided:
+    #
+    #   :plsql_blocks => whenever or not to consider PL/SQL blocks when the current
+    #                    statement is detected. By default is true.
+    #   :sqlplus_commands => whenever or not to consider SQLPLUS commands when
+    #                        trying to detect the current statement
+    #
+    # @return [Hash] a hash with the following keys: :statement => the current statement 
+    #   which corresponds to the provided position, :range => the statement boundaries
+    #   within the whole script
+    def self.current_statement(script_content, position=0, params = {})
+      opts = {
+        :plsql_blocks => true,
+        :sqlplus_commands => true
+      }.merge(params)
+      start_pos = 0
+      end_pos = 0
+
+      walker = PLSQLWalker.new(script_content)
+
+      walker.register_spot(Parser::SEMI_COLON_TERMINATOR) do |scanner|
+        type = Parser.statement_type(scanner.string[(start_pos..scanner.pos)])
+        if type
+          if opts[:plsql_blocks] && type != 'SQLPLUS'
+            #this is a plsql block, eat till the slash terminator
+            unless scanner.scan_until(Parser::SLASH_TERMINATOR)
+              #it's an invalid statement
+              scanner.terminate
+            end
+          end
+        end
+        if (start_pos..scanner.pos).include?(position)
+          # include the terminator
+          end_pos = scanner.pos
+          scanner.terminate
+        else
+          start_pos = scanner.pos
+        end
+      end
+
+      walker.register_spot(Parser::SLASH_TERMINATOR) do |scanner|
+        if (start_pos..scanner.pos).include?(position)
+          # include the terminator
+          end_pos = scanner.pos
+          scanner.terminate
+        else
+          start_pos = scanner.pos
+        end
+      end
+
+      if opts[:sqlplus_commands]
+        walker.register_spot(Parser::SQLPLUS_TERMINATOR) do |scanner|
+          type = Parser.statement_type(scanner.string[(start_pos..scanner.pos)])
+          if type && type == 'SQLPLUS' && (start_pos..scanner.pos).include?(position)
+            end_pos = scanner.pos - scanner.matched.length
+            scanner.terminate
+          end
+        end
+      end
+
+      walker.walk
+      {:statement => script_content[(start_pos...end_pos)], :range => (start_pos...end_pos)}
+
+    end
+
+  end
+
+end

data/lib/vorax/parser/plsql_structure.rb

@@ -0,0 +1,158 @@
+# encoding: utf-8
+
+require 'tree'
+
+module Vorax
+
+  module Parser
+
+    class Region
+
+      attr_accessor :start_pos, :end_pos
+      attr_reader :name, :type
+
+      def initialize(name, type, start_pos = 0, end_pos = 0)
+        @name = name
+        @type = type
+        @start_pos = start_pos
+        @end_pos = end_pos
+      end
+
+      def to_s
+        "#{@name}[#{@type}]: #{@start_pos}"
+      end
+
+    end
+
+    class PlsqlStructure
+
+      PLSQL_SPEC = /(?:\bpackage\b|\btype\b)/i unless defined?(PLSQL_SPEC)
+      SUBPROG = /(?:\bfunction\b|\bprocedure\b)/i unless defined?(SUBPROG)
+      BEGIN_MODULE = /(?:\bbegin\b)/i unless defined?(BEGIN_MODULE)
+      END_MODULE = /(?:\bend\b)/i unless defined?(END_MODULE)
+      FOR_STMT = /(?:\bfor\b)/i unless defined?(FOR_STMT)
+      LOOP_STMT = /(?:\bfor\b)/i unless defined?(LOOP_STMT)
+      IF_STMT = /(?:\bfor\b)/i unless defined?(LOOP_STMT)
+
+      attr_reader :text
+
+      def initialize(text)
+        @text = text
+        @root = Tree::TreeNode.new("root", nil)
+        @walker = PLSQLWalker.new(text)
+        @level = 0
+        @current_parent = @root
+        @begin_level = 0
+        register_spots()
+        @walker.walk
+      rescue Exception => e
+        # be prepare for any nasting parse error.
+        # Failing here is kind of usual, having in mind
+        # that we often parse incomplete code.
+        Vorax.debug(e.to_s)
+      end
+
+      def tree
+        @root
+      end
+      
+    private
+
+      def assign_parent(node)
+        @current_parent = node
+      end
+
+      def register_spots
+        register_plsql_spec_spot()
+        register_slash_terminator_spot()
+        register_subprog_spot()
+        register_begin_spot()
+        register_end_spot()
+      end
+
+      def register_plsql_spec_spot
+        @walker.register_spot(PLSQL_SPEC) do |scanner|
+          if @level == 0
+            meta_data = Parser.plsql_def("#{scanner.matched}#{scanner.rest}")
+            if meta_data[:type] == 'SPEC' || meta_data[:type] == 'BODY'
+              # is it a spec or a body?
+              region = Region.new(meta_data[:name], meta_data[:type], scanner.pos)
+              assign_parent(@current_parent << Tree::TreeNode.new(region.to_s, region))
+              @level += 1
+            end
+          end
+        end
+      end
+
+      def register_slash_terminator_spot
+        @walker.register_spot(Parser::SLASH_TERMINATOR) do |scanner|
+          # this should apply to the last top level node
+          if @root.has_children?
+            if @root.children.last.content
+              @root.children.last.content.end_pos = scanner.pos
+            end
+            assign_parent(@root)
+            @level = 0
+          end
+        end
+      end
+
+      def register_subprog_spot
+        @walker.register_spot(SUBPROG) do |scanner|
+        subprog_name = scanner.peek(32)[/(?:"[^"]+")|(?:[A-Z0-9$_#]+)/i]
+          if scanner.matched =~ /function/i
+            subprog_type = 'FUNCTION'
+          elsif scanner.matched =~ /procedure/i
+            subprog_type = 'PROCEDURE'
+          end
+          start_pos = scanner.pos - scanner.matched.length
+          region = Region.new(subprog_name, subprog_type, scanner.pos)
+          node = Tree::TreeNode.new(region.to_s, region)
+          @current_parent << node
+          if @current_parent && @current_parent.content 
+            if @current_parent.content.type != 'SPEC'
+              @level += 1
+              assign_parent(node)
+            else
+              node.content.end_pos = node.content.start_pos
+            end
+          end
+        end
+      end
+
+      def register_begin_spot
+        @walker.register_spot(BEGIN_MODULE) do |scanner|
+          @begin_level += 1
+          if @begin_level > 1
+            # start a new region
+            region = Region.new('anonymous', 'BLOCK', scanner.pos)
+            @level += 1
+            assign_parent(@current_parent << Tree::TreeNode.new(region.to_s, region))
+          end
+        end
+      end
+
+      def register_end_spot
+        @walker.register_spot(END_MODULE) do |scanner|
+          # we have an "end" match. first of all check if it's not part
+          # of an conditional compiling "$end" definition
+          char_behind = scanner.string[scanner.pos - scanner.matched.length - 1, 1]
+          if char_behind != '$'
+            metadata = Parser.plsql_def("#{scanner.matched}#{scanner.rest}")
+            if metadata[:type] == 'END' && metadata[:end_def] > 0
+              @begin_level -= 1 if @begin_level > 0
+              @level -= 1 if @level > 0
+              if @current_parent.content
+                @current_parent.content.end_pos = (scanner.pos - 1) + (metadata[:end_def] - 1)
+              end
+              assign_parent(@current_parent.parent)
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/vorax/parser/plsql_walker.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+
+require 'strscan'
+
+module Vorax
+
+  module Parser
+
+    # A PLSQL string scanner which is looking for interesting points within
+    # the provided source code. This is used instead of a fully fledged parser
+    # for speed considerations.
+    class PLSQLWalker
+
+      BEGIN_ML_COMMENT = /\/\*/ unless defined?(BEGIN_ML_COMMENT)
+      END_ML_COMMENT = /\*\// unless defined?(END_ML_COMMENT)
+      BEGIN_SL_COMMENT = /--/ unless defined?(BEGIN_SL_COMMENT)
+      END_SL_COMMENT = Parser::END_LINE unless defined?(END_SL_COMMENT)
+      BEGIN_PLSQL_SPECIAL_QUOTING = /q'[!\[{(<]/ unless defined?(BEGIN_PLSQL_SPECIAL_QUOTING)
+      BEGIN_DOUBLE_QUOTING = /[\"]/ unless defined?(BEGIN_DOUBLE_QUOTING)
+      BEGIN_SINGLE_QUOTING = /[']/ unless defined?(BEGIN_SINGLE_QUOTING)
+
+      # Create a new parse walker.
+      #
+      # @param text [String] the text to be walked/parsed
+      # @param create_default_spots [boolean] whenever or not to create default 
+      #  detection spots: multiline comments, singleline comments and quoted literals
+      def initialize(text, create_default_spots=true)
+        @text = text
+        @matchers = []
+        @ss = StringScanner.new(text)
+        create_default_spots() if create_default_spots
+      end
+
+      # Returns the string scanner used for walking the string.
+      #
+      # @return [StringScanner] the string scanner
+      def scanner
+        @ss
+      end
+
+      # Register a new detection spot. The order of specifying these spots is important.
+      #
+      # @param pattern [Regexp] the spot regular expression
+      # @param callback [Procedure] what to do when this spot is detected. The registered
+      #   block is always called with the string scanner object. Please do not use "return"
+      #   to exit from the defined block.
+      def register_spot(pattern, &callback)
+        @matchers << {:pattern => pattern, :callback => callback}
+      end
+
+      # Walk the text and trigger the registered callbacks. It returns the text which was
+      # successfully walked.
+      def walk
+        global_matcher = Regexp.new(@matchers.map { |e| e[:pattern].to_s }.join('|'), 
+                                    Regexp::IGNORECASE)
+        while !@ss.eos?
+          if match = @ss.scan_until(global_matcher)
+            current_pos = @ss.pos
+            @matchers.each do |matcher|
+              if @ss.matched =~ matcher[:pattern]
+                matcher[:callback].call(@ss)
+              end
+            end
+          else
+            @ss.terminate
+          end
+        end
+      end
+
+      # Register a spot to walk a multiline comment.
+      def register_default_ml_comment_spot
+        register_spot(BEGIN_ML_COMMENT) do |scanner|
+          scanner.scan_until(END_ML_COMMENT)
+        end
+      end
+
+      # Register a spot to walk a single line comment.
+      def register_default_sl_comment_spot
+        register_spot(BEGIN_SL_COMMENT) do |scanner|
+          scanner.scan_until(END_SL_COMMENT)
+        end
+      end
+
+      # Register a spot to walk a plsql special quoting liternal.
+      def register_default_plsql_quoting_spot
+        register_spot(BEGIN_PLSQL_SPECIAL_QUOTING) do |scanner|
+          scanner.scan_until(END_SL_COMMENT)
+          if scanner.matched =~ /q'\[/
+            scanner.scan_until(/\]'/) 
+          elsif scanner.matched =~ /q'[{]/
+            scanner.scan_until(/[}]'/) 
+          elsif scanner.matched =~ /q'[(]/
+            scanner.scan_until(/[)]'/) 
+          elsif scanner.matched =~ /q'[<]/
+            scanner.scan_until(/[>]'/) 
+          end
+        end
+      end
+
+      # Register a spot to walk a double quoted literal.
+      def register_default_double_quoting_spot
+        register_spot(BEGIN_DOUBLE_QUOTING) do |scanner|
+          scanner.scan_until(/"/)
+        end
+      end
+
+      # Register a spot to walk a single quoted literal.
+      def register_default_single_quoting_spot
+        register_spot(BEGIN_SINGLE_QUOTING) do |scanner|
+          collector = ''
+          begin
+            if match = scanner.scan_until(/\'+/) 
+              collector << match
+            end
+          end while (scanner.matched != "'" && !scanner.eos?)
+          collector
+        end
+      end
+
+      private
+
+      def create_default_spots
+        # define a multiline comment spot
+        register_default_ml_comment_spot()
+
+        # define a single line comment spot
+        register_default_sl_comment_spot()
+
+        # define special PLSQL quotes spot
+        register_default_plsql_quoting_spot()
+
+        # register a double quoted string spot
+        register_default_double_quoting_spot()
+        
+        # register a single quoted string spot
+        register_default_single_quoting_spot()
+      end
+
+    end
+
+  end
+
+end