vorax 0.1.0pre

data/lib/vorax/parser/statement_inspector.rb~

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+module Vorax
+
+  module Parser
+
+    class StatementInspector
+
+      def initialize(statement)
+        @statement = statement
+      end
+
+      def type
+        @type ||= Statement.new.type(statement)
+      end
+
+      def data_source
+        descriptor.refs
+      end
+
+      def recursive_data_source(refs, collect, position)
+        inner = refs.find { |r| r.respond_to?(:range) && r.range.include?(position) }
+        collect.unshift(refs).flatten!
+        if inner
+          desc = Parser::Alias.new
+          desc.walk(inner.base)
+          data_source(desc.refs, collect, position - inner_expr.range.first)
+        else
+          return collect
+        end
+      end
+
+      def query_fields
+        @query_fields ||= Column.new.walk(descriptor.query_fields)
+      end
+
+      private
+
+      def descriptor
+        unless @desc
+          @desc = Parser::Alias.new
+          @desc.walk(@statement)
+        end
+        @desc
+      end
+      
+    end
+
+  end
+  
+end
+

data/lib/vorax/parser/stmt_inspector.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+module Vorax
+
+  module Parser
+
+    # A class used to gather metadata information for an SQL statement. This is needed
+    # especially for implementing VoraX code completion.
+    class StmtInspector
+
+      # Creates a new statement inspector.
+      #
+      # @param statement [String] the statement to be inspected
+      def initialize(statement)
+        @statement = statement
+      end
+
+      # Get the type of the statement
+      #
+      # (see Parser.statement_type)
+      def type
+        @type ||= Parser.statement_type(statement)
+      end
+
+      # Get all tableref/exprref for this statement, taking into account
+      # the current position within that statement.
+      #
+      # @param position [int] the current position.
+      # @return an array of TableRef/ExprRef objects
+      def data_source(position=0)
+        recursive_data_source(descriptor.refs, position)
+      end
+
+      # If it's a query, the corresponding columns are returned.
+      #
+      # @return an array of columns
+      def query_fields
+        @query_fields ||= Column.new.walk(descriptor.query_fields)
+      end
+
+      # Find the provided alias for the statement, taking into account the
+      # current position.
+      #
+      # @param name [String] the alias name
+      # @param position [int] the current position
+      def find_alias(name, position=0)
+        data_source(position).find { |r| r.pointer && r.pointer.upcase == name.upcase }
+      end
+
+      private
+
+      def descriptor
+        unless @desc
+          @desc = Parser::Alias.new
+          @desc.walk(@statement)
+        end
+        @desc
+      end
+
+      def recursive_data_source(refs, position, collect = [])
+        inner = refs.find { |r| r.respond_to?(:range) && r.range.include?(position) }
+        collect.unshift(refs).flatten!
+        if inner
+          desc = Parser::Alias.new
+          desc.walk(inner.base)
+          recursive_data_source(desc.refs, position - inner.range.first, collect)
+        else
+          return collect
+        end
+      end
+
+      
+    end
+
+  end
+  
+end
+

data/lib/vorax/parser/target_ref.rb

@@ -0,0 +1,110 @@
+# encoding: UTF-8
+
+module Vorax
+
+  module Parser
+
+    # An abstraction for a table reference within an SQL statement. This class is used
+    # by the StmtInspector to model the FROM clause of a query or the target table of
+    # an INSERT or UPDATE.
+    class TableRef
+
+      attr_reader :base, :pointer
+
+      # Creates a new TableRef object.
+      #
+      # @param base [String] is the actual table/view name from the SQL statement.
+      # @param pointer [String] is the alias of the table, if there's any
+      def initialize(base, pointer = nil)
+        @base = base
+        @pointer = pointer
+      end
+
+      # Return the columns of this table ref.
+      #
+      # @return the columns of the table, in an unexpanded form (e.g. tbl.*)
+      def columns
+        ["#{@base}.*"]
+      end
+
+      # Compare too table ref objects.
+      #
+      # param obj [TableRef] the other tableref used for comparison.
+      def ==(obj)
+        self.base == obj.base && self.pointer == obj.pointer
+      end
+
+    end
+
+    # This class is used to model a reference within a SQL statement, given as an
+    # expression (e.g. "select * from (select * from dual);")
+    class ExprRef
+
+      attr_reader :base, :range, :pointer
+
+      # Creates a new ExprRef object.
+      #
+      # @param base [String] the actual expresion
+      # @param range [Range] the bounderies of this expression within the parent statement
+      # @param pointer [String] the alias of the expresion, if there is any
+      def initialize(base, range, pointer = nil)
+        @base = base
+        @range = range
+        @pointer = pointer
+      end
+
+      # Get all columns for this expression.
+      #
+      # @return all columns of the query expression
+      def columns
+        collect = []
+        recursive_columns(@base, collect)
+      end
+      
+      # Compare too table ref objects.
+      #
+      # param obj [TableRef] the other tableref used for comparison.
+      def ==(obj)
+        self.base == obj.base && self.range == obj.range && self.pointer == obj.pointer
+      end
+
+      private
+
+      def recursive_columns(statement, collect)
+        inspector = StmtInspector.new(statement)
+        columns_data = inspector.query_fields
+        columns_data.each do |column|
+          if column =~ /([a-z0-9#$_]+\.)?\*/i
+            #might be an alias
+            alias_name = column[/[a-z0-9#$_]+/]
+            ds = []
+            if alias_name
+              src = inspector.data_source.find do |r| 
+                (r.pointer && r.pointer.upcase == alias_name.upcase) || (r.base.upcase == alias_name.upcase)
+              end
+              ds << src if src
+            elsif column == '*'
+              ds = inspector.data_source
+            end
+            if ds.size > 0
+              ds.each do |source|
+                if source.respond_to?(:range)
+                  recursive_columns(source.base, collect)
+                else
+                  collect << "#{source.base}.*"
+                end
+              end
+            end
+          else
+            collect << column
+          end
+        end
+        return collect
+      end
+
+    end
+
+  end
+
+end
+

data/lib/vorax/sqlplus.rb

@@ -0,0 +1,281 @@
+# encoding: UTF-8
+
+module Vorax
+
+  # Provides integration with Oracle SqlPlus CLI tool.
+  class Sqlplus
+
+    # the tail size of the output chunk to be searched
+    # for the output END_MARKER.
+    TAIL_LENGTH = 100 unless defined?(TAIL_LENGTH)
+    private_constant :TAIL_LENGTH
+
+    attr_reader :bin_file, :default_funnel_name, :process
+    
+    # Creates a new sqlplus instance.
+    #
+    # @param bin_file [String] the path to the SqlPlus executable. By
+    #   default is "sqlplus", which requires that the executable to be
+    #   in $PATH.
+    def initialize(bin_file = "sqlplus")
+      @bin_file = bin_file
+      @busy = false
+      @start_marker, @end_marker, @cancel_marker = [2.chr, 3.chr, 4.chr]
+      @process = ChildProcess.build(@bin_file, "/nolog")
+      # On Unix we may abort the currently executing query by sending a
+      # INT signal to the Sqlplus process, but we need access to the 
+      # send_term private method.
+      class << @process; public :send_signal; end if ChildProcess.unix?
+      @process.duplex = true
+      @process.detach = true
+      @process.io.inherit!
+      @io_read, @io_write = VoraxIO.pipe
+      @process.io.stdout = @io_write
+      @process.start
+      @process.io.stdin.sync = true
+      @current_funnel = nil
+      @default_convertor_name = nil
+      @registered_convertors = {:vertical => Output::VerticalConvertor,
+                                :pagezip => Output::PagezipConvertor,
+                                :tablezip => Output::TablezipConvertor}
+    end
+
+    # Set the default convertor for the output returned by sqlplus.
+    #
+    # @param convertor_name [Symbol] the default funnel name. The
+    #   valid values are: :vertical, :pagezip and :tablezip
+    def default_convertor=(convertor_name)
+      Vorax.debug("default_convertor=#{convertor_name.inspect}")
+      @default_convertor_name = convertor_name
+    end
+
+    # Register a new convertor for the sqlplus output.
+    # 
+    # @param convertor_name [Symbol] the name of the convertor (key)
+    # @param convertor_class [Class] the class which implements the
+    #   convertor. It must be a subclass of BaseConvertor.
+    def register_convertor(convertor_name, convertor_class)
+      @registered_convertors[convertor_name] = convertor_class
+    end
+
+    # Execute an sqlplus command.
+    #
+    # @param command [String] the command to be executed.
+    # @param params [Hash] additional parameters. You may use
+    #   the following options:
+    #     :prep => a string with commands to be executed just before
+    #              running the provided command. For example, you may
+    #              choose to set some sqlplus options.
+    #     :post => a string with commands to be executed after the
+    #              provided command was run. Here it's a good place
+    #              to put commands that restores some options affected
+    #              by the executed command.
+    #     :convertor => the convertor used to convert the output received
+    #                from Sqlplus. By default it's the convertor set with
+    #                "default_convertor=" method.
+    #     :pack_file => the file name into which the command(s) to be
+    #                   executed are wrapped into and then sent for
+    #                   execution to sqlplus using '@<file_name>'. 
+    #                   Providing this option may prove to be a good
+    #                   thing for big commands. If this parameter is
+    #                   not provided then the command is sent directly
+    #                   to the input IO of the sqlplus process.
+    def exec(command, params = {})
+      Vorax.debug("exec: command=#{command.inspect} params=#{params.inspect}")
+      raise AnotherExecRunning if busy?
+      @tail = ""
+      opts = {
+        :prep => nil,
+        :post => nil,
+        :convertor => @default_convertor_name,
+        :pack_file => nil,
+      }.merge(params)
+      @busy = true
+      @look_for = @start_marker
+      prepare_funnel(opts[:convertor])
+      if @current_funnel && @current_funnel.is_a?(Output::HTMLFunnel)
+        # all HTML funnels expects html format
+        send_text("set markup html on\n")
+      else
+        send_text("set markup html off\n")
+      end
+      if opts[:pack_file]
+        send_text("@#{pack(command, opts)}\n")
+      else
+        send_text("#{opts[:prep]}\n") if opts[:prep]
+        capture { send_text("#{command}\n") }
+        send_text("#{opts[:post]}\n") if opts[:post]
+      end
+    end
+
+    # Send a text directly to the stdin of the sqlplus process.
+    #
+    # @param text [String] the text to be sent to sqlplus
+    def send_text(text)
+      @process.io.stdin.print(text)
+    end
+
+    # Check if the sqlplus process is busy executing something.
+    #
+    # @return true if the sqlplus is busy executing something, 
+    #   false otherwise
+    def busy?
+      @busy
+    end
+
+    # Check if the output of a previous executed sqlpus command
+    # was completely fetched out.
+    #
+    # @return true if the whole output was fetched, false otherwise
+    def eof?
+      not busy?
+    end
+
+    # Read the output spit by sqlplus process. If there is any default
+    # convertor, the returned output will be formatted according to
+    # that convertor.
+    #
+    # @param bytes [int] the maximum output chunk size
+    # @return the output chunk
+    def read_output(bytes=4086)
+      output = ""
+      raw_output = @tail
+      begin
+        raw_output << (@io_read.read_nonblock(bytes) || '').gsub(/\r/, '')
+      rescue Errno::EAGAIN
+      end
+      if raw_output
+        #p raw_output if raw_output
+        if @tail.empty?
+          # The logic of tail: when SET ECHO ON is used, the PRO <end_marker>
+          # statement will be also part of the output. The user will end up
+          # with something like "SQL> pro" as the last line. The solution would
+          # be to search for "pro <end_marker>" and to stop just before "pro", but
+          # the chunk may be split in the middle of the "pro <end_marker>" text and
+          # this is something which depends on the output of the executed
+          # statement (e.g. bytes=4086, output=4084 => chunk="...SQL> p"). The
+          # workaround is to delay sending the last TAIL_LENGTH characters until
+          # the next fetch.
+          raw_output, @tail = raw_output.slice!(0...raw_output.length-TAIL_LENGTH), raw_output
+        else
+          @tail = ''
+        end
+        scanner = StringScanner.new(raw_output)
+        while not scanner.eos?
+          if @look_for == @start_marker
+            if text = scanner.scan_until(/#{@look_for}/)
+              if text !~ /pro #{@look_for}/
+                # Only if it's not part of a PROMPT sqlplus command.
+                # This might happen when the "echo" sqlplus option
+                # is ON and the begin marker is included into the
+                # sql pack file. Because we are using big chunks to
+                # read data it's very unlikely that the echoing of the
+                # prompt command to be split in the middle.
+                @look_for = @end_marker
+              end
+            else
+              scanner.terminate
+            end
+          end
+          if @look_for == @end_marker
+            output = scanner.scan(/[^#{@look_for}]*/)
+            if scanner.scan(/#{@look_for}/)
+              output.gsub!(/\n.*?pro \z/, "")
+              # end of output reached
+              scanner.terminate
+              @busy = false
+            end
+          end
+        end
+      end
+      chunk = output.force_encoding('UTF-8')
+      if @current_funnel && !chunk.empty?
+        # nokogiri may be confused about those unclosed <p> tags
+        # sqlplus emits, so it's better to get rid of them and use
+        # <br> instead. 
+        @current_funnel.write(br_only(chunk))
+        chunk = @current_funnel.read
+      end
+      return chunk
+    end
+
+    # Cancel the currently executing statement. This is supported on Unix
+    # only. On Windows there's no way to send a CTRL+C signal to Sqlplus
+    # without aborting the process. There's an old enhancement request on 
+    # Oracle support: 
+    #
+    # Bug 8890996: ENH: CONTROL-C SHOULD NOT EXIT WINDOWS CONSOLE SQLPLUS
+    #
+    # So, as soon as we have some fixes from the Oracle guys I will come
+    # back to this method.
+    def cancel
+      raise PlatformNotSupported if ChildProcess.windows?
+      if busy?
+        @process.send_signal 'INT'
+        mark_cancel
+        # read until the cancel marker
+        raw_output = ""
+        until raw_output =~ /#{@cancel_marker}/
+          begin
+            raw_output = @io_read.read_nonblock(1024)
+            yield if block_given?
+          rescue Errno::EAGAIN
+            sleep 0.1
+          end
+        end
+        @busy = false
+      end
+    end
+
+    # Kill the sqlplus process.
+    def terminate
+      @process.stop
+    end
+
+    private
+
+    def br_only(chunk)
+      # be prepared for chunks with <p> tag broken in the middle
+      chunk.gsub(/<p>/, "<br>").gsub(/<p\z/, "<br").gsub(/\Ap>/, "br>")
+    end
+
+    def prepare_funnel(convertor_name)
+      convertor = @registered_convertors[convertor_name]
+      if convertor
+        @current_funnel = Output::HTMLFunnel.new(convertor.new)
+      else
+        @current_funnel = nil
+      end
+    end
+
+    def capture
+      @process.io.stdin.puts("#pro #{@start_marker}")
+      yield
+      @process.io.stdin.puts("#pro #{@end_marker}")
+      @process.io.stdin.puts(".")
+    end
+
+    def mark_cancel
+      @process.io.stdin.puts
+      @process.io.stdin.puts("pro #{@cancel_marker}")
+    end
+
+    def pack(command, opts)
+      pack_file = opts[:pack_file]
+      if pack_file
+        File.open(pack_file, 'w') do |f|
+          f.puts opts[:prep]
+          f.puts "#pro #@start_marker"
+          f.puts command.strip
+          f.puts "#pro #@end_marker"
+          f.puts "."
+          f.puts opts[:post]
+        end
+      end
+      pack_file
+    end
+
+
+  end
+
+end

data/lib/vorax/version.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+module Vorax
+
+  VERSION = '0.1.0pre' unless defined?(VERSION)
+
+end

data/lib/vorax/vorax_io.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+
+module Vorax
+
+  # Implements an IO pipe to link the stdin and stdout handlers of the
+  # sqlplus process to the ends of this pipe. This is required for Windows
+  # processes only. On Unix it's enough to use a builtin IO object.
+  class VoraxIO < IO
+
+    # A proxy for the original read_nonblock method
+    alias :old_read_nonblock :read_nonblock
+
+    # Creates a new IO.
+    def initialize(*args)
+      super(*args)
+      if ChildProcess.windows?
+        require 'Win32API'
+        @hFile = ChildProcess::Windows::Lib.get_osfhandle(fileno)
+        peek_params = [
+          'L', # handle to pipe to copy from
+          'L', # pointer to data buffer
+          'L', # size, in bytes, of data buffer
+          'L', # pointer to number of bytes read
+          'P', # pointer to total number of bytes available
+          'L'] # pointer to unread bytes in this message
+        @peekNamedPipe = Win32API.new("kernel32", "PeekNamedPipe", peek_params, 'I')
+        read_params = [
+            'L', # handle of file to read
+            'P', # pointer to buffer that receives data
+            'L', # number of bytes to read
+            'P', # pointer to number of bytes read
+            'L'] #pointer to structure for data
+        @readFile = Win32API.new("kernel32", "ReadFile", read_params, 'I')
+      end
+    end
+
+    # Read in nonblock mode from the pipe.
+    #
+    # @param bytes [int] the number of bytes to be read at once
+    # @see IO.read_nonblock
+    def read_nonblock(bytes)
+      if ChildProcess.windows?
+        read_file(peek)
+      else
+        old_read_nonblock(bytes)
+      end
+    end
+
+    private
+
+    def peek
+      available = [0].pack('I')
+      if @peekNamedPipe.Call(@hFile, 0, 0, 0, available, 0).zero?
+        raise IOError, 'Named pipe unavailable'
+      end
+      available.unpack('I')[0]
+    end
+
+    def read_file(bytes)
+      if bytes > 0
+        number = [0].pack('I')
+        buffer = ' ' * bytes
+        return '' if @readFile.call(@hFile, buffer, bytes, number, 0).zero?
+        buffer[0...number.unpack('I')[0]]
+      end
+    end
+
+  end
+
+end

data/lib/vorax.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+
+require 'rubygems' unless defined? Gem
+require 'logger'
+require 'childprocess'
+require 'antlr3'
+
+# The main Vorax namespace. Everything related to VoraX is part
+# of this module.
+module Vorax
+
+  # Sets the logger to be used for debug purposes.
+  # @param logger [Logger] the logger object.
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  # Get the current logger.
+  def self.logger
+    @logger
+  end
+
+  # Log a debug entry.
+  # @param message [String] the message to be logged.
+  def self.debug(message)
+    if @logger
+      @logger.add(Logger::DEBUG, nil, 'rby') { message }
+    end
+  end
+ 
+  # Raised when another SqlPlus is already executing.
+  class AnotherExecRunning < StandardError; end
+
+  # Raised when the platform is not supported (see cancel)
+  class PlatformNotSupported < StandardError; end
+
+end
+
+require 'vorax/version.rb'
+require 'vorax/vorax_io.rb'
+require 'vorax/sqlplus.rb'
+require 'vorax/base_funnel.rb'
+require 'vorax/output/html_funnel.rb'
+require 'vorax/output/html_convertor.rb'
+require 'vorax/output/vertical_convertor.rb'
+require 'vorax/output/zip_convertor.rb'
+require 'vorax/output/pagezip_convertor.rb'
+require 'vorax/output/tablezip_convertor.rb'
+require 'vorax/parser/parser.rb'
+require 'vorax/parser/conn_string.rb'
+require 'vorax/parser/target_ref.rb'
+require 'vorax/parser/stmt_inspector.rb'
+require 'vorax/parser/plsql_walker.rb'
+require 'vorax/parser/plsql_structure.rb'
+require 'vorax/parser/grammars/statement.rb'
+require 'vorax/parser/grammars/alias.rb'
+require 'vorax/parser/grammars/column.rb'
+require 'vorax/parser/grammars/package_spec.rb'
+require 'vorax/parser/grammars/plsql_def.rb'
+require 'vorax/parser/body_split.rb'

data/spec/column_spec.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+include Vorax
+
+describe 'column' do
+
+  it 'should work with multiple columns' do
+    Parser::Column.new.walk('col1, col2, f(a, b, c) x, 3+5, t.*, owner.tab.col y').should eq(["col1", "col2", "x", "t.*", "y"])
+  end
+
+  it 'should work with one column' do
+    Parser::Column.new.walk('col1').should eq(["col1"])
+  end
+
+  it 'should work with one column with alias' do
+    Parser::Column.new.walk('col1 as x').should eq(["x"])
+    Parser::Column.new.walk('col1 y').should eq(["y"])
+  end
+
+  it 'should work with referenced columns' do
+    Parser::Column.new.walk('tab.col1').should eq(["tab.col1"])
+    Parser::Column.new.walk('owner.tab.col2').should eq(["owner.tab.col2"])
+    Parser::Column.new.walk('"owner"."tab wow".col2').should eq(['"owner"."tab wow".col2'])
+  end
+
+  it 'should work with expressions' do
+    Parser::Column.new.walk('(select 1 from dual) x, 3+2').should eq(["x"])
+  end
+
+  it 'should ignore functions without alias' do
+    Parser::Column.new.walk('f(1, 2, 3), my_func(col)').should eq([])
+  end
+
+  it 'should work with nested expressions' do
+    Parser::Column.new.walk('f(1, g(2), x(3, 2, f(10))), my_func(col)').should eq([])
+  end
+
+end
+
+