each_sql 0.2.5 → 0.3.0

data/CHANGELOG.markdown

@@ -1,5 +1,19 @@
+### 0.3.0 / 2012/03/10
+* Internal implementation revised.
+ * At first, I thought this would be trivial, 
+   that I didn't need a real parser for just breaking SQL scripts
+   into individual executable units.
+   I couldn't be more wrong. Codes for handling a few exceptional cases
+   soon piled up and became unmaintainable.
+   The new version now employs Citrus parser for processing SQL scripts.
+   The output is not backward-compatible, for instance, comments before and after
+   each execution block are trimmed out.
+* Supports PostgreSQL (experimental)
+* `delimiter` command works for all types
+
 ### 0.2.5 / 2011/09/01
 * Can pass block directly to EachSQL(script)
+
 ```ruby
 EachSQL(script) do |sql|
   # ...

data/Gemfile

@@ -2,11 +2,13 @@ source "http://rubygems.org"
 # Add dependencies required to use your gem here.
 # Example:
 #   gem "activesupport", ">= 2.3.5"
+gem 'citrus', '~> 2.4.1'
+gem 'erubis', '~> 2.7.0'
+gem 'quote_unquote', '~> 0.1.1'
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.2"
-  gem "rcov", ">= 0"
 end

data/Gemfile.lock

@@ -1,14 +1,15 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    citrus (2.4.1)
+    erubis (2.7.0)
     git (1.2.5)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
-    rake (0.9.2)
-    rcov (0.9.10)
-    rcov (0.9.10-java)
+    quote_unquote (0.1.1)
+    rake (0.9.2.2)
 
 PLATFORMS
   java
@@ -16,5 +17,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
+  citrus (~> 2.4.1)
+  erubis (~> 2.7.0)
   jeweler (~> 1.6.2)
-  rcov
+  quote_unquote (~> 0.1.1)

data/README.markdown

@@ -1,13 +1,16 @@
-# each_sql
+each_sql
+========
 
-Enumerate each SQL statement in the given SQL script.
+Enumerate executable blocks in the given SQL script.
 
-## Installation
+Installation
+------------
 ```
 gem install 'each_sql'
 ```
 
-## Example
+Example
+-------
 ### Basic
 ```ruby
 require 'each_sql'
@@ -35,16 +38,24 @@ end
 EachSQL(plsql_script, :oracle).each do |sql|
   # ...
 end
+
+# For PostgreSQL scripts
+EachSQL(plpgsql_script, :postgres).each do |sql|
+  # ...
+end
 ```
 
-## TODO
-- More/better tests.
-- pgplsql support.
+TODO
+----
+- More tests.
+- Support for other RDBMSs
 
-## Warning
+Warning
+-------
 Stored procedure handling is at best incomplete. Use it at your own risk.
 
-## Contributing to each_sql
+Contributing to each_sql
+------------------------
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
@@ -54,8 +65,9 @@ Stored procedure handling is at best incomplete. Use it at your own risk.
 * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
-## Copyright
+Copyright
+---------
 
-Copyright (c) 2011 Junegunn Choi. See LICENSE.txt for
+Copyright (c) 2012 Junegunn Choi. See LICENSE.txt for
 further details.
 

data/Rakefile

@@ -32,14 +32,6 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
-end
-
 task :default => :test
 
 require 'rake/rdoctask'

data/VERSION

@@ -1 +1 @@
-0.2.5
+0.3.0

data/each_sql.gemspec

@@ -43,16 +43,13 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.2"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
     else
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.2"])
-      s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.2"])
-    s.add_dependency(%q<rcov>, [">= 0"])
   end
 end
 

data/lib/each_sql.rb

@@ -1,126 +1,62 @@
 # encoding: UTF-8
 # Junegunn Choi (junegunn.c@gmail.com)
 
+require 'rubygems'
 require 'each_sql/each_sql'
+require 'each_sql/parser'
 
 # Shortcut method for creating a Enumerable EachSQL object for the given input.
 # @param[String] input Input script.
 # @param[Symbol] The type of the input SQL script. :default, :mysql, and :oracle (or :plsql)
-# @return[EachSQL] Enumerable 
+# @yield[String] Executable SQL statement or block.
+# @return[Array] Array of executable SQL statements and blocks.
 def EachSQL input, type = :default
-  esql = EachSQL.new(input, EachSQL::Defaults[type])
+  esql   = EachSQL.new(type)
+  ret    = []
+  result = {}
+
+  process = lambda {
+    return if esql.empty?
+    result = esql.shift
+    sqls   = result[:sqls]
+    sqls.each do |sql|
+      if block_given?
+        yield sql
+      else
+        ret << sql
+      end
+    end
+  }
 
-  if block_given?
-    esql.each do |sql|
-      yield sql
+  input.to_s.each_line do |line|
+    case line
+    when /^\s*delimiter\s+(\S+)/i
+      process.call
+      if esql.empty?
+        esql.delimiter = $1
+      else
+        esql << line
+      end
+    when /#{Regexp.escape esql.delimiter}/
+      esql << line
+      process.call
+    else
+      esql << line
     end
-  else
-    esql
   end
-end
-
-class EachSQL
-  # EachSQL::Default Hash is a set of pre-defined parsing rules
-  # - :default: Default parsing rules for vendor-independent SQL scripts
-  # - :mysql:   Parsing rules for MySQL scripts. Understands `delimiter' statements.
-  # - :oracle:  Parsing rules for Oracle scripts. Removes trailing slashes after begin-end blocks.
-  Defaults = {
-    :default => {
-    :delimiter => /;+/,
-    :blocks => {
-      /`/          => /`/,
-      /"/          => /"/,
-      /'/          => /'/,
-      /\/\*[^+]/   => /\*\//,
-      /--+/        => $/,
-    },
-    :nesting_blocks => {
-      /\bdeclare.*?;\s*?begin\b/im => /;\s*?end\b/i,
-      /\bbegin\b/i => /;\s*?end\b/i,
-    },
-    :nesting_context => [
-      /\A\s*(begin|declare|create\b[^;]+?\b(procedure|function|trigger|package))\b/im
-    ],
-      :callbacks => {},
-      :ignore    => [],
-      :replace   => {},
-      # Let's assume we don't change delimiters within usual sql scripts
-      :strip_delimiter => lambda { |obj, stmt| stmt.sub(/\A;+/, '').sub(/;+\Z/, '') }
-    },
 
-    :mysql => {
-      :delimiter => /;+|delimiter\s+\S+/i,
-      :blocks => {
-        /`/          => /`/,
-        /"/          => /"/,
-        /'/          => /'/,
-        /\/\*[^+]/   => /\*\//,
-        /--+/        => $/,
-      },
-      :nesting_blocks => {
-        /\bbegin\b/i => /\bend\b/i
-      },
-      :nesting_context => [
-          /\A\s*(begin|create\b[^;]+?\b(procedure|function|trigger))\b/im
-      ],
-      # We need to change delimiter on `delimiter' command
-      :callbacks => {
-        /^\s*delimiter\s+(\S+)/i => lambda { |obj, stmt, md|
-          new_delimiter = Regexp.new(Regexp.escape md[1])
-          obj.delimiter = /(#{new_delimiter})+|delimiter\s+\S+/i
-          obj.delimiter_string = md[1]
-        }
-      },
-      :ignore => [
-        /^delimiter\s+\S+$/i
-      ],
-        :replace => {},
-        :strip_delimiter => lambda { |obj, stmt|
-        stmt.gsub(/(#{Regexp.escape(obj.delimiter_string || ';')})+\Z/, '')
-      }
-    },
+  if !esql.empty?
+    process.call
+  end
 
-    :oracle => {
-      :delimiter => /;+/,
-      :blocks => {
-        /`/          => /`/,
-        /"/          => /"/,
-        /'/          => /'/,
-        /\/\*[^+]/   => /\*\//,
-        /--+/        => $/,
-      },
-      :nesting_blocks => {
-        /\bbegin\b/i => /\bend\b/i,
-        /\bdeclare.*?;\s*?begin\b/im => {
-        :closer => %r{;\s*/}m,
-        # Stops immediately
-        :pop => true
-        },
-        /\bcreate[^;]+?\b(procedure|function|trigger|package)\b/im => {
-          :closer => %r{;\s*/}m,
-          # Stops immediately
-          :pop => true
-        }
-      },
-      :nesting_context => [
-        /\A\s*(\/\s*)*(begin|declare|create\b[^;]+?\b(procedure|function|trigger|package))\b/im
-      ],
-        :callbacks => {
-        /\Abegin\b/ => lambda { |obj, stmt, md|
-         # Oracle needs this
-         stmt << ';' if stmt !~ /;\Z/
-        }
-      },
-      :ignore => [],
-      :replace => { %r[\A/] => '' },
-      :strip_delimiter => lambda { |obj, stmt|
-        stmt.gsub(/(#{stmt =~ /;\s*\// ? '/' : ';'})+\Z/, '')
-      }
-    }
-  }
-  Defaults[:plsql] = Defaults[:oracle] # alias
+  if sql = result[:leftover]
+    if block_given?
+      yield sql
+    else
+      ret << sql
+    end
+  end
 
-  # Freeze the Hash
-  Defaults.freeze
+  ret
 end
 

data/lib/each_sql/each_sql.rb

@@ -1,173 +1,108 @@
-# encoding: UTF-8
-# Junegunn Choi (junegunn.c@gmail.com)
+require 'stringio'
 
 # Enumerable EachSQL object.
 class EachSQL
   include Enumerable
 
-  def initialize input, options
-    raise NotImplementedError.new if options.nil?
-    # immutables
-    @org_input = input && input.sub(/\A#{[65279].pack('U*')}/, '') # BOM
-    @options = options
-    @blocks = @options[:blocks]
-    @nblocks = @options[:nesting_blocks]
-    @all_blocks = @blocks.merge @nblocks
-  end
-
-  def each
-    return nil if @org_input.nil? || @org_input.empty?
-    @input = @org_input.dup
+  # @param[Symbol] type RDBMS type: :default|:mysql|:oracle|:postgres
+  def initialize type
+    @type   = type
+    @data   = ''
+    @sqls   = []
 
-    # Zero out comments and string literals to simplify subsequent parsing
-    @input_c = zero_out @org_input
-
-    @delimiter = @options[:delimiter]
-    while @input && @input.length > 0
-      # Extract a statement
-      statement = next_statement
+    self.delimiter = ';'
+  end
 
-      # When a non-empty statement is found
-      statement = @options[:strip_delimiter].call self, statement if @options[:strip_delimiter]
-      if statement.length > 0
-        # Apply replacements
-        @options[:replace].each do |k, v|
-          statement.gsub!(k, v)
-        end
-        statement.strip!
+  # @param[String] delim SQL delimiter
+  # @return[EachSQL]
+  def delimiter= delim
+    @delim = delim
+    @parser = EachSQL::Parser.parser_for @type, delim
+    self
+  end
 
-        # Process callbacks
-        @options[:callbacks].each do |pattern, callback|
-          #md = statement.match pattern
-          md = zero_out(statement).strip.match pattern
-          callback.call self, statement, md if md
-        end
+  # @return[String] Current delimiter.
+  def delimiter
+    @delim
+  end
 
-        # Ignore
-        if (@options[:ignore] || []).all? { |ipat| statement !~ ipat } && statement.empty? == false
-          yield statement
-          @prev_statement = statement
-        end
-      end
+  # Appends the given String to the buffer.
+  # @param[String] input String to append
+  def << input
+    if input
+      @data << input.sub(/\A#{[65279].pack('U*')}/, '') # BOM (FIXME)
     end
-    nil
+    self
   end
 
-  # To change delimiter while parsing the input
-  attr_accessor :delimiter, :delimiter_string
-
-  private
-  def zero_out input
-    output = input.dup
-    idx = 0
-    # Look for the closest block
-    while true
-      block_start, opener_length, opener, closer = @blocks.map { |opener, closer|
-        md = match output, opener, idx
-        [md && md[:begin], md && md[:length], opener, closer]
-      }.reject { |e| e.first.nil? }.min_by(&:first)
-      break if block_start.nil?
-
-      md = match output, closer, block_start + opener_length
-      idx = block_end = md ? md[:end] : (output.length-1)
-
-      output[block_start...block_end] = ' ' * (block_end - block_start)
-    end
-    output
+  # Parses current buffer and returns the result in Hash.
+  # :sqls is an Array of processed executable SQL blocks,
+  # :leftover is the unparsed trailing data
+  # @return [Hash]
+  def shift
+    result   = @parser.parse @data
+    @data    = result.captures[:leftover].join
+    leftover = strip_sql(@data)
+    {
+      :sqls =>
+        result.captures[:execution_block].map { |b| strip_sql b },
+      :leftover => leftover.empty? ? nil : leftover
+    }
   end
 
-  def next_statement
-    @cur = 0
+  # Return is the buffer is empty
+  # @return [Boolean]
+  def empty?
+    @data.gsub(/\s/, '').empty?
+  end
 
-    while process_next_block != :done
+  # Parses the buffer and enumerates through the executable blocks.
+  # @yield [String]
+  # @return [NilClass]
+  def each
+    result = shift
+    sqls   = (result[:sqls] + result[:leftover]).
+              map { |sql| strip_sql(sql) }.
+              reject(&:empty?)
+    sqls.each do |sql|
+      yield sql
     end
-
-    ret = @input[0...@cur].strip
-    @input   =   @input[@cur..-1]
-    @input_c = @input_c[@cur..-1]
-    return ret
   end
 
-  def process_next_block expect = nil
-    # Look for the closest delimiter
-    md = match @input_c, @delimiter, @cur
-    delim_start = md ? md[:begin] : @input.length
-    delim_end   = md ? md[:end]   : @input.length
-
-    # Look for the closest block depending on the current context
-    target_blocks = 
-      if @options[:nesting_context].any? {|pat| @input_c.match pat }
-        @all_blocks
-      else
-        @blocks
-      end
-
-    block_start, body_start, opener, closer = target_blocks.map { |opener, closer|
-      closer = closer[:closer] if closer.is_a? Hash
-      md = match @input_c, opener, @cur
-      [md && md[:begin], md && md[:end], opener, closer]
-    }.reject { |e| e.first.nil? }.min_by(&:first)
-
-    # If we're nested, look for the parent's closer as well
-    if expect && (md = match @input_c, expect, @cur) &&
-      (block_start.nil? || md[:begin] < block_start)
-
-      @cur = md[:end]
-      return :nest_closer
+  private
+  def strip_sql sql
+    # Preprocess
+    case @type
+    when :oracle
+      sql = sql.sub(/\A[\s\/]+/, '').sub(/[\s\/]+\Z/, '')
     end
 
-    # No block until the next delimiter
-    if block_start.nil? || block_start > delim_start
-      @cur = delim_end
-      return :done
+    # FIXME: Infinite loop?
+    # sql = sql.gsub(
+    #         /
+    #          (?:
+    #            (?:\A(?:#{Regexp.escape @delim}|[\s]+)+)
+    #            |
+    #            (?:(?:#{Regexp.escape @delim}|[\s]+)+\Z)
+    #          )+
+    #         /x, '')
+    prev_sql = nil
+    delim = Regexp.escape @delim
+    while prev_sql != sql
+      prev_sql = sql
+      sql = sql.strip.sub(/\A(?:#{delim})+/, '').sub(/(?:#{delim})+\Z/, '')
     end
 
-    # We found a block. Look for the end of it
-    @cur = body_start
-
-    # If nesting block, we go deeper
-    if @nblocks.keys.include? opener
-      while true
-        ret = process_next_block(closer)
-        break if ret == :nest_closer
-        throw_exception(closer) if @cur >= @input.length - 1
+    # Postprocess
+    case @type
+    when :oracle
+      if sql =~ /\bend(\s+\S+)?\Z/i
+        sql = sql + ';'
       end
-      return :done if @nblocks[opener].is_a?(Hash) && @nblocks[opener][:pop]
-
-      # If non-nesting block, just skip through it
-    else
-      skip_through_block closer
     end
 
-    return :continue
+    sql
   end
 
-  # For Ruby 1.8 compatibility
-  def match str, pat, idx
-    md = str[idx..-1].match(pat)
-    return nil if md.nil?
-
-    result = {
-      :begin => md && (md.begin(0) + idx),
-      :length => md && md[0].length,
-      :end => md && (md.end(0) + idx)
-    }
-    result
-  end
-
-  def skip_through_block closer
-    md = match @input_c, closer, @cur
-    throw_exception(closer) if md.nil?
-
-    @cur = md[:end]
-  end
-
-  def throw_exception closer
-    raise ArgumentError.new(
-      "Unclosed block: was expecting #{closer.inspect} " +
-      "while processing #{(@input[0, 60] + ' ... ').inspect}" + 
-      (@prev_statement ?
-       " after #{@prev_statement.inspect}" : ""))
-  end
 end#EachSQL