each_sql 0.1.0 → 0.2.0

data/CHANGELOG.rdoc

@@ -0,0 +1,7 @@
+=== 0.2.0 / 2011/06/17
+* Second release. Handles more cases.
+* Ruby 1.8 compatible
+
+=== 0.1.0 / 2011/06/15
+* Initial release. Turned out to be very flawed :p
+

data/Gemfile.lock

@@ -8,8 +8,10 @@ GEM
       rake
     rake (0.8.7)
     rcov (0.9.9)
+    rcov (0.9.9-java)
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES

data/README.rdoc

@@ -31,9 +31,7 @@ Enumerate each SQL statement in the given SQL script.
 - pgplsql support.
 
 == Warning
-Should be good enough for regular SQLs, but it is not guaranteed to parse stored procedures correctly.
-I thought this would be simple when I first started, but actually it wasn't, and the code now is a mess.
-Use it at your own risk.
+Stored procedure handling is at best incomplete. Use it at your own risk.
 
 == Contributing to each_sql
  

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/each_sql/each_sql.rb

@@ -8,7 +8,7 @@ class EachSQL
 	def initialize input, options
 		raise NotImplementedError.new if options.nil?
 		# immutables
-		@org_input = input
+		@org_input = input.sub(/\A#{[65279].pack('U*')}/, '') # BOM
 		@options = options
 		@blocks = @options[:blocks]
 		@nblocks = @options[:nesting_blocks]
@@ -16,21 +16,17 @@ class EachSQL
 	end
 
 	def each
-		@input = @org_input
+		# Phase 1: comment out input
+		@input = @org_input.dup
+		@input_c = zero_out @org_input
+
 		return nil if @input.nil? || @input.empty?
 
 		@delimiter = @options[:delimiter]
 
-		reset_cursor
-		while @input
-			# First look for next delimiter, this is to reduce the search space for blocks.
-			extend_scope
-
-			# We're done. Finished. Period. Out!
-			break if scope.empty?
-
+		while @input && @input.length > 0
 			# Extract a statement
-			statement = extract_statement
+			statement = next_statement
 
 			# When a non-empty statement is found
 			statement = @options[:strip_delimiter].call self, statement if @options[:strip_delimiter]
@@ -48,7 +44,8 @@ class EachSQL
 				end
 
 				# Ignore
-				if (@options[:ignore] || []).all? { |ipat| statement !~ ipat }
+				if statement.length > 0 && 
+						(@options[:ignore] || []).all? { |ipat| statement !~ ipat }
 					yield statement
 					@prev_statement = statement
 				end
@@ -59,75 +56,84 @@ class EachSQL
 
 	attr_accessor :delimiter, :delimiter_string
 private
+	def zero_out input
+		output = input.dup
+		idx = 0
+		# Look for the closest block
+		while true
+			block_start, opener_length, opener, closer = @blocks.map { |opener, closer|
+				md = match output, opener, idx
+				[md && md[:begin], md && md[:length], opener, closer]
+			}.reject { |e| e.first.nil? }.min_by(&:first)
+			break if block_start.nil?
 
-	def extract_statement
-		while process_next_block != :not_found
-		end
-
-		ret = scope.strip
-		@input = @input[@next_head..-1]
-		reset_cursor
-		return ret
-	end
-
-	def reset_cursor
-		@cur = @from = @to = 0
-	end
+			md = match output, closer, block_start + opener_length
+			idx = block_end = md ? md[:end] : (output.length-1)
 
-	def scope
-		@to ? @input[0, @to] : @input
+			output[block_start...block_end] = ' ' * (block_end - block_start)
+		end
+		output
 	end
 
-	def extend_scope
-		md = @input.match @delimiter, @to
+	def next_statement
+		@cur = 0
 
-		if md
-			@to = md.begin(0) + md[0].length
-			@next_head = @to
-		else
-			@to = @input.length
-			@next_head = @input.length
+		while process_next_block != :done
 		end
-		#puts "Extended: #{scope.inspect} #{@input[@next_head..-1]}"
+
+		ret = @input[0...@cur].strip
+		@input   = @input[@cur..-1]
+		@input_c = @input_c[@cur..-1]
+		return ret
 	end
 
 	def process_next_block expect = nil
+		# Look for the closest delimiter
+		md = match @input_c, @delimiter, @cur
+		delim_start = md ? md[:begin] : @input.length
+		delim_end   = md ? md[:end] : @input.length
+
 		# Look for the closest block
-		block_start, opener_length, opener, closer = @all_blocks.map { |opener, closer|
+		target_blocks = 
+			if @options[:nesting_context].any? {|pat| @input_c.match pat }
+				@all_blocks
+			else
+				@blocks
+			end
+
+		block_start, body_start, opener, closer = target_blocks.map { |opener, closer|
 			closer = closer[:closer] if closer.is_a? Hash
-			md = scope.match(opener, @cur)
-			[md && md.begin(0), md && md[0].length, opener, closer]
+			md = match @input_c, opener, @cur
+			[md && md[:begin], md && md[:end], opener, closer]
 		}.reject { |e| e.first.nil? }.min_by(&:first)
 
-		# p [scope, scope[@cur..-1], expect, scope.index(expect, @cur), block_start] if expect
-		# We found a block, but after the end of the nesting block
-		if expect &&
-				(prev_end = scope.index(expect, @cur)) && 
-				(block_start.nil? || prev_end <= block_start)
-			skip_through_block expect, prev_end == block_start
-			return :end_nest
+		# If we're nested, look for the parent's closer as well
+		if expect && (md = match @input_c, expect, @cur) &&
+				(block_start.nil? || md[:begin] < block_start)
+
+			@cur = md[:end]
+			return :nest_closer
 		end
 	
-		# If no block in this scope
-		return :not_found if block_start.nil?
+		# No block until the next delimiter
+		if block_start.nil? || block_start > delim_start
+			@cur = delim_end
+			return :done
+		end
+
+		# #####################################
 
 		# We found a block. Look for the end of it
-		@cur = block_start + opener_length
+		@cur = body_start
 
 		# If nesting block, we go deeper
 		if @nblocks.keys.include? opener
-			@prev_delimiter = @delimiter
-			if @nblocks[opener].is_a? Hash
-				@delimiter = @nblocks[opener][:delimiter] || @delimiter
-			end
 			while true
 				ret = process_next_block(closer)
-
-				break if ret == :end_nest
-				extend_scope if ret == :not_found
-				throw_exception(closer) if scope.length == @input.length
+				break if ret == :nest_closer
+				throw_exception(closer) if @cur >= @input.length - 1
 			end
-			@delimiter = @prev_delimiter
+			return :done if @nblocks[opener].is_a?(Hash) && @nblocks[opener][:pop]
 
 		# If non-nesting block, just skip through it
 		else
@@ -137,22 +143,29 @@ private
 		return :continue
 	end
 
-	def skip_through_block closer, rewind_delimiter = false
-		md = @input.match closer, @cur
-		block_end = md && md.begin(0)
+	def match str, pat, idx
+		md = str[idx..-1].match(pat)
+		return nil if md.nil?
 
-		throw_exception(closer) if block_end.nil?
+		result = {
+			:begin => md && (md.begin(0) + idx),
+			:length => md && md[0].length,
+			:end => md && (md.end(0) + idx)
+		}
+		result
+	end
 
-		@cur = block_end + (rewind_delimiter ? 0 : md[0].length)
-		while @cur > scope.length
-			extend_scope
-		end
+	def skip_through_block closer
+		md = match @input_c, closer, @cur
+		throw_exception(closer) if md.nil?
+
+		@cur = md[:end]
 	end
 
 	def throw_exception closer
 		raise ArgumentError.new(
 				"Unclosed block: was expecting #{closer.inspect} " +
-				"while processing #{$/ + scope.inspect}" + 
+				"while processing #{(@input[0, 60] + ' ... ').inspect}" + 
 				(@prev_statement ?
 					" after #{@prev_statement.inspect}" : ""))
 	end

data/lib/each_sql.rb

@@ -20,14 +20,19 @@ class EachSQL
 		:default => {
 			:delimiter => /;+/,
 			:blocks => {
+				/`/          => /`/,
+				/"/          => /"/,
 				/'/          => /'/,
 				/\/\*[^+]/   => /\*\//,
 				/--+/        => $/,
 			},
 			:nesting_blocks => {
-				/\bdeclare\b/i => /\bbegin\b/i,
-				/\bbegin\b/i => /\bend\b/i
+				/\bdeclare.*?;\s*?begin\b/im => /;\s*?end\b/i,
+				/\bbegin\b/i => /;\s*?end\b/i,
 			},
+			:nesting_context => [
+				/\A\s*(begin|declare|create\b[^;]+?\b(procedure|function|trigger|package))\b/im
+			],
 			:callbacks => {},
 			:ignore    => [],
 			:replace   => {},
@@ -38,6 +43,8 @@ class EachSQL
 		:mysql => {
 			:delimiter => /;+|delimiter\s+\S+/i,
 			:blocks => {
+				/`/          => /`/,
+				/"/          => /"/,
 				/'/          => /'/,
 				/\/\*[^+]/   => /\*\//,
 				/--+/        => $/,
@@ -45,11 +52,14 @@ class EachSQL
 			:nesting_blocks => {
 				/\bbegin\b/i => /\bend\b/i
 			},
+			:nesting_context => [
+				/\A\s*(begin|create\b[^;]+?\b(procedure|function|trigger))\b/im
+			],
 			# We need to change delimiter on `delimiter' command
 			:callbacks => {
 				/^\s*delimiter\s+(\S+)/i => lambda { |obj, stmt, md|
 					new_delimiter = Regexp.new(Regexp.escape md[1])
-					obj.delimiter = /#{new_delimiter}+|delimiter\s+\S+/i
+					obj.delimiter = /(#{new_delimiter})+|delimiter\s+\S+/i
 					obj.delimiter_string = md[1]
 				}
 			},
@@ -58,29 +68,45 @@ class EachSQL
 			],
 			:replace => {},
 			:strip_delimiter => lambda { |obj, stmt|
-				stmt.chomp(obj.delimiter_string || ';')
+				stmt.gsub(/(#{Regexp.escape(obj.delimiter_string || ';')})+\Z/, '')
 			}
 		},
 
 		:oracle => {
 			:delimiter => /;+/,
 			:blocks => {
+				/`/          => /`/,
+				/"/          => /"/,
 				/'/          => /'/,
 				/\/\*[^+]/   => /\*\//,
 				/--+/        => $/,
 			},
 			:nesting_blocks => {
 				/\bbegin\b/i => /\bend\b/i,
-				/\bcreate[^;]*\b(procedure|function|trigger|package)\b/im => {
+				/\bdeclare.*?;\s*?begin\b/im => {
 					:closer => %r{;\s*/}m,
-					:delimiter => /;\s*\//
+					# Stops immediately
+					:pop => true
+				},
+				/\bcreate[^;]+?\b(procedure|function|trigger|package)\b/im => {
+					:closer => %r{;\s*/}m,
+					# Stops immediately
+					:pop => true
+				}
+			},
+			:nesting_context => [
+				/\A\s*(begin|declare|create\b[^;]+?\b(procedure|function|trigger|package))\b/im
+			],
+			:callbacks => {
+				/\Abegin\b/ => lambda { |obj, stmt, md|
+					# Oracle needs this
+					stmt << ';' if stmt !~ /;\Z/
 				}
 			},
-			:callbacks => {},
 			:ignore => [],
-			:replace => {},
+			:replace => { %r[\A/] => '' },
 			:strip_delimiter => lambda { |obj, stmt| obj 
-				stmt.chomp( stmt =~ /;\s*\// ? '/' : ';' )
+				stmt.gsub(/(#{stmt =~ /;\s*\// ? '/' : ';'})+\Z/, '')
 			}
 		}
 	}

data/test/test_each_sql.rb

@@ -1,24 +1,22 @@
+# encoding: UTF-8
+
 $LOAD_PATH << File.dirname(__FILE__)
 require 'helper'
 
 class TestEachSql < Test::Unit::TestCase
 	def setup
 	@sql = [
-"select * from a",
-"select
-	*
-from
-	b",
-"select 'abc', 'abc;', 'abc''', 'abc/*', 'abc--' from c",
-
-"select
-	/*+ help */ *
-from
-	d",
-"select * from /* block comment ; */ e",
-"select * 
-from -- line comment ; /* ;; */
-f",
+"-------------- begin-end block;
+declare
+	/* end; */
+	/* begin */
+	null;
+	null;
+	null;
+begin
+	/* end */
+	null;
+end",
 "-------------- begin-end block;
 begin
 	-- begin-end block;
@@ -34,16 +32,21 @@ begin
 	-- end
 	/* end */
 end",
-"-------------- begin-end block;
-declare
-	/* end; */
-	/* begin */
-	null;
-	null;
-	null;
-begin
-	/* end */
-end",
+"select * from a",
+"select
+	*
+from
+	b",
+"select 'abc', 'abc;', 'abc''', 'abc/*', 'abc--' from c",
+
+"select
+	/*+ help */ *
+from
+	d",
+"select * from /* block comment ; */ e",
+"select * 
+from -- line comment ; /* ;; */
+f",
 "-------------- begin-end block;
 declare
 	/* end; */
@@ -66,13 +69,79 @@ begin
 	/* end */
 end",
 "select * from dual",
-"select * from dual"]
+"select b `begin` from dual",
+'select b "begin" from dual',
+'select 
+	begin , begin.* from begin'
+]
 
-	@oracle = "
+	@oracle = [
+'select * from dual',
+'create /* procedure */ sequence a',
+"create package something as
+	procedure log;
+	procedure log;
+	procedure log;
+end something;",
+"Create or replace Procedure tmmp(p1 number default 'begin', p2 number) as
+    str number(8, 2) := 1 / 4;
+begin
+	1 / 2;
+	begin
+		1 / 4;
+		null;
+	end;
+exception
+    when others then
+        raise;
+end;",
+"declare
+	a int;
+begin
+	1 / 2;
+	begin
+		1 / 4;
+		null;
+	end;
+exception
+    when others then
+        raise;
+end;",
+"begin
+	null;
+end;",
+"select * from dual",
+"select begin, end, create, procedure, end, from dual",
+"select * from dual"
+	]
+		
+	@oracle_script = "
 select * from dual;
-Create or replace Procedure tmmp(p1 number, p2 number) as
+create /* procedure */ sequence a;
+create package something as
+	procedure log;
+	procedure log;
+	procedure log;
+end something;
+/
+
+Create or replace Procedure tmmp(p1 number default 'begin', p2 number) as
     str number(8, 2) := 1 / 4;
 begin
+	1 / 2;
+	begin
+		1 / 4;
+		null;
+	end;
+exception
+    when others then
+        raise;
+end;
+/
+declare
+	a int;
+begin
+	1 / 2;
 	begin
 		1 / 4;
 		null;
@@ -82,15 +151,53 @@ exception
         raise;
 end;
 /
-select * from dual;"
+begin
+	null;
+end;
+/
+select * from dual;
+;
+;
+;
 
-	@mysql = "
+
+;;;;;;
+;
+
+select begin, end, create, procedure, end, from dual;
+select * from dual;
+"
+
+	@mysql = [
+"drop procedure if exists proc",
+"create procedure proc(p1 int, p2 int)
+begin
+    null;
+	begin
+		null;
+	end;
+end",
+"drop procedure if exists proc2",
+"create procedure proc(p1 int, p2 int)
+begin
+    null;
+
+end",
+"select * from dual",
+"select b `begin` from dual",
+'select b "begin" from dual',
+'select 
+	begin , begin.* from begin'
+	]
+	@mysql_script = "
 delimiter //
 drop procedure if exists proc //
 create procedure proc(p1 int, p2 int)
 begin
     null;
-
+	begin
+		null;
+	end;
 end //
 delimiter ;
 
@@ -103,7 +210,11 @@ begin
 end $$
 delimiter ;
 
-select * from dual;"
+select * from dual;
+select b `begin` from dual;
+select b \"begin\" from dual;
+select 
+	begin , begin.* from begin"
 	end
 
 	def test_sql
@@ -117,16 +228,18 @@ select * from dual;"
 	end
 	
 	def test_oracle
-		EachSQL(@oracle, :oracle).each_with_index do |sql,idx|
+		EachSQL(@oracle_script, :oracle).each_with_index do |sql,idx|
 			puts sql
 			puts '-' * 40
+			assert_equal @oracle[idx], sql
 		end
 	end
 
 	def test_mysql
-		EachSQL(@mysql, :mysql).each_with_index do |sql,idx|
+		EachSQL(@mysql_script, :mysql).each_with_index do |sql,idx|
 			puts sql
 			puts '-' * 40
+			assert_equal @mysql[idx], sql
 		end
 	end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: each_sql
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-06-15 00:00:00.000000000Z
+date: 2011-06-16 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2161738720 !ruby/object:Gem::Requirement
+  requirement: &2161248000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2161738720
+  version_requirements: *2161248000
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2161738240 !ruby/object:Gem::Requirement
+  requirement: &2161247520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.2
   type: :development
   prerelease: false
-  version_requirements: *2161738240
+  version_requirements: *2161247520
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &2161737760 !ruby/object:Gem::Requirement
+  requirement: &2161247040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161737760
+  version_requirements: *2161247040
 description: Enumerate through each SQL statement in SQL scripts.
 email: junegunn.c@gmail.com
 executables: []
@@ -53,6 +53,7 @@ extra_rdoc_files:
 - README.rdoc
 files:
 - .document
+- CHANGELOG.rdoc
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -79,7 +80,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4303771573274782197
+      hash: 2241318967940806787
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: