sql2avro 0.2.0 → 0.3.0

data/lib/interface/mysql.rb

@@ -47,7 +47,7 @@ class MySql < DbInterface
     end
   end
 
-  def data(table, min_id, max_id)
+  def data(table, min_id, max_id, &block)
     columns = nil
     rows = []
 
@@ -100,11 +100,8 @@ class MySql < DbInterface
         end
       end
 
-      rows << row
+      block.call(row)
     end
-
-    # TODO: stream this data out rather than return all in one batch.
-    rows
   end
 
   def sql_schema(table)

data/lib/sql2avro.rb

@@ -52,7 +52,7 @@ module Sql2Avro
     begin
       json_file = "#{filename}.json"
       File.open(json_file, 'w') do |f|
-        interface.data(table, min_id, max_id_this_batch).each do |datum|
+        interface.data(table, min_id, max_id_this_batch) do |datum|
           Yajl::Encoder.encode(datum, f)
           f.write "\n"
         end
@@ -61,7 +61,7 @@ module Sql2Avro
       cmd = "java -jar #{AVRO_TOOLS_PATH} fromjson --codec snappy --schema '#{schema}' #{json_file} > #{filename}"
       `#{cmd}`
 
-      `rm #{json_file}`
+      #`rm #{json_file}`
     rescue Exception => e
       retval[:error] = "#{e}\n\n#{e.backtrace}"
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sql2avro
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-06-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70241121702160 !ruby/object:Gem::Requirement
+  requirement: &70363993723780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241121702160
+  version_requirements: *70363993723780
 description: sql2avro extracts data from a specified SQL database table and transforms
   it into an Avro file with a schema based on the database table's schema. The intended
   use case is to incrementally load data out of an SQL database and into HDFS for
@@ -52,7 +52,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4019534732048256908
+      hash: 1878707749575910576
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -61,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4019534732048256908
+      hash: 1878707749575910576
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10