sql2avro 0.1.1 → 0.2.0

data/Makefile

@@ -1,4 +1,4 @@
-sql2avro-0.1.1.gem: sql2avro.gemspec
+sql2avro-*.gem: sql2avro.gemspec
 	bundle exec gem build $<
 
 vendor/avro-tools-1.7.4.jar:

data/lib/interface/mysql.rb

@@ -143,7 +143,7 @@ class MySql < DbInterface
     cmd = %{
       mysql \\
         --batch \\
-        --execute="#{sql}" \\
+        --execute="SET NAMES 'utf8'; #{sql}" \\
         --host #{db_host} \\
         --user #{username} \\
         --password=#{password} \\

data/lib/sql2avro.rb

@@ -27,7 +27,7 @@ module Sql2Avro
   # table is the table to pull from.
   #
   # min_id specifies the value of the id column from which to start.
-  def Sql2Avro.avroize(database_config, table, min_id)
+  def Sql2Avro.avroize(database_config, table, min_id, max_rows_per_batch=nil)
     raise "Database interface not specified." if !database_config.has_key? 'adapter'
     raise "Database interface not supported: #{database_config['adapter']}" if database_config['adapter'] != 'mysql'
 
@@ -35,19 +35,24 @@ module Sql2Avro
 
     schema = Yajl::Encoder.encode(interface.schema(table))
     max_id = interface.max_id(table)
+    max_id_this_batch = if max_rows_per_batch.nil?
+      max_id
+    else
+      [max_id, min_id + max_rows_per_batch].min
+    end
 
     date, time, zone = Time.now.utc.to_s.split
-    filename = "#{table}.#{date}T#{time}Z.#{min_id}.#{max_id}.avro"
+    filename = "#{table}.#{date}T#{time}Z.#{min_id}.#{max_id_this_batch}.avro"
 
     retval = {
-      max_id: max_id,
+      max_id: max_id_this_batch,
       path: filename
     }
 
     begin
       json_file = "#{filename}.json"
       File.open(json_file, 'w') do |f|
-        interface.data(table, min_id, max_id).each do |datum|
+        interface.data(table, min_id, max_id_this_batch).each do |datum|
           Yajl::Encoder.encode(datum, f)
           f.write "\n"
         end
@@ -57,8 +62,8 @@ module Sql2Avro
       `#{cmd}`
 
       `rm #{json_file}`
-    rescue
-      retval[:error] = $!.to_s
+    rescue Exception => e
+      retval[:error] = "#{e}\n\n#{e.backtrace}"
     end
 
     retval

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sql2avro
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-06-21 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70268699282980 !ruby/object:Gem::Requirement
+  requirement: &70241121702160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70268699282980
+  version_requirements: *70241121702160
 description: sql2avro extracts data from a specified SQL database table and transforms
   it into an Avro file with a schema based on the database table's schema. The intended
   use case is to incrementally load data out of an SQL database and into HDFS for
@@ -52,7 +52,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -933336779340994961
+      hash: -4019534732048256908
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -61,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -933336779340994961
+      hash: -4019534732048256908
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10