sql2avro 0.3.1 → 0.3.2

data/lib/interface/mysql.rb

@@ -148,10 +148,14 @@ class MySql < DbInterface
         #{db_name}
     }
 
-    Open3.popen3(cmd) do |i, o, e|
+    Open3.popen3(cmd) do |i, o, e, wait_thr|
       while (line = o.gets)
         block.call(line.chop.split(MYSQL_BATCH_SEP))
       end
+
+      if !wait_thr.value.success?
+        raise "Error querying mysql\n\nQuery: #{cmd}\nSTDERR: #{e.read}\nSTATUS: #{wait_thr.value}"
+      end
     end
   end
 

data/lib/sql2avro.rb

@@ -29,7 +29,7 @@ module Sql2Avro
   # min_id specifies the value of the id column from which to start.
   def Sql2Avro.avroize(database_config, table, min_id, max_rows_per_batch=nil)
     raise "Database interface not specified." if !database_config.has_key? 'adapter'
-    raise "Database interface not supported: #{database_config['adapter']}" if database_config['adapter'] != 'mysql'
+    raise "Database interface not supported: #{database_config['adapter']}" unless ['mysql', 'mysql2'].include? database_config['adapter']
 
     interface = MySql.new(database_config)
 
@@ -50,6 +50,9 @@ module Sql2Avro
     }
 
     begin
+      prev_default_internal = Encoding.default_internal
+      Encoding.default_internal = nil
+
       json_file = "#{filename}.json"
       File.open(json_file, 'w') do |f|
         interface.data(table, min_id, max_id_this_batch) do |datum|
@@ -58,10 +61,18 @@ module Sql2Avro
         end
       end
 
+      Encoding.default_internal = prev_default_internal
+
       cmd = "java -jar #{AVRO_TOOLS_PATH} fromjson --codec snappy --schema '#{schema}' #{json_file} > #{filename}"
       `#{cmd}`
+      if !$?.success?
+        raise "Error converting JSON to Avro.\n\nCommand: #{cmd}\nStatus: #{$?}"
+      end
 
       `rm #{json_file}`
+      if !$?.success?
+        raise "Error deleting temporary JSON file #{json_file}"
+      end
     rescue Exception => e
       retval[:error] = "#{e}\n\n#{e.backtrace}"
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sql2avro
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-21 00:00:00.000000000Z
+date: 2013-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70328438994660 !ruby/object:Gem::Requirement
+  requirement: &70297282972760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70328438994660
+  version_requirements: *70297282972760
 description: sql2avro extracts data from a specified SQL database table and transforms
   it into an Avro file with a schema based on the database table's schema. The intended
   use case is to incrementally load data out of an SQL database and into HDFS for
@@ -52,7 +52,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3312347816830426626
+      hash: -3119210657153955613
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -61,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3312347816830426626
+      hash: -3119210657153955613
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10