sql2avro 0.1.0

data/Makefile

@@ -0,0 +1,6 @@
+vendor/avro-tools-1.7.4.jar:
+	curl http://www.us.apache.org/dist/avro/avro-1.7.4/java/avro-tools-1.7.4.jar > vendor/$@
+
+sql2avro-0.1.0.gem: sql2avro.gemspec
+	bundle exec gem build $<
+

data/lib/interface/interface.rb

@@ -0,0 +1,10 @@
+class DbInterface
+  def schema(table)
+    raise "Return Avro JSON schema for #{table}"
+  end
+
+  def data(table)
+    raise "Return Avro JSON data for #{table}"
+  end
+end
+

data/lib/interface/mysql.rb

@@ -0,0 +1,203 @@
+require_relative './interface'
+require 'open3'
+
+class MySql < DbInterface
+  MYSQL_BATCH_SEP = "\t"
+
+  # config is a hash with this form (like ActiveRecord's):
+  # {
+  #   host:     "localhost",
+  #   username: "myuser",
+  #   password: "mypass",
+  #   database: "somedatabase"
+  # }
+  #
+  def initialize(config)
+    @db_host = config['host']
+    @db_name = config['database']
+    @username = config['username']
+    @password = config['password']
+  end
+
+  def schema(table)
+    types = avro_types(table)
+
+    schema = {
+      type: "record",
+      name: table,
+      fields: []
+    }
+
+    types.each do |k,v|
+      schema[:fields] << { name: k, type: ['null', v] }
+    end
+
+    schema
+  end
+
+  def max_id(table)
+    header_seen = false
+    query("SELECT MAX(id) FROM #{table}") do |line|
+      unless header_seen
+        header_seen = true
+        next
+      end
+
+      return line.first.to_i
+    end
+  end
+
+  def data(table, min_id, max_id)
+    columns = nil
+    rows = []
+
+    types = avro_types(table)
+
+    sql = """
+      SELECT *
+      FROM #{table}
+      WHERE id >= #{min_id}
+        AND id <= #{max_id}
+    """
+    query(sql) do |line|
+      # Get header.
+      if columns.nil?
+        columns = line
+        next
+      end
+
+      # Construct row mapping column names to values of appropriate type.
+      row = (0...columns.length).each_with_object({}) do |i, h|
+        colname = columns[i]
+        value = line[i]
+
+        # NOTE: all non-null type values are wrapped in a mapping from type to value,
+        # because that's what the Avro spec requires; see:
+        #  - http://avro.apache.org/docs/current/spec.html#json_encoding
+        #  - http://mail-archives.apache.org/mod_mbox/avro-user/201304.mbox/%3CCD86687D.E892E%25scott@richrelevance.com%3E
+
+        # Handle nulls.
+        if value == "NULL"
+          h[columns[i]] = nil
+          next
+        end
+
+        # Perform any necessary typecasts.
+        type = types[colname]
+        h[colname] = case type
+        when 'boolean'
+          { type => value.to_i.zero? }
+        when 'int','long'
+          { type => value.to_i }
+        when 'float','double'
+          { type => value.to_f }
+        when 'bytes'
+          { type => value }
+        when 'string'
+          { type => value }
+        else
+          raise "Unsupported type: #{type}"
+        end
+      end
+
+      rows << row
+    end
+
+    # TODO: stream this data out rather than return all in one batch.
+    rows
+  end
+
+  def sql_schema(table)
+    header_seen = false
+    columns = {}
+
+    query("DESCRIBE #{table}") do |line|
+      if header_seen == false
+        header_seen = true
+        next
+      end
+
+      name, type = line[0], line[1]
+      columns[name] = type
+    end
+
+    columns
+  end
+
+  def avro_types(table)
+    mysql_types = sql_schema(table)
+
+    types = {}
+    mysql_types.each do |k,v|
+      types[k] = MySql.avro_type(v)
+    end
+
+    types
+  end
+
+  def query(sql, &block)
+    MySql.query(sql, @db_host, @db_name, @username, @password, &block)
+  end
+
+  def self.query(sql, db_host, db_name, username, password, &block)
+    cmd = %{
+      mysql \\
+        --batch \\
+        --execute="#{sql}" \\
+        --host #{db_host} \\
+        --user #{username} \\
+        --password=#{password} \\
+        --quick \\
+        #{db_name}
+    }
+
+    Open3.popen3(cmd) do |stdin, stdout, stderr|
+      while (line = stdout.gets)
+        block.call(line.chop.split(MYSQL_BATCH_SEP))
+      end
+    end
+  end
+
+  def self.avro_type(mysql_type)
+    # Refer to https://github.com/apache/sqoop/blob/trunk/src/java/org/apache/sqoop/manager/ConnManager.java#L172.
+
+    case mysql_type
+
+    # See https://dev.mysql.com/doc/refman/5.0/en/numeric-type-overview.html
+    when /tinyint\(1\)/, /bool/, /boolean/
+      'boolean'
+    when /tinyint/, /smallint/, /mediumint/, /integer/, /int/
+      'int'
+    when /bigint/, /serial/
+      'long'
+    when /decimal/, /dec/
+      'string'
+    when /float/
+      'float'
+    when /double/
+      'double'
+    when /varchar\(\d+\)/
+      'string'
+
+    # See https://dev.mysql.com/doc/refman/5.0/en/date-and-time-type-overview.html.
+    when /date/, /datetime/, /time/, /timestamp/
+      'string'
+    when /year/
+      'int'
+
+    # See https://dev.mysql.com/doc/refman/5.0/en/string-type-overview.html.
+    when /char/, /varchar/
+      'string'
+    when /binary/, /varbinary/
+      'bytes'
+    when /tinytext/, /text/, /longtext/
+      'string'
+    when /tinyblob/, /blob/, /longblob/
+      'bytes'
+    else
+      raise "Unsupported MySQL data type: #{mysql_type}"
+    end
+  end
+
+end
+

data/lib/sql2avro.rb

@@ -0,0 +1,63 @@
+require 'open3'
+require 'yaml'
+require 'yajl'
+require_relative 'interface/mysql'
+
+module Sql2Avro
+  AVRO_TOOLS_PATH = File.expand_path('../vendor/avro-tools-1.7.4.jar', __FILE__)
+
+
+  # Pulls data from the given database table starting from the given id.
+  #
+  # This function creates an Avro file as a side effect, and returns {
+  #   max_id: greatest ID that was pulled in,
+  #   path: filepath of the resulting avroized file
+  #   error: error message, if any; otherwise omitted
+  # }
+  #
+  # database_config is a hash with this form (like ActiveRecord's):
+  # {
+  #   adapter:  "mysql",
+  #   host:     "localhost",
+  #   username: "myuser",
+  #   password: "mypass",
+  #   database: "somedatabase"
+  # }
+  #
+  # table is the table to pull from.
+  #
+  # min_id specifies the value of the id column from which to start.
+  def Sql2Avro.avroize(database_config, table, min_id)
+    raise "Database interface not specified." if !database_config.has_key? 'adapter'
+    raise "Database interface not supported: #{database_config['adapter']}" if database_config['adapter'] != 'mysql'
+
+    interface = MySql.new(database_config)
+
+    schema = Yajl::Encoder.encode(interface.schema(table))
+    max_id = interface.max_id(table)
+
+    date, time, zone = Time.now.utc.to_s.split
+    filename = "#{table}.#{date}T#{time}Z.#{min_id}.#{max_id}.avro"
+
+    retval = {
+      max_id: max_id,
+      path: filename
+    }
+
+    begin
+      Open3.popen3("java -jar #{AVRO_TOOLS_PATH} fromjson --codec snappy --schema '#{schema}' /dev/stdin > #{filename}") do |stdin, stdout, stderr, wait_thr|
+        $stdout = stdout
+
+        interface.data(table, min_id, max_id).each do |datum|
+          Yajl::Encoder.encode(datum, stdin)
+          stdin.write "\n"
+        end
+      end
+    rescue
+      retval[:error] = $!
+    end
+
+    retval
+  end
+end
+

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: sql2avro
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Mason Simon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-20 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &70282006753460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70282006753460
+description: sql2avro extracts data from a specified SQL database table and transforms
+  it into an Avro file with a schema based on the database table's schema. The intended
+  use case is to incrementally load data out of an SQL database and into HDFS for
+  analysis via Hadoop.
+email:
+- mason@verbasoftware.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/interface/interface.rb
+- lib/interface/mysql.rb
+- lib/sql2avro.rb
+- vendor/avro-tools-1.7.4.jar
+- Makefile
+homepage: https://github.com/Verba/sql2avro
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1867060671403551677
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1867060671403551677
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Tool for pulling data from SQL database tables into Avro files.
+test_files: []