samidare 0.0.11 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eaced8c4bed3cea616f56ffe1bd1bf72776f103c
-  data.tar.gz: d3ef165a96055bc1dd51fbafef03268a76550ba1
+  metadata.gz: e2e4fe3e66626b0fd4eaf311505f35fd0b6c72e5
+  data.tar.gz: bf4ffdd752f991a74f55541a40068c2f560cc774
 SHA512:
-  metadata.gz: bb7a324305a0c9221fbcc505e91ff12f7171a9584d549502b9c1a55c755cb92a6fce0da381e11b044c35c24c29bd1a9ae97466a2caadb81e61261916e4f873de
-  data.tar.gz: 2db8446c60c1be68741871681c686c5a4fb3e08a9c087fa3f96ca9cdce8d1527e5abacd2d052c3b31e784ecee45c250d98b6c70f2baf4389367735f07115dae3
+  metadata.gz: cec779faaaefadbd2731fd3010ea735fadae9823231bf4307d3c7d1ff7d1eb29804563812d15129611937da6eb5adce08af9bcef0eef709d758d2bd5f9fd126e
+  data.tar.gz: 5959e5303709334e485676f55c522f435b0d03ea461f2e4cba7cf085c9b034118ca1ee0522af418290164d94c5f6759a6aa5e5130755d44e6ac7eceacb189263

data/lib/samidare.rb

@@ -1,58 +1,27 @@
 require 'samidare/version'
 require 'samidare/embulk_utility'
+require 'samidare/embulk'
+require 'samidare/mysql'
 
 module Samidare
   class EmbulkClient
-    def generate_config(config)
-      Samidare::EmbulkUtility::ConfigGenerator.new(config).generate_config
+    def generate_config(bq_config)
+      Samidare::EmbulkUtility::ConfigGenerator.new.generate_config(database_configs, bq_config)
     end
 
-    def run(config, target_tables = [])
-      db_infos = Samidare::EmbulkUtility::DBInfo.generate_db_infos
-      table_infos = Samidare::EmbulkUtility::TableInfo.generate_table_infos
-      db_infos.keys.each do |db_name|
-        embulk_run(db_name, db_infos[db_name]['bq_dataset'], table_infos[db_name], config, target_tables)
-      end
+    def run(bq_config, target_table_names = [])
+      error_tables = Samidare::Embulk.new.run(
+        database_configs,
+        Samidare::MySQL::TableConfig.generate_table_configs,
+        bq_config,
+        target_table_names)
+      # return batch status(true: all tables success)
+      error_tables.size == 0
     end
 
     private
-    def embulk_run(db_name, bq_dataset, tables, config, target_tables)
-      process_times = []
-      big_query = Samidare::BigQueryUtility.new(config)
-      tables.each do |table|
-        next unless target_table?(table.name, target_tables)
-
-        start_time = Time.now
-        log "table: #{table.name} - start"
-
-        begin
-          big_query.delete_table(bq_dataset, table.name)
-          log "table: #{table.name} - deleted"
-        rescue
-          log "table: #{table.name} - does not exist"
-        end
-
-        cmd = "embulk run #{config['config_dir']}/#{db_name}/#{table.name}.yml"
-        log "cmd: #{cmd}"
-        result = system(cmd) ? 'success' : 'error'
-
-        process_time = Time.now - start_time
-        message = "table: #{table.name} - result: #{result}  #{sprintf('%10.1f', process_time)}sec"
-        log message
-        process_times << message
-      end
-      log '------------------------------------'
-      log "db_name: #{db_name}"
-      process_times.each { |process_time| log process_time }
-    end
-
-    def target_table?(table, target_tables)
-      return true if target_tables.empty?
-      target_tables.include?(table)
-    end
-
-    def log(message)
-      puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
+    def database_configs
+      YAML.load_file('database.yml')
     end
   end
 end

data/lib/samidare/bigquery_utility.rb

@@ -39,33 +39,33 @@ module Samidare
       @current_date = Date.today
     end
 
-    def self.generate_schema(column_infos)
-      json_body = column_infos.map { |column_info| column_info.to_json }.join(",\n")
+    def self.generate_schema(columns)
+      json_body = columns.map { |column| column.to_json }.join(",\n")
       "[\n" + json_body + "\n]\n"
     end
 
-    def self.generate_sql(table_info, column_infos)
-      columns = column_infos.map { |column_info| column_info.converted_value }
+    def self.generate_sql(table_config, columns)
+      columns = columns.map { |column| column.converted_value }
       sql = "SELECT " + columns.join(",")
-      sql << " FROM #{table_info.name}"
-      sql << " WHERE #{table_info.condition}" if table_info.condition
+      sql << " FROM #{table_config.name}"
+      sql << " WHERE #{table_config.condition}" if table_config.condition
       sql << "\n"
       sql
     end
 
-    def generate_embulk_config(db_name, db_info, table_info, column_infos)
-      host = db_info['host']
-      user = db_info['username']
-      password = db_info['password']
-      database = db_info['database']
-      query = Samidare::BigQueryUtility.generate_sql(table_info, column_infos)
+    def generate_embulk_config(db_name, database_config, table_config, columns)
+      host = database_config['host']
+      user = database_config['username']
+      password = database_config['password']
+      database = database_config['database']
+      query = Samidare::BigQueryUtility.generate_sql(table_config, columns)
       project = @config['project_id']
       p12_keyfile_path = @config['key']
       service_account_email = @config['service_email']
-      dataset = db_info['bq_dataset']
-      table_name = actual_table_name(table_info.name, db_info['daily_snapshot'] || table_info.daily_snapshot)
-      schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_info.name}.json"
-      path_prefix = "/var/tmp/embulk_#{db_name}_#{table_info.name}"
+      dataset = database_config['bq_dataset']
+      table_name = actual_table_name(table_config.name, database_config['daily_snapshot'] || table_config.daily_snapshot)
+      schema_path = "#{@config['schema_dir']}/#{db_name}/#{table_config.name}.json"
+      path_prefix = "/var/tmp/embulk_#{db_name}_#{table_config.name}"
 
       ERB.new(CONTENTS).result(binding)
     end

data/lib/samidare/embulk.rb

@@ -0,0 +1,57 @@
+module Samidare
+  class Embulk
+    def run(database_configs, all_table_configs, bq_config, target_table_names = [])
+      error_tables = []
+      database_configs.keys.each do |db_name|
+        table_configs = target_table_configs(all_table_configs[db_name], target_table_names)
+        error_tables << run_by_database(db_name, table_configs, database_configs[db_name]['bq_dataset'], bq_config)
+      end
+      error_tables
+    end
+
+    def target_table_configs(table_configs, target_table_names)
+      return table_configs if target_table_names.empty?
+      table_configs.select { |table_config| target_table_names.include?(table_config.name) }
+    end
+
+    private
+    def run_by_database(db_name, table_configs, bq_dataset, bq_config)
+      process_times = []
+      error_tables = []
+      big_query = Samidare::BigQueryUtility.new(bq_config)
+      table_configs.each do |table_config|
+        start_time = Time.now
+        log "table: #{table_config.name} - start"
+
+        begin
+          big_query.delete_table(bq_dataset, table_config.name)
+          log "table: #{table_config.name} - deleted"
+        rescue
+          log "table: #{table_config.name} - does not exist"
+        end
+
+        cmd = "embulk run #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml"
+        log "cmd: #{cmd}"
+        if system(cmd)
+          result = 'success'
+        else
+          result = 'error'
+          error_tables << table_config.name
+        end
+
+        process_time = "table: #{table_config.name} - result: #{result}  #{sprintf('%10.1f', Time.now - start_time)}sec"
+        log process_time
+        process_times << process_time
+      end
+      log '------------------------------------'
+      log "db_name: #{db_name}"
+      process_times.each { |process_time| log process_time }
+
+      error_tables
+    end
+
+    def log(message)
+      puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
+    end
+  end
+end

data/lib/samidare/embulk_utility.rb

@@ -1,36 +1,28 @@
-require 'mysql2-cs-bind'
-require 'json'
-require 'yaml'
-require 'fileutils'
-require 'samidare/bigquery_utility'
-
 module Samidare
   module EmbulkUtility
     class ConfigGenerator
-      def initialize(bq_config)
-        @bq_config = bq_config.dup
-      end
-
-      def generate_config
-        db_infos = EmbulkUtility::DBInfo.generate_db_infos
-        table_infos = EmbulkUtility::TableInfo.generate_table_infos
-        bq_utility = BigQueryUtility.new(@bq_config)
+      def generate_config(database_configs, bq_config)
+        bq_utility = BigQueryUtility.new(bq_config)
 
-        db_infos.keys.each do |db_name|
-          db_info = db_infos[db_name]
-          table_info = table_infos[db_name]
-          mysql_client = EmbulkUtility::MySQLClient.new(db_info)
+        database_configs.keys.each do |db_name|
+          database_config = database_configs[db_name]
+          table_configs = all_table_configs[db_name]
+          mysql_client = MySQL::MySQLClient.new(database_config)
 
-          table_info.each do |table|
+          table_configs.each do |table_config|
             write(
-              "#{@bq_config['schema_dir']}/#{db_name}",
-              "#{table.name}.json",
-              mysql_client.generate_bq_schema(table.name)
+              "#{bq_config['schema_dir']}/#{db_name}",
+              "#{table_config.name}.json",
+              mysql_client.generate_bq_schema(table_config.name)
             )
             write(
-              "#{@bq_config['config_dir']}/#{db_name}",
-              "#{table.name}.yml",
-              bq_utility.generate_embulk_config(db_name, db_info, table, mysql_client.column_infos(table.name))
+              "#{bq_config['config_dir']}/#{db_name}",
+              "#{table_config.name}.yml",
+              bq_utility.generate_embulk_config(
+                db_name,
+                database_config,
+                table_config,
+                mysql_client.columns(table_config.name))
             )
           end
         end
@@ -41,122 +33,9 @@ module Samidare
         FileUtils.mkdir_p(directory) unless FileTest.exist?(directory)
         File.write("#{directory}/#{file_name}", content)
       end
-    end
-
-    class MySQLClient
-      COLUMN_INFO_SQL = <<-SQL
-        SELECT column_name, data_type
-        FROM INFORMATION_SCHEMA.COLUMNS
-        WHERE table_schema = ?
-        AND table_name = ?
-        ORDER BY ordinal_position
-      SQL
-
-      def initialize(db_info)
-        @db_info = db_info
-      end
-
-      def client
-        @client ||= Mysql2::Client.new(
-          :host => @db_info['host'],
-          :username => @db_info['username'],
-          :password => @db_info['password'],
-          :database => @db_info['database'])
-      end
-
-      def generate_bq_schema(table_name)
-        infos = column_infos(table_name)
-        BigQueryUtility.generate_schema(infos)
-      end
 
-      def column_infos(table_name)
-        rows = client.xquery(COLUMN_INFO_SQL, @db_info['database'], table_name)
-        rows.map { |row| ColumnInfo.new(row['column_name'], row['data_type']) }
-      end
-    end
-
-    class DBInfo
-      def self.generate_db_infos
-        configs = YAML.load_file('database.yml')
-        configs
-      end
-    end
-
-    class TableInfo
-      def initialize(config)
-        @config = config.dup
-      end
-
-      def self.generate_table_infos
-        configs = YAML.load_file('table.yml')
-        configs.each_with_object({}) do |(db, db_info), table_infos|
-          table_infos[db] = db_info['tables'].map { |config| TableInfo.new(config) }
-          table_infos
-        end
-      end
-
-      def name
-        @config['name']
-      end
-
-      def daily_snapshot
-        @config['daily_snapshot'] || false
-      end
-
-      def condition
-        @config['condition']
-      end
-    end
-
-    class ColumnInfo
-      attr_reader :column_name, :data_type
-
-      TYPE_MAPPINGS = {
-        'int' => 'integer',
-        'tinyint' => 'integer',
-        'smallint' => 'integer',
-        'mediumint' => 'integer',
-        'bigint' => 'integer',
-        'float' => 'float',
-        'double' => 'float',
-        'decimal' => 'float',
-        'char' => 'string',
-        'varchar' => 'string',
-        'tinytext' => 'string',
-        'text' => 'string',
-        'date' => 'timestamp',
-        'datetime' => 'timestamp',
-        'timestamp' => 'timestamp'
-      }
-
-      def initialize(column_name, data_type)
-        @column_name = column_name
-        @data_type = data_type
-      end
-
-      def bigquery_data_type
-        TYPE_MAPPINGS[@data_type]
-      end
-
-      def converted_value
-        if bigquery_data_type == 'timestamp'
-          # time zone translate to UTC
-          "UNIX_TIMESTAMP(#{escaped_column_name}) AS #{escaped_column_name}"
-        elsif data_type == 'tinyint'
-          # for MySQL tinyint(1) problem
-          "CAST(#{escaped_column_name} AS signed) AS #{escaped_column_name}"
-        else
-          escaped_column_name
-        end
-      end
-
-      def to_json(*a)
-        { "name" => @column_name, "type" => bigquery_data_type }.to_json(*a)
-      end
-
-      private
-      def escaped_column_name
-        "`#{@column_name}`"
+      def all_table_configs
+        @all_table_configs ||= MySQL::TableConfig.generate_table_configs
       end
     end
   end

data/lib/samidare/mysql.rb

@@ -0,0 +1,117 @@
+require 'mysql2-cs-bind'
+require 'json'
+require 'yaml'
+require 'fileutils'
+require 'samidare/bigquery_utility'
+
+module Samidare
+  module MySQL
+    class MySQLClient
+      COLUMN_SQL = <<-SQL
+        SELECT column_name, data_type
+        FROM INFORMATION_SCHEMA.COLUMNS
+        WHERE table_schema = ?
+        AND table_name = ?
+        ORDER BY ordinal_position
+      SQL
+
+      def initialize(database_config)
+        @database_config = database_config
+      end
+
+      def client
+        @client ||= Mysql2::Client.new(
+          :host => @database_config['host'],
+          :username => @database_config['username'],
+          :password => @database_config['password'],
+          :database => @database_config['database'])
+      end
+
+      def generate_bq_schema(table_name)
+        infos = columns(table_name)
+        BigQueryUtility.generate_schema(infos)
+      end
+
+      def columns(table_name)
+        rows = client.xquery(COLUMN_SQL, @database_config['database'], table_name)
+        rows.map { |row| Column.new(row['column_name'], row['data_type']) }
+      end
+    end
+
+    class TableConfig
+      attr_reader :name, :daily_snapshot, :condition
+
+      def initialize(config)
+        @name = config['name']
+        @daily_snapshot = config['daily_snapshot'] || false
+        @condition = config['condition']
+      end
+
+      def self.generate_table_configs(file_path = 'table.yml')
+        configs = YAML.load_file(file_path)
+        configs.each_with_object({}) do |(db, database_config), table_configs|
+          table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
+          table_configs
+        end
+      end
+
+      def ==(another)
+        self.instance_variables.all? do |v|
+          self.instance_variable_get(v) == another.instance_variable_get(v)
+        end
+      end
+    end
+
+    class Column
+      attr_reader :column_name, :data_type
+
+      TYPE_MAPPINGS = {
+        'int' => 'integer',
+        'tinyint' => 'integer',
+        'smallint' => 'integer',
+        'mediumint' => 'integer',
+        'bigint' => 'integer',
+        'float' => 'float',
+        'double' => 'float',
+        'decimal' => 'float',
+        'char' => 'string',
+        'varchar' => 'string',
+        'tinytext' => 'string',
+        'text' => 'string',
+        'date' => 'timestamp',
+        'datetime' => 'timestamp',
+        'timestamp' => 'timestamp'
+      }
+
+      def initialize(column_name, data_type)
+        @column_name = column_name
+        @data_type = data_type
+      end
+
+      def bigquery_data_type
+        TYPE_MAPPINGS[@data_type]
+      end
+
+      def converted_value
+        if bigquery_data_type == 'timestamp'
+          # time zone translate to UTC
+          "UNIX_TIMESTAMP(#{escaped_column_name}) AS #{escaped_column_name}"
+        elsif data_type == 'tinyint'
+          # for MySQL tinyint(1) problem
+          "CAST(#{escaped_column_name} AS signed) AS #{escaped_column_name}"
+        else
+          escaped_column_name
+        end
+      end
+
+      def to_json(*a)
+        { "name" => @column_name, "type" => bigquery_data_type }.to_json(*a)
+      end
+
+      private
+      def escaped_column_name
+        "`#{@column_name}`"
+      end
+    end
+  end
+end

data/lib/samidare/version.rb

@@ -1,3 +1,3 @@
 module Samidare
-  VERSION = "0.0.11"
+  VERSION = "0.1.0"
 end

data/spec/samidare/bigquery_utility_spec.rb

@@ -4,12 +4,12 @@ require 'timecop'
 
 describe Samidare::BigQueryUtility do
   describe '.generate_schema' do
-    subject { Samidare::BigQueryUtility.generate_schema(column_infos) }
+    subject { Samidare::BigQueryUtility.generate_schema(columns) }
 
-    let(:column_infos) { [
-      Samidare::EmbulkUtility::ColumnInfo.new('id', 'int'),
-      Samidare::EmbulkUtility::ColumnInfo.new('name', 'varchar'),
-      Samidare::EmbulkUtility::ColumnInfo.new('created_at', 'datetime')
+    let(:columns) { [
+      Samidare::MySQL::Column.new('id', 'int'),
+      Samidare::MySQL::Column.new('name', 'varchar'),
+      Samidare::MySQL::Column.new('created_at', 'datetime')
       ] }
     let(:schema_json) {
       <<-JSON.unindent
@@ -24,22 +24,22 @@ describe Samidare::BigQueryUtility do
   end
 
   describe '.generate_sql' do
-    subject { Samidare::BigQueryUtility.generate_sql(table_info, column_infos) }
+    subject { Samidare::BigQueryUtility.generate_sql(table_config, columns) }
 
-    let(:column_infos) { [
-      Samidare::EmbulkUtility::ColumnInfo.new('id', 'int'),
-      Samidare::EmbulkUtility::ColumnInfo.new('name', 'varchar'),
-      Samidare::EmbulkUtility::ColumnInfo.new('created_at', 'datetime')
+    let(:columns) { [
+      Samidare::MySQL::Column.new('id', 'int'),
+      Samidare::MySQL::Column.new('name', 'varchar'),
+      Samidare::MySQL::Column.new('created_at', 'datetime')
       ] }
 
     context 'no condition' do
-      let(:table_info) { Samidare::EmbulkUtility::TableInfo.new({ 'name' => 'simple' }) }
+      let(:table_config) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple' }) }
       let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple\n" }
       it { expect(subject).to eq sql }
     end
 
     context 'has condition' do
-      let(:table_info) { Samidare::EmbulkUtility::TableInfo.new({ 'name' => 'simple', 'condition' => 'created_at >= CURRENT_DATE() - INTERVAL 3 MONTH' }) }
+      let(:table_config) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple', 'condition' => 'created_at >= CURRENT_DATE() - INTERVAL 3 MONTH' }) }
       let(:sql) { "SELECT `id`,`name`,UNIX_TIMESTAMP(`created_at`) AS `created_at` FROM simple WHERE created_at >= CURRENT_DATE() - INTERVAL 3 MONTH\n" }
       it { expect(subject).to eq sql }
     end

data/spec/samidare/embulk_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Samidare::Embulk do
+  describe '#target_table_configs' do
+    subject { Samidare::Embulk.new(nil, nil).target_table_configs(table_configs, target_table_names) }
+
+    context 'all tables' do
+      let(:table_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge' }) }
+      let(:table_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
+      let(:table_configs) { [table_hoge, table_fuga] }
+      let(:target_table_names) { [] }
+      it { expect(subject).to match(table_configs) }
+    end
+
+    context 'target table selected' do
+      let(:table_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge' }) }
+      let(:table_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
+      let(:table_configs) { [table_hoge, table_fuga] }
+      let(:target_table_names) { ['hoge'] }
+      it { expect(subject).to match([table_hoge]) }
+    end
+  end
+end

data/spec/samidare/embulk_utility_spec.rb

@@ -1,15 +1,15 @@
 require 'spec_helper'
 
-describe Samidare::EmbulkUtility::ColumnInfo do
-  let(:column_info) { Samidare::EmbulkUtility::ColumnInfo.new(column_name, data_type) }
+describe Samidare::MySQL::Column do
+  let(:column) { Samidare::MySQL::Column.new(column_name, data_type) }
   let(:column_name) { 'id' }
   let(:data_type) { 'int' }
 
-  it { expect(column_info.column_name).to eq 'id' }
-  it { expect(column_info.data_type).to eq 'int' }
+  it { expect(column.column_name).to eq 'id' }
+  it { expect(column.data_type).to eq 'int' }
 
   describe '#bigquery_data_type' do
-    subject { column_info.bigquery_data_type }
+    subject { column.bigquery_data_type }
 
     context 'int' do
       let(:data_type) { 'int' }
@@ -88,7 +88,7 @@ describe Samidare::EmbulkUtility::ColumnInfo do
   end
 
   describe '#converted_value' do
-    subject { column_info.converted_value }
+    subject { column.converted_value }
 
     context 'datetime' do
       let(:column_name) { 'create_at' }
@@ -110,7 +110,7 @@ describe Samidare::EmbulkUtility::ColumnInfo do
   end
 
   describe '#to_json' do
-    subject { column_info.to_json }
+    subject { column.to_json }
 
     let(:column_name) { 'id' }
     let(:data_type) { 'int' }

data/spec/samidare/mysql_spec.rb

@@ -0,0 +1,135 @@
+require 'spec_helper'
+
+describe Samidare::MySQL::TableConfig do
+
+  describe '.generate_table_configs' do
+    subject { Samidare::MySQL::TableConfig.generate_table_configs('spec/support/table.yml') }
+    let(:db01_hoge) { Samidare::MySQL::TableConfig.new({ 'name' => 'hoge', 'daily_snapshot' => true }) }
+    let(:db01_simple) { Samidare::MySQL::TableConfig.new({ 'name' => 'simple' }) }
+    let(:db02_fuga) { Samidare::MySQL::TableConfig.new({ 'name' => 'fuga' }) }
+    let(:db02_with_condition) { Samidare::MySQL::TableConfig.new({ 'name' => 'with_condition', 'condition' => 'created_at < CURRENT_DATE()' }) }
+
+    it { expect(subject['db01'][0]).to eq db01_hoge }
+    it { expect(subject['db01'][1]).to eq db01_simple }
+    it { expect(subject['db02'][0]).to eq db02_fuga }
+    it { expect(subject['db02'][1]).to eq db02_with_condition }
+  end
+end
+
+describe Samidare::MySQL::Column do
+  let(:column) { Samidare::MySQL::Column.new(column_name, data_type) }
+  let(:column_name) { 'id' }
+  let(:data_type) { 'int' }
+
+  it { expect(column.column_name).to eq 'id' }
+  it { expect(column.data_type).to eq 'int' }
+
+  describe '#bigquery_data_type' do
+    subject { column.bigquery_data_type }
+
+    context 'int' do
+      let(:data_type) { 'int' }
+      it { expect(subject).to eq 'integer' }
+    end
+
+    context 'tinyint' do
+      let(:data_type) { 'tinyint' }
+      it { expect(subject).to eq 'integer' }
+    end
+
+    context 'smallint' do
+      let(:data_type) { 'smallint' }
+      it { expect(subject).to eq 'integer' }
+    end
+
+    context 'mediumint' do
+      let(:data_type) { 'mediumint' }
+      it { expect(subject).to eq 'integer' }
+    end
+
+    context 'bigint' do
+      let(:data_type) { 'bigint' }
+      it { expect(subject).to eq 'integer' }
+    end
+
+    context 'float' do
+      let(:data_type) { 'float' }
+      it { expect(subject).to eq 'float' }
+    end
+
+    context 'double' do
+      let(:data_type) { 'double' }
+      it { expect(subject).to eq 'float' }
+    end
+
+    context 'decimal' do
+      let(:data_type) { 'decimal' }
+      it { expect(subject).to eq 'float' }
+    end
+
+    context 'char' do
+      let(:data_type) { 'char' }
+      it { expect(subject).to eq 'string' }
+    end
+
+    context 'varchar' do
+      let(:data_type) { 'varchar' }
+      it { expect(subject).to eq 'string' }
+    end
+
+    context 'tinytext' do
+      let(:data_type) { 'tinytext' }
+      it { expect(subject).to eq 'string' }
+    end
+
+    context 'text' do
+      let(:data_type) { 'text' }
+      it { expect(subject).to eq 'string' }
+    end
+
+    context 'date' do
+      let(:data_type) { 'date' }
+      it { expect(subject).to eq 'timestamp' }
+    end
+
+    context 'datetime' do
+      let(:data_type) { 'datetime' }
+      it { expect(subject).to eq 'timestamp' }
+    end
+
+    context 'timestamp' do
+      let(:data_type) { 'timestamp' }
+      it { expect(subject).to eq 'timestamp' }
+    end
+  end
+
+  describe '#converted_value' do
+    subject { column.converted_value }
+
+    context 'datetime' do
+      let(:column_name) { 'create_at' }
+      let(:data_type) { 'datetime' }
+      it { expect(subject).to eq 'UNIX_TIMESTAMP(`create_at`) AS `create_at`' }
+    end
+
+    context 'int' do
+      let(:column_name) { 'id' }
+      let(:data_type) { 'int' }
+      it { expect(subject).to eq '`id`' }
+    end
+
+    context 'varchar' do
+      let(:column_name) { 'explanation' }
+      let(:data_type) { 'varchar' }
+      it { expect(subject).to eq '`explanation`' }
+    end
+  end
+
+  describe '#to_json' do
+    subject { column.to_json }
+
+    let(:column_name) { 'id' }
+    let(:data_type) { 'int' }
+    it { expect(subject).to eq '{"name":"id","type":"integer"}' }
+  end
+end

data/spec/samidare_spec.rb

@@ -1,30 +1,7 @@
-require 'spec_helper'
-
-describe Samidare do
-  it 'has a version number' do
-    expect(Samidare::VERSION).not_to be nil
-  end
-
-  describe Samidare::EmbulkClient do
-    describe '#target_table?' do
-      subject { Samidare::EmbulkClient.new.send(:target_table?, table, target_tables) }
-      context 'target_tables is empty' do
-        let(:table) { 'hoge' }
-        let(:target_tables) { [] }
-        it { expect(subject).to be_truthy }
-      end
-
-      context 'is included' do
-        let(:table) { 'hoge' }
-        let(:target_tables) { ['hoge'] }
-        it { expect(subject).to be_truthy }
-      end
-
-      context 'is not included' do
-        let(:table) { 'hoge' }
-        let(:target_tables) { ['fuga'] }
-        it { expect(subject).to be_falsy }
-      end
-    end
-  end
-end
+require 'spec_helper'
+
+describe Samidare do
+  it 'has a version number' do
+    expect(Samidare::VERSION).not_to be nil
+  end
+end

data/spec/support/databe.yml

@@ -0,0 +1,13 @@
+db01:
+  host: localhost
+  username: root
+  password:
+  database: embulk
+  bq_dataset: mysql
+
+db02:
+  host: localhost
+  username: root
+  password:
+  database: embulk2
+  bq_dataset: mysql2

data/spec/support/table.yml

@@ -0,0 +1,11 @@
+db01:
+  tables:
+    - name: hoge
+      daily_snapshot: true
+    - name: simple
+
+db02:
+  tables:
+    - name: fuga
+    - name: with_condition
+      condition: created_at < CURRENT_DATE()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: samidare
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.1.0
 platform: ruby
 authors:
 - Ryoji Kobori
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2015-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -179,13 +179,19 @@ files:
 - Rakefile
 - lib/samidare.rb
 - lib/samidare/bigquery_utility.rb
+- lib/samidare/embulk.rb
 - lib/samidare/embulk_utility.rb
+- lib/samidare/mysql.rb
 - lib/samidare/version.rb
 - samidare.gemspec
 - spec/samidare/bigquery_utility_spec.rb
+- spec/samidare/embulk_spec.rb
 - spec/samidare/embulk_utility_spec.rb
+- spec/samidare/mysql_spec.rb
 - spec/samidare_spec.rb
 - spec/spec_helper.rb
+- spec/support/databe.yml
+- spec/support/table.yml
 homepage: https://github.com/cobot00/samidare
 licenses:
 - MIT
@@ -212,6 +218,10 @@ specification_version: 4
 summary: Embulk utility for MySQL to BigQuery
 test_files:
 - spec/samidare/bigquery_utility_spec.rb
+- spec/samidare/embulk_spec.rb
 - spec/samidare/embulk_utility_spec.rb
+- spec/samidare/mysql_spec.rb
 - spec/samidare_spec.rb
 - spec/spec_helper.rb
+- spec/support/databe.yml
+- spec/support/table.yml