samidare 0.1.2 → 0.2.0

data/lib/samidare/embulk.rb

@@ -1,61 +1,61 @@
-module Samidare
-  class Embulk
-    def run(database_configs, all_table_configs, bq_config, target_table_names = [])
-      error_tables = []
-      database_configs.keys.each do |db_name|
-        table_configs = target_table_configs(all_table_configs[db_name], target_table_names)
-        error_tables = error_tables + run_by_database(
-          db_name,
-          table_configs,
-          database_configs[db_name]['bq_dataset'],
-          bq_config)
-      end
-      error_tables
-    end
-
-    def target_table_configs(table_configs, target_table_names)
-      return table_configs if target_table_names.empty?
-      table_configs.select { |table_config| target_table_names.include?(table_config.name) }
-    end
-
-    private
-    def run_by_database(db_name, table_configs, bq_dataset, bq_config)
-      process_times = []
-      error_tables = []
-      big_query = Samidare::BigQueryUtility.new(bq_config)
-      table_configs.each do |table_config|
-        start_time = Time.now
-        log "table: #{table_config.name} - start"
-
-        begin
-          big_query.delete_table(bq_dataset, table_config.name)
-          log "table: #{table_config.name} - deleted"
-        rescue
-          log "table: #{table_config.name} - does not exist"
-        end
-
-        cmd = "embulk run #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml"
-        log "cmd: #{cmd}"
-        if system(cmd)
-          result = 'success'
-        else
-          result = 'error'
-          error_tables << table_config.name
-        end
-
-        process_time = "table: #{table_config.name} - result: #{result}  #{sprintf('%10.1f', Time.now - start_time)}sec"
-        log process_time
-        process_times << process_time
-      end
-      log '------------------------------------'
-      log "db_name: #{db_name}"
-      process_times.each { |process_time| log process_time }
-
-      error_tables
-    end
-
-    def log(message)
-      puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
-    end
-  end
+module Samidare
+  class Embulk
+    def run(database_configs, all_table_configs, bq_config, target_table_names = [])
+      error_tables = []
+      database_configs.keys.each do |db_name|
+        table_configs = target_table_configs(all_table_configs[db_name], target_table_names)
+        error_tables = error_tables + run_by_database(
+          db_name,
+          table_configs,
+          database_configs[db_name]['bq_dataset'],
+          bq_config)
+      end
+      error_tables
+    end
+
+    def target_table_configs(table_configs, target_table_names)
+      return table_configs if target_table_names.empty?
+      table_configs.select { |table_config| target_table_names.include?(table_config.name) }
+    end
+
+    private
+    def run_by_database(db_name, table_configs, bq_dataset, bq_config)
+      process_times = []
+      error_tables = []
+      big_query = Samidare::BigQueryUtility.new(bq_config)
+      table_configs.each do |table_config|
+        start_time = Time.now
+        log "table: #{table_config.name} - start"
+
+        begin
+          big_query.delete_table(bq_dataset, table_config.name)
+          log "table: #{table_config.name} - deleted"
+        rescue
+          log "table: #{table_config.name} - does not exist"
+        end
+
+        cmd = "embulk run #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml"
+        log "cmd: #{cmd}"
+        if system(cmd)
+          result = 'success'
+        else
+          result = 'error'
+          error_tables << table_config.name
+        end
+
+        process_time = "table: #{table_config.name} - result: #{result}  #{sprintf('%10.1f', Time.now - start_time)}sec"
+        log process_time
+        process_times << process_time
+      end
+      log '------------------------------------'
+      log "db_name: #{db_name}"
+      process_times.each { |process_time| log process_time }
+
+      error_tables
+    end
+
+    def log(message)
+      puts "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
+    end
+  end
 end

data/lib/samidare/embulk_utility.rb

@@ -1,42 +1,42 @@
-module Samidare
-  module EmbulkUtility
-    class ConfigGenerator
-      def generate_config(database_configs, bq_config)
-        bq_utility = BigQueryUtility.new(bq_config)
-
-        database_configs.keys.each do |db_name|
-          database_config = database_configs[db_name]
-          table_configs = all_table_configs[db_name]
-          mysql_client = MySQL::MySQLClient.new(database_config)
-
-          table_configs.each do |table_config|
-            write(
-              "#{bq_config['schema_dir']}/#{db_name}",
-              "#{table_config.name}.json",
-              mysql_client.generate_bq_schema(table_config.name)
-            )
-            write(
-              "#{bq_config['config_dir']}/#{db_name}",
-              "#{table_config.name}.yml",
-              bq_utility.generate_embulk_config(
-                db_name,
-                database_config,
-                table_config,
-                mysql_client.columns(table_config.name))
-            )
-          end
-        end
-      end
-
-      private
-      def write(directory, file_name, content)
-        FileUtils.mkdir_p(directory) unless FileTest.exist?(directory)
-        File.write("#{directory}/#{file_name}", content)
-      end
-
-      def all_table_configs
-        @all_table_configs ||= MySQL::TableConfig.generate_table_configs
-      end
-    end
-  end
-end
+module Samidare
+  module EmbulkUtility
+    class ConfigGenerator
+      def generate_config(database_configs, bq_config)
+        bq_utility = BigQueryUtility.new(bq_config)
+
+        database_configs.keys.each do |db_name|
+          database_config = database_configs[db_name]
+          table_configs = all_table_configs[db_name]
+          mysql_client = MySQL::MySQLClient.new(database_config)
+
+          table_configs.each do |table_config|
+            write(
+              "#{bq_config['schema_dir']}/#{db_name}",
+              "#{table_config.name}.json",
+              mysql_client.generate_bq_schema(table_config.name)
+            )
+            write(
+              "#{bq_config['config_dir']}/#{db_name}",
+              "#{table_config.name}.yml",
+              bq_utility.generate_embulk_config(
+                db_name,
+                database_config,
+                table_config,
+                mysql_client.columns(table_config.name))
+            )
+          end
+        end
+      end
+
+      private
+      def write(directory, file_name, content)
+        FileUtils.mkdir_p(directory) unless FileTest.exist?(directory)
+        File.write("#{directory}/#{file_name}", content)
+      end
+
+      def all_table_configs
+        @all_table_configs ||= MySQL::TableConfig.generate_table_configs
+      end
+    end
+  end
+end

data/lib/samidare/mysql.rb

@@ -1,117 +1,117 @@
-require 'mysql2-cs-bind'
-require 'json'
-require 'yaml'
-require 'fileutils'
-require 'samidare/bigquery_utility'
-
-module Samidare
-  module MySQL
-    class MySQLClient
-      COLUMN_SQL = <<-SQL
-        SELECT column_name, data_type
-        FROM INFORMATION_SCHEMA.COLUMNS
-        WHERE table_schema = ?
-        AND table_name = ?
-        ORDER BY ordinal_position
-      SQL
-
-      def initialize(database_config)
-        @database_config = database_config
-      end
-
-      def client
-        @client ||= Mysql2::Client.new(
-          :host => @database_config['host'],
-          :username => @database_config['username'],
-          :password => @database_config['password'],
-          :database => @database_config['database'])
-      end
-
-      def generate_bq_schema(table_name)
-        infos = columns(table_name)
-        BigQueryUtility.generate_schema(infos)
-      end
-
-      def columns(table_name)
-        rows = client.xquery(COLUMN_SQL, @database_config['database'], table_name)
-        rows.map { |row| Column.new(row['column_name'], row['data_type']) }
-      end
-    end
-
-    class TableConfig
-      attr_reader :name, :daily_snapshot, :condition
-
-      def initialize(config)
-        @name = config['name']
-        @daily_snapshot = config['daily_snapshot'] || false
-        @condition = config['condition']
-      end
-
-      def self.generate_table_configs(file_path = 'table.yml')
-        configs = YAML.load_file(file_path)
-        configs.each_with_object({}) do |(db, database_config), table_configs|
-          table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
-          table_configs
-        end
-      end
-
-      def ==(another)
-        self.instance_variables.all? do |v|
-          self.instance_variable_get(v) == another.instance_variable_get(v)
-        end
-      end
-    end
-
-    class Column
-      attr_reader :column_name, :data_type
-
-      TYPE_MAPPINGS = {
-        'int' => 'integer',
-        'tinyint' => 'integer',
-        'smallint' => 'integer',
-        'mediumint' => 'integer',
-        'bigint' => 'integer',
-        'float' => 'float',
-        'double' => 'float',
-        'decimal' => 'float',
-        'char' => 'string',
-        'varchar' => 'string',
-        'tinytext' => 'string',
-        'text' => 'string',
-        'date' => 'timestamp',
-        'datetime' => 'timestamp',
-        'timestamp' => 'timestamp'
-      }
-
-      def initialize(column_name, data_type)
-        @column_name = column_name
-        @data_type = data_type
-      end
-
-      def bigquery_data_type
-        TYPE_MAPPINGS[@data_type]
-      end
-
-      def converted_value
-        if bigquery_data_type == 'timestamp'
-          # time zone translate to UTC
-          "UNIX_TIMESTAMP(#{escaped_column_name}) AS #{escaped_column_name}"
-        elsif data_type == 'tinyint'
-          # for MySQL tinyint(1) problem
-          "CAST(#{escaped_column_name} AS signed) AS #{escaped_column_name}"
-        else
-          escaped_column_name
-        end
-      end
-
-      def to_json(*a)
-        { "name" => @column_name, "type" => bigquery_data_type }.to_json(*a)
-      end
-
-      private
-      def escaped_column_name
-        "`#{@column_name}`"
-      end
-    end
-  end
-end
+require 'mysql2-cs-bind'
+require 'json'
+require 'yaml'
+require 'fileutils'
+require 'samidare/bigquery_utility'
+
+module Samidare
+  module MySQL
+    class MySQLClient
+      COLUMN_SQL = <<-SQL
+        SELECT column_name, data_type
+        FROM INFORMATION_SCHEMA.COLUMNS
+        WHERE table_schema = ?
+        AND table_name = ?
+        ORDER BY ordinal_position
+      SQL
+
+      def initialize(database_config)
+        @database_config = database_config
+      end
+
+      def client
+        @client ||= Mysql2::Client.new(
+          :host => @database_config['host'],
+          :username => @database_config['username'],
+          :password => @database_config['password'],
+          :database => @database_config['database'])
+      end
+
+      def generate_bq_schema(table_name)
+        infos = columns(table_name)
+        BigQueryUtility.generate_schema(infos)
+      end
+
+      def columns(table_name)
+        rows = client.xquery(COLUMN_SQL, @database_config['database'], table_name)
+        rows.map { |row| Column.new(row['column_name'], row['data_type']) }
+      end
+    end
+
+    class TableConfig
+      attr_reader :name, :daily_snapshot, :condition
+
+      def initialize(config)
+        @name = config['name']
+        @daily_snapshot = config['daily_snapshot'] || false
+        @condition = config['condition']
+      end
+
+      def self.generate_table_configs(file_path = 'table.yml')
+        configs = YAML.load_file(file_path)
+        configs.each_with_object({}) do |(db, database_config), table_configs|
+          table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
+          table_configs
+        end
+      end
+
+      def ==(another)
+        self.instance_variables.all? do |v|
+          self.instance_variable_get(v) == another.instance_variable_get(v)
+        end
+      end
+    end
+
+    class Column
+      attr_reader :column_name, :data_type
+
+      TYPE_MAPPINGS = {
+        'int' => 'integer',
+        'tinyint' => 'integer',
+        'smallint' => 'integer',
+        'mediumint' => 'integer',
+        'bigint' => 'integer',
+        'float' => 'float',
+        'double' => 'float',
+        'decimal' => 'float',
+        'char' => 'string',
+        'varchar' => 'string',
+        'tinytext' => 'string',
+        'text' => 'string',
+        'date' => 'timestamp',
+        'datetime' => 'timestamp',
+        'timestamp' => 'timestamp'
+      }
+
+      def initialize(column_name, data_type)
+        @column_name = column_name
+        @data_type = data_type
+      end
+
+      def bigquery_data_type
+        TYPE_MAPPINGS[@data_type]
+      end
+
+      def converted_value
+        if bigquery_data_type == 'timestamp'
+          # time zone translate to UTC
+          "UNIX_TIMESTAMP(#{escaped_column_name}) AS #{escaped_column_name}"
+        elsif data_type == 'tinyint'
+          # for MySQL tinyint(1) problem
+          "CAST(#{escaped_column_name} AS signed) AS #{escaped_column_name}"
+        else
+          escaped_column_name
+        end
+      end
+
+      def to_json(*a)
+        { "name" => @column_name, "type" => bigquery_data_type }.to_json(*a)
+      end
+
+      private
+      def escaped_column_name
+        "`#{@column_name}`"
+      end
+    end
+  end
+end

data/lib/samidare/version.rb

@@ -1,3 +1,3 @@
-module Samidare
-  VERSION = "0.1.2"
-end
+module Samidare
+  VERSION = "0.2.0"
+end

data/samidare.gemspec

@@ -25,9 +25,9 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'unindent', '1.0'
   spec.add_dependency 'mysql2-cs-bind', '0.0.6'
-  spec.add_dependency 'embulk-output-bigquery', '0.1.7'
-  spec.add_dependency 'embulk-input-mysql', '0.6.0'
-  spec.add_dependency 'embulk-parser-jsonl', '0.0.1'
+  spec.add_dependency 'embulk-output-bigquery', '0.4.3'
+  spec.add_dependency 'embulk-input-mysql', '0.8.2'
+  spec.add_dependency 'embulk-parser-jsonl', '0.2.0'
   spec.add_dependency 'embulk-formatter-jsonl', '0.1.4'
   spec.add_dependency 'bigquery', '0.8.3'
 end