kiba-plus 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ab51f248407db0efeea72628dc83f639fa952f4
-  data.tar.gz: e1d93e552d093ea421c1330ee2d1dbcf3f563c0d
+  metadata.gz: b37f1fbad737a5141060feae03a0d99b98b20d42
+  data.tar.gz: f72a505eaf75bf1f24be63de334b63ecd955c881
 SHA512:
-  metadata.gz: 55cfec8dcfcad2f4224bcaf3d37d6aa2e7ca7831786cf0b099ec8800f4b3843b6d19e2ff06a8c57af11c10f8d91be3bf920c5410a3895bb3646f5228065e9471
-  data.tar.gz: 5468e321b8b4dbfb9d8174a41964e7b79c8d5d1c8af9129b751621d02759629bb170ef61c51999483df1b260900e096229ea7d5420e9bdbb3db421f51515c356
+  metadata.gz: 6f22ebbaa54bcf735a58235c9485342763aa8bcff593d292b6d055897db79a9963ffae44d71bdd9612f20cce76de77d2f6dbcdb04c0fc2b8369fd33734cf2f98
+  data.tar.gz: e7088d77e6e03bb90445f0424a72532fc264ef87503f17fb7e4468bd7d481de3b17fc9994c06b4aa042d66dcbde00c40cf6d68497748028e3c2102b7e61ba1dc

data/.gitignore

@@ -42,3 +42,6 @@ build/
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+/test/pg_copy_tmp/*
+!/test/pg_copy_tmp/.keep

data/.travis.yml

@@ -1,4 +1,25 @@
 language: ruby
+
 rvm:
   - 2.2.4
-before_install: gem install bundler -v 1.11.2
+
+services:
+  - mysql
+  - postgresql
+
+env:
+  - MYSQL2_SRC_CONNECT_URL=mysql2://travis@localhost/kiba_plus_src_test
+  - MYSQL2_DEST_CONNECT_URL=mysql2://travis@localhost/kiba_plus_dest_test
+  - PG_SRC_CONNECT_URL=postgresql://postgres@localhost/kiba_plus_src_test
+  - PG_DEST_CONNECT_URL=postgresql://postgres@localhost/kiba_plus_dest_test
+
+before_install:
+  - gem install bundler -v 1.11.2
+
+before_script:
+  - mysql -e 'create database kiba_plus_src_test;'
+  - mysql -e 'create database kiba_plus_dest_test;'
+  - psql -c 'create database kiba_plus_src_test;' -U postgres
+  - psql -c 'create database kiba_plus_dest_test;' -U postgres
+
+script: bundle exec rake test

data/README.md

@@ -4,6 +4,8 @@ Kiba enhancement for Ruby ETL. It connects to various data sources including rel
 # Usage
 
 ```ruby
+# /tmp/customer_mysql_to_pg.etl
+
 require 'kiba/plus'
 
 SOURCE_URL = 'mysql://root@localhost/shopperplus'
@@ -27,22 +29,33 @@ post_process do
 end
 ```
 
-Execute:
+Execute in shell:
 
 ```shell
-bundle exec kiba customer_mysql_to_pg.etl
-```
-
-Output:
+$ bundle exec kiba /tmp/customer_mysql_to_pg.etl
 
-```
 # Output:
 # I, [2016-05-16T01:53:36.832565 #87909]  INFO -- : TRUNCATE TABLE customers;
 # I, [2016-05-16T01:53:36.841770 #87909]  INFO -- : COPY customers (id, email, first_name, last_name) FROM STDIN WITH DELIMITER ',' NULL '\N' CSV
 # Insert total: 428972
 ```
 
-More Examples(TODO).
+Execute in ruby script:
+
+```ruby
+require 'kiba'
+
+job_definition = Kiba.parse(IO.read('/tmp/customer_mysql_to_pg.etl'), '/tmp/customer_mysql_to_pg.etl')
+Kiba.run(job_definition)
+```
+
+# Examples
+
+* [CSV to MySQL](https://github.com/hooopo/kiba-plus/blob/master/examples/customer_csv_to_mysql.etl)
+* [CSV to PG](https://github.com/hooopo/kiba-plus/blob/master/examples/customer_csv_to_pg.etl)
+* [MySQL to CSV](https://github.com/hooopo/kiba-plus/blob/master/examples/customer_mysql_to_csv.etl)
+* [MySQL to PG](https://github.com/hooopo/kiba-plus/blob/master/examples/customer_mysql_to_pg.etl)
+* [MySQL incremental to PG](https://github.com/hooopo/kiba-plus/blob/master/examples/incremental_insert.etl)
 
 # Main Feature
 
@@ -73,6 +86,7 @@ More Examples(TODO).
 Add this line to your application's Gemfile:
 
 ```ruby
+gem 'kiba'
 gem 'kiba-plus'
 ```
 
@@ -84,10 +98,21 @@ Or install it yourself as:
 
     $ gem install kiba-plus
 
-## Usage
+## Development
 
+First of all, Please run the following code in shell.
 
-## Development
+```bash
+
+$ mysql -e 'create database kiba_plus_src_test;'
+
+$ mysql -e 'create database kiba_plus_dest_test;'
+
+$ psql -c 'create database kiba_plus_src_test;' -U postgres
+
+$ psql -c 'create database kiba_plus_dest_test;' -U postgres
+
+```
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 

data/examples/customer_csv_to_mysql.etl

@@ -11,6 +11,6 @@ destination Kiba::Plus::Destination::MysqlBulk, { :connect_url => DEST_URL,
                                 :incremental => false
                               }
 post_process do
-  result = Mysql2::Client.new(connect_hash(DEST_URL)).query("SELECT COUNT(*) AS num FROM customers")
+  result = Mysql2::Client.new(mysql2_connect_hash(DEST_URL)).query("SELECT COUNT(*) AS num FROM customers")
   puts "Insert total: #{result.first['num']}"
 end

data/examples/customer_mysql_to_csv.etl

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
-DEST_URL = 'mysql://root@localhost/crm2_dev'
+SOURCE_URL = 'mysql://root@localhost/shopperplus'
 
 source Kiba::Plus::Source::Mysql, :connect_url => SOURCE_URL,
                            :query => %Q{SELECT id, email, 'hooopo' AS first_name, 'Wang' AS last_name FROM customers}

data/kiba-plus.gemspec

@@ -26,10 +26,15 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+
   spec.add_runtime_dependency "kiba", "~> 0.6"
   spec.add_runtime_dependency "mysql2", "~> 0.4"
   spec.add_runtime_dependency "pg", "~> 0.18"
+
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency 'database_cleaner', '~> 1.5.3'
+  spec.add_development_dependency 'sequel', '~> 4.34'
+  spec.add_development_dependency 'pry'
 end

data/lib/kiba/plus/destination/csv.rb

@@ -2,17 +2,17 @@ require 'csv'
 
 module Kiba::Plus::Destination
   class Csv
-    attr_reader :options
+    attr_reader :options, :csv
 
     def initialize(options = {})
       @options = options
       @options.assert_valid_keys(
         :output_file,
+        :mode,
         :row_sep,
         :col_sep,
         :force_quotes,
-        :quote_char,
-        :mode
+        :quote_char
       )
       @csv = CSV.open(output_file, mode, {
         :col_sep => col_sep,
@@ -22,36 +22,39 @@ module Kiba::Plus::Destination
       })
     end
 
-    def mode
-      options.fetch(:mode, "w")
+    def write(row)
+      @csv << row.values
     end
 
-    def output_file
-      options.fetch(:output_file)
+    def close
+      @csv.close
     end
 
-    def col_sep
-      options.fetch(:col_sep, ",")
-    end
+    private
 
-    def quote_char
-      options.fetch(:quote_char, '"')
+    def output_file
+      options.fetch(:output_file)
     end
 
-    def force_quotes
-      options.fetch(:force_quotes, false)
+    def mode
+      options.fetch(:mode, "w")
     end
 
     def row_sep
       options.fetch(:row_sep, "\n")
     end
 
-    def write(row)
-      @csv << row.values
+    def col_sep
+      options.fetch(:col_sep, ",")
     end
 
-    def close
-      @csv.close
+    def force_quotes
+      options.fetch(:force_quotes, false)
+    end
+
+    def quote_char
+      options.fetch(:quote_char, '"')
     end
+
   end
 end

data/lib/kiba/plus/destination/mysql.rb

@@ -3,7 +3,7 @@ require 'mysql2'
 module Kiba::Plus::Destination
   class Mysql
     include Kiba::Plus::Helper
-    attr_reader :options
+    attr_reader :options, :client
 
     def initialize(options = {})
       @options = options
@@ -12,8 +12,8 @@ module Kiba::Plus::Destination
         :prepare_sql,
         :columns
       )
-      @client = Mysql2::Client.new(connect_hash(connect_url))
-      @pre_stmt = @client.prepare(prepare_sql)
+      @client = Mysql2::Client.new(mysql2_connect_hash(connect_url))
+      init
     end
 
     def write(row)
@@ -21,6 +21,7 @@ module Kiba::Plus::Destination
     rescue => e
       Kiba::Plus.logger.error "ERROR for #{row}"
       Kiba::Plus.logger.error e.message
+      raise e
     end
 
     def close
@@ -30,6 +31,10 @@ module Kiba::Plus::Destination
 
     private
 
+    def init
+      @pre_stmt = @client.prepare(prepare_sql)
+    end
+
     def connect_url
       options.fetch(:connect_url)
     end

data/lib/kiba/plus/destination/mysql_bulk.rb

@@ -3,45 +3,53 @@ require 'mysql2'
 module Kiba::Plus::Destination
   class MysqlBulk
     include Kiba::Plus::Helper
-    attr_reader :options
+    attr_reader :options, :client
 
     def initialize(options = {})
       @options = options
-      @options.assert_valid_keys(:table_name,
-                                 :columns,
-                                 :input_file,
-                                 :connect_url,
-                                 :truncate,
-                                 :incremental,
-                                 :delimited_by,
-                                 :enclosed_by,
-                                 :ignore_lines
-                                 )
-
-      @client = Mysql2::Client.new(connect_hash(connect_url).merge(local_infile: true))
+      @options.assert_valid_keys(
+        :connect_url,
+        :table_name,
+        :columns,
+        :truncate,
+        :incremental,
+        :input_file,
+        :ignore_input_file_header,
+        :delimited_by,
+        :enclosed_by,
+        :ignore_lines
+      )
+
+      @client = Mysql2::Client.new(mysql2_connect_hash(connect_url).merge(local_infile: true))
     end
 
-    def connect_url
-      options.fetch(:connect_url)
+    def write(row)
+      # blank!
     end
 
-    def table_name
-      options.fetch(:table_name)
-    end
+    def close
+      if truncate
+        sql = truncate_sql
+        Kiba::Plus.logger.info sql
+        client.query(sql)
+      end
 
-    def delimited_by
-      options.fetch(:delimited_by, ",")
-    end
+      sql = bulk_sql
+      Kiba::Plus.logger.info sql
+      client.query(sql)
 
-    def enclosed_by
-      options.fetch(:enclosed_by, '"')
+      client.close
+      @client = nil
     end
 
-    def ignore_lines
-      options.fetch(:ignore_lines, 0)
+    private
+
+    def connect_url
+      options.fetch(:connect_url)
     end
 
-    def write(row)
+    def table_name
+      options.fetch(:table_name)
     end
 
     def columns
@@ -60,27 +68,46 @@ module Kiba::Plus::Destination
       options.fetch(:input_file)
     end
 
-    def close
-      if truncate
-        truncate_sql = "TRUNCATE TABLE #{table_name};"
-        Kiba::Plus.logger.info truncate_sql
-        @client.query(truncate_sql)
-      end
+    def ignore_input_file_header
+      !!options.fetch(:ignore_input_file_header, false)
+    end
+
+    def delimited_by
+      options.fetch(:delimited_by, ",")
+    end
+
+    def enclosed_by
+      options.fetch(:enclosed_by, '"')
+    end
+
+    def ignore_lines
+      options.fetch(:ignore_lines, 0).to_i
+    end
+
+    def real_ignore_lines
+      lines = ignore_lines
+      lines += 1 if ignore_input_file_header
+      lines
+    end
 
-      bulk_sql = <<-SQL
-        LOAD DATA LOCAL INFILE '#{input_file}'
-          REPLACE INTO TABLE #{table_name}
-          FIELDS
+    def truncate_sql
+      sql = "TRUNCATE TABLE #{table_name}"
+      format_sql sql
+    end
+
+    def bulk_sql
+      sql = <<-SQL
+      LOAD DATA LOCAL INFILE '#{input_file}'
+        REPLACE
+        INTO TABLE #{table_name}
+        FIELDS
           TERMINATED BY '#{delimited_by}'
           ENCLOSED BY '#{enclosed_by}'
-          IGNORE #{ignore_lines} LINES
-          (#{columns.join(',')})
+        IGNORE #{real_ignore_lines} LINES
+        (#{columns.join(',')})
       SQL
-      Kiba::Plus.logger.info bulk_sql
-      @client.query(bulk_sql)
-
-      @client.close
-      @client = nil
+      format_sql sql
     end
+
   end
 end

data/lib/kiba/plus/destination/pg.rb

@@ -2,18 +2,20 @@ require 'pg'
 
 module Kiba::Plus::Destination
   class Pg
-    attr_reader :options
+    attr_reader :options, :conn
 
     def initialize(options = {})
       @options = options
       @options.assert_valid_keys(
         :connect_url,
+        :schema,
         :prepare_name,
         :prepare_sql,
         :columns
       )
       @conn = PG.connect(connect_url)
-      @conn.prepare(prepare_name, prepare_sql)
+      @conn.exec "SET search_path TO %s" % [ options.fetch(:schema) ] unless options.fetch(:schema).empty?
+      init
     end
 
     def write(row)
@@ -23,6 +25,7 @@ module Kiba::Plus::Destination
       Kiba::Plus.logger.error "ERROR for #{row}"
       Kiba::Plus.logger.error ex.message
       # Maybe, write to db table or file
+      raise ex
     end
 
     def close
@@ -32,6 +35,10 @@ module Kiba::Plus::Destination
 
     private
 
+    def init
+      @conn.prepare(prepare_name, prepare_sql)
+    end
+
     def connect_url
       options.fetch(:connect_url)
     end

data/lib/kiba/plus/destination/pg_bulk.rb

@@ -2,20 +2,48 @@ require_relative 'pg_bulk_utils'
 module Kiba::Plus::Destination
   class PgBulk
     include PgBulkUtils
-    attr_reader :options
+    include Kiba::Plus::Helper
+    attr_reader :options, :conn
 
     def initialize(options = {})
       @options = options
       @options.assert_valid_keys(
-       :connect_url,
-       :input_file,
-       :table_name,
-       :columns,
-       :truncate,
-       :incremental,
-       :unique_by
-       )
+        :connect_url,
+        :table_name,
+        :columns,
+        :truncate,
+        :incremental,
+        :unique_by,
+        :input_file,
+        :ignore_input_file_header
+      )
       @conn = PG.connect(connect_url)
+
+      init
+    end
+
+    def write(row)
+      # blank!
+    end
+
+    def close
+      if incremental
+        truncate_staging_table
+        create_staging_table
+        copy_to_staging_table
+        delete_before_insert
+        merge_to_target_table
+        truncate_staging_table
+      else
+        copy_to_target_table
+      end
+      @conn.close
+      @conn = nil
+    end
+
+    private
+
+    def init
       if truncate
         truncate_staging_table
         truncate_target_table
@@ -30,10 +58,6 @@ module Kiba::Plus::Destination
       options.fetch(:table_name)
     end
 
-    def input_file
-      options.fetch(:input_file)
-    end
-
     def columns
       options.fetch(:columns)
     end
@@ -50,35 +74,53 @@ module Kiba::Plus::Destination
       options.fetch(:unique_by, :id)
     end
 
+    def input_file
+      options.fetch(:input_file)
+    end
+
+    def ignore_input_file_header
+      !!options.fetch(:ignore_input_file_header, false)
+    end
+
     def copy_to_target_table
-      sql = "COPY #{table_name} (#{columns.join(', ')}) FROM '#{File.expand_path(input_file)}' WITH DELIMITER ',' NULL '\\N' CSV"
+      sql = copy_to_target_table_sql
       Kiba::Plus.logger.info sql
       @conn.exec(sql)
     end
 
     def copy_to_staging_table
-      sql = "COPY #{staging_table_name} (#{columns.join(', ')}) FROM '#{File.expand_path(input_file)}' WITH DELIMITER ',' NULL '\\N' CSV"
+      sql = copy_to_staging_table_sql
       Kiba::Plus.logger.info sql
       @conn.exec(sql)
     end
 
-    def write(row)
-      # blank!
+    def copy_to_target_table_sql
+      sql = <<-SQL
+      COPY #{table_name} (#{columns.join(', ')})
+        FROM '#{File.expand_path(input_file)}'
+          WITH
+            #{ignore_input_file_header ? 'HEADER' : ''}
+            DELIMITER ','
+            NULL '\\N'
+            CSV
+      SQL
+
+      format_sql sql
     end
 
-    def close
-      if incremental
-        truncate_staging_table
-        create_staging_table
-        copy_to_staging_table
-        delete_before_insert
-        merge_to_target_table
-        truncate_staging_table
-      else
-        copy_to_target_table
-      end
-      @conn.close
-      @conn = nil
+    def copy_to_staging_table_sql
+      sql = <<-SQL
+      COPY #{staging_table_name} (#{columns.join(', ')})
+        FROM '#{File.expand_path(input_file)}'
+          WITH
+            #{ignore_input_file_header ? 'HEADER' : ''}
+            DELIMITER ','
+            NULL '\\N'
+            CSV
+      SQL
+
+      format_sql sql
     end
+
   end
 end

data/lib/kiba/plus/destination/pg_bulk2.rb

@@ -4,19 +4,54 @@ require_relative 'pg_bulk_utils'
 module Kiba::Plus::Destination
   class PgBulk2
     include PgBulkUtils
-    attr_reader :options
+    include Kiba::Plus::Helper
+    attr_reader :options, :conn
 
     def initialize(options = {})
       @options = options
-      @options.assert_valid_keys(:table_name,
-                                 :columns,
-                                 :connect_url,
-                                 :truncate,
-                                 :incremental,
-                                 :unique_by
-                                 )
+      @options.assert_valid_keys(
+        :connect_url,
+        :table_name,
+        :columns,
+        :truncate,
+        :incremental,
+        :unique_by
+      )
 
       @conn = PG.connect(connect_url)
+
+      init
+    end
+
+    def write(row)
+      begin
+        @conn.put_copy_data CSV.generate_line(row.values_at(*columns))
+      rescue Exception => err
+        errmsg = "%s while copy data: %s" % [ err.class.name, err.message ]
+        @conn.put_copy_end( errmsg )
+        Kiba::Plus.logger.error @conn.get_result
+        raise
+      end
+    end
+
+    def close
+      @conn.put_copy_end
+      @conn.get_last_result
+      if incremental
+        delete_before_insert
+        merge_to_target_table
+        truncate_staging_table
+      end
+    rescue
+      raise
+    ensure
+      @conn.close
+      @conn = nil
+    end
+
+    private
+
+    def init
       if truncate
         truncate_staging_table
         truncate_target_table
@@ -24,12 +59,12 @@ module Kiba::Plus::Destination
       if incremental
         truncate_staging_table
         create_staging_table
-        sql = "COPY #{staging_table_name} (#{columns.join(', ')}) FROM STDIN WITH DELIMITER ',' NULL '\\N' CSV"
+        sql = bulk_sql_with_incremental
       else
-        sql = "COPY #{table_name} (#{columns.join(', ')}) FROM STDIN WITH DELIMITER ',' NULL '\\N' CSV"
+        sql = bulk_sql_with_non_incremental
       end
       Kiba::Plus.logger.info sql
-      @res  = @conn.exec(sql)
+      @conn.exec(sql)
     end
 
     def connect_url
@@ -40,17 +75,6 @@ module Kiba::Plus::Destination
       options.fetch(:table_name)
     end
 
-    def write(row)
-      begin
-        @conn.put_copy_data CSV.generate_line(row.values_at(*columns))
-      rescue Exception => err
-        errmsg = "%s while copy data: %s" % [ err.class.name, err.message ]
-        @conn.put_copy_end( errmsg )
-        Kiba::Plus.logger.error @conn.get_result
-        raise
-      end
-    end
-
     def columns
       options.fetch(:columns)
     end
@@ -67,19 +91,29 @@ module Kiba::Plus::Destination
       options.fetch(:unique_by, :id)
     end
 
-    def close
-      @conn.put_copy_end
-      @conn.get_last_result
-      if incremental
-        delete_before_insert
-        merge_to_target_table
-        truncate_staging_table
-      end
-    rescue
-      raise
-    ensure
-      @conn.close
-      @conn = nil
+    def bulk_sql_with_incremental
+      sql = <<-SQL
+      COPY #{staging_table_name} (#{columns.join(', ')})
+        FROM STDIN
+          WITH
+            DELIMITER ','
+            NULL '\\N'
+            CSV
+      SQL
+      format_sql sql
     end
+
+    def bulk_sql_with_non_incremental
+      sql = <<-SQL
+      COPY #{table_name} (#{columns.join(', ')})
+        FROM STDIN
+          WITH
+            DELIMITER ','
+            NULL '\\N'
+            CSV
+      SQL
+      format_sql sql
+    end
+
   end
 end

data/lib/kiba/plus/destination/pg_bulk_utils.rb

@@ -1,39 +1,72 @@
 module Kiba::Plus::Destination
   module PgBulkUtils
+
+    private
+
     def staging_table_name
       table_name + "_staging"
     end
 
     def create_staging_table
-      sql = "CREATE TABLE IF NOT EXISTS #{staging_table_name} (LIKE #{table_name} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES)"
+      sql = create_staging_table_sql
       Kiba::Plus.logger.info sql
       @conn.exec(sql)
     end
 
+    def create_staging_table_sql
+      sql = <<-SQL
+      CREATE TABLE IF NOT EXISTS #{staging_table_name} (
+        LIKE #{table_name} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES
+      )
+      SQL
+      format_sql sql
+    end
+
     def truncate_staging_table
-      truncate_sql = "TRUNCATE TABLE #{staging_table_name}"
-      Kiba::Plus.logger.info truncate_sql
-      @conn.exec(truncate_sql) rescue nil
+      sql = truncate_staging_table_sql
+      Kiba::Plus.logger.info sql
+      @conn.exec(sql) rescue nil
+    end
+
+    def truncate_staging_table_sql
+      sql = "TRUNCATE TABLE #{staging_table_name}"
+      format_sql sql
     end
 
     def truncate_target_table
-      truncate_sql = "TRUNCATE TABLE #{table_name};"
-      Kiba::Plus.logger.info truncate_sql
-      @conn.exec(truncate_sql)
+      sql = truncate_target_table_sql
+      Kiba::Plus.logger.info sql
+      @conn.exec(sql)
+    end
+
+    def truncate_target_table_sql
+      sql = "TRUNCATE TABLE #{table_name}"
+      format_sql sql
     end
 
-    # TODO add where condition to speed up deleting.
     def delete_before_insert
-      where = Array(unique_by).map{|x| ["#{staging_table_name}.#{x}", "#{table_name}.#{x}"].join(" = ") }.join(" AND ")
-      sql = "DELETE FROM #{table_name} USING #{staging_table_name} WHERE #{where}"
+      sql = delete_before_insert_sql
       Kiba::Plus.logger.info sql
       @conn.exec(sql)
     end
 
+    # TODO add where condition to speed up deleting.
+    def delete_before_insert_sql
+      where = Array(unique_by).map{|x| ["#{staging_table_name}.#{x}", "#{table_name}.#{x}"].join(" = ") }.join(" AND ")
+      sql = "DELETE FROM #{table_name} USING #{staging_table_name} WHERE #{where}"
+      format_sql sql
+    end
+
     def merge_to_target_table
-      sql = "INSERT INTO #{table_name} (SELECT * FROM #{staging_table_name})"
+      sql = merge_to_target_table_sql
       Kiba::Plus.logger.info sql
       @conn.exec(sql)
     end
+
+    def merge_to_target_table_sql
+      sql = "INSERT INTO #{table_name} (SELECT * FROM #{staging_table_name})"
+      format_sql sql
+    end
+
   end
 end

data/lib/kiba/plus/helper.rb

@@ -2,13 +2,15 @@ require 'uri'
 module Kiba
   module Plus
     module Helper
-      def connect_hash(url)
+      def mysql2_connect_hash(url)
+        return url if url.is_a?(Hash)
+
         u = URI.parse(url)
         {
           host: u.host,
+          port: u.port,
           username: u.user,
           password: u.password,
-          port: u.port,
           database: u.path[1..-1]
         }
       end
@@ -17,6 +19,11 @@ module Kiba
         u = URI.parse(url)
         u.scheme
       end
+
+      def format_sql(sql)
+        sql.to_s.gsub(/[\n][\s]*[\n]/, "\n")
+      end
+
     end
   end
 end

data/lib/kiba/plus/job.rb

@@ -15,7 +15,7 @@ module Kiba
         @options.assert_valid_keys(:connect_url, :job_id, :job_name, :start_at, :completed_at)
         url = URI.parse(connect_url)
         if url.scheme =~ /mysql/i
-          @client = Mysql2::Client.new(connect_hash(connect_url))
+          @client = Mysql2::Client.new(mysql2_connect_hash(connect_url))
         elsif url.scheme =~ /postgres/i
           @client = PG.connect(connect_url)
         else

data/lib/kiba/plus/source/mysql.rb

@@ -10,41 +10,29 @@ module Kiba
       def initialize(options = {})
         @options = options
         @options.assert_valid_keys(
-          :query,
-          :output,
-          :last_pull_at,
-          :incremental,
-          :connect_url
-          )
-        @client = Mysql2::Client.new(connect_hash(connect_url))
+          :connect_url,
+          :query
+        )
+        @client = Mysql2::Client.new(mysql2_connect_hash(connect_url))
       end
 
       def each
+        Kiba::Plus.logger.info query
         results = client.query(query, as: :hash, symbolize_keys: true, stream: true)
         results.each do |row|
           yield(row)
         end
       end
 
-      def query
-        options.fetch(:query)
-      end
-
-      def output
-        options.fetch(:output)
-      end
-
-      def last_pull_at
-        options[:last_pull_at]
-      end
-
-      def incremental
-        options.fetch(:incremental, true)
-      end
+      private
 
       def connect_url
         options.fetch(:connect_url)
       end
+
+      def query
+        options.fetch(:query)
+      end
     end
   end
 end

data/lib/kiba/plus/source/pg.rb

@@ -0,0 +1,40 @@
+require 'pg'
+require 'uri'
+
+module Kiba
+  module Plus::Source
+    class Pg
+      include Kiba::Plus::Helper
+      attr_reader :options, :client
+
+      def initialize(options = {})
+        @options = options
+        @options.assert_valid_keys(
+          :connect_url,
+          :schema,
+          :query
+        )
+        @client = PG.connect(connect_url)
+        @client.exec "SET search_path TO %s" % [ options.fetch(:schema) ] unless options.fetch(:schema).empty?
+      end
+
+      def each
+        Kiba::Plus.logger.info query
+        results = client.query(query)
+        results.each do |row|
+          yield(row)
+        end
+      end
+
+      private
+
+      def connect_url
+        options.fetch(:connect_url)
+      end
+
+      def query
+        options.fetch(:query)
+      end
+    end
+  end
+end

data/lib/kiba/plus/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Plus
-    VERSION = "0.1.2"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-plus
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Hooopo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-21 00:00:00.000000000 Z
+date: 2017-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -94,6 +94,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.34'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.34'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: It connects to various data sources including relational, non-relational,
   and flat file, cloud services and HTTP resources. It has flexible load strategies
   including insert, bulk load and upsert.
@@ -135,6 +177,7 @@ files:
 - lib/kiba/plus/job.rb
 - lib/kiba/plus/logger.rb
 - lib/kiba/plus/source/mysql.rb
+- lib/kiba/plus/source/pg.rb
 - lib/kiba/plus/version.rb
 homepage: https://github.com/hooopo/kiba-plus
 licenses:
@@ -157,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Kiba enhancement for Ruby ETL