kiba-plus 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39c5c1848a724492f99fb2472d5743e7c0848d35
-  data.tar.gz: 547f4f57babb41d3cd664fe873bd55f730ff4ce0
+  metadata.gz: 3ab51f248407db0efeea72628dc83f639fa952f4
+  data.tar.gz: e1d93e552d093ea421c1330ee2d1dbcf3f563c0d
 SHA512:
-  metadata.gz: 778e4bad4aa95d371625962431790a74fbee12077dc4406cb962ca975629dbab850e2241141656802d71196dfaed0b8598b5bcc130e16af180c058c02b8a1e38
-  data.tar.gz: 45a9389410c08e7eedc594370b9ca70d1656662b87c13f70589cc86597ccc3470a12318499011a98521dff44f21dd154dbdc4716282bf18dafd11c524953f80a
+  metadata.gz: 55cfec8dcfcad2f4224bcaf3d37d6aa2e7ca7831786cf0b099ec8800f4b3843b6d19e2ff06a8c57af11c10f8d91be3bf920c5410a3895bb3646f5228065e9471
+  data.tar.gz: 5468e321b8b4dbfb9d8174a41964e7b79c8d5d1c8af9129b751621d02759629bb170ef61c51999483df1b260900e096229ea7d5420e9bdbb3db421f51515c356

data/examples/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../
   specs:
-    kiba-plus (0.1.0)
+    kiba-plus (0.1.2)
       kiba (~> 0.6)
       mysql2 (~> 0.4)
       pg (~> 0.18)

data/examples/incremental_insert.etl

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+require_relative 'init'
+
+SOURCE_URL = 'mysql://root@localhost/shopperplus'
+
+DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'
+
+pre_process do
+  @job_id = Kiba::Plus::Job.new(
+    :connect_url => DEST_URL,
+    :start_at => Time.now,
+    :job_name => "customer"
+    ).start
+end
+
+last_pull_at = Kiba::Plus::Job.new(
+  :connect_url => DEST_URL,
+  :job_name => "customer"
+).last_pull_at
+
+source Kiba::Plus::Source::Mysql, { :connect_url => SOURCE_URL,
+                           :query => %Q{SELECT id, email, 'hooopo' AS first_name, 'Wang' AS last_name FROM customers WHERE updated_at > '#{last_pull_at.to_s}'},
+                           :last_pull_at => last_pull_at,
+                           :incremental => true
+                         }
+
+destination Kiba::Plus::Destination::PgBulk2, { :connect_url => DEST_URL,
+                                :table_name => "customers",
+                                :truncate => false,
+                                :columns => [:id, :email, :first_name, :last_name],
+                                :incremental => true,
+                                :unique_by => :id
+                              }
+
+post_process do
+  Kiba::Plus::Job.new(
+    :connect_url => DEST_URL,
+    :job_id => @job_id,
+    :job_name => "customer"
+  ).complete
+  result = PG.connect(DEST_URL).query("SELECT COUNT(*) AS num FROM customers")
+  puts "Insert total: #{result.first['num']}"
+end
+
+# Output:
+# I, [2016-05-16T01:53:36.832565 #87909]  INFO -- : TRUNCATE TABLE customers;
+# I, [2016-05-16T01:53:36.841770 #87909]  INFO -- : COPY customers (id, email, first_name, last_name) FROM STDIN WITH DELIMITER ',' NULL '\N' CSV
+# Insert total: 428972

data/lib/kiba/plus/destination/pg_bulk.rb

@@ -1,5 +1,7 @@
+require_relative 'pg_bulk_utils'
 module Kiba::Plus::Destination
   class PgBulk
+    include PgBulkUtils
     attr_reader :options
 
     def initialize(options = {})
@@ -48,32 +50,6 @@ module Kiba::Plus::Destination
       options.fetch(:unique_by, :id)
     end
 
-    def write(row)
-      # blank!
-    end
-
-    def staging_table_name
-      table_name + "_staging"
-    end
-
-    def create_staging_table
-      sql = "CREATE TABLE IF NOT EXISTS #{staging_table_name} (LIKE #{table_name} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES)"
-      Kiba::Plus.logger.info sql
-      @conn.exec(sql)
-    end
-
-    def truncate_staging_table
-      truncate_sql = "TRUNCATE TABLE #{staging_table_name}"
-      Kiba::Plus.logger.info truncate_sql
-      @conn.exec(truncate_sql) rescue nil
-    end
-
-    def truncate_target_table
-      truncate_sql = "TRUNCATE TABLE #{table_name};"
-      Kiba::Plus.logger.info truncate_sql
-      @conn.exec(truncate_sql)
-    end
-
     def copy_to_target_table
       sql = "COPY #{table_name} (#{columns.join(', ')}) FROM '#{File.expand_path(input_file)}' WITH DELIMITER ',' NULL '\\N' CSV"
       Kiba::Plus.logger.info sql
@@ -86,18 +62,8 @@ module Kiba::Plus::Destination
       @conn.exec(sql)
     end
 
-    # TODO add where condition to speed up deleting.
-    def delete_before_insert
-      where = Array(unique_by).map{|x| ["#{staging_table_name}.#{x}", "#{table_name}.#{x}"].join(" = ") }.join(" AND ")
-      sql = "DELETE FROM #{table_name} USING #{staging_table_name} WHERE #{where}"
-      Kiba::Plus.logger.info sql
-      @conn.exec(sql)
-    end
-
-    def merge_to_target_table
-      sql = "INSERT INTO #{table_name} (SELECT * FROM #{staging_table_name})"
-      Kiba::Plus.logger.info sql
-      @conn.exec(sql)
+    def write(row)
+      # blank!
     end
 
     def close

data/lib/kiba/plus/destination/pg_bulk2.rb

@@ -1,7 +1,9 @@
 require 'pg'
 require 'csv'
+require_relative 'pg_bulk_utils'
 module Kiba::Plus::Destination
   class PgBulk2
+    include PgBulkUtils
     attr_reader :options
 
     def initialize(options = {})
@@ -10,16 +12,22 @@ module Kiba::Plus::Destination
                                  :columns,
                                  :connect_url,
                                  :truncate,
-                                 :incremental
+                                 :incremental,
+                                 :unique_by
                                  )
 
       @conn = PG.connect(connect_url)
-      truncate_sql = "TRUNCATE TABLE #{table_name};"
       if truncate
-        Kiba::Plus.logger.info truncate_sql
-        @conn.exec(truncate_sql)
+        truncate_staging_table
+        truncate_target_table
+      end
+      if incremental
+        truncate_staging_table
+        create_staging_table
+        sql = "COPY #{staging_table_name} (#{columns.join(', ')}) FROM STDIN WITH DELIMITER ',' NULL '\\N' CSV"
+      else
+        sql = "COPY #{table_name} (#{columns.join(', ')}) FROM STDIN WITH DELIMITER ',' NULL '\\N' CSV"
       end
-      sql = "COPY #{table_name} (#{columns.join(', ')}) FROM STDIN WITH DELIMITER ',' NULL '\\N' CSV"
       Kiba::Plus.logger.info sql
       @res  = @conn.exec(sql)
     end
@@ -55,9 +63,18 @@ module Kiba::Plus::Destination
       options.fetch(:incremental, true)
     end
 
+    def unique_by
+      options.fetch(:unique_by, :id)
+    end
+
     def close
       @conn.put_copy_end
       @conn.get_last_result
+      if incremental
+        delete_before_insert
+        merge_to_target_table
+        truncate_staging_table
+      end
     rescue
       raise
     ensure

data/lib/kiba/plus/destination/pg_bulk_utils.rb

@@ -0,0 +1,39 @@
+module Kiba::Plus::Destination
+  module PgBulkUtils
+    def staging_table_name
+      table_name + "_staging"
+    end
+
+    def create_staging_table
+      sql = "CREATE TABLE IF NOT EXISTS #{staging_table_name} (LIKE #{table_name} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES)"
+      Kiba::Plus.logger.info sql
+      @conn.exec(sql)
+    end
+
+    def truncate_staging_table
+      truncate_sql = "TRUNCATE TABLE #{staging_table_name}"
+      Kiba::Plus.logger.info truncate_sql
+      @conn.exec(truncate_sql) rescue nil
+    end
+
+    def truncate_target_table
+      truncate_sql = "TRUNCATE TABLE #{table_name};"
+      Kiba::Plus.logger.info truncate_sql
+      @conn.exec(truncate_sql)
+    end
+
+    # TODO add where condition to speed up deleting.
+    def delete_before_insert
+      where = Array(unique_by).map{|x| ["#{staging_table_name}.#{x}", "#{table_name}.#{x}"].join(" = ") }.join(" AND ")
+      sql = "DELETE FROM #{table_name} USING #{staging_table_name} WHERE #{where}"
+      Kiba::Plus.logger.info sql
+      @conn.exec(sql)
+    end
+
+    def merge_to_target_table
+      sql = "INSERT INTO #{table_name} (SELECT * FROM #{staging_table_name})"
+      Kiba::Plus.logger.info sql
+      @conn.exec(sql)
+    end
+  end
+end

data/lib/kiba/plus/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Plus
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-plus
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Hooopo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-17 00:00:00.000000000 Z
+date: 2016-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -119,6 +119,7 @@ files:
 - examples/customer_mysql_to_csv.etl
 - examples/customer_mysql_to_pg.etl
 - examples/data/customer.csv
+- examples/incremental_insert.etl
 - examples/init.rb
 - examples/sources/customer.rb
 - kiba-plus.gemspec
@@ -129,6 +130,7 @@ files:
 - lib/kiba/plus/destination/pg.rb
 - lib/kiba/plus/destination/pg_bulk.rb
 - lib/kiba/plus/destination/pg_bulk2.rb
+- lib/kiba/plus/destination/pg_bulk_utils.rb
 - lib/kiba/plus/helper.rb
 - lib/kiba/plus/job.rb
 - lib/kiba/plus/logger.rb