kiba-plus 0.1.12 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2c182fb620f80bbd0703399075d2aafd870e65b
-  data.tar.gz: cee85bda9ae9987d1d934ddc027cb9f2bd4e4c47
+  metadata.gz: bbe35735f2f3a6127a9ab2c5d95729f9a7e5a6fa
+  data.tar.gz: 8d975f78c52c8e3b0515c7403aeeddcd61d63c50
 SHA512:
-  metadata.gz: 7c84b2095c49737f7d69eba1880a83b1bc8185acb070fcb58491d7c35633591d258c2373fef4de1179964c47e57fe1a380affb01e5eca3cbcba76009fd298bf7
-  data.tar.gz: 8c76f6b6264e4024c2c72ca582ef496756a2683765012a252b9e7df7dbae9d4db4abd08b06d2f0f2555a7f0010304f5d2d8d96bcdb4d4960ea34f567e9488984
+  metadata.gz: a4065afb5ecf4fc54aba299173dad3fc628f365bd273eab056178e04bfeb618b446dfa4203e0ca7fb73864c3ea1a0ff32a3bf2e88a4e1a392bef732020a2bd1f
+  data.tar.gz: 879cede2ab9990971ed4a263a61365cff64a99838cc0c42fc58abecf83640b791c88973c6f1aad6821938ebe96e3adfbb223d2a3efc4aaa0ea02a3e2db74a8e0

data/README.md

@@ -8,6 +8,9 @@ Kiba enhancement for Ruby ETL. It connects to various data sources including rel
 
 require 'kiba/plus'
 
+require 'kiba/plus/source/mysql'
+require 'kiba/plus/destination/pg_bulk2'
+
 SOURCE_URL = 'mysql://root@localhost/shopperplus'
 
 DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'
@@ -120,4 +123,4 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kiba-plus.
+Bug reports and pull requests are welcome on GitHub at https://github.com/hooopo/kiba-plus.

data/examples/customer_csv_to_mysql.etl

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
+require 'kiba/plus/destination/mysql_bulk'
+
 DEST_URL   = 'mysql://root@localhost/crm2_dev'
 
 destination Kiba::Plus::Destination::MysqlBulk, { :connect_url => DEST_URL,

data/examples/customer_csv_to_pg.etl

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
+require 'kiba/plus/destination/pg_bulk'
+
 DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'
 
 destination Kiba::Plus::Destination::PgBulk, { :connect_url => DEST_URL,

data/examples/customer_mysql_to_csv.etl

@@ -1,6 +1,9 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
+require 'kiba/plus/source/mysql'
+require 'kiba/plus/destination/csv'
+
 SOURCE_URL = 'mysql://root@localhost/shopperplus'
 
 source Kiba::Plus::Source::Mysql, :connect_url => SOURCE_URL,

data/examples/customer_mysql_to_pg.etl

@@ -1,6 +1,9 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
+require 'kiba/plus/source/mysql'
+require 'kiba/plus/destination/pg_bulk2'
+
 SOURCE_URL = 'mysql://root@localhost/shopperplus'
 
 DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'

data/examples/incremental_insert.etl

@@ -1,6 +1,10 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
+require 'kiba/plus/job'
+require 'kiba/plus/source/mysql'
+require 'kiba/plus/destination/pg_bulk2'
+
 SOURCE_URL = 'mysql://root@localhost/shopperplus'
 
 DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'

data/kiba-plus.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "kiba", "~> 0.6"
+  spec.add_runtime_dependency "kiba", "~> 1.0.0"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/kiba/plus/destination/csv.rb

@@ -57,4 +57,4 @@ module Kiba::Plus::Destination
     end
 
   end
-end
+end

data/lib/kiba/plus/destination/pg_bulk.rb

@@ -29,12 +29,12 @@ module Kiba::Plus::Destination
 
     def close
       if incremental
-        truncate_staging_table
+        drop_staging_table
         create_staging_table
         copy_to_staging_table
         delete_before_insert
         merge_to_target_table
-        truncate_staging_table
+        drop_staging_table
       else
         copy_to_target_table
       end
@@ -46,7 +46,7 @@ module Kiba::Plus::Destination
 
     def init
       if truncate
-        truncate_staging_table
+        drop_staging_table
         truncate_target_table
       end
     end

data/lib/kiba/plus/destination/pg_bulk2.rb

@@ -46,7 +46,7 @@ module Kiba::Plus::Destination
       if incremental
         delete_before_insert
         merge_to_target_table
-        truncate_staging_table
+        drop_staging_table
       end
     rescue
       raise
@@ -59,11 +59,11 @@ module Kiba::Plus::Destination
 
     def init
       if truncate
-        truncate_staging_table
+        drop_staging_table
         truncate_target_table
       end
       if incremental
-        truncate_staging_table
+        drop_staging_table
         create_staging_table
         sql = bulk_sql_with_incremental
       else

data/lib/kiba/plus/destination/pg_bulk_utils.rb

@@ -22,14 +22,14 @@ module Kiba::Plus::Destination
       format_sql sql
     end
 
-    def truncate_staging_table
-      sql = truncate_staging_table_sql
+    def drop_staging_table
+      sql = drop_staging_table_sql
       Kiba::Plus.logger.info sql
       @conn.exec(sql) rescue nil
     end
 
-    def truncate_staging_table_sql
-      sql = "TRUNCATE TABLE #{staging_table_name}"
+    def drop_staging_table_sql
+      sql = "DROP TABLE IF EXISTS #{staging_table_name}"
       format_sql sql
     end
 
@@ -69,4 +69,4 @@ module Kiba::Plus::Destination
     end
 
   end
-end
+end

data/lib/kiba/plus/helper.rb

@@ -26,4 +26,4 @@ module Kiba
 
     end
   end
-end
+end

data/lib/kiba/plus/logger.rb

@@ -9,4 +9,4 @@ module Kiba
       @logger = logger
     end
   end
-end
+end

data/lib/kiba/plus/source/mysql.rb

@@ -22,7 +22,7 @@ module Kiba
 
       def each
         Kiba::Plus.logger.info query
-        results = client.query(query, as: :hash, symbolize_keys: true, stream: true)
+        results = client.query(query, as: :hash, symbolize_keys: true, stream: true, cache_rows: false)
         results.each do |row|
           yield(row)
         end

data/lib/kiba/plus/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Plus
-    VERSION = "0.1.12"
+    VERSION = "0.1.13"
   end
 end

data/lib/kiba/plus.rb

@@ -17,4 +17,4 @@ class Hash
       end
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-plus
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.1.13
 platform: ruby
 authors:
 - Hooopo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-08-14 00:00:00.000000000 Z
+date: 2017-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement