kiba-plus 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d724aaf93a4a7d042ef123f220fe43427a382993
-  data.tar.gz: 3a2b31601585be0e49b7cacf426f01c2a91f1a8b
+  metadata.gz: 39c5c1848a724492f99fb2472d5743e7c0848d35
+  data.tar.gz: 547f4f57babb41d3cd664fe873bd55f730ff4ce0
 SHA512:
-  metadata.gz: 75b623813154f0983795517fd5d1740629326e263f7c14ba1d50acf2bae64d5ba552c5a5346b4d9a3714b1b07c963d8472a0bbc040c3b8b7a04eefa3d9084c3c
-  data.tar.gz: 13d50fe54a0c0616c8c8217540f8fb131b480bc8fbb189cbb8b15ca622c929085b2d12c0e72d8de2d2056706f6d88668ec27221ae51dc6a5f19abbb26281e83d
+  metadata.gz: 778e4bad4aa95d371625962431790a74fbee12077dc4406cb962ca975629dbab850e2241141656802d71196dfaed0b8598b5bcc130e16af180c058c02b8a1e38
+  data.tar.gz: 45a9389410c08e7eedc594370b9ca70d1656662b87c13f70589cc86597ccc3470a12318499011a98521dff44f21dd154dbdc4716282bf18dafd11c524953f80a

data/examples/customer_csv_to_mysql.etl

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require_relative 'init'
+
+DEST_URL   = 'mysql://root@localhost/crm2_dev'
+
+destination Kiba::Plus::Destination::MysqlBulk, { :connect_url => DEST_URL,
+                                :table_name => "customers",
+                                :input_file => File.expand_path(File.dirname(__FILE__) + "/data/customer.csv"),
+                                :truncate => true,
+                                :columns => [:id, :email, :first_name, :last_name],
+                                :incremental => false
+                              }
+post_process do
+  result = Mysql2::Client.new(connect_hash(DEST_URL)).query("SELECT COUNT(*) AS num FROM customers")
+  puts "Insert total: #{result.first['num']}"
+end

data/examples/customer_csv_to_pg.etl

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require_relative 'init'
+
+DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'
+
+destination Kiba::Plus::Destination::PgBulk, { :connect_url => DEST_URL,
+                                :table_name => "customers",
+                                :input_file => File.expand_path(File.dirname(__FILE__) + "/data/customer.csv"),
+                                :truncate => true,
+                                :columns => [:id, :email, :first_name, :last_name],
+                                :incremental => false
+                              }
+post_process do
+  result = PG.connect(DEST_URL).query("SELECT COUNT(*) AS num FROM customers")
+  puts "Insert total: #{result.first['num']}"
+end

data/examples/customer_mysql_to_csv.etl

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 require_relative 'init'
 
-SOURCE_URL = 'mysql://root@localhost/shopperplus'
+DEST_URL = 'mysql://root@localhost/crm2_dev'
 
 source Kiba::Plus::Source::Mysql, :connect_url => SOURCE_URL,
                            :query => %Q{SELECT id, email, 'hooopo' AS first_name, 'Wang' AS last_name FROM customers}

data/examples/data/customer.csv

@@ -0,0 +1,10 @@
+224862,hello@excite.com,hooopo,Wang
+251732,hello@hotmail.ca,hooopo,Wang
+30797,hello@excite.com,hooopo,Wang
+59629,hello@rogers.com,hooopo,Wang
+133665,"hello@live.ca",hooopo,Wang
+199481,"hello@acanac.net",hooopo,Wang
+288980,hello@gmail.com,hooopo,Wang
+147067,hello@qc.aira.com,hooopo,Wang
+339972,hello@nahlene.com,hooopo,Wang
+307171,hello@gmail.com,hooopo,Wang

data/examples/init.rb

@@ -1,5 +1,7 @@
 Bundler.require(:default)
 
+include Kiba::Plus::Helper
+
 source_files = File.expand_path(File.dirname(__FILE__) + "/sources/*.rb")
 destination_files = File.expand_path(File.dirname(__FILE__) + "/destinations/*.rb")
 

data/kiba-plus.gemspec

@@ -9,9 +9,9 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Hooopo"]
   spec.email         = ["hoooopo@gmail.com"]
 
-  spec.summary       = %q{kiba plus}
-  spec.description   = %q{kiba plus}
-  spec.homepage      = ""
+  spec.summary       = %q{Kiba enhancement for Ruby ETL}
+  spec.description   = %q{It connects to various data sources including relational, non-relational, and flat file, cloud services and HTTP resources. It has flexible load strategies including insert, bulk load and upsert.}
+  spec.homepage      = "https://github.com/hooopo/kiba-plus"
   spec.license       = "MIT"
 
   # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or

data/lib/kiba/plus/destination/csv.rb

@@ -6,16 +6,44 @@ module Kiba::Plus::Destination
 
     def initialize(options = {})
       @options = options
-      @options.assert_valid_keys(:output_file)
-      @csv = CSV.open(output_file, 'w', {col_sep: delimiter})
+      @options.assert_valid_keys(
+        :output_file,
+        :row_sep,
+        :col_sep,
+        :force_quotes,
+        :quote_char,
+        :mode
+      )
+      @csv = CSV.open(output_file, mode, {
+        :col_sep => col_sep,
+        :quote_char => quote_char,
+        :force_quotes => force_quotes,
+        :row_sep => row_sep
+      })
+    end
+
+    def mode
+      options.fetch(:mode, "w")
     end
 
     def output_file
       options.fetch(:output_file)
     end
 
-    def delimiter
-      ","
+    def col_sep
+      options.fetch(:col_sep, ",")
+    end
+
+    def quote_char
+      options.fetch(:quote_char, '"')
+    end
+
+    def force_quotes
+      options.fetch(:force_quotes, false)
+    end
+
+    def row_sep
+      options.fetch(:row_sep, "\n")
     end
 
     def write(row)

data/lib/kiba/plus/destination/mysql_bulk.rb

@@ -12,7 +12,10 @@ module Kiba::Plus::Destination
                                  :input_file,
                                  :connect_url,
                                  :truncate,
-                                 :incremental
+                                 :incremental,
+                                 :delimited_by,
+                                 :enclosed_by,
+                                 :ignore_lines
                                  )
 
       @client = Mysql2::Client.new(connect_hash(connect_url).merge(local_infile: true))
@@ -26,6 +29,18 @@ module Kiba::Plus::Destination
       options.fetch(:table_name)
     end
 
+    def delimited_by
+      options.fetch(:delimited_by, ",")
+    end
+
+    def enclosed_by
+      options.fetch(:enclosed_by, '"')
+    end
+
+    def ignore_lines
+      options.fetch(:ignore_lines, 0)
+    end
+
     def write(row)
     end
 
@@ -52,7 +67,15 @@ module Kiba::Plus::Destination
         @client.query(truncate_sql)
       end
 
-      bulk_sql = "LOAD DATA LOCAL INFILE '#{input_file}' REPLACE INTO TABLE #{table_name} FIELDS TERMINATED BY ', ' (#{columns.join(',')})"
+      bulk_sql = <<-SQL
+        LOAD DATA LOCAL INFILE '#{input_file}'
+          REPLACE INTO TABLE #{table_name}
+          FIELDS
+          TERMINATED BY '#{delimited_by}'
+          ENCLOSED BY '#{enclosed_by}'
+          IGNORE #{ignore_lines} LINES
+          (#{columns.join(',')})
+      SQL
       Kiba::Plus.logger.info bulk_sql
       @client.query(bulk_sql)
 

data/lib/kiba/plus/version.rb

@@ -1,5 +1,5 @@
 module Kiba
   module Plus
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiba-plus
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Hooopo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-15 00:00:00.000000000 Z
+date: 2016-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kiba
@@ -94,7 +94,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-description: kiba plus
+description: It connects to various data sources including relational, non-relational,
+  and flat file, cloud services and HTTP resources. It has flexible load strategies
+  including insert, bulk load and upsert.
 email:
 - hoooopo@gmail.com
 executables: []
@@ -112,8 +114,11 @@ files:
 - bin/setup
 - examples/Gemfile
 - examples/Gemfile.lock
+- examples/customer_csv_to_mysql.etl
+- examples/customer_csv_to_pg.etl
 - examples/customer_mysql_to_csv.etl
 - examples/customer_mysql_to_pg.etl
+- examples/data/customer.csv
 - examples/init.rb
 - examples/sources/customer.rb
 - kiba-plus.gemspec
@@ -129,7 +134,7 @@ files:
 - lib/kiba/plus/logger.rb
 - lib/kiba/plus/source/mysql.rb
 - lib/kiba/plus/version.rb
-homepage: ''
+homepage: https://github.com/hooopo/kiba-plus
 licenses:
 - MIT
 metadata:
@@ -153,5 +158,5 @@ rubyforge_project:
 rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: kiba plus
+summary: Kiba enhancement for Ruby ETL
 test_files: []