kiba-plus 0.1.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Hooopo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,98 @@
+# kiba-plus
+Kiba enhancement for Ruby ETL. It connects to various data sources including relational, non-relational, and flat file, cloud services and HTTP resources. It has flexible load strategies including insert, bulk load and upsert.
+
+# Usage
+
+```ruby
+require 'kiba/plus'
+
+SOURCE_URL = 'mysql://root@localhost/shopperplus'
+
+DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'
+
+source Kiba::Plus::Source::Mysql, { :connect_url => SOURCE_URL,
+                           :query => %Q{SELECT id, email, 'hooopo' AS first_name, 'Wang' AS last_name FROM customers}
+                         }
+
+destination Kiba::Plus::Destination::PgBulk2, { :connect_url => DEST_URL,
+                                :table_name => "customers",
+                                :truncate => true,
+                                :columns => [:id, :email, :first_name, :last_name],
+                                :incremental => false
+                              }
+
+post_process do
+  result = PG.connect(DEST_URL).query("SELECT COUNT(*) AS num FROM customers")
+  puts "Insert total: #{result.first['num']}"
+end
+```
+
+Execute:
+
+```shell
+bundle exec kiba customer_mysql_to_pg.etl
+```
+
+Output:
+
+```
+# Output:
+# I, [2016-05-16T01:53:36.832565 #87909]  INFO -- : TRUNCATE TABLE customers;
+# I, [2016-05-16T01:53:36.841770 #87909]  INFO -- : COPY customers (id, email, first_name, last_name) FROM STDIN WITH DELIMITER ',' NULL '\N' CSV
+# Insert total: 428972
+```
+
+More Examples(TODO).
+
+# Main Feature
+
+* Csv Source
+* MySQL Source
+* Postgresql Source
+* Citus Source
+* Greenplus Source
+* MongoDB Source (TODO)
+* Elastic Source (TODO)
+* Redshift Source (TODO)
+
+* Csv Destination
+* MySQL Destination
+* Postgresql Destination
+* Citus Destination
+* Greenplus Destination
+* MongoDB Destination (TODO)
+* Elastic Destination (TODO)
+* Redshift Destination (TODO)
+
+* Bulk Load for large dataset
+* Upsert for MySQL & Postgresql
+* Incremental Update
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kiba-plus'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kiba-plus
+
+## Usage
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kiba-plus.

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kiba/plus"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gem 'kiba-plus', :path => "../"
+gem 'pry'

data/examples/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: ../
+  specs:
+    kiba-plus (0.1.0)
+      kiba (~> 0.6)
+      mysql2 (~> 0.4)
+      pg (~> 0.18)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.1)
+    kiba (0.6.1)
+    method_source (0.8.2)
+    mysql2 (0.4.4)
+    pg (0.18.4)
+    pry (0.10.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  kiba-plus!
+  pry
+
+BUNDLED WITH
+   1.11.2

data/examples/customer_mysql_to_csv.etl

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+require_relative 'init'
+
+SOURCE_URL = 'mysql://root@localhost/shopperplus'
+
+source Kiba::Plus::Source::Mysql, :connect_url => SOURCE_URL,
+                           :query => %Q{SELECT id, email, 'hooopo' AS first_name, 'Wang' AS last_name FROM customers}
+
+destination Kiba::Plus::Destination::Csv, :output_file => "/tmp/customer_csv.csv"
+
+post_process do
+  puts %x{head -n 10 /tmp/customer_csv.csv}
+end

data/examples/customer_mysql_to_pg.etl

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+require_relative 'init'
+
+SOURCE_URL = 'mysql://root@localhost/shopperplus'
+
+DEST_URL   = 'postgresql://hooopo@localhost:5432/crm2_dev'
+
+source Kiba::Plus::Source::Mysql, { :connect_url => SOURCE_URL,
+                           :query => %Q{SELECT id, email, 'hooopo' AS first_name, 'Wang' AS last_name FROM customers}
+                         }
+
+destination Kiba::Plus::Destination::PgBulk2, { :connect_url => DEST_URL,
+                                :table_name => "customers",
+                                :truncate => true,
+                                :columns => [:id, :email, :first_name, :last_name],
+                                :incremental => false
+                              }
+
+post_process do
+  result = PG.connect(DEST_URL).query("SELECT COUNT(*) AS num FROM customers")
+  puts "Insert total: #{result.first['num']}"
+end
+
+# Output:
+# I, [2016-05-16T01:53:36.832565 #87909]  INFO -- : TRUNCATE TABLE customers;
+# I, [2016-05-16T01:53:36.841770 #87909]  INFO -- : COPY customers (id, email, first_name, last_name) FROM STDIN WITH DELIMITER ',' NULL '\N' CSV
+# Insert total: 428972

data/examples/init.rb

@@ -0,0 +1,8 @@
+Bundler.require(:default)
+
+source_files = File.expand_path(File.dirname(__FILE__) + "/sources/*.rb")
+destination_files = File.expand_path(File.dirname(__FILE__) + "/destinations/*.rb")
+
+[source_files, destination_files].each do |files|
+  Dir.glob(files).each {|f| require(f);puts "import #{f}"}
+end

data/kiba-plus.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kiba/plus/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kiba-plus"
+  spec.version       = Kiba::Plus::VERSION
+  spec.authors       = ["Hooopo"]
+  spec.email         = ["hoooopo@gmail.com"]
+
+  spec.summary       = %q{kiba plus}
+  spec.description   = %q{kiba plus}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_runtime_dependency "kiba", "~> 0.6"
+  spec.add_runtime_dependency "mysql2", "~> 0.4"
+  spec.add_runtime_dependency "pg", "~> 0.18"
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/lib/kiba/plus.rb

@@ -0,0 +1,30 @@
+require_relative "plus/version"
+require_relative "plus/helper"
+require_relative "plus/source/mysql"
+require_relative "plus/destination/csv"
+
+require_relative "plus/destination/mysql"
+require_relative "plus/destination/mysql_bulk"
+
+require_relative "plus/destination/pg"
+require_relative "plus/destination/pg_bulk"
+require_relative "plus/destination/pg_bulk2"
+
+require_relative "plus/job"
+require_relative 'plus/logger'
+
+module Kiba
+  module Plus
+  end
+end
+
+class Hash
+  def assert_valid_keys(*valid_keys)
+    valid_keys.flatten!
+    each_key do |k|
+      unless valid_keys.include?(k)
+        raise ArgumentError.new("Unknown key: #{k.inspect}. Valid keys are: #{valid_keys.map(&:inspect).join(', ')}")
+      end
+    end
+  end
+end

data/lib/kiba/plus/destination/csv.rb

@@ -0,0 +1,29 @@
+require 'csv'
+
+module Kiba::Plus::Destination
+  class Csv
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options
+      @options.assert_valid_keys(:output_file)
+      @csv = CSV.open(output_file, 'w', {col_sep: delimiter})
+    end
+
+    def output_file
+      options.fetch(:output_file)
+    end
+
+    def delimiter
+      ","
+    end
+
+    def write(row)
+      @csv << row.values
+    end
+
+    def close
+      @csv.close
+    end
+  end
+end

data/lib/kiba/plus/destination/mysql.rb

@@ -0,0 +1,45 @@
+require 'mysql2'
+
+module Kiba::Plus::Destination
+  class Mysql
+    include Kiba::Plus::Helper
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options
+      @options.assert_valid_keys(
+        :connect_url,
+        :prepare_sql,
+        :columns
+      )
+      @client = Mysql2::Client.new(connect_hash(connect_url))
+      @pre_stmt = @client.prepare(prepare_sql)
+    end
+
+    def write(row)
+      @pre_stmt.execute(*row.values_at(*columns))
+    rescue => e
+      Kiba::Plus.logger.error "ERROR for #{row}"
+      Kiba::Plus.logger.error e.message
+    end
+
+    def close
+      @client.close
+      @client = nil
+    end
+
+    private
+
+    def connect_url
+      options.fetch(:connect_url)
+    end
+
+    def prepare_sql
+      options.fetch(:prepare_sql)
+    end
+
+    def columns
+      options.fetch(:columns)
+    end
+  end
+end

data/lib/kiba/plus/destination/mysql_bulk.rb

@@ -0,0 +1,63 @@
+require 'mysql2'
+
+module Kiba::Plus::Destination
+  class MysqlBulk
+    include Kiba::Plus::Helper
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options
+      @options.assert_valid_keys(:table_name,
+                                 :columns,
+                                 :input_file,
+                                 :connect_url,
+                                 :truncate,
+                                 :incremental
+                                 )
+
+      @client = Mysql2::Client.new(connect_hash(connect_url).merge(local_infile: true))
+    end
+
+    def connect_url
+      options.fetch(:connect_url)
+    end
+
+    def table_name
+      options.fetch(:table_name)
+    end
+
+    def write(row)
+    end
+
+    def columns
+      options.fetch(:columns)
+    end
+
+    def truncate
+      options.fetch(:truncate, false)
+    end
+
+    def incremental
+      options.fetch(:incremental, true)
+    end
+
+    def input_file
+      options.fetch(:input_file)
+    end
+
+    def close
+      if truncate
+        truncate_sql = "TRUNCATE TABLE #{table_name};"
+        Kiba::Plus.logger.info truncate_sql
+        @client.query(truncate_sql)
+      end
+
+      bulk_sql = "LOAD DATA LOCAL INFILE '#{input_file}' REPLACE INTO TABLE #{table_name} FIELDS TERMINATED BY ', ' (#{columns.join(',')})"
+      Kiba::Plus.logger.info bulk_sql
+      @client.query(bulk_sql)
+
+      @client.close
+      @client = nil
+    end
+  end
+end