redshift-connector 4.3.0

data/lib/redshift-connector/version.rb

@@ -0,0 +1,3 @@
+module RedshiftConnector
+  VERSION = '4.3.0'
+end

data/test/all.rb

@@ -0,0 +1,3 @@
+Dir.glob("#{__dir__}/**/test_*.rb").each do |path|
+  load path
+end

data/test/config.rb

@@ -0,0 +1,13 @@
+$TEST_SCHEMA = 'aamine'
+
+module RedshiftConnector
+  Exporter.default_data_source = Redshift
+
+  S3Bucket.add(
+    'redshift-copy-buffer',
+    bucket: 'redshift-copy-buffer',
+    prefix: 'development',
+    iam_role: 'arn:aws:iam::789035092620:role/RedshiftDevelopers',
+    default: true
+  )
+end

data/test/config.rb.example

@@ -0,0 +1,18 @@
+module RedshiftConnector
+  # For test only
+  $TEST_SCHEMA = 'test'
+
+  Exporter.default_data_source = Redshift
+
+  S3Bucket.add(
+    'ENTRY_NAME',
+    bucket: 'YOUR_BUCKET_NAME',
+    prefix: 'development',
+    # When using IAM role
+    iam_role: 'arn:aws:iam::NNNNNNNNNNNN:role/RRRRRRRRR',
+    # When using explicit access key
+    #access_key_id: 'AAAAAAAAAA',
+    #secret_access_key: 'SSSSSSSSSS',
+    default: true
+  )
+end

data/test/database.yml

@@ -0,0 +1,15 @@
+mysql:
+  adapter: mysql2
+  host: localhost
+  username: minero-aoki
+  database: test
+  encoding: utf8
+
+redshift:
+  adapter: redshift
+  host: dwh.ckpd.co
+  port: 5439
+  database: production
+  username: aamine
+  password: "3edCVfr$"
+  encoding: utf8

data/test/database.yml.example

@@ -0,0 +1,15 @@
+mysql:
+  adapter: mysql2
+  host: localhost
+  username: USER_NAME
+  database: test
+  encoding: utf8
+
+redshift:
+  adapter: redshift
+  host: HOST_NAME
+  port: 5439
+  database: DATABASE_NAME
+  username: USER_NAME
+  password: PASSWORD
+  encoding: utf8

data/test/foreach.rb

@@ -0,0 +1,5 @@
+require_relative 'helper'
+
+RedshiftConnector.foreach(schema: 'tabemiru', table: 'items', query: 'select * from tabemiru.items where id < 10') do |row|
+  p row
+end

data/test/helper.rb

@@ -0,0 +1,25 @@
+require 'active_record'
+require 'yaml'
+require 'logger'
+
+YAML.load_file("#{__dir__}/database.yml").each do |name, ent|
+  ActiveRecord::Base.configurations[name] = ent
+end
+
+class BaseConn < ActiveRecord::Base
+  establish_connection :mysql
+  self.abstract_class = true
+end
+class ItemPv < BaseConn
+  connection
+end
+class Redshift < ActiveRecord::Base
+  establish_connection :redshift
+end
+
+require 'redshift-connector'
+# This IS REQUIRED to ensure to load mysql2 driver, at least outside of Rails.
+ActiveRecord::Import.require_adapter 'mysql2'
+require_relative 'config'
+
+RedshiftConnector.logger = Logger.new($stderr)

data/test/item_pvs.ct.mysql

@@ -0,0 +1,11 @@
+create table item_pvs
+( id bigint(20) primary key
+, data_date date not null
+, item_id int(11) not null
+, pv bigint(20) not null default 0
+, uu bigint(20) not null default 0
+)
+;
+
+create unique index item_pvs_idx on item_pvs (id, data_date, item_id) using btree
+;

data/test/item_pvs.ct.redshift

@@ -0,0 +1,9 @@
+create table item_pvs
+( id bigint encode lzo
+, data_date date encode lzo
+, item_id integer encode lzo
+, pv bigint encode lzo
+, uu bigint encode lzo
+)
+sortkey (data_date, item_id)
+;

data/test/reader/test_redshift_csv.rb

@@ -0,0 +1,30 @@
+require 'test/unit'
+require 'redshift-connector/reader'
+
+module RedshiftConnector
+  module Reader
+    class TestRedshiftCSV < Test::Unit::TestCase
+      def parse_row(line)
+        r = RedshiftCSV.new(nil)
+        r.parse_row(line, 1)
+      end
+
+      def test_parse_row
+        assert_equal ['xxx', 'yyyy', 'zzz'],
+          parse_row(%Q("xxx","yyyy","zzz"\n))
+
+        assert_equal ['xxx', 'yyyy', 'zzz'],
+          parse_row(%Q( "xxx" , "yyyy","zzz"\t\n))
+
+        assert_equal ['x,x', "y\r\ny", 'z"z', 'a\\a'],
+          parse_row(%Q("x\\,x","y\\r\\ny","z\\"z","a\\\\a"\n))
+
+        assert_equal ['981179', '2017-01-07', '6', 'show', '99', '3'],
+          parse_row(%Q("981179","2017-01-07","6","show","99","3"\r\n))
+
+        assert_equal ['981179', '2017-01-07', '6', '852', 'show', '{"page"=>"4"}', '1', '1'],
+          parse_row(%Q("981179","2017-01-07","6","852","show","{\\"page\\"=>\\"4\\"}","1","1"\n))
+      end
+    end
+  end
+end

data/test/test_connector.rb

@@ -0,0 +1,148 @@
+require_relative 'helper'
+require 'test/unit'
+
+class TestConnector < Test::Unit::TestCase
+  def test_connector_upsert
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_delta(
+      schema: $TEST_SCHEMA,
+      table: 'item_pvs',
+
+      txn_id: data_date,
+      condition: %Q(data_date = date '#{data_date}'),
+
+      columns: %w[id data_date item_id pv uu],
+      upsert_columns: %w[pv uu],
+      filter: -> (id, data_date, item_id, pv, uu) {
+        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+      }
+    )
+    job.execute
+  end
+
+  def test_connector_delete_insert
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_delta(
+      schema: $TEST_SCHEMA,
+      table: 'item_pvs',
+
+      txn_id: data_date,
+      condition: %Q(data_date = date '#{data_date}'),
+      delete_cond: %Q(data_date = date '#{data_date}'),
+
+      columns: %w[id data_date item_id pv uu],
+      filter: -> (id, data_date, item_id, pv, uu) {
+        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+      }
+    )
+    job.execute
+  end
+
+  def test_dup_options
+    data_date = '2016-11-03'
+    assert_raise(ArgumentError) {
+      RedshiftConnector.transport_delta(
+        schema: $TEST_SCHEMA,
+        table: 'item_pvs',
+
+        txn_id: data_date,
+        condition: %Q(data_date = date '#{data_date}'),
+        delete_cond: %Q(data_date = date '#{data_date}'),
+
+        # Conflicts with delete_cond option
+        upsert_columns: %w[pv uu],
+
+        columns: %w[id data_date item_id pv uu],
+        filter: -> (id, data_date, item_id, pv, uu) {
+          [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+        }
+      )
+    }
+  end
+
+  def test_no_required_option
+    data_date = '2016-11-03'
+    assert_raise(ArgumentError) {
+      RedshiftConnector.transport_delta(
+        schema: $TEST_SCHEMA,
+        table: 'item_pvs',
+
+        txn_id: data_date,
+        condition: %Q(data_date = date '#{data_date}'),
+
+        columns: %w[id data_date item_id pv uu],
+        filter: -> (id, data_date, item_id, pv, uu) {
+          [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+        }
+      )
+    }
+  end
+
+  def test_connector_rebuild_truncate
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_all(
+      strategy: 'truncate',
+      schema: $TEST_SCHEMA,
+      table: 'item_pvs',
+      txn_id: data_date,
+      columns: %w[id data_date item_id pv uu],
+      filter: -> (id, data_date, item_id, pv, uu) {
+        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+      }
+    )
+    job.execute
+  end
+
+  def test_connector_rebuild_rename
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_all(
+      strategy: 'rename',
+      schema: $TEST_SCHEMA,
+      table: 'item_pvs',
+      txn_id: data_date,
+      columns: %w[id data_date item_id pv uu],
+      filter: -> (id, data_date, item_id, pv, uu) {
+        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+      }
+    )
+    job.execute
+  end
+
+  def test_connector_src_dest_table
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_delta(
+      schema: $TEST_SCHEMA,
+      src_table: 'item_pvs',
+      dest_table: 'item_pvs',
+
+      txn_id: data_date,
+      condition: %Q(data_date = date '#{data_date}'),
+
+      columns: %w[id data_date item_id pv uu],
+      upsert_columns: %w[pv uu],
+      filter: -> (id, data_date, item_id, pv, uu) {
+        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+      }
+    )
+    job.execute
+  end
+
+  def test_connector_missing_src_dest
+    data_date = '2016-11-03'
+    assert_raise(ArgumentError) {
+      RedshiftConnector.transport_delta(
+        schema: $TEST_SCHEMA,
+        src_table: 'item_pvs',
+
+        txn_id: data_date,
+        condition: %Q(data_date = date '#{data_date}'),
+
+        columns: %w[id data_date item_id pv uu],
+        upsert_columns: %w[pv uu],
+        filter: -> (id, data_date, item_id, pv, uu) {
+          [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
+        }
+      )
+    }
+  end
+end

data/test/test_reader.rb

@@ -0,0 +1,10 @@
+require 'test/unit'
+require 'redshift-connector/reader'
+
+module RedshiftConnector
+  class TestReader < Test::Unit::TestCase
+    def test_get
+      assert_equal Reader::RedshiftCSV, Reader.get(:redshift_csv)
+    end
+  end
+end

data/test/test_s3_import.rb

@@ -0,0 +1,32 @@
+require_relative 'helper'
+require 'test/unit'
+require 'redshift-connector'
+
+class TestS3Import < Test::Unit::TestCase
+  def test_import_delta_tsv
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_delta_from_s3(
+      prefix: "#{$TEST_SCHEMA}_export/item_pvs_tsv/#{data_date}/item_pvs.tsv.",
+      format: :tsv,
+
+      table: 'item_pvs',
+      columns: %w[id data_date item_id pv uu],
+      upsert_columns: %w[pv uu]
+    )
+    job.execute
+  end
+
+  def test_import_all
+    data_date = '2016-11-03'
+    job = RedshiftConnector.transport_all_from_s3(
+      strategy: 'truncate',
+
+      prefix: "#{$TEST_SCHEMA}_export/item_pvs_tsv/#{data_date}/item_pvs.tsv.",
+      format: :tsv,
+
+      table: 'item_pvs',
+      columns: %w[id data_date item_id pv uu]
+    )
+    job.execute
+  end
+end

metadata

@@ -0,0 +1,190 @@
+--- !ruby/object:Gem::Specification
+name: redshift-connector
+version: !ruby/object:Gem::Version
+  version: 4.3.0
+platform: ruby
+authors:
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: activerecord4-redshift-adapter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.0
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: redshift-connector is a bulk data connector for Rails (ActiveRecord).
+email: aamine@loveruby.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/redshift-connector.rb
+- lib/redshift-connector/connector.rb
+- lib/redshift-connector/exporter.rb
+- lib/redshift-connector/importer.rb
+- lib/redshift-connector/importer/activerecord-import.rb
+- lib/redshift-connector/importer/insert_delta.rb
+- lib/redshift-connector/importer/rebuild_rename.rb
+- lib/redshift-connector/importer/rebuild_truncate.rb
+- lib/redshift-connector/importer/upsert.rb
+- lib/redshift-connector/logger.rb
+- lib/redshift-connector/query.rb
+- lib/redshift-connector/reader.rb
+- lib/redshift-connector/reader/abstract.rb
+- lib/redshift-connector/reader/csv.rb
+- lib/redshift-connector/reader/exception.rb
+- lib/redshift-connector/reader/redshift_csv.rb
+- lib/redshift-connector/reader/tsv.rb
+- lib/redshift-connector/s3_bucket.rb
+- lib/redshift-connector/s3_data_file.rb
+- lib/redshift-connector/s3_data_file_bundle.rb
+- lib/redshift-connector/version.rb
+- test/all.rb
+- test/config.rb
+- test/config.rb.example
+- test/database.yml
+- test/database.yml.example
+- test/foreach.rb
+- test/helper.rb
+- test/item_pvs.ct.mysql
+- test/item_pvs.ct.redshift
+- test/reader/test_redshift_csv.rb
+- test/test_connector.rb
+- test/test_reader.rb
+- test/test_s3_import.rb
+homepage: https://github.com/aamine/redshift-connector
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Redshift bulk data connector
+test_files: []