redshift-connector 7.0.2 → 7.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4869945b94bf612693b98677cc3e4e9387f42533
-  data.tar.gz: f920bd5ee6f860615cbfa4926b5a82f59642235c
+SHA256:
+  metadata.gz: 6d05034adedce4cea33f24328428eb4823e89224eba62cd3548bf41e329cfe5d
+  data.tar.gz: 85b98e33059fcac9e50e505f9eea6cfc53665cf31b66a1de1fc676199150687d
 SHA512:
-  metadata.gz: ca4ba82a836040efab25cd5c3484aab7fbc9881833bcb6dc1c8afb391d2b00348b577ac1a722e3d964e9f57a62a602bce38bf6e97917da9b095fa06fe622e477
-  data.tar.gz: 505f1dd2b14ee460e46c274232d673bad58bd55b88a4558225a54c292177765b72f07514dfc0e59f5f7de4e8b78f373a0ea4c13dfcbc7404e03586e1c4431fa8
+  metadata.gz: 6ab6e0528d17951481902ed37679f110bed1718848a17cb962598524e5439e3784dc46b9293222fdf93e0c74ab820d58243e92fc3268172e91019140e99d92dc
+  data.tar.gz: 9800067fa5f06ea4268702519650b1155cdb71c212852402e7cf539f9e25a9408a2987f2e77b97ac6551dd8f5d7511e12fca10d9ebabfcaf07653f5f1feac464

data/.gitignore

@@ -0,0 +1,20 @@
+# Packaging
+*.gem
+/spec/reports/
+/spec/examples.txt
+
+# Documents
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+# Bundler
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+Gemfile.lock
+
+# Gem Specific
+test/database.yml
+test/config.rb

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+gemspec
+
+# We need explicit version specification (4 or 5) here,
+# to resolve version dependencies correctly.
+gem 'activerecord', '~> 5.0'
+gem 'mysql2'

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016,2017 Minero Aoki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/RELEASE.md

@@ -0,0 +1,76 @@
+# Release Note
+
+## version 7.2.1
+
+- no change.
+
+## version 7.2.0
+
+- Removes aws-sdk dependency
+
+## version 7.0.2
+
+- [fix] RedshiftConnector.foreach did not work
+
+## version 7.0.1
+
+- [fix] RedshiftConnector.transport_delta_from_s3, .transport_all_from_s3 were wrongly dropped, restore them.
+
+## version 7.0.0
+
+- [INCOMPATIBLE] Library hierarchy changed: redshift-connector/* -> redshift_connector/*.  redshift-connector.rb still exists as an entry point for bundler.
+- [new] Exporter becomes pluggable.  You can implement your own exporter data source instead of ActiveRecord.
+
+## version 6.0.0
+
+- version number change only.
+
+## version 5.6.0
+
+- Unifies version 4.x (supports Rails 4) and 5.x (supports Rails 5).
+
+## version 4.5.0 / 5.5.0
+
+- [new] Separates S3 access layer to another gem: redshift-connector-data_file
+
+## version 4.4.1 / 5.4.1
+
+- [new] New option enable_sort for Connector.foreach, to enforce global sorting.
+
+## version 4.4.0 / 5.4.0
+
+- [CHANGE] Drops export-only-once feature (and FORCE environment switch), it is not so useful.
+  Exporter now always exports data.
+
+## version 4.3.2 / 5.3.2
+
+- [new] Allows reading from S3 signed URL (for separated export/import processes)
+
+## version 4.3.1 / 5.3.1
+
+- First release for Rails 5 series.
+- [fix] Add option for AWS multi-regions support
+
+## version 4.3.0
+
+- [new] New method RedshiftConnector.foreach to read rows with UNLOAD
+
+## version 4.2.0
+
+- [new] New methods RedshiftConnector.transport_delta_from_s3, .transport_all_from_s3 to read from S3
+ 
+## version 4.1.0
+
+- [new] Introduces rebuild operator.  New facade method Connector.transport_all.
+  
+## version 4.0.2
+  
+- [fix] Correctly parses UNLOAD-generated CSV (dangerous characeters are escaped by backslash).
+  
+## version 4.0.1
+  
+- [new] Allow configure the default logger by RedshiftConnector.logger=.
+  
+## version 4.0.0
+  
+First release for Rails 4 series.

data/Rakefile

@@ -0,0 +1,3 @@
+task :test do
+  load "#{__dir__}/test/all.rb"
+end

data/lib/redshift_connector/version.rb

@@ -1,3 +1,3 @@
 module RedshiftConnector
-  VERSION = '7.0.2'
+  VERSION = '7.2.1'
 end

data/redshift-connector.gemspec

@@ -0,0 +1,26 @@
+require_relative 'lib/redshift_connector/version'
+
+Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.name = 'redshift-connector'
+  s.version = RedshiftConnector::VERSION
+  s.summary = 'Redshift bulk data connector'
+  s.description = 'redshift-connector is a bulk data connector for Rails (ActiveRecord).'
+  s.license = 'MIT'
+
+  s.author = ['Minero Aoki']
+  s.email = 'aamine@loveruby.net'
+  s.homepage = 'https://github.com/bricolages/redshift-connector'
+
+  s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  s.require_path = 'lib'
+
+  s.required_ruby_version = '>= 2.1.0'
+  s.add_dependency 'redshift-connector-data_file', '>= 7.0.0'
+  s.add_dependency 'activerecord'
+  s.add_dependency 'activerecord-redshift'
+  s.add_dependency 'pg', '~> 0.18.0'
+  s.add_dependency 'activerecord-import'
+  s.add_development_dependency 'test-unit'
+  s.add_development_dependency 'rake'
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector
 version: !ruby/object:Gem::Version
-  version: 7.0.2
+  version: 7.2.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-22 00:00:00.000000000 Z
+date: 2018-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redshift-connector-data_file
@@ -80,20 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: aws-sdk
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -108,20 +94,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,7 +114,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- Gemfile
+- LICENSE
 - README.md
+- RELEASE.md
+- Rakefile
 - lib/redshift-connector.rb
 - lib/redshift_connector.rb
 - lib/redshift_connector/active_record_data_source.rb
@@ -161,18 +138,7 @@ files:
 - lib/redshift_connector/importer/upsert.rb
 - lib/redshift_connector/query.rb
 - lib/redshift_connector/version.rb
-- test/all.rb
-- test/config.rb
-- test/config.rb.example
-- test/database.yml
-- test/database.yml.example
-- test/foreach.rb
-- test/helper.rb
-- test/item_pvs.ct.mysql
-- test/item_pvs.ct.redshift
-- test/reader/test_redshift_csv.rb
-- test/test_connector.rb
-- test/test_s3_import.rb
+- redshift-connector.gemspec
 homepage: https://github.com/bricolages/redshift-connector
 licenses:
 - MIT
@@ -193,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Redshift bulk data connector

data/test/all.rb

@@ -1,3 +0,0 @@
-Dir.glob("#{__dir__}/**/test_*.rb").each do |path|
-  load path
-end

data/test/config.rb

@@ -1,13 +0,0 @@
-$TEST_SCHEMA = 'aamine'
-
-module RedshiftConnector
-  Exporter.default_data_source = RedshiftConnector::ActiveRecordDataSource.new(Redshift)
-
-  S3Bucket.add(
-    'redshift-copy-buffer',
-    bucket: 'redshift-copy-buffer',
-    prefix: 'development',
-    iam_role: 'arn:aws:iam::789035092620:role/RedshiftDevelopers',
-    default: true
-  )
-end

data/test/config.rb.example

@@ -1,18 +0,0 @@
-module RedshiftConnector
-  # For test only
-  $TEST_SCHEMA = 'test'
-
-  Exporter.default_data_source = Redshift
-
-  S3Bucket.add(
-    'ENTRY_NAME',
-    bucket: 'YOUR_BUCKET_NAME',
-    prefix: 'development',
-    # When using IAM role
-    iam_role: 'arn:aws:iam::NNNNNNNNNNNN:role/RRRRRRRRR',
-    # When using explicit access key
-    #access_key_id: 'AAAAAAAAAA',
-    #secret_access_key: 'SSSSSSSSSS',
-    default: true
-  )
-end

data/test/database.yml

@@ -1,15 +0,0 @@
-mysql:
-  adapter: mysql2
-  host: localhost
-  username: minero-aoki
-  database: test
-  encoding: utf8
-
-redshift:
-  adapter: redshift
-  host: dwh.ckpd.co
-  port: 5439
-  database: production
-  username: aamine
-  password: "3edCVfr$"
-  encoding: utf8

data/test/database.yml.example

@@ -1,15 +0,0 @@
-mysql:
-  adapter: mysql2
-  host: localhost
-  username: USER_NAME
-  database: test
-  encoding: utf8
-
-redshift:
-  adapter: redshift
-  host: HOST_NAME
-  port: 5439
-  database: DATABASE_NAME
-  username: USER_NAME
-  password: PASSWORD
-  encoding: utf8

data/test/foreach.rb

@@ -1,5 +0,0 @@
-require_relative 'helper'
-
-RedshiftConnector.foreach(schema: 'tabemiru', table: 'items', query: 'select id from tabemiru.items where id < 50 order by 1', enable_sort: true) do |row|
-  p row
-end

data/test/helper.rb

@@ -1,25 +0,0 @@
-require 'active_record'
-require 'yaml'
-require 'logger'
-
-YAML.load_file("#{__dir__}/database.yml").each do |name, ent|
-  ActiveRecord::Base.configurations[name] = ent
-end
-
-class BaseConn < ActiveRecord::Base
-  establish_connection :mysql
-  self.abstract_class = true
-end
-class ItemPv < BaseConn
-  connection
-end
-class Redshift < ActiveRecord::Base
-  establish_connection :redshift
-end
-
-require 'redshift-connector'
-# This IS REQUIRED to ensure to load mysql2 driver, at least outside of Rails.
-ActiveRecord::Import.require_adapter 'mysql2'
-require_relative 'config'
-
-RedshiftConnector.logger = Logger.new($stderr)

data/test/item_pvs.ct.mysql

@@ -1,11 +0,0 @@
-create table item_pvs
-( id bigint(20) primary key
-, data_date date not null
-, item_id int(11) not null
-, pv bigint(20) not null default 0
-, uu bigint(20) not null default 0
-)
-;
-
-create unique index item_pvs_idx on item_pvs (id, data_date, item_id) using btree
-;

data/test/item_pvs.ct.redshift

@@ -1,9 +0,0 @@
-create table item_pvs
-( id bigint encode lzo
-, data_date date encode lzo
-, item_id integer encode lzo
-, pv bigint encode lzo
-, uu bigint encode lzo
-)
-sortkey (data_date, item_id)
-;

data/test/reader/test_redshift_csv.rb

@@ -1,30 +0,0 @@
-require 'test/unit'
-require 'redshift-connector/data_file'
-
-module RedshiftConnector
-  module Reader
-    class TestRedshiftCSV < Test::Unit::TestCase
-      def parse_row(line)
-        r = RedshiftCSV.new(nil)
-        r.parse_row(line, 1)
-      end
-
-      def test_parse_row
-        assert_equal ['xxx', 'yyyy', 'zzz'],
-          parse_row(%Q("xxx","yyyy","zzz"\n))
-
-        assert_equal ['xxx', 'yyyy', 'zzz'],
-          parse_row(%Q( "xxx" , "yyyy","zzz"\t\n))
-
-        assert_equal ['x,x', "y\r\ny", 'z"z', 'a\\a'],
-          parse_row(%Q("x\\,x","y\\r\\ny","z\\"z","a\\\\a"\n))
-
-        assert_equal ['981179', '2017-01-07', '6', 'show', '99', '3'],
-          parse_row(%Q("981179","2017-01-07","6","show","99","3"\r\n))
-
-        assert_equal ['981179', '2017-01-07', '6', '852', 'show', '{"page"=>"4"}', '1', '1'],
-          parse_row(%Q("981179","2017-01-07","6","852","show","{\\"page\\"=>\\"4\\"}","1","1"\n))
-      end
-    end
-  end
-end

data/test/test_connector.rb

@@ -1,148 +0,0 @@
-require_relative 'helper'
-require 'test/unit'
-
-class TestConnector < Test::Unit::TestCase
-  def test_connector_upsert
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_delta(
-      schema: $TEST_SCHEMA,
-      table: 'item_pvs',
-
-      txn_id: data_date,
-      condition: %Q(data_date = date '#{data_date}'),
-
-      columns: %w[id data_date item_id pv uu],
-      upsert_columns: %w[pv uu],
-      filter: -> (id, data_date, item_id, pv, uu) {
-        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-      }
-    )
-    job.execute
-  end
-
-  def test_connector_delete_insert
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_delta(
-      schema: $TEST_SCHEMA,
-      table: 'item_pvs',
-
-      txn_id: data_date,
-      condition: %Q(data_date = date '#{data_date}'),
-      delete_cond: %Q(data_date = date '#{data_date}'),
-
-      columns: %w[id data_date item_id pv uu],
-      filter: -> (id, data_date, item_id, pv, uu) {
-        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-      }
-    )
-    job.execute
-  end
-
-  def test_dup_options
-    data_date = '2016-11-03'
-    assert_raise(ArgumentError) {
-      RedshiftConnector.transport_delta(
-        schema: $TEST_SCHEMA,
-        table: 'item_pvs',
-
-        txn_id: data_date,
-        condition: %Q(data_date = date '#{data_date}'),
-        delete_cond: %Q(data_date = date '#{data_date}'),
-
-        # Conflicts with delete_cond option
-        upsert_columns: %w[pv uu],
-
-        columns: %w[id data_date item_id pv uu],
-        filter: -> (id, data_date, item_id, pv, uu) {
-          [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-        }
-      )
-    }
-  end
-
-  def test_no_required_option
-    data_date = '2016-11-03'
-    assert_raise(ArgumentError) {
-      RedshiftConnector.transport_delta(
-        schema: $TEST_SCHEMA,
-        table: 'item_pvs',
-
-        txn_id: data_date,
-        condition: %Q(data_date = date '#{data_date}'),
-
-        columns: %w[id data_date item_id pv uu],
-        filter: -> (id, data_date, item_id, pv, uu) {
-          [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-        }
-      )
-    }
-  end
-
-  def test_connector_rebuild_truncate
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_all(
-      strategy: 'truncate',
-      schema: $TEST_SCHEMA,
-      table: 'item_pvs',
-      txn_id: data_date,
-      columns: %w[id data_date item_id pv uu],
-      filter: -> (id, data_date, item_id, pv, uu) {
-        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-      }
-    )
-    job.execute
-  end
-
-  def test_connector_rebuild_rename
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_all(
-      strategy: 'rename',
-      schema: $TEST_SCHEMA,
-      table: 'item_pvs',
-      txn_id: data_date,
-      columns: %w[id data_date item_id pv uu],
-      filter: -> (id, data_date, item_id, pv, uu) {
-        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-      }
-    )
-    job.execute
-  end
-
-  def test_connector_src_dest_table
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_delta(
-      schema: $TEST_SCHEMA,
-      src_table: 'item_pvs',
-      dest_table: 'item_pvs',
-
-      txn_id: data_date,
-      condition: %Q(data_date = date '#{data_date}'),
-
-      columns: %w[id data_date item_id pv uu],
-      upsert_columns: %w[pv uu],
-      filter: -> (id, data_date, item_id, pv, uu) {
-        [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-      }
-    )
-    job.execute
-  end
-
-  def test_connector_missing_src_dest
-    data_date = '2016-11-03'
-    assert_raise(ArgumentError) {
-      RedshiftConnector.transport_delta(
-        schema: $TEST_SCHEMA,
-        src_table: 'item_pvs',
-
-        txn_id: data_date,
-        condition: %Q(data_date = date '#{data_date}'),
-
-        columns: %w[id data_date item_id pv uu],
-        upsert_columns: %w[pv uu],
-        filter: -> (id, data_date, item_id, pv, uu) {
-          [id.to_i, data_date, item_id.to_i, pv.to_i, uu.to_i]
-        }
-      )
-    }
-  end
-end

data/test/test_s3_import.rb

@@ -1,32 +0,0 @@
-require_relative 'helper'
-require 'test/unit'
-require 'redshift-connector'
-
-class TestS3Import < Test::Unit::TestCase
-  def test_import_delta_tsv
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_delta_from_s3(
-      prefix: "#{$TEST_SCHEMA}_export/item_pvs_tsv/#{data_date}/item_pvs.tsv.",
-      format: :tsv,
-
-      table: 'item_pvs',
-      columns: %w[id data_date item_id pv uu],
-      upsert_columns: %w[pv uu]
-    )
-    job.execute
-  end
-
-  def test_import_all
-    data_date = '2016-11-03'
-    job = RedshiftConnector.transport_all_from_s3(
-      strategy: 'truncate',
-
-      prefix: "#{$TEST_SCHEMA}_export/item_pvs_tsv/#{data_date}/item_pvs.tsv.",
-      format: :tsv,
-
-      table: 'item_pvs',
-      columns: %w[id data_date item_id pv uu]
-    )
-    job.execute
-  end
-end