redshift_extractor 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: aa8e751f08bdd4b304276a2c3689340736df59cc
4
+ data.tar.gz: 1e81e2a83c9a9f753e5a72a46656fe7b51ccc8fc
5
+ SHA512:
6
+ metadata.gz: eb63d429a7eb62e67ce033b6b33462978801cb98d9c6cab2f2fbf17c783313ca68928b9517adfe47cd6279882bb627ef859e16c810d47cee0a8b116f2507d79c
7
+ data.tar.gz: 4dfc53bdfc08aa9fb03d6736f80b9e100369a25a8a0273d95d92b6c8f0848c251d8a53a2f248c17f85327f345c4902ecc33ae93e40daf7ae0fee7844f3699ecd
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
data/.travis.yml ADDED
@@ -0,0 +1,4 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2.2
4
+ before_install: gem install bundler -v 1.10.6
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in redshift_extractor.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 MrPowers
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,41 @@
1
+ # RedshiftExtractor
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/redshift_extractor`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'redshift_extractor'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install redshift_extractor
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/MrPowers/redshift_extractor.
36
+
37
+
38
+ ## License
39
+
40
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
41
+
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "redshift_extractor"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/setup ADDED
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,21 @@
1
+ module RedshiftExtractor; class Copy
2
+
3
+ attr_reader :aws_access_key_id, :aws_secret_access_key, :data_source, :table_name
4
+
5
+ def initialize(args)
6
+ @aws_access_key_id = args.fetch(:aws_access_key_id)
7
+ @aws_secret_access_key = args.fetch(:aws_secret_access_key)
8
+ @data_source = args.fetch(:data_source)
9
+ @table_name = args.fetch(:table_name)
10
+ end
11
+
12
+ def copy_sql
13
+ "copy #{table_name} from '#{data_source}' credentials '#{credentials}' manifest dateformat 'auto' timeformat 'auto' blanksasnull emptyasnull escape gzip removequotes delimiter '|';"
14
+ end
15
+
16
+ def credentials
17
+ "aws_access_key_id=#{aws_access_key_id};aws_secret_access_key=#{aws_secret_access_key}"
18
+ end
19
+
20
+ end; end
21
+
@@ -0,0 +1,14 @@
1
+ module RedshiftExtractor; class Drop
2
+
3
+ attr_reader :table_name
4
+
5
+ def initialize(args)
6
+ @table_name = args.fetch(:table_name)
7
+ end
8
+
9
+ def drop_sql
10
+ "drop table if exists #{table_name};"
11
+ end
12
+
13
+ end; end
14
+
@@ -0,0 +1,60 @@
1
+ module RedshiftExtractor; class Extractor
2
+
3
+ def initialize(args)
4
+ @args = args
5
+ end
6
+
7
+ def config
8
+ @config ||= OpenStruct.new(@args)
9
+ end
10
+
11
+ def run
12
+ unload
13
+ drop
14
+ create
15
+ copy
16
+ end
17
+
18
+ private
19
+
20
+ def unloader
21
+ Unload.new(
22
+ aws_access_key_id: config.aws_access_key_id,
23
+ aws_secret_access_key: config.aws_secret_access_key,
24
+ s3_destination: config.unload_s3_destination,
25
+ select_sql: config.unload_select_sql
26
+ )
27
+ end
28
+
29
+ def unload
30
+ source_connection.exec(unloader.unload_sql)
31
+ end
32
+
33
+ def drop
34
+ dropper = Drop.new(table_name: config.table_name)
35
+ destination_connection.exec(dropper.drop_sql)
36
+ end
37
+
38
+ def create
39
+ destination_connection.exec(config.create_sql)
40
+ end
41
+
42
+ def copy
43
+ copier = Copy.new(
44
+ aws_access_key_id: config.aws_access_key_id,
45
+ aws_secret_access_key: config.aws_secret_access_key,
46
+ data_source: config.copy_data_source,
47
+ table_name: config.table_name
48
+ )
49
+ destination_connection.exec(copier.copy_sql)
50
+ end
51
+
52
+ def destination_connection
53
+ PGconn.connect(config.database_config_destination)
54
+ end
55
+
56
+ def source_connection
57
+ PGconn.connect(config.database_config_source)
58
+ end
59
+
60
+ end; end
@@ -0,0 +1,25 @@
1
+ module RedshiftExtractor; class Unload
2
+
3
+ attr_reader :aws_access_key_id, :aws_secret_access_key, :s3_destination, :select_sql
4
+
5
+ def initialize(args)
6
+ @aws_access_key_id = args.fetch(:aws_access_key_id)
7
+ @aws_secret_access_key = args.fetch(:aws_secret_access_key)
8
+ @s3_destination = args.fetch(:s3_destination)
9
+ @select_sql = args.fetch(:select_sql)
10
+ end
11
+
12
+ def unload_sql
13
+ "UNLOAD('#{escaped_extract_sql}') to '#{s3_destination}' CREDENTIALS '#{credentials}' MANIFEST GZIP ADDQUOTES ESCAPE;"
14
+ end
15
+
16
+ def escaped_extract_sql
17
+ select_sql.gsub("'", "\\\\'")
18
+ end
19
+
20
+ def credentials
21
+ "aws_access_key_id=#{aws_access_key_id};aws_secret_access_key=#{aws_secret_access_key}"
22
+ end
23
+
24
+ end; end
25
+
@@ -0,0 +1,3 @@
1
+ module RedshiftExtractor
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,16 @@
1
+ require 'ostruct'
2
+
3
+ require "redshift_extractor/version"
4
+
5
+ require 'pg'
6
+
7
+ def require_all(pattern)
8
+ root = File.expand_path("../", File.dirname(__FILE__))
9
+ Dir.glob("#{root}/#{pattern}/**/*.rb").sort.each { |path| require path }
10
+ end
11
+
12
+ require_all("lib/redshift_extractor")
13
+
14
+ module RedshiftExtractor
15
+ end
16
+
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'redshift_extractor/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "redshift_extractor"
8
+ spec.version = RedshiftExtractor::VERSION
9
+ spec.authors = ["MrPowers"]
10
+ spec.email = ["matthewkevinpowers@gmail.com"]
11
+
12
+ spec.summary = %q{Uses the Redshift UNLOAD and COPY commands to move data from one Redshift cluster to another}
13
+ spec.homepage = "https://github.com/mrpowers/redshift_extractor"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = "exe"
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.10"
22
+ spec.add_development_dependency "rake", "~> 10.0"
23
+ spec.add_development_dependency "rspec"
24
+
25
+ spec.add_dependency "pg", "0.18.3"
26
+ end
metadata ADDED
@@ -0,0 +1,117 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: redshift_extractor
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - MrPowers
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2015-10-27 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.10'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.10'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pg
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.18.3
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 0.18.3
69
+ description:
70
+ email:
71
+ - matthewkevinpowers@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".rspec"
78
+ - ".travis.yml"
79
+ - Gemfile
80
+ - LICENSE.txt
81
+ - README.md
82
+ - Rakefile
83
+ - bin/console
84
+ - bin/setup
85
+ - lib/redshift_extractor.rb
86
+ - lib/redshift_extractor/copy.rb
87
+ - lib/redshift_extractor/drop.rb
88
+ - lib/redshift_extractor/extractor.rb
89
+ - lib/redshift_extractor/unload.rb
90
+ - lib/redshift_extractor/version.rb
91
+ - redshift_extractor.gemspec
92
+ homepage: https://github.com/mrpowers/redshift_extractor
93
+ licenses:
94
+ - MIT
95
+ metadata: {}
96
+ post_install_message:
97
+ rdoc_options: []
98
+ require_paths:
99
+ - lib
100
+ required_ruby_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - ">="
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ required_rubygems_version: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ requirements: []
111
+ rubyforge_project:
112
+ rubygems_version: 2.4.5
113
+ signing_key:
114
+ specification_version: 4
115
+ summary: Uses the Redshift UNLOAD and COPY commands to move data from one Redshift
116
+ cluster to another
117
+ test_files: []