redshift_extractor 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/redshift_extractor.rb +1 -0
- data/lib/redshift_extractor/extractor.rb +11 -28
- data/lib/redshift_extractor/version.rb +1 -1
- data/redshift_extractor.gemspec +1 -0
- metadata +16 -4
- data/lib/redshift_extractor/copy.rb +0 -24
- data/lib/redshift_extractor/drop.rb +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 75a7c818e209b911997eaffb71754a90093eabec
|
4
|
+
data.tar.gz: 55661e198a7a48e55743a9f91629c5b5ab85fa7a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6713dfa0680b463e0974faad511ae5b94a01c86668fe3929116761e5c24b89baea291779fdefee14c6b83a5febb8a96a3e40e21c60cdf19bd11b0bc79321513c
|
7
|
+
data.tar.gz: c8f852a026538cf5c500ea327f839dd6b4c375a0924ed8ed48a94fb5496994b81757b7536e46ed5fdbf1d87735c21cb667f2fae0d2e276fd6b46493eb11fd92a
|
data/lib/redshift_extractor.rb
CHANGED
@@ -10,8 +10,6 @@ module RedshiftExtractor; class Extractor
|
|
10
10
|
|
11
11
|
def run
|
12
12
|
unload
|
13
|
-
drop
|
14
|
-
create
|
15
13
|
copy
|
16
14
|
end
|
17
15
|
|
@@ -30,37 +28,22 @@ module RedshiftExtractor; class Extractor
|
|
30
28
|
source_connection.exec(unloader.unload_sql)
|
31
29
|
end
|
32
30
|
|
33
|
-
def
|
34
|
-
|
35
|
-
destination_schema: config.destination_schema,
|
36
|
-
destination_table: config.destination_table
|
37
|
-
)
|
38
|
-
end
|
39
|
-
|
40
|
-
def drop
|
41
|
-
destination_connection.exec(dropper.drop_sql)
|
42
|
-
end
|
43
|
-
|
44
|
-
def create
|
45
|
-
destination_connection.exec(config.create_sql)
|
31
|
+
def copy
|
32
|
+
copier.run
|
46
33
|
end
|
47
34
|
|
48
35
|
def copier
|
49
|
-
|
36
|
+
args = {
|
37
|
+
schema: config.destination_schema,
|
38
|
+
table: config.destination_table,
|
39
|
+
create_sql: config.create_sql,
|
50
40
|
aws_access_key_id: config.aws_access_key_id,
|
51
41
|
aws_secret_access_key: config.aws_secret_access_key,
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
def copy
|
59
|
-
destination_connection.exec(copier.copy_sql)
|
60
|
-
end
|
61
|
-
|
62
|
-
def destination_connection
|
63
|
-
PGconn.connect(config.database_config_destination)
|
42
|
+
s3_path: config.copy_data_source,
|
43
|
+
db_config: config.database_config_destination,
|
44
|
+
copy_command_options: "manifest dateformat 'auto' timeformat 'auto' blanksasnull emptyasnull escape gzip removequotes delimiter '|';"
|
45
|
+
}
|
46
|
+
RedshiftCopier::Copy.new(args)
|
64
47
|
end
|
65
48
|
|
66
49
|
def source_connection
|
data/redshift_extractor.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redshift_extractor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- MrPowers
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - '='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 0.18.3
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: redshift_copier
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.2.0
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.2.0
|
69
83
|
description:
|
70
84
|
email:
|
71
85
|
- matthewkevinpowers@gmail.com
|
@@ -83,8 +97,6 @@ files:
|
|
83
97
|
- bin/console
|
84
98
|
- bin/setup
|
85
99
|
- lib/redshift_extractor.rb
|
86
|
-
- lib/redshift_extractor/copy.rb
|
87
|
-
- lib/redshift_extractor/drop.rb
|
88
100
|
- lib/redshift_extractor/extractor.rb
|
89
101
|
- lib/redshift_extractor/unload.rb
|
90
102
|
- lib/redshift_extractor/version.rb
|
@@ -1,24 +0,0 @@
|
|
1
|
-
module RedshiftExtractor; class Copy
|
2
|
-
|
3
|
-
attr_reader :aws_access_key_id, :aws_secret_access_key, :data_source, :destination_schema, :destination_table
|
4
|
-
|
5
|
-
def initialize(args)
|
6
|
-
@aws_access_key_id = args.fetch(:aws_access_key_id)
|
7
|
-
@aws_secret_access_key = args.fetch(:aws_secret_access_key)
|
8
|
-
@data_source = args.fetch(:data_source)
|
9
|
-
@destination_schema = args.fetch(:destination_schema)
|
10
|
-
@destination_table = args.fetch(:destination_table)
|
11
|
-
end
|
12
|
-
|
13
|
-
def copy_sql
|
14
|
-
"copy #{destination_schema}.#{destination_table} from '#{data_source}'"\
|
15
|
-
" credentials '#{credentials}'"\
|
16
|
-
" manifest dateformat 'auto' timeformat 'auto' blanksasnull emptyasnull escape gzip removequotes delimiter '|';"
|
17
|
-
end
|
18
|
-
|
19
|
-
def credentials
|
20
|
-
"aws_access_key_id=#{aws_access_key_id};aws_secret_access_key=#{aws_secret_access_key}"
|
21
|
-
end
|
22
|
-
|
23
|
-
end; end
|
24
|
-
|
@@ -1,15 +0,0 @@
|
|
1
|
-
module RedshiftExtractor; class Drop
|
2
|
-
|
3
|
-
attr_reader :destination_schema, :destination_table
|
4
|
-
|
5
|
-
def initialize(args)
|
6
|
-
@destination_schema = args.fetch(:destination_schema)
|
7
|
-
@destination_table = args.fetch(:destination_table)
|
8
|
-
end
|
9
|
-
|
10
|
-
def drop_sql
|
11
|
-
"drop table if exists #{destination_schema}.#{destination_table};"
|
12
|
-
end
|
13
|
-
|
14
|
-
end; end
|
15
|
-
|