redshift_extractor 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -4
- data/lib/redshift_extractor/copy.rb +4 -3
- data/lib/redshift_extractor/drop.rb +4 -3
- data/lib/redshift_extractor/extractor.rb +6 -2
- data/lib/redshift_extractor/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: efa20918ea071a3450d6a62cd675e89744f76321
|
4
|
+
data.tar.gz: 3fb762b159c2d28203d75a5f87b63b101511ba4f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 02377cc2c4d433b56a09f6054bdba298349cb8aed7177f33c5cacad100f4edb03fa29679bfcb24bc5900e8110331bdf465b87f0b72fd5ad02c111b308ed914d5
|
7
|
+
data.tar.gz: cc0b7c84cd915d6ec25215f2cd025c73878045e68e0a1f898280d0f18e6a6e89a70160b4c634dc1de428477e668ada029f7de3bb9baf51ebc575b7cb415e4a4e
|
data/README.md
CHANGED
@@ -18,7 +18,7 @@ One database connection is established with the source database to UNLOAD the da
|
|
18
18
|
|
19
19
|
## Running the Code
|
20
20
|
|
21
|
-
The RedshiftExtractor::Extractor class is instantiated with a long hash of arguments.
|
21
|
+
The `RedshiftExtractor::Extractor` class is instantiated with a long hash of arguments ([sorry Sandi Metz!](https://robots.thoughtbot.com/sandi-metz-rules-for-developers)).
|
22
22
|
|
23
23
|
```ruby
|
24
24
|
args = {
|
@@ -26,7 +26,8 @@ args = {
|
|
26
26
|
database_config_destination: "database_config_destination",
|
27
27
|
unload_s3_destination: "unload_s3_destination",
|
28
28
|
unload_select_sql: "unload_select_sql",
|
29
|
-
|
29
|
+
destination_schema: "destination_schema",
|
30
|
+
destination_table: "destination_table",
|
30
31
|
create_sql: "create_sql",
|
31
32
|
copy_data_source: "copy_data_source",
|
32
33
|
aws_access_key_id: "aws_access_key_id",
|
@@ -56,9 +57,9 @@ Here is a description of the parameters:
|
|
56
57
|
|
57
58
|
- unload_select_sql: A SQL SELECT query that will be run on the source table
|
58
59
|
|
59
|
-
-
|
60
|
+
- destination_schema, destination_table: The table that will be dropped, recreated, and populated with data from the COPY command
|
60
61
|
|
61
|
-
- create_sql: The SQL that creates the
|
62
|
+
- create_sql: The SQL that creates the destination_schema.destination_table table (this SQL is run to recreate the table in the step above)
|
62
63
|
|
63
64
|
- copy_data_source: This is typically `"#{unload_s3_destination}manifest"`. The UNLOAD command automatically creates a manifest file that can be used by the COPY command to load the data.
|
64
65
|
|
@@ -1,16 +1,17 @@
|
|
1
1
|
module RedshiftExtractor; class Copy
|
2
2
|
|
3
|
-
attr_reader :aws_access_key_id, :aws_secret_access_key, :data_source, :
|
3
|
+
attr_reader :aws_access_key_id, :aws_secret_access_key, :data_source, :destination_schema, :destination_table
|
4
4
|
|
5
5
|
def initialize(args)
|
6
6
|
@aws_access_key_id = args.fetch(:aws_access_key_id)
|
7
7
|
@aws_secret_access_key = args.fetch(:aws_secret_access_key)
|
8
8
|
@data_source = args.fetch(:data_source)
|
9
|
-
@
|
9
|
+
@destination_schema = args.fetch(:destination_schema)
|
10
|
+
@destination_table = args.fetch(:destination_table)
|
10
11
|
end
|
11
12
|
|
12
13
|
def copy_sql
|
13
|
-
"copy #{
|
14
|
+
"copy #{destination_schema}.#{destination_table} from '#{data_source}'"\
|
14
15
|
" credentials '#{credentials}'"\
|
15
16
|
" manifest dateformat 'auto' timeformat 'auto' blanksasnull emptyasnull escape gzip removequotes delimiter '|';"
|
16
17
|
end
|
@@ -1,13 +1,14 @@
|
|
1
1
|
module RedshiftExtractor; class Drop
|
2
2
|
|
3
|
-
attr_reader :
|
3
|
+
attr_reader :destination_schema, :destination_table
|
4
4
|
|
5
5
|
def initialize(args)
|
6
|
-
@
|
6
|
+
@destination_schema = args.fetch(:destination_schema)
|
7
|
+
@destination_table = args.fetch(:destination_table)
|
7
8
|
end
|
8
9
|
|
9
10
|
def drop_sql
|
10
|
-
"drop table if exists #{
|
11
|
+
"drop table if exists #{destination_schema}.#{destination_table};"
|
11
12
|
end
|
12
13
|
|
13
14
|
end; end
|
@@ -31,7 +31,10 @@ module RedshiftExtractor; class Extractor
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def dropper
|
34
|
-
Drop.new(
|
34
|
+
Drop.new(
|
35
|
+
destination_schema: config.destination_schema,
|
36
|
+
destination_table: config.destination_table
|
37
|
+
)
|
35
38
|
end
|
36
39
|
|
37
40
|
def drop
|
@@ -47,7 +50,8 @@ module RedshiftExtractor; class Extractor
|
|
47
50
|
aws_access_key_id: config.aws_access_key_id,
|
48
51
|
aws_secret_access_key: config.aws_secret_access_key,
|
49
52
|
data_source: config.copy_data_source,
|
50
|
-
|
53
|
+
destination_schema: config.destination_schema,
|
54
|
+
destination_table: config.destination_table
|
51
55
|
)
|
52
56
|
end
|
53
57
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redshift_extractor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- MrPowers
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|