flare-up 0.3 → 0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/Gemfile.lock +1 -1
- data/README.md +8 -6
- data/lib/flare_up.rb +1 -0
- data/lib/flare_up/boot.rb +5 -3
- data/lib/flare_up/cli.rb +3 -6
- data/lib/flare_up/emitter.rb +38 -0
- data/lib/flare_up/stl_load_error_fetcher.rb +1 -1
- data/lib/flare_up/version.rb +1 -1
- data/spec/lib/flare_up/boot_spec.rb +4 -0
- data/spec/lib/flare_up/cli_spec.rb +25 -1
- data/spec/lib/flare_up/emitter_spec.rb +3 -0
- data/spec/lib/flare_up/stl_load_error_fetcher_spec.rb +1 -1
- metadata +4 -1
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MDY0ZDc2M2NlMjg4ZjQ2YzllZGYyNTkxYWI0ZGRkMDE4N2I2OTJhNQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ZWZjNmM0NDAyMDY2NjU0YThiYzAwNDljODAzODllMzlhYzcyNmMwZg==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NTYyMDZiYzBlZDY3YjcyMzdiZTVhYjZiZmUzYWIzYmM4NWM0NzAxODRmYjcz
|
10
|
+
NDdiODM2NzdhMWMzMTFkZmIzYmVmNWY2Njg0MmIwYTRiZjRkOTNjOWMyZjcw
|
11
|
+
NjAxM2MxNDU4NmJjNjJlZWI0MGI2NmI0MzlmNDZmZDM4YTZjMzQ=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MDExZjBkNmNlOGExNTU5YWY4ZTFhZjI3ZjdmNDUxY2Y1NDE3ZDc4ZTdiYjVk
|
14
|
+
MjlmZjlhNWUyM2JjMjNkMDIyZTYxNGI3MTUzM2Q4OTdmMzVhNGJhMzZkNWY5
|
15
|
+
MjBlNTU0ZDA1YTI1YTIzMWIwNjNkNWUwZjlhYWRlYTJmNzllN2M=
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -28,12 +28,14 @@ Usage:
|
|
28
28
|
flare-up copy DATA_SOURCE REDSHIFT_ENDPOINT DATABASE TABLE
|
29
29
|
|
30
30
|
Options:
|
31
|
-
[--aws-access-key=AWS_ACCESS_KEY]
|
32
|
-
[--aws-secret-key=AWS_SECRET_KEY]
|
33
|
-
[--redshift-username=REDSHIFT_USERNAME]
|
34
|
-
[--redshift-password=REDSHIFT_PASSWORD]
|
35
|
-
[--column-list=one two three]
|
36
|
-
[--copy-options=COPY_OPTIONS]
|
31
|
+
[--aws-access-key=AWS_ACCESS_KEY] # Required unless ENV['AWS_ACCESS_KEY_ID'] is set.
|
32
|
+
[--aws-secret-key=AWS_SECRET_KEY] # Required unless ENV['AWS_SECRET_ACCESS_KEY'] is set.
|
33
|
+
[--redshift-username=REDSHIFT_USERNAME] # Required unless ENV['REDSHIFT_USERNAME'] is set.
|
34
|
+
[--redshift-password=REDSHIFT_PASSWORD] # Required unless ENV['REDSHIFT_PASSWORD'] is set.
|
35
|
+
[--column-list=one two three] # A space-separated list of columns, should your DATA_SOURCE require it
|
36
|
+
[--copy-options=COPY_OPTIONS] # Appended to the end of the COPY command; enclose "IN QUOTES"
|
37
|
+
[--colorize-output], [--no-colorize-output] # Should Flare-up colorize its output?
|
38
|
+
# Default: true
|
37
39
|
```
|
38
40
|
|
39
41
|
## Sample Usage
|
data/lib/flare_up.rb
CHANGED
data/lib/flare_up/boot.rb
CHANGED
@@ -8,9 +8,10 @@ module FlareUp
|
|
8
8
|
copy = create_copy_command(options)
|
9
9
|
|
10
10
|
begin
|
11
|
+
Emitter.info("Executing command: #{copy.get_command}")
|
11
12
|
handle_load_errors(copy.execute(conn))
|
12
13
|
rescue CopyCommandError => e
|
13
|
-
|
14
|
+
Emitter.error(e.message)
|
14
15
|
CLI.bailout(1)
|
15
16
|
end
|
16
17
|
|
@@ -40,10 +41,11 @@ module FlareUp
|
|
40
41
|
# TODO: How can we test this?
|
41
42
|
def self.handle_load_errors(stl_load_errors)
|
42
43
|
return if stl_load_errors.empty?
|
43
|
-
|
44
|
+
Emitter.error("There was an error processing the COPY command. Displaying the last (#{stl_load_errors.length}) errors.")
|
44
45
|
stl_load_errors.each do |e|
|
45
|
-
|
46
|
+
Emitter.error(e.pretty_print)
|
46
47
|
end
|
48
|
+
CLI.bailout(1)
|
47
49
|
end
|
48
50
|
|
49
51
|
end
|
data/lib/flare_up/cli.rb
CHANGED
@@ -16,6 +16,7 @@ https://github.com/sharethrough/flare-up/blob/v#{FlareUp::VERSION}/README.md
|
|
16
16
|
option :redshift_password, :type => :string, :desc => "Required unless ENV['REDSHIFT_PASSWORD'] is set."
|
17
17
|
option :column_list, :type => :array, :desc => 'A space-separated list of columns, should your DATA_SOURCE require it'
|
18
18
|
option :copy_options, :type => :string, :desc => "Appended to the end of the COPY command; enclose \"IN QUOTES\""
|
19
|
+
option :colorize_output, :type => :boolean, :desc => 'Should Flare-up colorize its output?', :default => true
|
19
20
|
|
20
21
|
def copy(data_source, endpoint, database_name, table_name)
|
21
22
|
boot_options = {
|
@@ -32,10 +33,11 @@ https://github.com/sharethrough/flare-up/blob/v#{FlareUp::VERSION}/README.md
|
|
32
33
|
CLI.env_validator(boot_options, :redshift_username, 'REDSHIFT_USERNAME')
|
33
34
|
CLI.env_validator(boot_options, :redshift_password, 'REDSHIFT_PASSWORD')
|
34
35
|
rescue ArgumentError => e
|
35
|
-
|
36
|
+
Emitter.error(e.message)
|
36
37
|
CLI.bailout(1)
|
37
38
|
end
|
38
39
|
|
40
|
+
Emitter.store_options(boot_options)
|
39
41
|
Boot.boot(boot_options)
|
40
42
|
end
|
41
43
|
|
@@ -50,11 +52,6 @@ https://github.com/sharethrough/flare-up/blob/v#{FlareUp::VERSION}/README.md
|
|
50
52
|
exit(1)
|
51
53
|
end
|
52
54
|
|
53
|
-
# TODO: Extract
|
54
|
-
def self.output_error(message)
|
55
|
-
puts "\x1b[31m#{message}" unless ENV['TESTING']
|
56
|
-
end
|
57
|
-
|
58
55
|
end
|
59
56
|
|
60
57
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module FlareUp
|
2
|
+
|
3
|
+
class Emitter
|
4
|
+
|
5
|
+
RISKY_OPTIONS = [
|
6
|
+
:aws_access_key,
|
7
|
+
:aws_secret_key,
|
8
|
+
:redshift_username,
|
9
|
+
:redshift_password
|
10
|
+
]
|
11
|
+
|
12
|
+
# TODO: How do we test this?
|
13
|
+
def self.error(message)
|
14
|
+
$stderr.puts sanitize("\x1b[31m#{message}") unless ENV['TESTING']
|
15
|
+
end
|
16
|
+
|
17
|
+
# TODO: How do we test this?
|
18
|
+
def self.info(message)
|
19
|
+
$stdout.puts sanitize(message) unless ENV['TESTING']
|
20
|
+
end
|
21
|
+
|
22
|
+
# TODO: How do we test this?
|
23
|
+
def self.sanitize(message)
|
24
|
+
RISKY_OPTIONS.each do |risky_option|
|
25
|
+
message.gsub!(@BOOT_OPTIONS[risky_option], 'REDACTED')
|
26
|
+
end
|
27
|
+
message.gsub!(/\e\[(\d+)(;\d+)*m/, '') unless @BOOT_OPTIONS[:colorize_output]
|
28
|
+
message
|
29
|
+
end
|
30
|
+
|
31
|
+
# TODO: How do we test this?
|
32
|
+
def self.store_options(options)
|
33
|
+
@BOOT_OPTIONS = options
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
@@ -3,7 +3,7 @@ module FlareUp
|
|
3
3
|
class STLLoadErrorFetcher
|
4
4
|
|
5
5
|
def self.fetch_errors(connection)
|
6
|
-
query_result = connection.execute('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT
|
6
|
+
query_result = connection.execute('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 1')
|
7
7
|
errors = []
|
8
8
|
query_result.each do |row|
|
9
9
|
errors << STLLoadError.from_pg_results_row(row)
|
data/lib/flare_up/version.rb
CHANGED
@@ -3,6 +3,10 @@ describe FlareUp::Boot do
|
|
3
3
|
describe '.boot' do
|
4
4
|
let(:copy_command) { instance_double('FlareUp::CopyCommand') }
|
5
5
|
|
6
|
+
before do
|
7
|
+
allow(copy_command).to receive(:get_command)
|
8
|
+
end
|
9
|
+
|
6
10
|
context 'when there is an error' do
|
7
11
|
before do
|
8
12
|
expect(FlareUp::Boot).to receive(:create_copy_command).and_return(copy_command)
|
@@ -10,7 +10,8 @@ describe FlareUp::CLI do
|
|
10
10
|
:redshift_username => 'TEST_REDSHIFT_USERNAME',
|
11
11
|
:redshift_password => 'TEST_REDSHIFT_PASSWORD',
|
12
12
|
:aws_access_key => 'TEST_AWS_ACCESS_KEY',
|
13
|
-
:aws_secret_key => 'TEST_AWS_SECRET_KEY'
|
13
|
+
:aws_secret_key => 'TEST_AWS_SECRET_KEY',
|
14
|
+
:colorize_output => true
|
14
15
|
}
|
15
16
|
end
|
16
17
|
|
@@ -24,6 +25,11 @@ describe FlareUp::CLI do
|
|
24
25
|
|
25
26
|
describe '#copy' do
|
26
27
|
|
28
|
+
it 'should pass the options on the emitter' do
|
29
|
+
expect(FlareUp::Emitter).to receive(:store_options).with(required_options)
|
30
|
+
FlareUp::CLI.start(required_arguments)
|
31
|
+
end
|
32
|
+
|
27
33
|
context 'when no options are specified' do
|
28
34
|
it 'should boot with the proper options' do
|
29
35
|
expect(FlareUp::Boot).to receive(:boot).with(required_options)
|
@@ -45,6 +51,24 @@ describe FlareUp::CLI do
|
|
45
51
|
end
|
46
52
|
end
|
47
53
|
|
54
|
+
describe 'colorizing output' do
|
55
|
+
|
56
|
+
context 'when it is not specified' do
|
57
|
+
it 'should boot with the proper options' do
|
58
|
+
expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:colorize_output => true))
|
59
|
+
FlareUp::CLI.start(required_arguments)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
context 'when it is specified' do
|
64
|
+
it 'should boot with the proper options' do
|
65
|
+
expect(FlareUp::Boot).to receive(:boot).with(required_options.merge(:colorize_output => false))
|
66
|
+
FlareUp::CLI.start(required_arguments + ['--colorize_output=false'])
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
48
72
|
describe 'Redshift credentials' do
|
49
73
|
|
50
74
|
describe 'username' do
|
@@ -6,7 +6,7 @@ describe FlareUp::STLLoadErrorFetcher do
|
|
6
6
|
|
7
7
|
before do
|
8
8
|
expect(connection).to receive(:execute).
|
9
|
-
with('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT
|
9
|
+
with('SELECT * FROM stl_load_errors ORDER BY query DESC, line_number, position LIMIT 1').
|
10
10
|
and_return([
|
11
11
|
{
|
12
12
|
'err_reason' => 'TEST_REASON',
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flare-up
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.4'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert Slifka
|
@@ -104,6 +104,7 @@ files:
|
|
104
104
|
- lib/flare_up/cli.rb
|
105
105
|
- lib/flare_up/connection.rb
|
106
106
|
- lib/flare_up/copy_command.rb
|
107
|
+
- lib/flare_up/emitter.rb
|
107
108
|
- lib/flare_up/env_wrap.rb
|
108
109
|
- lib/flare_up/stl_load_error.rb
|
109
110
|
- lib/flare_up/stl_load_error_fetcher.rb
|
@@ -117,6 +118,7 @@ files:
|
|
117
118
|
- spec/lib/flare_up/cli_spec.rb
|
118
119
|
- spec/lib/flare_up/connection_spec.rb
|
119
120
|
- spec/lib/flare_up/copy_command_spec.rb
|
121
|
+
- spec/lib/flare_up/emitter_spec.rb
|
120
122
|
- spec/lib/flare_up/stl_load_error_fetcher_spec.rb
|
121
123
|
- spec/lib/flare_up/stl_load_error_spec.rb
|
122
124
|
- spec/spec_helper.rb
|
@@ -148,6 +150,7 @@ test_files:
|
|
148
150
|
- spec/lib/flare_up/cli_spec.rb
|
149
151
|
- spec/lib/flare_up/connection_spec.rb
|
150
152
|
- spec/lib/flare_up/copy_command_spec.rb
|
153
|
+
- spec/lib/flare_up/emitter_spec.rb
|
151
154
|
- spec/lib/flare_up/stl_load_error_fetcher_spec.rb
|
152
155
|
- spec/lib/flare_up/stl_load_error_spec.rb
|
153
156
|
- spec/spec_helper.rb
|