masamune 0.17.15 → 0.18.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/masamune/actions/aws_emr.rb +5 -5
- data/lib/masamune/actions/hadoop_filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_streaming.rb +3 -3
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/postgres.rb +5 -5
- data/lib/masamune/actions/s3cmd.rb +1 -1
- data/lib/masamune/cached_filesystem.rb +1 -1
- data/lib/masamune/commands/aws_emr.rb +11 -11
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +2 -2
- data/lib/masamune/commands/hive.rb +19 -19
- data/lib/masamune/commands/postgres.rb +19 -19
- data/lib/masamune/commands/postgres_admin.rb +13 -13
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/configuration.rb +36 -52
- data/lib/masamune/environment.rb +4 -3
- data/lib/masamune/method_logger.rb +4 -0
- data/lib/masamune/schema/store.rb +17 -5
- data/lib/masamune/thor.rb +1 -3
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +4 -4
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +1 -1
- data/spec/masamune/actions/hadoop_streaming_spec.rb +5 -5
- data/spec/masamune/actions/hive_spec.rb +4 -4
- data/spec/masamune/actions/postgres_spec.rb +2 -2
- data/spec/masamune/actions/s3cmd_spec.rb +1 -1
- data/spec/masamune/commands/aws_emr_spec.rb +1 -1
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +1 -1
- data/spec/masamune/commands/hadoop_streaming_spec.rb +1 -1
- data/spec/masamune/commands/hive_spec.rb +1 -1
- data/spec/masamune/commands/postgres_admin_spec.rb +15 -2
- data/spec/masamune/commands/postgres_spec.rb +1 -1
- data/spec/masamune/commands/s3cmd_spec.rb +1 -1
- data/spec/masamune/configuration_spec.rb +96 -1
- data/spec/masamune/environment_spec.rb +28 -1
- data/spec/masamune/schema/catalog_spec.rb +1 -1
- data/spec/masamune/tasks/aws_emr_thor_spec.rb +2 -2
- data/spec/masamune/tasks/hive_thor_spec.rb +7 -7
- data/spec/masamune/tasks/postgres_thor_spec.rb +4 -4
- data/spec/masamune/tasks/shell_thor_spec.rb +1 -1
- data/spec/masamune/thor_spec.rb +8 -13
- data/spec/support/masamune/example_group.rb +4 -3
- data/spec/support/masamune/job_example_group.rb +26 -27
- data/spec/support/masamune/mock_command.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +8 -10
- data/spec/support/masamune/step_example_group.rb +25 -28
- data/spec/support/masamune/task_example_group.rb +75 -28
- data/spec/support/masamune/thor_mute.rb +4 -2
- data/spec/support/rspec/example/action_example_group.rb +5 -3
- data/spec/support/rspec/example/task_example_group.rb +4 -67
- data/spec/support/rspec/example/transform_example_group.rb +7 -5
- metadata +16 -2
@@ -20,38 +20,85 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
+
require_relative 'shared_example_group'
|
24
|
+
|
23
25
|
module Masamune::TaskExampleGroup
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
Array.wrap(command).each do |cmd|
|
41
|
-
described_class.start([cmd, *(default_options + options)].compact)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
26
|
+
extend ActiveSupport::Concern
|
27
|
+
|
28
|
+
include Masamune::ExampleGroup
|
29
|
+
include Masamune::SharedExampleGroup
|
30
|
+
include Masamune::Actions::Filesystem
|
31
|
+
include Masamune::Actions::Hive
|
32
|
+
include Masamune::Transform::DenormalizeTable
|
33
|
+
|
34
|
+
shared_examples 'general usage' do
|
35
|
+
it 'exits with status code 0 and prints general usage' do
|
36
|
+
expect { execute_command }.to raise_error { |e|
|
37
|
+
expect(e).to be_a(SystemExit)
|
38
|
+
expect(e.status).to eq(0)
|
39
|
+
}
|
40
|
+
expect(stdout.string).to match(/^Commands:/)
|
41
|
+
expect(stderr.string).to be_blank
|
46
42
|
end
|
47
43
|
end
|
48
44
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
45
|
+
shared_examples 'command usage' do
|
46
|
+
it 'exits with status code 0 and prints command usage' do
|
47
|
+
expect { execute_command }.to raise_error { |e|
|
48
|
+
expect(e).to be_a(SystemExit)
|
49
|
+
expect(e.status).to eq(0)
|
50
|
+
}
|
51
|
+
expect(stdout.string).to match(/^Usage:/)
|
52
|
+
expect(stdout.string).to match(/^Options:/)
|
53
|
+
expect(stderr.string).to be_blank
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
shared_examples 'executes with success' do
|
58
|
+
it 'exits with status code 0' do
|
59
|
+
expect { execute_command }.to raise_error { |e|
|
60
|
+
expect(e).to be_a(SystemExit)
|
61
|
+
expect(e.status).to eq(0)
|
62
|
+
}
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
shared_examples 'raises Thor::MalformattedArgumentError' do |message|
|
67
|
+
it { expect { execute_command }.to raise_error Thor::MalformattedArgumentError, message }
|
68
|
+
end
|
69
|
+
|
70
|
+
shared_examples 'raises Thor::RequiredArgumentMissingError' do |message|
|
71
|
+
it { expect { execute_command }.to raise_error Thor::RequiredArgumentMissingError, message }
|
72
|
+
end
|
73
|
+
|
74
|
+
shared_context 'task_fixture' do |context_options = {}|
|
75
|
+
include_context 'job_fixture', context_options
|
76
|
+
|
77
|
+
let(:execute_command_times) { !ENV['MASAMUNE_FASTER_SPEC'] && context_options.fetch(:idempotent, false) ? 2 : 1 }
|
78
|
+
end
|
79
|
+
|
80
|
+
included do
|
81
|
+
let!(:default_options) { configuration.as_options }
|
82
|
+
|
83
|
+
let(:thor_class) { described_class }
|
84
|
+
let(:command) { nil }
|
85
|
+
let(:options) { [] }
|
86
|
+
let!(:stdout) { StringIO.new }
|
87
|
+
let!(:stderr) { StringIO.new }
|
88
|
+
|
89
|
+
let(:execute_command_times) { 1 }
|
90
|
+
|
91
|
+
before do
|
92
|
+
thor_class.send(:include, Masamune::ThorMute)
|
93
|
+
end
|
94
|
+
|
95
|
+
subject(:execute_command) do
|
96
|
+
capture(stdout: stdout, stderr: stderr, enable: !default_options.include?('--debug')) do
|
97
|
+
execute_command_times.times do
|
98
|
+
thor_class.start([command, *(default_options + options)].compact)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
55
102
|
end
|
56
103
|
end
|
57
104
|
|
@@ -22,8 +22,10 @@
|
|
22
22
|
|
23
23
|
# Silence noisy and uninformative create_command method
|
24
24
|
module Masamune::ThorMute
|
25
|
-
|
26
|
-
|
25
|
+
extend ActiveSupport::Concern
|
26
|
+
|
27
|
+
included do
|
28
|
+
instance_eval do
|
27
29
|
def create_command(*a)
|
28
30
|
tmp_stdout = $stdout
|
29
31
|
$stdout = StringIO.new
|
@@ -21,9 +21,11 @@
|
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
23
|
module ActionExampleGroup
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
extend ActiveSupport::Concern
|
25
|
+
|
26
|
+
included do
|
27
|
+
let(:run_dir) { Dir.mktmpdir('masamune') }
|
28
|
+
before do
|
27
29
|
instance.environment = Masamune::ExampleGroup
|
28
30
|
Masamune::ExampleGroup.filesystem.add_path(:run_dir, run_dir)
|
29
31
|
end
|
@@ -21,78 +21,15 @@
|
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
23
|
module TaskExampleGroup
|
24
|
-
|
25
|
-
tmp_stdout = $stdout
|
26
|
-
$stdout = stdout
|
27
|
-
tmp_stderr = $stderr
|
28
|
-
$stderr = stderr
|
29
|
-
yield
|
30
|
-
ensure
|
31
|
-
$stdout = tmp_stdout
|
32
|
-
$stderr = tmp_stderr
|
33
|
-
end
|
34
|
-
|
35
|
-
shared_examples 'general usage' do
|
36
|
-
it 'exits with status code 0 and prints general usage' do
|
37
|
-
expect { cli_invocation }.to raise_error { |e|
|
38
|
-
expect(e).to be_a(SystemExit)
|
39
|
-
expect(e.status).to eq(0)
|
40
|
-
}
|
41
|
-
expect(stdout.string).to match(/^Commands:/)
|
42
|
-
expect(stderr.string).to be_blank
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
shared_examples 'command usage' do
|
47
|
-
it 'exits with status code 0 and prints command usage' do
|
48
|
-
expect { cli_invocation }.to raise_error { |e|
|
49
|
-
expect(e).to be_a(SystemExit)
|
50
|
-
expect(e.status).to eq(0)
|
51
|
-
}
|
52
|
-
expect(stdout.string).to match(/^Usage:/)
|
53
|
-
expect(stdout.string).to match(/^Options:/)
|
54
|
-
expect(stderr.string).to be_blank
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
shared_examples 'executes with success' do
|
59
|
-
it 'exits with status code 0' do
|
60
|
-
expect { cli_invocation }.to raise_error { |e|
|
61
|
-
expect(e).to be_a(SystemExit)
|
62
|
-
expect(e.status).to eq(0)
|
63
|
-
}
|
64
|
-
end
|
65
|
-
end
|
24
|
+
extend ActiveSupport::Concern
|
66
25
|
|
67
|
-
|
68
|
-
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, message }
|
69
|
-
end
|
26
|
+
include Masamune::TaskExampleGroup
|
70
27
|
|
71
|
-
|
72
|
-
|
73
|
-
end
|
74
|
-
|
75
|
-
def self.included(base)
|
76
|
-
base.before :all do
|
28
|
+
included do
|
29
|
+
before :all do
|
77
30
|
ENV['THOR_DEBUG'] = '1'
|
78
31
|
Masamune::Actions::DataFlow.reset_module!
|
79
32
|
end
|
80
|
-
|
81
|
-
base.let(:thor_class) { described_class }
|
82
|
-
base.let(:command) { nil }
|
83
|
-
base.let(:options) { {} }
|
84
|
-
base.let!(:stdout) { StringIO.new }
|
85
|
-
base.let!(:stderr) { StringIO.new }
|
86
|
-
|
87
|
-
base.before do
|
88
|
-
thor_class.send(:include, Masamune::ThorMute)
|
89
|
-
end
|
90
|
-
|
91
|
-
base.subject(:cli_invocation) do
|
92
|
-
capture(stdout, stderr) do
|
93
|
-
thor_class.start([command, *options].compact)
|
94
|
-
end
|
95
|
-
end
|
96
33
|
end
|
97
34
|
end
|
98
35
|
|
@@ -21,11 +21,13 @@
|
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
23
|
module TransformExampleGroup
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
extend ActiveSupport::Concern
|
25
|
+
|
26
|
+
included do
|
27
|
+
let(:transform) { Object.new.extend(described_class) }
|
28
|
+
let(:environment) { double }
|
29
|
+
let(:catalog) { Masamune::Schema::Catalog.new(environment) }
|
30
|
+
after do
|
29
31
|
catalog.clear!
|
30
32
|
end
|
31
33
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masamune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.18.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Andrews
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -136,6 +136,20 @@ dependencies:
|
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: hashie
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
139
153
|
- !ruby/object:Gem::Dependency
|
140
154
|
name: rake
|
141
155
|
requirement: !ruby/object:Gem::Requirement
|