mortar 0.15.29 → 0.15.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/mortar/conf/luigi/logging.ini +1 -1
- data/lib/mortar/local/controller.rb +13 -11
- data/lib/mortar/local/params.rb +2 -0
- data/lib/mortar/templates/project/luigiscripts/client.cfg.template +1 -0
- data/lib/mortar/templates/project/luigiscripts/luigiscript.py +108 -95
- data/lib/mortar/updater.rb +5 -1
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/auth_spec.rb +6 -2
- data/spec/mortar/command/local_spec.rb +51 -41
- data/spec/mortar/local/controller_spec.rb +20 -14
- data/spec/mortar/local/params_spec.rb +4 -0
- data/spec/mortar/updater_spec.rb +5 -5
- metadata +7 -7
|
@@ -90,7 +90,7 @@ EOF
|
|
|
90
90
|
if !auth.has_credentials
|
|
91
91
|
error(NO_AWS_KEYS_ERROR_MESSAGE)
|
|
92
92
|
else
|
|
93
|
-
vars = fetch_aws_keys(auth
|
|
93
|
+
vars = fetch_aws_keys(auth)
|
|
94
94
|
if vars['aws_access_key_id'] && vars['aws_secret_access_key']
|
|
95
95
|
set_aws_keys(vars['aws_access_key_id'], vars['aws_secret_access_key'])
|
|
96
96
|
else
|
|
@@ -101,9 +101,8 @@ EOF
|
|
|
101
101
|
end
|
|
102
102
|
|
|
103
103
|
# Fetches AWS Keys based on auth
|
|
104
|
-
def fetch_aws_keys(auth
|
|
105
|
-
|
|
106
|
-
project_name = base.options[:project] || project.name
|
|
104
|
+
def fetch_aws_keys(auth)
|
|
105
|
+
project_name = ENV['MORTAR_PROJECT_NAME']
|
|
107
106
|
return auth.api.get_config_vars(project_name).body['config']
|
|
108
107
|
end
|
|
109
108
|
|
|
@@ -111,11 +110,20 @@ EOF
|
|
|
111
110
|
ENV['AWS_ACCESS_KEY'] = aws_access_key
|
|
112
111
|
ENV['AWS_SECRET_KEY'] = aws_secret_key
|
|
113
112
|
end
|
|
114
|
-
|
|
113
|
+
|
|
114
|
+
def set_project_name(base)
|
|
115
|
+
project = base.project
|
|
116
|
+
project_name = base.options[:project] || project.name
|
|
117
|
+
ENV['MORTAR_PROJECT_NAME'] = project_name
|
|
118
|
+
end
|
|
115
119
|
# Main entry point to perform installation and configuration necessary
|
|
116
120
|
# to run pig on the users local machine
|
|
117
121
|
def install_and_configure(pig_version=nil, command=nil, install_sqoop=false)
|
|
118
122
|
#To support old watchtower plugins we'll accept nil pig_version
|
|
123
|
+
base = Mortar::Command::Base.new
|
|
124
|
+
set_project_name(base)
|
|
125
|
+
require_aws_keys()
|
|
126
|
+
|
|
119
127
|
if pig_version.nil?
|
|
120
128
|
pig_version = Mortar::PigVersion::Pig09.new
|
|
121
129
|
end
|
|
@@ -204,7 +212,6 @@ README
|
|
|
204
212
|
|
|
205
213
|
# Main entry point for user running a pig script
|
|
206
214
|
def run(pig_script, pig_version, pig_parameters)
|
|
207
|
-
require_aws_keys
|
|
208
215
|
install_and_configure(pig_version, 'run')
|
|
209
216
|
pig = Mortar::Local::Pig.new()
|
|
210
217
|
pig.run_script(pig_script, pig_version, pig_parameters)
|
|
@@ -212,7 +219,6 @@ README
|
|
|
212
219
|
|
|
213
220
|
# Main entry point for illustrating a pig alias
|
|
214
221
|
def illustrate(pig_script, pig_alias, pig_version, pig_parameters, skip_pruning, no_browser)
|
|
215
|
-
require_aws_keys
|
|
216
222
|
install_and_configure(pig_version, 'illustrate')
|
|
217
223
|
pig = Mortar::Local::Pig.new()
|
|
218
224
|
pig.illustrate_alias(pig_script, pig_alias, skip_pruning, no_browser, pig_version, pig_parameters)
|
|
@@ -231,7 +237,6 @@ README
|
|
|
231
237
|
end
|
|
232
238
|
|
|
233
239
|
def run_luigi(pig_version, luigi_script, luigi_script_parameters, project_config_parameters)
|
|
234
|
-
require_aws_keys
|
|
235
240
|
install_and_configure(pig_version, 'luigi')
|
|
236
241
|
py = Mortar::Local::Python.new()
|
|
237
242
|
unless py.run_stillson_luigi_client_cfg_expansion(luigi_script, project_config_parameters)
|
|
@@ -241,7 +246,6 @@ README
|
|
|
241
246
|
end
|
|
242
247
|
|
|
243
248
|
def sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
|
|
244
|
-
require_aws_keys
|
|
245
249
|
install_and_configure(pig_version, 'sqoop', true)
|
|
246
250
|
sqoop = Mortar::Local::Sqoop.new()
|
|
247
251
|
options[:dbtable] = dbtable
|
|
@@ -249,7 +253,6 @@ README
|
|
|
249
253
|
end
|
|
250
254
|
|
|
251
255
|
def sqoop_export_query(pig_version, connstr, query, s3dest, options)
|
|
252
|
-
require_aws_keys
|
|
253
256
|
install_and_configure(pig_version, 'sqoop', true)
|
|
254
257
|
sqoop = Mortar::Local::Sqoop.new()
|
|
255
258
|
options[:sqlquery] = sqoop.prep_query(query)
|
|
@@ -257,7 +260,6 @@ README
|
|
|
257
260
|
end
|
|
258
261
|
|
|
259
262
|
def sqoop_export_incremental(pig_version, connstr, dbtable, column, max_value, s3dest, options)
|
|
260
|
-
require_aws_keys
|
|
261
263
|
install_and_configure(pig_version, 'sqoop', true)
|
|
262
264
|
sqoop = Mortar::Local::Sqoop.new()
|
|
263
265
|
options[:dbtable] = dbtable
|
data/lib/mortar/local/params.rb
CHANGED
|
@@ -49,6 +49,8 @@ module Mortar
|
|
|
49
49
|
params['AWS_SECRET_KEY'] = ENV['AWS_SECRET_KEY']
|
|
50
50
|
params['AWS_SECRET_ACCESS_KEY'] = ENV['AWS_SECRET_KEY']
|
|
51
51
|
params['aws_secret_access_key'] = ENV['AWS_SECRET_KEY']
|
|
52
|
+
|
|
53
|
+
params['MORTAR_PROJECT_NAME'] = ENV['MORTAR_PROJECT_NAME']
|
|
52
54
|
|
|
53
55
|
param_list = params.map do |k,v|
|
|
54
56
|
{"name" => k, "value" => v}
|
|
@@ -4,131 +4,144 @@ from luigi.s3 import S3Target, S3PathTask
|
|
|
4
4
|
|
|
5
5
|
from mortar.luigi import mortartask
|
|
6
6
|
|
|
7
|
+
import logging
|
|
8
|
+
|
|
7
9
|
"""
|
|
8
|
-
|
|
10
|
+
Luigi is a powerful, easy-to-use framework for building data pipelines.
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
This is an example Luigi script to get you started. This script has a
|
|
13
|
+
'fill in the blank' interaction. Feel free to alter it to build your pipeline.
|
|
12
14
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
In this example we will run a Pig script, and then shutdown any clusters associated
|
|
16
|
+
with that script. We will do that by running the ShutdownClusters Task,
|
|
17
|
+
which is dependent on RunMyExamplePigScript Task. This means the cluster will only
|
|
18
|
+
shutdown after RunMyExamplePigScript (where the data transformation happens)
|
|
19
|
+
Task is completed.
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
+
For full tutorials and in-depth Luigi documentation, visit:
|
|
22
|
+
https://help.mortardata.com/technologies/luigi
|
|
21
23
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
+
To Run:
|
|
25
|
+
mortar luigi luigiscripts/<%= project_name %>_luigi.py \
|
|
24
26
|
--output-base-path "s3://mortar-example-output-data/<your_username_here>/<%= project_name %>"
|
|
25
27
|
"""
|
|
26
28
|
|
|
27
29
|
MORTAR_PROJECT = '<%= project_name %>'
|
|
28
30
|
|
|
31
|
+
"""
|
|
32
|
+
This logger outputs logs to Mortar Logs. An example of it's usage can be seen
|
|
33
|
+
in the ShutdownClusters function.
|
|
34
|
+
"""
|
|
35
|
+
logger = logging.getLogger('luigi-interface')
|
|
36
|
+
|
|
29
37
|
# helper function
|
|
30
38
|
def create_full_path(base_path, sub_path):
|
|
31
|
-
|
|
39
|
+
return '%s/%s' % (base_path, sub_path)
|
|
32
40
|
|
|
33
41
|
class RunMyExamplePigScript(mortartask.MortarProjectPigscriptTask):
|
|
34
|
-
"""
|
|
35
|
-
This is a Luigi Task that extends MortarProjectPigscriptTask to run a Pig
|
|
36
|
-
script on Mortar.
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
"""
|
|
40
|
-
The location in S3 where the output of the Mortar job will be written.
|
|
41
|
-
"""
|
|
42
|
-
output_base_path = luigi.Parameter()
|
|
43
|
-
|
|
44
|
-
"""
|
|
45
|
-
Default cluster size to use for running Mortar jobs. A cluster size of 0
|
|
46
|
-
will run in Mortar's local mode. This is a fast (and free!) way to run jobs
|
|
47
|
-
on small data samples. Cluster sizes >= 2 will run on a Hadoop cluster.
|
|
48
|
-
"""
|
|
49
|
-
cluster_size = luigi.IntParameter(default=0)
|
|
50
|
-
|
|
51
|
-
"""
|
|
52
|
-
Path to input data being analyzed using the Pig script.
|
|
53
|
-
"""
|
|
54
|
-
input_path = luigi.Parameter(default ='s3://mortar-example-data/tutorial/excite.log.bz2')
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def requires(self):
|
|
58
42
|
"""
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
exists on S3 before we run the script. S3PathTask validates that the
|
|
62
|
-
specified file or directory exists on S3.
|
|
43
|
+
This is a Luigi Task that extends MortarProjectPigscriptTask to run a Pig
|
|
44
|
+
script on Mortar.
|
|
63
45
|
"""
|
|
64
|
-
return [S3PathTask(self.input_path)]
|
|
65
46
|
|
|
66
|
-
def project(self):
|
|
67
47
|
"""
|
|
68
|
-
|
|
48
|
+
The location in S3 where the output of the Mortar job will be written.
|
|
69
49
|
"""
|
|
70
|
-
|
|
50
|
+
output_base_path = luigi.Parameter()
|
|
71
51
|
|
|
72
|
-
def script_output(self):
|
|
73
|
-
"""
|
|
74
|
-
The script_output method is how you define where the output from this task
|
|
75
|
-
will be stored. Luigi will check this output location before starting any
|
|
76
|
-
tasks that depend on this task.
|
|
77
|
-
"""
|
|
78
|
-
return[S3Target(self.output_base_path + '/pigoutput')]
|
|
79
|
-
|
|
80
|
-
def token_path(self):
|
|
81
52
|
"""
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
where those tokens will be written.
|
|
53
|
+
Default cluster size to use for running Mortar jobs. A cluster size of 0
|
|
54
|
+
will run in Mortar's local mode. This is a fast (and free!) way to run jobs
|
|
55
|
+
on small data samples. Cluster sizes >= 2 will run on a Hadoop cluster.
|
|
86
56
|
"""
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def parameters(self):
|
|
57
|
+
cluster_size = luigi.IntParameter(default=0)
|
|
58
|
+
|
|
90
59
|
"""
|
|
91
|
-
|
|
60
|
+
Path to input data being analyzed using the Pig script.
|
|
92
61
|
"""
|
|
93
|
-
|
|
62
|
+
input_path = luigi.Parameter(default ='s3://mortar-example-data/tutorial/excite.log.bz2')
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def requires(self):
|
|
66
|
+
"""
|
|
67
|
+
The requires method specifies a list of other Tasks that must be complete
|
|
68
|
+
for this Task to run. In this case, we want to require that our input
|
|
69
|
+
exists on S3 before we run the script. S3PathTask validates that the
|
|
70
|
+
specified file or directory exists on S3.
|
|
71
|
+
"""
|
|
72
|
+
return [S3PathTask(self.input_path)]
|
|
73
|
+
|
|
74
|
+
def project(self):
|
|
75
|
+
"""
|
|
76
|
+
Name of Mortar Project to run.
|
|
77
|
+
"""
|
|
78
|
+
return MORTAR_PROJECT
|
|
79
|
+
|
|
80
|
+
def script_output(self):
|
|
81
|
+
"""
|
|
82
|
+
The script_output method is how you define where the output from this task
|
|
83
|
+
will be stored. Luigi will check this output location before starting any
|
|
84
|
+
tasks that depend on this task.
|
|
85
|
+
"""
|
|
86
|
+
return[S3Target(self.output_base_path + '/pigoutput')]
|
|
87
|
+
|
|
88
|
+
def token_path(self):
|
|
89
|
+
"""
|
|
90
|
+
Luigi manages dependencies between tasks by checking for the existence of
|
|
91
|
+
files. When one task finishes it writes out a 'token' file that will
|
|
92
|
+
trigger the next task in the dependency graph. This is the base path for
|
|
93
|
+
where those tokens will be written.
|
|
94
|
+
"""
|
|
95
|
+
return self.output_base_path
|
|
96
|
+
|
|
97
|
+
def parameters(self):
|
|
98
|
+
"""
|
|
99
|
+
Parameters for this pig job.
|
|
100
|
+
"""
|
|
101
|
+
return {'INPUT_PATH': self.input_path,
|
|
94
102
|
'OUTPUT_PATH': self.output_base_path + '/pigoutput'}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
103
|
+
|
|
104
|
+
def script(self):
|
|
105
|
+
"""
|
|
106
|
+
Name of Pig script to run.
|
|
107
|
+
"""
|
|
108
|
+
return '<%= project_name %>'
|
|
101
109
|
|
|
102
110
|
|
|
103
111
|
class ShutdownClusters(mortartask.MortarClusterShutdownTask):
|
|
104
|
-
"""
|
|
105
|
-
When the pipeline is completed, this task shuts down all active clusters not
|
|
106
|
-
currently running jobs. As this task is only shutting down clusters and not
|
|
107
|
-
generating any output data, this S3 location is used to store a 'token' file
|
|
108
|
-
indicating when the task has been completed.
|
|
109
|
-
"""
|
|
110
|
-
output_base_path = luigi.Parameter()
|
|
111
|
-
|
|
112
|
-
def requires(self):
|
|
113
112
|
"""
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
113
|
+
When the pipeline is completed, this task shuts down all active clusters not
|
|
114
|
+
currently running jobs. As this task is only shutting down clusters and not
|
|
115
|
+
generating any output data, this S3 location is used to store a 'token' file
|
|
116
|
+
indicating when the task has been completed.
|
|
117
117
|
"""
|
|
118
|
-
|
|
118
|
+
output_base_path = luigi.Parameter()
|
|
119
119
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
120
|
+
def requires(self):
|
|
121
|
+
"""
|
|
122
|
+
The ShutdownClusters task is dependent on RunMyExamplePigScript because a
|
|
123
|
+
cluster should not shut down until all the tasks are completed. You can
|
|
124
|
+
think of this as saying 'shut down my cluster after running my task'.
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# This is an example of emitting log messages.
|
|
129
|
+
logger.info('My Log Message!')
|
|
130
|
+
|
|
131
|
+
return RunMyExamplePigScript(output_base_path = self.output_base_path)
|
|
132
|
+
|
|
133
|
+
def output(self):
|
|
134
|
+
"""
|
|
135
|
+
This output statement is needed because ShutdownClusters has no actual
|
|
136
|
+
output. We write a token with the class name to S3 to know that this task
|
|
137
|
+
has completed and it does not need to be run again.
|
|
138
|
+
"""
|
|
139
|
+
return [S3Target((create_full_path(self.output_base_path, 'ShutdownClusters')))]
|
|
127
140
|
|
|
128
141
|
if __name__ == "__main__":
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
142
|
+
"""
|
|
143
|
+
The final task in your pipeline, which will in turn pull in any dependencies
|
|
144
|
+
need to be run should be called in the main method. In this case ShutdownClusters
|
|
145
|
+
is being called.
|
|
146
|
+
"""
|
|
147
|
+
luigi.run(main_task_cls= ShutdownClusters)
|
data/lib/mortar/updater.rb
CHANGED
|
@@ -28,7 +28,11 @@ module Mortar
|
|
|
28
28
|
def self.get_newest_version
|
|
29
29
|
begin
|
|
30
30
|
require "excon"
|
|
31
|
-
gem_data = Mortar::Helpers.json_decode(
|
|
31
|
+
gem_data = Mortar::Helpers.json_decode(
|
|
32
|
+
Excon.get('https://rubygems.org/api/v1/gems/mortar.json',
|
|
33
|
+
{:connect_timeout => CONNECT_TIMEOUT,
|
|
34
|
+
:read_timeout => READ_TIMEOUT,
|
|
35
|
+
:ssl_version => :TLSv1}).body)
|
|
32
36
|
gem_data.default = "0.0.0"
|
|
33
37
|
gem_data['version']
|
|
34
38
|
rescue Exception => e
|
data/lib/mortar/version.rb
CHANGED
data/spec/mortar/auth_spec.rb
CHANGED
|
@@ -34,7 +34,9 @@ module Mortar
|
|
|
34
34
|
|
|
35
35
|
FakeFS.activate!
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
my_mode_output = Object.new
|
|
38
|
+
stub(my_mode_output).mode {"0600".to_i(8)}
|
|
39
|
+
stub(my_mode_output).readable? {true}
|
|
38
40
|
stub(FakeFS::File).stat {my_mode_output}
|
|
39
41
|
stub(FakeFS::FileUtils).chmod
|
|
40
42
|
stub(FakeFS::File).readlines do |path|
|
|
@@ -95,7 +97,9 @@ module Mortar
|
|
|
95
97
|
stub(@cli).check
|
|
96
98
|
#@cli.should_receive(:check_for_associated_ssh_key)
|
|
97
99
|
@cli.reauthorize
|
|
98
|
-
Netrc.read(@cli.netrc_path)["api.#{@cli.host}"]
|
|
100
|
+
netrc_response = Netrc.read(@cli.netrc_path)["api.#{@cli.host}"]
|
|
101
|
+
netrc_response[0].should == 'one'
|
|
102
|
+
netrc_response[1].should == 'two'
|
|
99
103
|
end
|
|
100
104
|
|
|
101
105
|
it "prompts for github_username when user doesn't have one." do
|
|
@@ -161,60 +161,72 @@ PARAMS
|
|
|
161
161
|
end
|
|
162
162
|
|
|
163
163
|
context("configure") do
|
|
164
|
-
|
|
165
164
|
it "errors if the project root doesn't exist or we can't cd there" do
|
|
166
165
|
stderr, stdout = execute("local:configure --project-root /foo/baz")
|
|
167
166
|
stderr.should == " ! No such directory /foo/baz\n"
|
|
168
167
|
end
|
|
169
168
|
|
|
170
169
|
it "errors if java can't be found" do
|
|
171
|
-
|
|
172
|
-
|
|
170
|
+
ENV['AWS_ACCESS_KEY'] = "foo"
|
|
171
|
+
ENV['AWS_SECRET_KEY'] = "BAR"
|
|
172
|
+
with_git_initialized_project do |p|
|
|
173
|
+
any_instance_of(Mortar::Local::Java) do |j|
|
|
174
|
+
stub(j).check_install.returns(false)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
stderr, stdout = execute("local:configure")
|
|
178
|
+
stderr.should == Mortar::Local::Controller::NO_JAVA_ERROR_MESSAGE.gsub(/^/, " ! ")
|
|
173
179
|
end
|
|
174
|
-
stderr, stdout = execute("local:configure")
|
|
175
|
-
stderr.should == Mortar::Local::Controller::NO_JAVA_ERROR_MESSAGE.gsub(/^/, " ! ")
|
|
176
180
|
end
|
|
177
181
|
|
|
178
182
|
it "errors if python can't be found" do
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
183
|
+
ENV['AWS_ACCESS_KEY'] = "foo"
|
|
184
|
+
ENV['AWS_SECRET_KEY'] = "BAR"
|
|
185
|
+
with_git_initialized_project do |p|
|
|
186
|
+
any_instance_of(Mortar::Local::Java) do |j|
|
|
187
|
+
stub(j).check_install.returns(true)
|
|
188
|
+
end
|
|
189
|
+
any_instance_of(Mortar::Local::Pig) do |j|
|
|
190
|
+
stub(j).install_pig.returns(true)
|
|
191
|
+
stub(j).install_lib.returns(true)
|
|
192
|
+
end
|
|
193
|
+
any_instance_of(Mortar::Local::Python) do |j|
|
|
194
|
+
stub(j).check_or_install.returns(false)
|
|
195
|
+
end
|
|
196
|
+
stderr, stdout = execute("local:configure")
|
|
197
|
+
stderr.should == Mortar::Local::Controller::NO_PYTHON_ERROR_MESSAGE.gsub(/^/, " ! ")
|
|
188
198
|
end
|
|
189
|
-
stderr, stdout = execute("local:configure")
|
|
190
|
-
stderr.should == Mortar::Local::Controller::NO_PYTHON_ERROR_MESSAGE.gsub(/^/, " ! ")
|
|
191
199
|
end
|
|
192
200
|
|
|
193
201
|
it "checks for java, installs pig/python, and configures a virtualenv" do
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
202
|
+
ENV['AWS_ACCESS_KEY'] = "foo"
|
|
203
|
+
ENV['AWS_SECRET_KEY'] = "BAR"
|
|
204
|
+
with_git_initialized_project do |p|
|
|
205
|
+
any_instance_of(Mortar::Local::Java) do |j|
|
|
206
|
+
mock(j).check_install.returns(true)
|
|
207
|
+
end
|
|
208
|
+
any_instance_of(Mortar::Local::Pig) do |j|
|
|
209
|
+
mock(j).install_pig.with_any_args.returns(true)
|
|
210
|
+
stub(j).install_lib.returns(true)
|
|
211
|
+
end
|
|
212
|
+
any_instance_of(Mortar::Local::Python) do |j|
|
|
213
|
+
mock(j).check_or_install.returns(true)
|
|
214
|
+
mock(j).check_virtualenv.returns(true)
|
|
215
|
+
mock(j).setup_project_python_environment.returns(true)
|
|
216
|
+
end
|
|
217
|
+
any_instance_of(Mortar::Local::Jython) do |j|
|
|
218
|
+
mock(j).install_or_update.returns(true)
|
|
219
|
+
end
|
|
220
|
+
any_instance_of(Mortar::Local::Sqoop) do |j|
|
|
221
|
+
mock(j).install_or_update.returns(true)
|
|
222
|
+
end
|
|
223
|
+
any_instance_of(Mortar::Local::Controller) do |j|
|
|
224
|
+
mock(j).write_local_readme
|
|
225
|
+
mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
|
|
226
|
+
end
|
|
227
|
+
stderr, stdout = execute("local:configure")
|
|
228
|
+
stderr.should == ""
|
|
215
229
|
end
|
|
216
|
-
stderr, stdout = execute("local:configure")
|
|
217
|
-
stderr.should == ""
|
|
218
230
|
end
|
|
219
231
|
|
|
220
232
|
# configure
|
|
@@ -307,7 +319,6 @@ STDERR
|
|
|
307
319
|
end
|
|
308
320
|
any_instance_of(Mortar::Local::Controller) do |u|
|
|
309
321
|
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig09),'luigi')
|
|
310
|
-
mock(u).require_aws_keys()
|
|
311
322
|
end
|
|
312
323
|
any_instance_of(Mortar::Command::Local) do |u|
|
|
313
324
|
mock(u).sync_code_with_cloud().returns("some-git-ref")
|
|
@@ -339,7 +350,6 @@ STDERR
|
|
|
339
350
|
end
|
|
340
351
|
any_instance_of(Mortar::Local::Controller) do |u|
|
|
341
352
|
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig09),'luigi')
|
|
342
|
-
mock(u).require_aws_keys()
|
|
343
353
|
end
|
|
344
354
|
any_instance_of(Mortar::Command::Local) do |u|
|
|
345
355
|
mock(u).sync_code_with_cloud().returns("some-git-ref")
|
|
@@ -29,6 +29,7 @@ module Mortar::Local
|
|
|
29
29
|
stub_core
|
|
30
30
|
ENV['AWS_ACCESS_KEY'] = "foo"
|
|
31
31
|
ENV['AWS_SECRET_KEY'] = "BAR"
|
|
32
|
+
ENV['MORTAR_PROJECT_NAME'] ='projectName'
|
|
32
33
|
end
|
|
33
34
|
|
|
34
35
|
context("aws keys") do
|
|
@@ -50,8 +51,7 @@ module Mortar::Local
|
|
|
50
51
|
ctrl = Mortar::Local::Controller.new
|
|
51
52
|
previous_stderr, $stderr = $stderr, StringIO.new
|
|
52
53
|
mock(Mortar::Auth).has_credentials{true}
|
|
53
|
-
|
|
54
|
-
mock(ctrl).fetch_aws_keys(Mortar::Auth, 'base'){{}}
|
|
54
|
+
mock(ctrl).fetch_aws_keys(Mortar::Auth){{}}
|
|
55
55
|
begin
|
|
56
56
|
expect { ctrl.require_aws_keys }.to raise_error(SystemExit)
|
|
57
57
|
$stderr.string.should eq(Mortar::Local::Controller::API_CONFIG_ERROR_MESSAGE.gsub(/^/, " ! "))
|
|
@@ -74,10 +74,9 @@ module Mortar::Local
|
|
|
74
74
|
it "sets fetches and sets aws keys if missing and logged in" do
|
|
75
75
|
ENV.delete('AWS_ACCESS_KEY')
|
|
76
76
|
ctrl = Mortar::Local::Controller.new
|
|
77
|
-
stub(Mortar::Command::Base).new {'base'}
|
|
78
77
|
stub(Mortar::Auth).has_credentials{true}
|
|
79
78
|
previous_stderr, $stderr = $stderr, StringIO.new
|
|
80
|
-
mock(ctrl).fetch_aws_keys(Mortar::Auth
|
|
79
|
+
mock(ctrl).fetch_aws_keys(Mortar::Auth){
|
|
81
80
|
{
|
|
82
81
|
"aws_access_key_id"=>"key_id",
|
|
83
82
|
"aws_secret_access_key"=>"secret"
|
|
@@ -93,26 +92,21 @@ module Mortar::Local
|
|
|
93
92
|
end
|
|
94
93
|
end
|
|
95
94
|
|
|
96
|
-
|
|
97
|
-
|
|
98
95
|
it "fetches aws keys" do
|
|
99
96
|
ctrl = Mortar::Local::Controller.new
|
|
100
|
-
auth = Mortar::Auth
|
|
101
|
-
|
|
97
|
+
auth = Mortar::Auth
|
|
102
98
|
|
|
103
99
|
with_blank_project do
|
|
104
|
-
base = Mortar::Command::Base.new
|
|
105
100
|
with_git_initialized_project do |p|
|
|
106
101
|
# stub api request
|
|
107
102
|
configs = {}
|
|
108
|
-
mock(Mortar::Auth.api).get_config_vars("
|
|
103
|
+
mock(Mortar::Auth.api).get_config_vars("projectName").returns(Excon::Response.new(:body => {"config" => configs}))
|
|
109
104
|
|
|
110
|
-
ctrl.fetch_aws_keys(auth
|
|
105
|
+
ctrl.fetch_aws_keys(auth).should eq(configs)
|
|
111
106
|
end
|
|
112
107
|
end
|
|
113
108
|
end
|
|
114
109
|
|
|
115
|
-
|
|
116
110
|
it "returns if they are not present but override is in place" do
|
|
117
111
|
ENV.delete('AWS_ACCESS_KEY')
|
|
118
112
|
ENV['MORTAR_IGNORE_AWS_KEYS'] = 'true'
|
|
@@ -130,8 +124,13 @@ module Mortar::Local
|
|
|
130
124
|
|
|
131
125
|
context("install_and_configure") do
|
|
132
126
|
it "supplied default pig version" do
|
|
127
|
+
ENV.delete('MORTAR_PROJECT_NAME')
|
|
133
128
|
ctrl = Mortar::Local::Controller.new
|
|
134
129
|
|
|
130
|
+
any_instance_of(Mortar::Command::Base) do |b|
|
|
131
|
+
mock(b).project.returns(Mortar::Project::Project)
|
|
132
|
+
mock(b).options.returns({:project =>'myproject'})
|
|
133
|
+
end
|
|
135
134
|
any_instance_of(Mortar::Local::Java) do |j|
|
|
136
135
|
mock(j).check_install.returns(true)
|
|
137
136
|
end
|
|
@@ -150,12 +149,18 @@ module Mortar::Local
|
|
|
150
149
|
mock(ctrl).write_local_readme
|
|
151
150
|
mock(ctrl).ensure_local_install_dirs_in_gitignore
|
|
152
151
|
ctrl.install_and_configure
|
|
152
|
+
ENV['MORTAR_PROJECT_NAME'].should eq('myproject')
|
|
153
153
|
end
|
|
154
154
|
|
|
155
155
|
it "install sqoop with command" do
|
|
156
|
+
ENV.delete('MORTAR_PROJECT_NAME')
|
|
156
157
|
command = 'command'
|
|
157
158
|
ctrl = Mortar::Local::Controller.new
|
|
158
159
|
|
|
160
|
+
any_instance_of(Mortar::Command::Base) do |b|
|
|
161
|
+
mock(b).project.returns(Mortar::Project::Project)
|
|
162
|
+
mock(b).options.returns({:project =>'myproject'})
|
|
163
|
+
end
|
|
159
164
|
any_instance_of(Mortar::Local::Java) do |j|
|
|
160
165
|
mock(j).check_install.returns(true)
|
|
161
166
|
end
|
|
@@ -177,6 +182,9 @@ module Mortar::Local
|
|
|
177
182
|
mock(ctrl).write_local_readme
|
|
178
183
|
mock(ctrl).ensure_local_install_dirs_in_gitignore
|
|
179
184
|
ctrl.install_and_configure(Mortar::PigVersion::Pig012.new, command, true)
|
|
185
|
+
begin
|
|
186
|
+
ENV['MORTAR_PROJECT_NAME'].should eq('myproject')
|
|
187
|
+
end
|
|
180
188
|
end
|
|
181
189
|
end
|
|
182
190
|
|
|
@@ -184,7 +192,6 @@ module Mortar::Local
|
|
|
184
192
|
|
|
185
193
|
it "checks for aws keys, checks depenendency installation, runs script" do
|
|
186
194
|
c = Mortar::Local::Controller.new
|
|
187
|
-
mock(c).require_aws_keys
|
|
188
195
|
mock(c).install_and_configure("0.9", "run")
|
|
189
196
|
test_script = "foobar-script"
|
|
190
197
|
the_parameters = []
|
|
@@ -199,7 +206,6 @@ module Mortar::Local
|
|
|
199
206
|
context("illustrate") do
|
|
200
207
|
it "checks for aws keys, checks depenendency installation, runs the illustrate process" do
|
|
201
208
|
c = Mortar::Local::Controller.new
|
|
202
|
-
mock(c).require_aws_keys
|
|
203
209
|
mock(c).install_and_configure("0.9", "illustrate")
|
|
204
210
|
test_script = "foobar-script"
|
|
205
211
|
script_alias = "some_alias"
|
|
@@ -42,6 +42,7 @@ module Mortar::Local
|
|
|
42
42
|
it "returns params for a logged-in user" do
|
|
43
43
|
ENV['AWS_ACCESS_KEY'] = "abc"
|
|
44
44
|
ENV['AWS_SECRET_KEY'] = "012"
|
|
45
|
+
ENV['MORTAR_PROJECT_NAME'] = "projectName"
|
|
45
46
|
|
|
46
47
|
# setup fake auth
|
|
47
48
|
stub_core
|
|
@@ -56,11 +57,13 @@ module Mortar::Local
|
|
|
56
57
|
get_param_value(params, "AWS_SECRET_KEY").should == "012"
|
|
57
58
|
get_param_value(params, "AWS_SECRET_ACCESS_KEY").should == "012"
|
|
58
59
|
get_param_value(params, "aws_secret_access_key").should == "012"
|
|
60
|
+
get_param_value(params, "MORTAR_PROJECT_NAME").should =="projectName"
|
|
59
61
|
end
|
|
60
62
|
|
|
61
63
|
it "returns params for a non-logged-in user" do
|
|
62
64
|
ENV['AWS_ACCESS_KEY'] = "abc"
|
|
63
65
|
ENV['AWS_SECRET_KEY'] = "012"
|
|
66
|
+
ENV['MORTAR_PROJECT_NAME'] = "projectName"
|
|
64
67
|
|
|
65
68
|
params = @params.automatic_parameters()
|
|
66
69
|
get_param_value(params, "MORTAR_EMAIL").should == "notloggedin@user.org"
|
|
@@ -73,6 +76,7 @@ module Mortar::Local
|
|
|
73
76
|
get_param_value(params, "AWS_SECRET_KEY").should == "012"
|
|
74
77
|
get_param_value(params, "AWS_SECRET_ACCESS_KEY").should == "012"
|
|
75
78
|
get_param_value(params, "aws_secret_access_key").should == "012"
|
|
79
|
+
get_param_value(params, "MORTAR_PROJECT_NAME").should =="projectName"
|
|
76
80
|
end
|
|
77
81
|
end
|
|
78
82
|
|
data/spec/mortar/updater_spec.rb
CHANGED
|
@@ -46,21 +46,21 @@ module Mortar
|
|
|
46
46
|
|
|
47
47
|
context "get ruby version" do
|
|
48
48
|
it "makes gem call" do
|
|
49
|
-
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT}) do
|
|
49
|
+
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT, :ssl_version => :TLSv1}) do
|
|
50
50
|
{:body => Mortar::API::OkJson.encode({"version" => "1.0.0"}), :status => 200}
|
|
51
51
|
end
|
|
52
52
|
Updater.get_newest_version.should == "1.0.0"
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
it "has no version field" do
|
|
56
|
-
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT}) do
|
|
56
|
+
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT, :ssl_version => :TLSv1}) do
|
|
57
57
|
{:body => Mortar::API::OkJson.encode({"no_version" => "none"}), :status => 200}
|
|
58
58
|
end
|
|
59
59
|
Updater.get_newest_version.should == "0.0.0"
|
|
60
60
|
end
|
|
61
61
|
|
|
62
62
|
it "has an exception" do
|
|
63
|
-
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT}) do
|
|
63
|
+
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT, :ssl_version => :TLSv1}) do
|
|
64
64
|
raise Exception
|
|
65
65
|
end
|
|
66
66
|
Updater.get_newest_version.should == "0.0.0"
|
|
@@ -69,7 +69,7 @@ module Mortar
|
|
|
69
69
|
|
|
70
70
|
context "update check" do
|
|
71
71
|
it "displays no message when we have a current version" do
|
|
72
|
-
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT}) do
|
|
72
|
+
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT, :ssl_version => :TLSv1}) do
|
|
73
73
|
{:body => Mortar::API::OkJson.encode({"version" => "1.0.0"}), :status => 200}
|
|
74
74
|
end
|
|
75
75
|
capture_stderr do
|
|
@@ -81,7 +81,7 @@ module Mortar
|
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
it "displays message when we have an outdated version" do
|
|
84
|
-
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT}) do
|
|
84
|
+
Excon.stub({:method => :get, :path => "/api/v1/gems/mortar.json", :connect_timeout => Mortar::Updater::CONNECT_TIMEOUT, :read_timeout => Mortar::Updater::READ_TIMEOUT, :ssl_version => :TLSv1}) do
|
|
85
85
|
{:body => Mortar::API::OkJson.encode({"version" => "2.0.0"}), :status => 200}
|
|
86
86
|
end
|
|
87
87
|
capture_stderr do
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mortar
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
hash:
|
|
4
|
+
hash: 31
|
|
5
5
|
prerelease:
|
|
6
6
|
segments:
|
|
7
7
|
- 0
|
|
8
8
|
- 15
|
|
9
|
-
-
|
|
10
|
-
version: 0.15.
|
|
9
|
+
- 30
|
|
10
|
+
version: 0.15.30
|
|
11
11
|
platform: ruby
|
|
12
12
|
authors:
|
|
13
13
|
- Mortar Data
|
|
@@ -15,7 +15,7 @@ autorequire:
|
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
17
|
|
|
18
|
-
date: 2014-10-
|
|
18
|
+
date: 2014-10-24 00:00:00 Z
|
|
19
19
|
dependencies:
|
|
20
20
|
- !ruby/object:Gem::Dependency
|
|
21
21
|
name: rdoc
|
|
@@ -41,12 +41,12 @@ dependencies:
|
|
|
41
41
|
requirements:
|
|
42
42
|
- - ~>
|
|
43
43
|
- !ruby/object:Gem::Version
|
|
44
|
-
hash:
|
|
44
|
+
hash: 41
|
|
45
45
|
segments:
|
|
46
46
|
- 0
|
|
47
47
|
- 8
|
|
48
|
-
-
|
|
49
|
-
version: 0.8.
|
|
48
|
+
- 11
|
|
49
|
+
version: 0.8.11
|
|
50
50
|
type: :runtime
|
|
51
51
|
version_requirements: *id002
|
|
52
52
|
- !ruby/object:Gem::Dependency
|