mortar 0.15.39 → 0.15.40
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/mortar/local/python.rb +4 -2
- data/lib/mortar/project.rb +34 -34
- data/lib/mortar/templates/script/runpython.sh +3 -2
- data/lib/mortar/templates/script/runstillson.sh +4 -4
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/local/python_spec.rb +21 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YTVjMGVkNTRjZGJjMjQyODI3Yjg3ZDkzMjE5NzU3ZThjZDE4NGNlZA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YTRkN2M3MDQ3YTA4MDY1MDMyOWRmY2YzYjc5NGRjZjYzM2IyNjk5Nw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MTk1ZTZmNGZlMjRmYWU3OGM4OTUzYmI0NTg1M2RlM2Q1Mjk2MmFlY2U1MWRh
|
10
|
+
MjVkZWZlZjhkYjE5MWMzYzFmNDhlN2QyZjZhNDhiYzIyMjM4NzliNTczZjUy
|
11
|
+
YjM0MzQ3NDI1NzRiZjAyY2I2YmZkYWFmNzdkNGQxMGI2NWFkN2U=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YzExMTA1YjgxMzU5NjQ3NTBlYjdlYzgyZTJlMzEyOGZhYWFmNjc3MjcyZTU5
|
14
|
+
OTVlZTA2NDc4ODA0ZWM5NDk4MGEwM2EyY2JkMDFhNzJmNDVmYTk2Nzg4NmQ0
|
15
|
+
MTU1ZDQ0NmZiNTZhYTEwM2U2YzUwYjIwYzNmNjYyNDk1M2ZhMjk=
|
data/lib/mortar/local/python.rb
CHANGED
@@ -36,6 +36,7 @@ class Mortar::Local::Python
|
|
36
36
|
|
37
37
|
@candidate_pythons = nil
|
38
38
|
|
39
|
+
|
39
40
|
# Execute either an installation of python or an inspection
|
40
41
|
# of the local system to see if a usable python is available
|
41
42
|
def check_or_install
|
@@ -330,7 +331,7 @@ class Mortar::Local::Python
|
|
330
331
|
parameters = merge_parameters(auto_params, project_config_parameters)
|
331
332
|
stillson_template_params = {
|
332
333
|
:parameters => parameters,
|
333
|
-
:
|
334
|
+
:luigiscripts_path => File.join(project_root, 'luigiscripts')
|
334
335
|
}
|
335
336
|
run_templated_script(stillson_command_script_template_path, stillson_template_params)
|
336
337
|
end
|
@@ -358,7 +359,8 @@ class Mortar::Local::Python
|
|
358
359
|
return {
|
359
360
|
:python_arugments => "",
|
360
361
|
:python_script => luigi_script.executable_path(),
|
361
|
-
:script_arguments => script_args.join(" ")
|
362
|
+
:script_arguments => script_args.join(" "),
|
363
|
+
:python_path => File.join(project_root, 'luigiscripts')
|
362
364
|
}
|
363
365
|
end
|
364
366
|
|
data/lib/mortar/project.rb
CHANGED
@@ -20,22 +20,22 @@ require 'pathname'
|
|
20
20
|
module Mortar
|
21
21
|
module Project
|
22
22
|
class ProjectError < RuntimeError; end
|
23
|
-
|
23
|
+
|
24
24
|
class Project
|
25
25
|
def self.required_directories
|
26
26
|
["macros", "pigscripts", "udfs"]
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
attr_accessor :name
|
30
30
|
attr_accessor :remote
|
31
31
|
attr_accessor :root_path
|
32
|
-
|
32
|
+
|
33
33
|
def initialize(name, root_path, remote)
|
34
34
|
@name = name
|
35
35
|
@root_path = root_path
|
36
36
|
@remote = remote
|
37
37
|
end
|
38
|
-
|
38
|
+
|
39
39
|
def python_udfs_path
|
40
40
|
File.join(@root_path, "udfs/python")
|
41
41
|
end
|
@@ -46,7 +46,7 @@ module Mortar
|
|
46
46
|
"python",
|
47
47
|
".py")
|
48
48
|
end
|
49
|
-
|
49
|
+
|
50
50
|
def pigscripts_path
|
51
51
|
File.join(@root_path, "pigscripts")
|
52
52
|
end
|
@@ -88,7 +88,7 @@ module Mortar
|
|
88
88
|
def sparkjobs_path
|
89
89
|
File.join(@root_path, "src/main/scala")
|
90
90
|
end
|
91
|
-
|
91
|
+
|
92
92
|
def sparkscripts_path
|
93
93
|
File.join(@root_path, "sparkscripts")
|
94
94
|
end
|
@@ -114,11 +114,11 @@ module Mortar
|
|
114
114
|
File.exists?(File.join(@root_path, ".mortar-project-remote"))
|
115
115
|
end
|
116
116
|
end
|
117
|
-
|
117
|
+
|
118
118
|
class ProjectEntity
|
119
|
-
|
119
|
+
|
120
120
|
include Enumerable
|
121
|
-
|
121
|
+
|
122
122
|
def initialize(path, name, filename_extension, optional=false)
|
123
123
|
@path = path
|
124
124
|
@name = name
|
@@ -126,27 +126,27 @@ module Mortar
|
|
126
126
|
@optional = optional
|
127
127
|
@elements = elements
|
128
128
|
end
|
129
|
-
|
129
|
+
|
130
130
|
def method_missing(method, *args)
|
131
131
|
method_name = method.to_s
|
132
132
|
return @elements[method_name] if @elements[method_name]
|
133
133
|
super
|
134
134
|
end
|
135
|
-
|
135
|
+
|
136
136
|
def each
|
137
137
|
@elements.each {|key, value| yield [key, value]}
|
138
138
|
end
|
139
|
-
|
139
|
+
|
140
140
|
def [](key)
|
141
141
|
@elements[key]
|
142
142
|
end
|
143
|
-
|
143
|
+
|
144
144
|
def keys
|
145
145
|
@elements.keys
|
146
146
|
end
|
147
|
-
|
147
|
+
|
148
148
|
protected
|
149
|
-
|
149
|
+
|
150
150
|
def element_name(element_path)
|
151
151
|
File.basename(element_path, @filename_extension)
|
152
152
|
end
|
@@ -163,7 +163,7 @@ module Mortar
|
|
163
163
|
if duplicates.length > 0
|
164
164
|
raise ProjectError, "Multiple scripts found with the same name. Each script must have a unique name regardless of what directory it is in. Duplicate script names found: #{duplicates.collect{|d| d[0]}}"
|
165
165
|
end
|
166
|
-
|
166
|
+
|
167
167
|
file_paths_hsh = scripts.flatten
|
168
168
|
return Hash[*file_paths_hsh]
|
169
169
|
else
|
@@ -171,12 +171,12 @@ module Mortar
|
|
171
171
|
end
|
172
172
|
return Hash[]
|
173
173
|
end
|
174
|
-
|
174
|
+
|
175
175
|
def element(path)
|
176
176
|
raise NotImplementedError, "Implement in subclass"
|
177
177
|
end
|
178
178
|
end
|
179
|
-
|
179
|
+
|
180
180
|
class PigScripts < ProjectEntity
|
181
181
|
def element(name, path)
|
182
182
|
PigScript.new(name, path)
|
@@ -206,12 +206,12 @@ module Mortar
|
|
206
206
|
Script.new(name, path)
|
207
207
|
end
|
208
208
|
end
|
209
|
-
|
209
|
+
|
210
210
|
class Script
|
211
|
-
|
211
|
+
|
212
212
|
attr_reader :name
|
213
213
|
attr_reader :path
|
214
|
-
|
214
|
+
|
215
215
|
def initialize(name, path)
|
216
216
|
@name = name
|
217
217
|
@path = path
|
@@ -228,43 +228,43 @@ module Mortar
|
|
228
228
|
p = Pathname.new(@path).dirname
|
229
229
|
rel_path = p.relative_path_from(Pathname.pwd).to_s
|
230
230
|
end
|
231
|
-
|
231
|
+
|
232
232
|
def to_s
|
233
233
|
code
|
234
234
|
end
|
235
235
|
end
|
236
236
|
|
237
237
|
class LuigiScript < Script
|
238
|
-
|
238
|
+
|
239
239
|
def executable_path
|
240
|
-
"
|
240
|
+
"#{self.rel_path}/#{self.name}.py"
|
241
241
|
end
|
242
|
-
|
242
|
+
|
243
243
|
end
|
244
244
|
|
245
245
|
class SparkScript < Script
|
246
|
-
|
246
|
+
|
247
247
|
def executable_path
|
248
248
|
"sparkscripts/#{self.name}"
|
249
249
|
end
|
250
|
-
|
250
|
+
|
251
251
|
end
|
252
|
-
|
252
|
+
|
253
253
|
class ControlScript < Script
|
254
|
-
|
254
|
+
|
255
255
|
def executable_path
|
256
256
|
"#{self.rel_path}/#{self.name}.pig"
|
257
257
|
end
|
258
|
-
|
258
|
+
|
259
259
|
end
|
260
|
-
|
260
|
+
|
261
261
|
class PigScript < Script
|
262
|
-
|
262
|
+
|
263
263
|
def executable_path
|
264
264
|
"#{self.rel_path}/#{self.name}.pig"
|
265
265
|
end
|
266
|
-
|
266
|
+
|
267
267
|
end
|
268
|
-
|
268
|
+
|
269
269
|
end
|
270
270
|
end
|
@@ -4,11 +4,12 @@ set -e
|
|
4
4
|
|
5
5
|
# Setup python environment
|
6
6
|
source <%= @local_install_dir %>/pythonenv/bin/activate
|
7
|
+
export PYTHONPATH=$PYTHONPATH:<%= @python_path %>
|
7
8
|
|
8
|
-
export LUIGI_CONFIG_PATH
|
9
|
+
export LUIGI_CONFIG_PATH=<%= @python_path %>/client.cfg
|
9
10
|
|
10
11
|
# Run Python
|
11
12
|
<%= @local_install_dir %>/pythonenv/bin/python \
|
12
13
|
<%= @python_arugments %> \
|
13
14
|
<%= @python_script %> \
|
14
|
-
<%= @script_arguments %>
|
15
|
+
<%= @script_arguments %>
|
@@ -6,10 +6,10 @@ set -e
|
|
6
6
|
source <%= @local_install_dir %>/pythonenv/bin/activate
|
7
7
|
|
8
8
|
# template config file
|
9
|
-
export LUIGI_CONFIG_TEMPLATE_PATH
|
9
|
+
export LUIGI_CONFIG_TEMPLATE_PATH=<%= @luigiscripts_path %>/client.cfg.template
|
10
10
|
|
11
11
|
# expanded config file
|
12
|
-
export LUIGI_CONFIG_PATH
|
12
|
+
export LUIGI_CONFIG_PATH=<%= @luigiscripts_path %>/client.cfg
|
13
13
|
|
14
14
|
# Setup parameters in environment variables
|
15
15
|
<% @parameters.each do |p| %>
|
@@ -17,8 +17,8 @@ export <%= p['name'] %>="<%= p['value'] %>";
|
|
17
17
|
<% end %>
|
18
18
|
|
19
19
|
# Run stillson to expand the configuration template
|
20
|
-
if [ -f "$LUIGI_CONFIG_TEMPLATE_PATH" ]
|
21
|
-
then
|
20
|
+
if [ -f "$LUIGI_CONFIG_TEMPLATE_PATH" ]
|
21
|
+
then
|
22
22
|
stillson "$LUIGI_CONFIG_TEMPLATE_PATH" -o $LUIGI_CONFIG_PATH
|
23
23
|
else
|
24
24
|
echo "No luigi client configuration template found in expected location $LUIGI_CONFIG_TEMPLATE_PATH. Not expanding."
|
data/lib/mortar/version.rb
CHANGED
@@ -131,19 +131,27 @@ module Mortar::Local
|
|
131
131
|
context "running python commands" do
|
132
132
|
|
133
133
|
it "Generates the appropriate tempate variables" do
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
134
|
+
FakeFS do
|
135
|
+
with_git_initialized_project do |p|
|
136
|
+
script_name = "some_luigi_script"
|
137
|
+
script_path = File.join(p.luigiscripts_path, "#{script_name}.py")
|
138
|
+
write_file(script_path)
|
139
|
+
luigi_script = Mortar::Project::LuigiScript.new(script_name, script_path)
|
140
|
+
args = %W{--myoption 2 --myotheroption 3}
|
141
|
+
py = Mortar::Local::Python.new
|
142
|
+
|
143
|
+
dir = Pathname.new(script_path).dirname
|
144
|
+
rel_path = dir.relative_path_from(Pathname.pwd).to_s + '/' + script_name + '.py'
|
145
|
+
|
146
|
+
expected_hash = {
|
147
|
+
:python_arugments => "",
|
148
|
+
:python_script => rel_path,
|
149
|
+
:script_arguments => "--local-scheduler --logging-conf-file #{py.luigi_logging_config_file_path} --myoption 2 --myotheroption 3",
|
150
|
+
:python_path => p.luigiscripts_path
|
151
|
+
}
|
152
|
+
|
153
|
+
expect(py.luigi_command_template_parameters(luigi_script, args)).to eq(expected_hash)
|
154
|
+
end
|
147
155
|
end
|
148
156
|
end
|
149
157
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mortar
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.15.
|
4
|
+
version: 0.15.40
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mortar Data
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-04-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rdoc
|