mortar 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +36 -0
- data/bin/mortar +13 -0
- data/lib/mortar.rb +23 -0
- data/lib/mortar/auth.rb +312 -0
- data/lib/mortar/cli.rb +54 -0
- data/lib/mortar/command.rb +267 -0
- data/lib/mortar/command/auth.rb +96 -0
- data/lib/mortar/command/base.rb +319 -0
- data/lib/mortar/command/clusters.rb +41 -0
- data/lib/mortar/command/describe.rb +97 -0
- data/lib/mortar/command/generate.rb +121 -0
- data/lib/mortar/command/help.rb +166 -0
- data/lib/mortar/command/illustrate.rb +97 -0
- data/lib/mortar/command/jobs.rb +174 -0
- data/lib/mortar/command/pigscripts.rb +45 -0
- data/lib/mortar/command/projects.rb +128 -0
- data/lib/mortar/command/validate.rb +94 -0
- data/lib/mortar/command/version.rb +42 -0
- data/lib/mortar/errors.rb +24 -0
- data/lib/mortar/generators/generator_base.rb +107 -0
- data/lib/mortar/generators/macro_generator.rb +37 -0
- data/lib/mortar/generators/pigscript_generator.rb +40 -0
- data/lib/mortar/generators/project_generator.rb +67 -0
- data/lib/mortar/generators/udf_generator.rb +28 -0
- data/lib/mortar/git.rb +233 -0
- data/lib/mortar/helpers.rb +488 -0
- data/lib/mortar/project.rb +156 -0
- data/lib/mortar/snapshot.rb +39 -0
- data/lib/mortar/templates/macro/macro.pig +14 -0
- data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
- data/lib/mortar/templates/pigscript/python_udf.py +13 -0
- data/lib/mortar/templates/project/Gemfile +3 -0
- data/lib/mortar/templates/project/README.md +8 -0
- data/lib/mortar/templates/project/gitignore +4 -0
- data/lib/mortar/templates/project/macros/gitkeep +0 -0
- data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
- data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
- data/lib/mortar/templates/udf/python_udf.py +13 -0
- data/lib/mortar/version.rb +20 -0
- data/lib/vendor/mortar/okjson.rb +598 -0
- data/lib/vendor/mortar/uuid.rb +312 -0
- data/spec/mortar/auth_spec.rb +156 -0
- data/spec/mortar/command/auth_spec.rb +46 -0
- data/spec/mortar/command/base_spec.rb +82 -0
- data/spec/mortar/command/clusters_spec.rb +61 -0
- data/spec/mortar/command/describe_spec.rb +135 -0
- data/spec/mortar/command/generate_spec.rb +139 -0
- data/spec/mortar/command/illustrate_spec.rb +140 -0
- data/spec/mortar/command/jobs_spec.rb +364 -0
- data/spec/mortar/command/pigscripts_spec.rb +70 -0
- data/spec/mortar/command/projects_spec.rb +165 -0
- data/spec/mortar/command/validate_spec.rb +119 -0
- data/spec/mortar/command_spec.rb +122 -0
- data/spec/mortar/git_spec.rb +278 -0
- data/spec/mortar/helpers_spec.rb +82 -0
- data/spec/mortar/project_spec.rb +76 -0
- data/spec/mortar/snapshot_spec.rb +46 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +278 -0
- data/spec/support/display_message_matcher.rb +68 -0
- metadata +259 -0
@@ -0,0 +1,140 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'spec_helper'
|
18
|
+
require 'fakefs/spec_helpers'
|
19
|
+
require 'mortar/command/illustrate'
|
20
|
+
require 'mortar/api/illustrate'
|
21
|
+
require 'launchy'
|
22
|
+
|
23
|
+
module Mortar::Command
|
24
|
+
describe Illustrate do
|
25
|
+
|
26
|
+
before(:each) do
|
27
|
+
stub_core
|
28
|
+
@git = Mortar::Git::Git.new
|
29
|
+
end
|
30
|
+
|
31
|
+
context("index") do
|
32
|
+
|
33
|
+
it "errors when an alias is not provided" do
|
34
|
+
with_git_initialized_project do |p|
|
35
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
36
|
+
stderr, stdout = execute("illustrate my_script", p)
|
37
|
+
stderr.should == <<-STDERR
|
38
|
+
! Usage: mortar illustrate PIGSCRIPT ALIAS
|
39
|
+
! Must specify PIGSCRIPT and ALIAS.
|
40
|
+
STDERR
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
it "errors when no remote exists in the project" do
|
45
|
+
with_git_initialized_project do |p|
|
46
|
+
@git.git('remote rm mortar')
|
47
|
+
p.remote = nil
|
48
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
49
|
+
stderr, stdout = execute("illustrate my_script my_alias", p, @git)
|
50
|
+
stderr.should == <<-STDERR
|
51
|
+
! Unable to find git remote for project myproject
|
52
|
+
STDERR
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
it "errors when requested pigscript cannot be found" do
|
57
|
+
with_git_initialized_project do |p|
|
58
|
+
stderr, stdout = execute("illustrate does_not_exist my_alias", p, @git)
|
59
|
+
stderr.should == <<-STDERR
|
60
|
+
! Unable to find pigscript does_not_exist
|
61
|
+
! No pigscripts found
|
62
|
+
STDERR
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
it "requests and reports on a successful illustrate" do
|
67
|
+
with_git_initialized_project do |p|
|
68
|
+
# stub api requests
|
69
|
+
illustrate_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
70
|
+
illustrate_url = "https://api.mortardata.com/illustrates/#{illustrate_id}"
|
71
|
+
parameters = ["name"=>"key", "value"=>"value" ]
|
72
|
+
|
73
|
+
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
74
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", is_a(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
75
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
76
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
77
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
78
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_READING_DATA, "status_description" => "Reading data"})).ordered
|
79
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PRUNING_DATA, "status_description" => "Pruning data"})).ordered
|
80
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
81
|
+
|
82
|
+
# stub launchy
|
83
|
+
mock(Launchy).open(illustrate_url) {Thread.new {}}
|
84
|
+
|
85
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
86
|
+
stderr, stdout = execute("illustrate my_script my_alias --polling_interval 0.05 -p key=value", p, @git)
|
87
|
+
stdout.should == <<-STDOUT
|
88
|
+
Taking code snapshot... done
|
89
|
+
Sending code snapshot to Mortar... done
|
90
|
+
Starting illustrate... done
|
91
|
+
|
92
|
+
\r\e[0KStatus: Pending... /\r\e[0KStatus: GATEWAY_STARTING... -\r\e[0KStatus: In progress... \\\r\e[0KStatus: Reading data... |\r\e[0KStatus: Pruning data... /\r\e[0KStatus: Succeeded
|
93
|
+
|
94
|
+
Results available at https://api.mortardata.com/illustrates/c571a8c7f76a4fd4a67c103d753e2dd5
|
95
|
+
Opening web browser to show results... done
|
96
|
+
STDOUT
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
it "requests and reports on a failed illustrate" do
|
101
|
+
with_git_initialized_project do |p|
|
102
|
+
# stub api requests
|
103
|
+
illustrate_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
104
|
+
|
105
|
+
error_message = "This is my error message\nWith multiple lines."
|
106
|
+
line_number = 23
|
107
|
+
column_number = 32
|
108
|
+
error_type = 'PigError'
|
109
|
+
|
110
|
+
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
111
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", is_a(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
112
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
113
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_FAILURE,
|
114
|
+
"error_message" => error_message,
|
115
|
+
"line_number" => line_number,
|
116
|
+
"column_number" => column_number,
|
117
|
+
"error_type" => error_type,
|
118
|
+
"status_description" => "Failed"})).ordered
|
119
|
+
|
120
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
121
|
+
stderr, stdout = execute("illustrate my_script my_alias --polling_interval 0.05", p, @git)
|
122
|
+
stdout.should == <<-STDOUT
|
123
|
+
Taking code snapshot... done
|
124
|
+
Sending code snapshot to Mortar... done
|
125
|
+
Starting illustrate... done
|
126
|
+
|
127
|
+
\r\e[0KStatus: Pending... /\r\e[0KStatus: Failed
|
128
|
+
|
129
|
+
STDOUT
|
130
|
+
stderr.should == <<-STDERR
|
131
|
+
! Illustrate failed with PigError at Line 23, Column 32:
|
132
|
+
!
|
133
|
+
! This is my error message
|
134
|
+
! With multiple lines.
|
135
|
+
STDERR
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
@@ -0,0 +1,364 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'spec_helper'
|
18
|
+
require 'fakefs/spec_helpers'
|
19
|
+
require 'mortar/command/jobs'
|
20
|
+
require 'mortar/api/jobs'
|
21
|
+
|
22
|
+
module Mortar::Command
|
23
|
+
describe Jobs do
|
24
|
+
|
25
|
+
before(:each) do
|
26
|
+
stub_core
|
27
|
+
@git = Mortar::Git::Git.new
|
28
|
+
end
|
29
|
+
|
30
|
+
context("index") do
|
31
|
+
end
|
32
|
+
|
33
|
+
context("run") do
|
34
|
+
it "handles singlejobcluster parameter" do
|
35
|
+
with_git_initialized_project do |p|
|
36
|
+
# stub api requests
|
37
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
38
|
+
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
39
|
+
cluster_size = 5
|
40
|
+
|
41
|
+
mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
42
|
+
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
43
|
+
:keepalive => false) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
44
|
+
|
45
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
46
|
+
stderr, stdout = execute("jobs:run my_script -1 --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
|
47
|
+
stdout.should == <<-STDOUT
|
48
|
+
Taking code snapshot... done
|
49
|
+
Sending code snapshot to Mortar... done
|
50
|
+
Requesting job execution... done
|
51
|
+
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
52
|
+
|
53
|
+
Job status can be viewed on the web at:
|
54
|
+
|
55
|
+
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
56
|
+
|
57
|
+
Or by running:
|
58
|
+
|
59
|
+
mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5
|
60
|
+
|
61
|
+
STDOUT
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
it "runs a job on a new cluster" do
|
66
|
+
with_git_initialized_project do |p|
|
67
|
+
# stub api requests
|
68
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
69
|
+
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
70
|
+
cluster_size = 5
|
71
|
+
|
72
|
+
mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
73
|
+
:parameters => match_array([{"name" => "FIRST_PARAM", "value" => "FOO"}, {"name" => "SECOND_PARAM", "value" => "BAR"}]),
|
74
|
+
:keepalive => true) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
75
|
+
|
76
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
77
|
+
stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST_PARAM=FOO -p SECOND_PARAM=BAR", p, @git)
|
78
|
+
stdout.should == <<-STDOUT
|
79
|
+
Taking code snapshot... done
|
80
|
+
Sending code snapshot to Mortar... done
|
81
|
+
Requesting job execution... done
|
82
|
+
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
83
|
+
|
84
|
+
Job status can be viewed on the web at:
|
85
|
+
|
86
|
+
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
87
|
+
|
88
|
+
Or by running:
|
89
|
+
|
90
|
+
mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5
|
91
|
+
|
92
|
+
STDOUT
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
it "runs a job on an existing cluster" do
|
97
|
+
with_git_initialized_project do |p|
|
98
|
+
# stub api requests
|
99
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
100
|
+
job_url = "http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5"
|
101
|
+
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
102
|
+
|
103
|
+
mock(Mortar::Auth.api).post_job_existing_cluster("myproject", "my_script", is_a(String), cluster_id, :parameters => []) {Excon::Response.new(:body => {"job_id" => job_id, "web_job_url" => job_url})}
|
104
|
+
|
105
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
106
|
+
stderr, stdout = execute("jobs:run my_script --clusterid e2790e7e8c7d48e39157238d58191346", p, @git)
|
107
|
+
stdout.should == <<-STDOUT
|
108
|
+
Taking code snapshot... done
|
109
|
+
Sending code snapshot to Mortar... done
|
110
|
+
Requesting job execution... done
|
111
|
+
job_id: c571a8c7f76a4fd4a67c103d753e2dd5
|
112
|
+
|
113
|
+
Job status can be viewed on the web at:
|
114
|
+
|
115
|
+
http://127.0.0.1:5000/jobs/job_detail?job_id=c571a8c7f76a4fd4a67c103d753e2dd5
|
116
|
+
|
117
|
+
Or by running:
|
118
|
+
|
119
|
+
mortar jobs:status c571a8c7f76a4fd4a67c103d753e2dd5
|
120
|
+
|
121
|
+
STDOUT
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
it "runs a job with parameter file" do
|
126
|
+
with_git_initialized_project do |p|
|
127
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
128
|
+
cluster_size = 5
|
129
|
+
keepalive = true
|
130
|
+
|
131
|
+
mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
132
|
+
:parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]),
|
133
|
+
:keepalive => true) {Excon::Response.new(:body => {"job_id" => job_id})}
|
134
|
+
|
135
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
136
|
+
|
137
|
+
parameters = <<PARAMS
|
138
|
+
FIRST=PIZZA
|
139
|
+
SECOND=LASAGNA
|
140
|
+
|
141
|
+
THIRD=BEAR
|
142
|
+
PARAMS
|
143
|
+
|
144
|
+
write_file(File.join(p.root_path, "params.ini"), parameters)
|
145
|
+
stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST=FOO -p SECOND=BAR --param-file params.ini", p, @git)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
it "runs a job with parameter file with comments" do
|
150
|
+
with_git_initialized_project do |p|
|
151
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
152
|
+
cluster_size = 5
|
153
|
+
keepalive = true
|
154
|
+
|
155
|
+
mock(Mortar::Auth.api).post_job_new_cluster("myproject", "my_script", is_a(String), cluster_size,
|
156
|
+
:parameters => match_array([{"name" => "FIRST", "value" => "FOO"}, {"name" => "SECOND", "value" => "BAR"}, {"name" => "THIRD", "value" => "BEAR\n"}]),
|
157
|
+
:keepalive => true) {Excon::Response.new(:body => {"job_id" => job_id})}
|
158
|
+
|
159
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
160
|
+
|
161
|
+
parameters = <<PARAMS
|
162
|
+
FIRST=PIZZA
|
163
|
+
SECOND=LASAGNA
|
164
|
+
; This is a test
|
165
|
+
|
166
|
+
THIRD=BEAR
|
167
|
+
PARAMS
|
168
|
+
|
169
|
+
write_file(File.join(p.root_path, "params.ini"), parameters)
|
170
|
+
stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST=FOO -p SECOND=BAR --param-file params.ini", p, @git)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
it "runs a job with malformed parameter file" do
|
175
|
+
with_git_initialized_project do |p|
|
176
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
177
|
+
cluster_size = 5
|
178
|
+
keepalive = true
|
179
|
+
|
180
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
181
|
+
|
182
|
+
parameters = <<PARAMS
|
183
|
+
FIRST=PIZZA
|
184
|
+
SECONDLASAGasNA
|
185
|
+
; This is a test
|
186
|
+
Natta
|
187
|
+
THIRD=BEAR
|
188
|
+
PARAMS
|
189
|
+
|
190
|
+
write_file(File.join(p.root_path, "params.ini"), parameters)
|
191
|
+
stderr, stdout = execute("jobs:run my_script --clustersize 5 -p FIRST=FOO -p SECOND=BAR --param-file params.ini", p, @git)
|
192
|
+
stderr.should == <<-STDERR
|
193
|
+
! Parameter file is malformed
|
194
|
+
STDERR
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
context("status") do
|
200
|
+
|
201
|
+
it "gets status for a completed, successful job" do
|
202
|
+
with_git_initialized_project do |p|
|
203
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
204
|
+
pigscript_name = "my_script"
|
205
|
+
project_name = "myproject"
|
206
|
+
status_code = Mortar::API::Jobs::STATUS_SUCCESS
|
207
|
+
progress = 100
|
208
|
+
outputs = [{'name'=> 'hottest_songs_of_the_decade',
|
209
|
+
'records' => 10,
|
210
|
+
'alias' => 'output_data',
|
211
|
+
'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data'},
|
212
|
+
{'name'=> 'hottest_songs_of_the_decade',
|
213
|
+
'records' => 100,
|
214
|
+
'alias' => 'output_data_2',
|
215
|
+
'location' => 's3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2'}]
|
216
|
+
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
217
|
+
start_timestamp = "2012-02-28T03:35:42.831000+00:00"
|
218
|
+
running_timestamp = "2012-02-28T03:41:52.613000+00:00"
|
219
|
+
stop_timestamp = "2012-02-28T03:44:52.613000+00:00"
|
220
|
+
parameters = {"my_param_1" => "value1", "MY_PARAM_2" => "3"}
|
221
|
+
|
222
|
+
mock(Mortar::Auth.api).get_job(job_id) {Excon::Response.new(:body => {"job_id" => job_id,
|
223
|
+
"pigscript_name" => pigscript_name,
|
224
|
+
"project_name" => project_name,
|
225
|
+
"status_code" => status_code,
|
226
|
+
"status_description" => "Success",
|
227
|
+
"progress" => progress,
|
228
|
+
"cluster_id" => cluster_id,
|
229
|
+
"start_timestamp" => start_timestamp,
|
230
|
+
"running_timestamp" => running_timestamp,
|
231
|
+
"stop_timestamp" => stop_timestamp,
|
232
|
+
"duration" => "6 mins",
|
233
|
+
"num_hadoop_jobs" => 4,
|
234
|
+
"num_hadoop_jobs_succeeded" => 4,
|
235
|
+
"parameters" => parameters,
|
236
|
+
"outputs" => outputs
|
237
|
+
})}
|
238
|
+
stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5", p, @git)
|
239
|
+
stdout.should == <<-STDOUT
|
240
|
+
=== myproject: my_script (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
|
241
|
+
cluster_id: e2790e7e8c7d48e39157238d58191346
|
242
|
+
hadoop jobs complete: 4.00 / 4.00
|
243
|
+
job began running at: 2012-02-28T03:41:52.613000+00:00
|
244
|
+
job finished at: 2012-02-28T03:44:52.613000+00:00
|
245
|
+
job run with parameters:
|
246
|
+
MY_PARAM_2: 3
|
247
|
+
my_param_1: value1
|
248
|
+
job running for: 6 mins
|
249
|
+
job submitted at: 2012-02-28T03:35:42.831000+00:00
|
250
|
+
outputs:
|
251
|
+
output_data:
|
252
|
+
location: s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data
|
253
|
+
records: 10
|
254
|
+
output_data_2:
|
255
|
+
location: s3n://my-bucket/my-folder/hottest_songs_of_the_decade/output_data_2
|
256
|
+
records: 100
|
257
|
+
progress: 100%
|
258
|
+
status: Success
|
259
|
+
STDOUT
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
it "gets status for a running job" do
|
264
|
+
with_git_initialized_project do |p|
|
265
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
266
|
+
pigscript_name = "my_script"
|
267
|
+
project_name = "myproject"
|
268
|
+
status_code = Mortar::API::Jobs::STATUS_RUNNING
|
269
|
+
progress = 55
|
270
|
+
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
271
|
+
start_timestamp = "2012-02-28T03:35:42.831000+00:00"
|
272
|
+
running_timestamp = "2012-02-28T03:41:52.613000+00:00"
|
273
|
+
parameters = {"my_param_1" => "value1", "MY_PARAM_2" => "3"}
|
274
|
+
|
275
|
+
mock(Mortar::Auth.api).get_job(job_id) {Excon::Response.new(:body => {"job_id" => job_id,
|
276
|
+
"pigscript_name" => pigscript_name,
|
277
|
+
"project_name" => project_name,
|
278
|
+
"status_code" => status_code,
|
279
|
+
"status_description" => "Running",
|
280
|
+
"progress" => progress,
|
281
|
+
"cluster_id" => cluster_id,
|
282
|
+
"start_timestamp" => start_timestamp,
|
283
|
+
"running_timestamp" => running_timestamp,
|
284
|
+
"duration" => "6 mins",
|
285
|
+
"num_hadoop_jobs" => 4,
|
286
|
+
"num_hadoop_jobs_succeeded" => 2,
|
287
|
+
"parameters" => parameters
|
288
|
+
})}
|
289
|
+
stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5", p, @git)
|
290
|
+
stdout.should == <<-STDOUT
|
291
|
+
=== myproject: my_script (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
|
292
|
+
cluster_id: e2790e7e8c7d48e39157238d58191346
|
293
|
+
hadoop jobs complete: 2.00 / 4.00
|
294
|
+
job began running at: 2012-02-28T03:41:52.613000+00:00
|
295
|
+
job run with parameters:
|
296
|
+
MY_PARAM_2: 3
|
297
|
+
my_param_1: value1
|
298
|
+
job running for: 6 mins
|
299
|
+
job submitted at: 2012-02-28T03:35:42.831000+00:00
|
300
|
+
progress: 55%
|
301
|
+
status: Running
|
302
|
+
STDOUT
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
it "gets status for a error job" do
|
307
|
+
with_git_initialized_project do |p|
|
308
|
+
job_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
309
|
+
pigscript_name = "my_script"
|
310
|
+
project_name = "myproject"
|
311
|
+
status_code = Mortar::API::Jobs::STATUS_EXECUTION_ERROR
|
312
|
+
progress = 55
|
313
|
+
cluster_id = "e2790e7e8c7d48e39157238d58191346"
|
314
|
+
start_timestamp = "2012-02-28T03:35:42.831000+00:00"
|
315
|
+
running_timestamp = "2012-02-28T03:41:52.613000+00:00"
|
316
|
+
stop_timestamp = "2012-02-28T03:45:52.613000+00:00"
|
317
|
+
parameters = {"my_param_1" => "value1", "MY_PARAM_2" => "3"}
|
318
|
+
error = {"message" => "An error occurred and here's some more info",
|
319
|
+
"type" => "RuntimeError",
|
320
|
+
"line_number" => 43,
|
321
|
+
"column_number" => 34}
|
322
|
+
mock(Mortar::Auth.api).get_job(job_id) {Excon::Response.new(:body => {"job_id" => job_id,
|
323
|
+
"pigscript_name" => pigscript_name,
|
324
|
+
"project_name" => project_name,
|
325
|
+
"status_code" => status_code,
|
326
|
+
"status_description" => "Execution error",
|
327
|
+
"progress" => progress,
|
328
|
+
"cluster_id" => cluster_id,
|
329
|
+
"start_timestamp" => start_timestamp,
|
330
|
+
"running_timestamp" => running_timestamp,
|
331
|
+
"stop_timestamp" => stop_timestamp,
|
332
|
+
"duration" => "6 mins",
|
333
|
+
"num_hadoop_jobs" => 4,
|
334
|
+
"num_hadoop_jobs_succeeded" => 0,
|
335
|
+
"parameters" => parameters,
|
336
|
+
"error" => error
|
337
|
+
})}
|
338
|
+
stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5", p, @git)
|
339
|
+
stdout.should == <<-STDOUT
|
340
|
+
=== myproject: my_script (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
|
341
|
+
cluster_id: e2790e7e8c7d48e39157238d58191346
|
342
|
+
error:
|
343
|
+
column_number: 34
|
344
|
+
line_number: 43
|
345
|
+
message: An error occurred and here's some more info
|
346
|
+
type: RuntimeError
|
347
|
+
hadoop jobs complete: 0.00 / 4.00
|
348
|
+
job began running at: 2012-02-28T03:41:52.613000+00:00
|
349
|
+
job finished at: 2012-02-28T03:45:52.613000+00:00
|
350
|
+
job run with parameters:
|
351
|
+
MY_PARAM_2: 3
|
352
|
+
my_param_1: value1
|
353
|
+
job running for: 6 mins
|
354
|
+
job submitted at: 2012-02-28T03:35:42.831000+00:00
|
355
|
+
progress: 55%
|
356
|
+
status: Execution error
|
357
|
+
STDOUT
|
358
|
+
end
|
359
|
+
end
|
360
|
+
|
361
|
+
|
362
|
+
end
|
363
|
+
end
|
364
|
+
end
|