mortar 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/mortar/command.rb +0 -1
- data/lib/mortar/command/fixtures.rb +100 -0
- data/lib/mortar/command/illustrate.rb +3 -1
- data/lib/mortar/generators/project_generator.rb +6 -0
- data/lib/mortar/helpers.rb +19 -1
- data/lib/mortar/project.rb +4 -0
- data/lib/mortar/templates/project/README.md +1 -1
- data/lib/mortar/templates/project/fixtures/gitkeep +0 -0
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/fixtures_spec.rb +136 -0
- data/spec/mortar/command/generate_spec.rb +4 -0
- data/spec/mortar/command/illustrate_spec.rb +30 -3
- data/spec/mortar/command/jobs_spec.rb +1 -1
- data/spec/mortar/git_spec.rb +2 -0
- metadata +146 -137
data/lib/mortar/command.rb
CHANGED
@@ -0,0 +1,100 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require "mortar/command/base"
|
18
|
+
|
19
|
+
# create a reusable fixture.
|
20
|
+
#
|
21
|
+
class Mortar::Command::Fixtures < Mortar::Command::Base
|
22
|
+
|
23
|
+
WARNING_NUM_ROWS = 50
|
24
|
+
|
25
|
+
#fixtures:head [INPUT_URL] [NUM_ROWS] [FIXTURE_NAME]
|
26
|
+
#
|
27
|
+
#Create a reusable fixture [FIXTURE_NAME] made up of [NUM_ROWS]
|
28
|
+
#number of rows from the head of the input file(s) at [INPUT_URL].
|
29
|
+
#
|
30
|
+
# Examples:
|
31
|
+
#
|
32
|
+
# $ mortar fixtures:head s3n://tbmmsd/*.tsv.* 100 samll_song_sample
|
33
|
+
#
|
34
|
+
def head
|
35
|
+
input_url = shift_argument
|
36
|
+
num_rows = shift_argument
|
37
|
+
fixture_name = shift_argument
|
38
|
+
unless input_url && num_rows && fixture_name
|
39
|
+
error("Usage: mortar fixtures:head INPUT_URL NUM_ROWS FIXTURE_NAME\nMust specifiy INPUT_URL, NUM_ROWS, and FIXTURE_NAME.")
|
40
|
+
end
|
41
|
+
if does_fixture_exist(fixture_name)
|
42
|
+
error("Fixture #{fixture_name} already exists.")
|
43
|
+
end
|
44
|
+
unless num_rows.to_i < WARNING_NUM_ROWS
|
45
|
+
warning("Creating fixtures with more than #{WARNING_NUM_ROWS} rows is not recommended. Large local fixtures may cause slowness when using Mortar.")
|
46
|
+
display
|
47
|
+
end
|
48
|
+
validate_arguments!
|
49
|
+
validate_git_based_project!
|
50
|
+
|
51
|
+
fixture_id = nil
|
52
|
+
action("Requesting fixture creation") do
|
53
|
+
fixture_id = api.post_fixture_limit(project.name, fixture_name, input_url, num_rows).body['fixture_id']
|
54
|
+
end
|
55
|
+
|
56
|
+
poll_for_fixture_results(fixture_id)
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def does_fixture_exist(fixture_name)
|
64
|
+
fixture_path = File.join(project.fixtures_path, fixture_name)
|
65
|
+
File.exists?(fixture_path)
|
66
|
+
end
|
67
|
+
|
68
|
+
def poll_for_fixture_results(fixture_id)
|
69
|
+
fixture_result = nil
|
70
|
+
display
|
71
|
+
ticking(polling_interval) do |ticks|
|
72
|
+
fixture_result = api.get_fixture(fixture_id).body
|
73
|
+
is_finished =
|
74
|
+
Mortar::API::Fixtures::STATUSES_COMPLETE.include?(fixture_result["status_code"])
|
75
|
+
|
76
|
+
redisplay("Status: %s %s" % [
|
77
|
+
fixture_result['status_description'] + (is_finished ? "" : "..."),
|
78
|
+
is_finished ? "" : spinner(ticks)],
|
79
|
+
is_finished) # only display newline on last message
|
80
|
+
if is_finished
|
81
|
+
display
|
82
|
+
break
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
case fixture_result['status_code']
|
87
|
+
when Mortar::API::Fixtures::STATUS_FAILED
|
88
|
+
error_message = "Fixture generation failed with #{fixture_result['error_type'] || 'error'}"
|
89
|
+
error_context = get_error_message_context(fixture_result['error_message'] || "")
|
90
|
+
error_message += ":\n\n#{fixture_result['error_message']}\n\n#{error_context}"
|
91
|
+
error(error_message)
|
92
|
+
when Mortar::API::Fixtures::STATUS_CREATED
|
93
|
+
fixture_result['sample_s3_urls'].each do |u|
|
94
|
+
download_to_file(u['url'], "fixtures/#{fixture_result['name']}/#{u['name']}")
|
95
|
+
display
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
@@ -27,6 +27,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
27
27
|
#
|
28
28
|
# Illustrate the effects and output of a pigscript.
|
29
29
|
#
|
30
|
+
# -s, --skippruning # Don't try to reduce the illustrate results to the smallest size possible.
|
30
31
|
# -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
|
31
32
|
# -f, --param-file PARAMFILE # Load pig parameter values from a file.
|
32
33
|
# --no_browser # Don't open the illustrate results automatically in the browser.
|
@@ -38,6 +39,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
38
39
|
def index
|
39
40
|
pigscript_name = shift_argument
|
40
41
|
alias_name = shift_argument
|
42
|
+
skip_pruning = options[:skippruning] ||= false
|
41
43
|
unless pigscript_name && alias_name
|
42
44
|
error("Usage: mortar illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
|
43
45
|
end
|
@@ -48,7 +50,7 @@ class Mortar::Command::Illustrate < Mortar::Command::Base
|
|
48
50
|
|
49
51
|
illustrate_id = nil
|
50
52
|
action("Starting illustrate") do
|
51
|
-
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, git_ref, :parameters => pig_parameters).body["illustrate_id"]
|
53
|
+
illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, skip_pruning, git_ref, :parameters => pig_parameters).body["illustrate_id"]
|
52
54
|
end
|
53
55
|
|
54
56
|
illustrate_result = nil
|
data/lib/mortar/helpers.rb
CHANGED
@@ -19,6 +19,7 @@
|
|
19
19
|
|
20
20
|
require 'fileutils'
|
21
21
|
require "vendor/mortar/okjson"
|
22
|
+
require "curb"
|
22
23
|
|
23
24
|
module Mortar
|
24
25
|
module Helpers
|
@@ -44,6 +45,13 @@ module Mortar
|
|
44
45
|
File.open(path, "w"){|f| f.write(str_data)}
|
45
46
|
end
|
46
47
|
|
48
|
+
def download_to_file(url, path, mkdir_p=true)
|
49
|
+
c = Curl::Easy.new(url)
|
50
|
+
c.on_progress {|dl_total, dl_now, ul_total, ul_now| redisplay("Downloading #{path}: #{dl_now.to_i} of #{dl_total.to_i}"); true }
|
51
|
+
c.perform
|
52
|
+
write_to_file(c.body_str, path, mkdir_p)
|
53
|
+
end
|
54
|
+
|
47
55
|
def display(msg="", new_line=true)
|
48
56
|
if new_line
|
49
57
|
puts(msg)
|
@@ -53,6 +61,16 @@ module Mortar
|
|
53
61
|
end
|
54
62
|
end
|
55
63
|
|
64
|
+
def warning(msg="", new_line=true)
|
65
|
+
message = "WARNING: #{msg}"
|
66
|
+
if new_line
|
67
|
+
display(message)
|
68
|
+
else
|
69
|
+
print(msg)
|
70
|
+
$stdout.flush
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
56
74
|
def redisplay(line, line_break = false)
|
57
75
|
display("\r\e[0K#{line}", line_break)
|
58
76
|
end
|
@@ -340,7 +358,7 @@ module Mortar
|
|
340
358
|
Mortar::Helpers.error_with_failure = false
|
341
359
|
end
|
342
360
|
$stderr.puts(" ! #{message}.")
|
343
|
-
$stderr.puts(" ! Search for help at:
|
361
|
+
$stderr.puts(" ! Search for help at: http://help.mortardata.com")
|
344
362
|
$stderr.puts(" ! Or report a bug at: https://github.com/mortardata/mortar/issues/new")
|
345
363
|
$stderr.puts
|
346
364
|
$stderr.puts(" Error: #{error.message} (#{error.class})")
|
data/lib/mortar/project.rb
CHANGED
@@ -4,4 +4,4 @@ Mortar is a platform-as-a-service for Hadoop. With Mortar, you can run jobs on
|
|
4
4
|
|
5
5
|
# Getting Started
|
6
6
|
|
7
|
-
For help getting started with Mortar, check out the [Mortar Help](http://
|
7
|
+
For help getting started with Mortar, check out the [Mortar Help](http://help.mortardata.com/) site.
|
File without changes
|
data/lib/mortar/version.rb
CHANGED
@@ -0,0 +1,136 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2012 Mortar Data Inc.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'spec_helper'
|
18
|
+
require 'fakefs/spec_helpers'
|
19
|
+
require 'mortar/command/fixtures'
|
20
|
+
require 'mortar/api/fixtures'
|
21
|
+
|
22
|
+
module Mortar::Command
|
23
|
+
describe Fixtures do
|
24
|
+
before(:each) do
|
25
|
+
stub_core
|
26
|
+
@git = Mortar::Git::Git.new
|
27
|
+
end
|
28
|
+
|
29
|
+
context("index") do
|
30
|
+
|
31
|
+
it "errors when missing command" do
|
32
|
+
with_git_initialized_project do |p|
|
33
|
+
stderr, stdout = execute("fixtures:head s3n://tbmmsd/*.tsv.* 5", p, @git)
|
34
|
+
stderr.should == <<-STDERR
|
35
|
+
! Usage: mortar fixtures:head INPUT_URL NUM_ROWS FIXTURE_NAME
|
36
|
+
! Must specifiy INPUT_URL, NUM_ROWS, and FIXTURE_NAME.
|
37
|
+
STDERR
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
it "requests and reports on a successful fixtures:head" do
|
42
|
+
with_git_initialized_project do |p|
|
43
|
+
fixture_id = "12345abcde"
|
44
|
+
name = "My_pet_fixture"
|
45
|
+
url = "s3://my_pet_fixture"
|
46
|
+
num_rows = "60"
|
47
|
+
|
48
|
+
sample_s3_urls = [ {'url' => "url1",
|
49
|
+
'name' => "url1_name"}]
|
50
|
+
|
51
|
+
mock(Mortar::Auth.api).post_fixture_limit(p.name, name, url, num_rows) {Excon::Response.new(:body => {"fixture_id" => fixture_id})}
|
52
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_PENDING, "status_description" => "Pending"})).ordered
|
53
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_CREATING, "status_description" => "Creating"})).ordered
|
54
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_SAVING, "status_description" => "Uploading"})).ordered
|
55
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_CREATED, "status_description" => "Success", "name" => name, "sample_s3_urls" => sample_s3_urls})).ordered
|
56
|
+
|
57
|
+
any_instance_of(Mortar::Command::Fixtures) do |base|
|
58
|
+
mock(base).download_to_file(sample_s3_urls[0]['url'], "fixtures/#{name}/#{sample_s3_urls[0]['name']}")
|
59
|
+
end
|
60
|
+
|
61
|
+
stderr, stdout = execute("fixtures:head #{url} #{num_rows} #{name} --polling_interval 0.05", p, @git)
|
62
|
+
|
63
|
+
stdout.should == <<-STDOUT
|
64
|
+
WARNING: Creating fixtures with more than 50 rows is not recommended. Large local fixtures may cause slowness when using Mortar.
|
65
|
+
|
66
|
+
Requesting fixture creation... done
|
67
|
+
|
68
|
+
\r\e[0KStatus: Pending... /\r\e[0KStatus: Creating... -\r\e[0KStatus: Uploading... \\\r\e[0KStatus: Success \n\n
|
69
|
+
STDOUT
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
it "requests and reports on a failed fixtures:head" do
|
75
|
+
with_git_initialized_project do |p|
|
76
|
+
fixture_id = "12345abcde"
|
77
|
+
name = "My_pet_fixture"
|
78
|
+
url = "s3://my_pet_fixture"
|
79
|
+
num_rows = "60"
|
80
|
+
|
81
|
+
sample_s3_urls = [ {'url' => "url1",
|
82
|
+
'name' => "url1_name"}]
|
83
|
+
|
84
|
+
mock(Mortar::Auth.api).post_fixture_limit(p.name, name, url, num_rows) {Excon::Response.new(:body => {"fixture_id" => fixture_id})}
|
85
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_PENDING, "status_description" => "Pending"})).ordered
|
86
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_CREATING, "status_description" => "Creating"})).ordered
|
87
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_SAVING, "status_description" => "Uploading"})).ordered
|
88
|
+
mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_FAILED,
|
89
|
+
"status_description" => "Failed",
|
90
|
+
"name" => name,
|
91
|
+
"error_message" => "This is an error message.",
|
92
|
+
"error_type" => "UserError" })).ordered
|
93
|
+
|
94
|
+
stderr, stdout = execute("fixtures:head #{url} #{num_rows} #{name} --polling_interval 0.05", p, @git)
|
95
|
+
|
96
|
+
stdout.should == <<-STDOUT
|
97
|
+
WARNING: Creating fixtures with more than 50 rows is not recommended. Large local fixtures may cause slowness when using Mortar.
|
98
|
+
|
99
|
+
Requesting fixture creation... done
|
100
|
+
|
101
|
+
\r\e[0KStatus: Pending... /\r\e[0KStatus: Creating... -\r\e[0KStatus: Uploading... \\\r\e[0KStatus: Failed \n
|
102
|
+
STDOUT
|
103
|
+
|
104
|
+
stderr.should == <<-STDERR
|
105
|
+
! Fixture generation failed with UserError:
|
106
|
+
!
|
107
|
+
! This is an error message.
|
108
|
+
STDERR
|
109
|
+
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
it "tries to create a fixture in an existing directory" do
|
115
|
+
with_git_initialized_project do |p|
|
116
|
+
fixture_id = "12345abcde"
|
117
|
+
name = "My_pet_fixture"
|
118
|
+
url = "s3://my_pet_fixture"
|
119
|
+
num_rows = "60"
|
120
|
+
|
121
|
+
fixtures_dir = File.join(Dir.pwd, "fixtures", name)
|
122
|
+
FileUtils.mkdir_p(fixtures_dir)
|
123
|
+
|
124
|
+
stderr, stdout = execute("fixtures:head #{url} #{num_rows} #{name} --polling_interval 0.05", p, @git)
|
125
|
+
|
126
|
+
stderr.should == <<-STDERR
|
127
|
+
! Fixture #{name} already exists.
|
128
|
+
STDERR
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
|
133
|
+
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -32,11 +32,13 @@ describe Mortar::Command::Generate do
|
|
32
32
|
stderr, stdout = execute("generate:project Test")
|
33
33
|
File.exists?("Test").should be_true
|
34
34
|
File.exists?("Test/macros").should be_true
|
35
|
+
File.exists?("Test/fixtures").should be_true
|
35
36
|
File.exists?("Test/pigscripts").should be_true
|
36
37
|
File.exists?("Test/udfs").should be_true
|
37
38
|
File.exists?("Test/README.md").should be_true
|
38
39
|
File.exists?("Test/Gemfile").should be_false
|
39
40
|
File.exists?("Test/macros/.gitkeep").should be_true
|
41
|
+
File.exists?("Test/fixtures/.gitkeep").should be_true
|
40
42
|
File.exists?("Test/pigscripts/Test.pig").should be_true
|
41
43
|
File.exists?("Test/udfs/python/Test.py").should be_true
|
42
44
|
|
@@ -82,11 +84,13 @@ STDERR
|
|
82
84
|
stderr, stdout = execute("new Test")
|
83
85
|
File.exists?("Test").should be_true
|
84
86
|
File.exists?("Test/macros").should be_true
|
87
|
+
File.exists?("Test/fixtures").should be_true
|
85
88
|
File.exists?("Test/pigscripts").should be_true
|
86
89
|
File.exists?("Test/udfs").should be_true
|
87
90
|
File.exists?("Test/README.md").should be_true
|
88
91
|
File.exists?("Test/Gemfile").should be_false
|
89
92
|
File.exists?("Test/macros/.gitkeep").should be_true
|
93
|
+
File.exists?("Test/fixtures/.gitkeep").should be_true
|
90
94
|
File.exists?("Test/pigscripts/Test.pig").should be_true
|
91
95
|
File.exists?("Test/udfs/python/Test.py").should be_true
|
92
96
|
|
@@ -71,7 +71,7 @@ STDERR
|
|
71
71
|
parameters = ["name"=>"key", "value"=>"value" ]
|
72
72
|
|
73
73
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
74
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", is_a(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
74
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
75
75
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
76
76
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
77
77
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_PROGRESS, "status_description" => "In progress"})).ordered
|
@@ -105,7 +105,7 @@ STDOUT
|
|
105
105
|
parameters = ["name"=>"key", "value"=>"value" ]
|
106
106
|
|
107
107
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
108
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", is_a(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
108
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
109
109
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
110
110
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
111
111
|
|
@@ -118,6 +118,33 @@ Starting illustrate... done
|
|
118
118
|
|
119
119
|
\r\e[0KStatus: Pending... /\r\e[0KStatus: Succeeded
|
120
120
|
|
121
|
+
Results available at https://api.mortardata.com/illustrates/c571a8c7f76a4fd4a67c103d753e2dd5
|
122
|
+
STDOUT
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
|
127
|
+
it "requests and reports on a successful illustrate that skips pruning" do
|
128
|
+
with_git_initialized_project do |p|
|
129
|
+
# stub api requests
|
130
|
+
illustrate_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
131
|
+
illustrate_url = "https://api.mortardata.com/illustrates/#{illustrate_id}"
|
132
|
+
parameters = ["name"=>"key", "value"=>"value" ]
|
133
|
+
|
134
|
+
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
135
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", true, is_a(String), :parameters => parameters) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
136
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
137
|
+
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_SUCCESS, "status_description" => "Succeeded", "web_result_url" => illustrate_url})).ordered
|
138
|
+
|
139
|
+
write_file(File.join(p.pigscripts_path, "my_script.pig"))
|
140
|
+
stderr, stdout = execute("illustrate my_script my_alias --polling_interval 0.05 -p key=value -s --no_browser", p, @git)
|
141
|
+
stdout.should == <<-STDOUT
|
142
|
+
Taking code snapshot... done
|
143
|
+
Sending code snapshot to Mortar... done
|
144
|
+
Starting illustrate... done
|
145
|
+
|
146
|
+
\r\e[0KStatus: Pending... /\r\e[0KStatus: Succeeded
|
147
|
+
|
121
148
|
Results available at https://api.mortardata.com/illustrates/c571a8c7f76a4fd4a67c103d753e2dd5
|
122
149
|
STDOUT
|
123
150
|
end
|
@@ -134,7 +161,7 @@ STDOUT
|
|
134
161
|
error_type = 'PigError'
|
135
162
|
|
136
163
|
# These don't test the validity of the error message, it only tests that the CLI can handle a message returned from the server
|
137
|
-
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", is_a(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
164
|
+
mock(Mortar::Auth.api).post_illustrate("myproject", "my_script", "my_alias", false, is_a(String), :parameters => []) {Excon::Response.new(:body => {"illustrate_id" => illustrate_id})}
|
138
165
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
139
166
|
mock(Mortar::Auth.api).get_illustrate(illustrate_id, :exclude_result => true).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Illustrate::STATUS_FAILURE,
|
140
167
|
"error_message" => error_message,
|
@@ -419,7 +419,7 @@ STDOUT
|
|
419
419
|
"parameters" => parameters,
|
420
420
|
"outputs" => outputs
|
421
421
|
}))
|
422
|
-
stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 -p", p, @git)
|
422
|
+
stderr, stdout = execute("jobs:status c571a8c7f76a4fd4a67c103d753e2dd5 -p --polling_interval 0.05", p, @git)
|
423
423
|
stdout.should == <<-STDOUT
|
424
424
|
\r[/] Status: [=> ] 0% Complete (0.00 / 4.00 MapReduce jobs finished)\r\e[0K=== myproject: my_script (job_id: c571a8c7f76a4fd4a67c103d753e2dd5)
|
425
425
|
cluster_id: e2790e7e8c7d48e39157238d58191346
|
data/spec/mortar/git_spec.rb
CHANGED
metadata
CHANGED
@@ -1,165 +1,183 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: mortar
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.4
|
5
5
|
prerelease:
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 3
|
9
|
-
- 3
|
10
|
-
version: 0.3.3
|
11
6
|
platform: ruby
|
12
|
-
authors:
|
7
|
+
authors:
|
13
8
|
- Mortar Data
|
14
9
|
autorequire:
|
15
10
|
bindir: bin
|
16
11
|
cert_chain: []
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
- !ruby/object:Gem::Dependency
|
12
|
+
date: 2012-10-25 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
21
15
|
name: mortar-api-ruby
|
22
|
-
|
23
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
24
17
|
none: false
|
25
|
-
requirements:
|
18
|
+
requirements:
|
26
19
|
- - ~>
|
27
|
-
- !ruby/object:Gem::Version
|
28
|
-
|
29
|
-
segments:
|
30
|
-
- 0
|
31
|
-
- 3
|
32
|
-
- 0
|
33
|
-
version: 0.3.0
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.3.1
|
34
22
|
type: :runtime
|
35
|
-
version_requirements: *id001
|
36
|
-
- !ruby/object:Gem::Dependency
|
37
|
-
name: netrc
|
38
23
|
prerelease: false
|
39
|
-
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.3.1
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: netrc
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
40
33
|
none: false
|
41
|
-
requirements:
|
34
|
+
requirements:
|
42
35
|
- - ~>
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
-
hash: 9
|
45
|
-
segments:
|
46
|
-
- 0
|
47
|
-
- 7
|
48
|
-
- 5
|
36
|
+
- !ruby/object:Gem::Version
|
49
37
|
version: 0.7.5
|
50
38
|
type: :runtime
|
51
|
-
|
52
|
-
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 0.7.5
|
46
|
+
- !ruby/object:Gem::Dependency
|
53
47
|
name: launchy
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 2.1.1
|
54
|
+
type: :runtime
|
54
55
|
prerelease: false
|
55
|
-
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
56
57
|
none: false
|
57
|
-
requirements:
|
58
|
+
requirements:
|
58
59
|
- - ~>
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
hash: 9
|
61
|
-
segments:
|
62
|
-
- 2
|
63
|
-
- 1
|
64
|
-
- 1
|
60
|
+
- !ruby/object:Gem::Version
|
65
61
|
version: 2.1.1
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: curb
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.8.1
|
66
70
|
type: :runtime
|
67
|
-
version_requirements: *id003
|
68
|
-
- !ruby/object:Gem::Dependency
|
69
|
-
name: excon
|
70
71
|
prerelease: false
|
71
|
-
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.8.1
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: excon
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
72
81
|
none: false
|
73
|
-
requirements:
|
82
|
+
requirements:
|
74
83
|
- - ~>
|
75
|
-
- !ruby/object:Gem::Version
|
76
|
-
hash: 43
|
77
|
-
segments:
|
78
|
-
- 0
|
79
|
-
- 15
|
80
|
-
- 4
|
84
|
+
- !ruby/object:Gem::Version
|
81
85
|
version: 0.15.4
|
82
86
|
type: :development
|
83
|
-
version_requirements: *id004
|
84
|
-
- !ruby/object:Gem::Dependency
|
85
|
-
name: fakefs
|
86
87
|
prerelease: false
|
87
|
-
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ~>
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 0.15.4
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: fakefs
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
88
97
|
none: false
|
89
|
-
requirements:
|
90
|
-
- -
|
91
|
-
- !ruby/object:Gem::Version
|
92
|
-
|
93
|
-
segments:
|
94
|
-
- 0
|
95
|
-
version: "0"
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
96
102
|
type: :development
|
97
|
-
version_requirements: *id005
|
98
|
-
- !ruby/object:Gem::Dependency
|
99
|
-
name: gem-release
|
100
103
|
prerelease: false
|
101
|
-
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: gem-release
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
102
113
|
none: false
|
103
|
-
requirements:
|
104
|
-
- -
|
105
|
-
- !ruby/object:Gem::Version
|
106
|
-
|
107
|
-
segments:
|
108
|
-
- 0
|
109
|
-
version: "0"
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
110
118
|
type: :development
|
111
|
-
version_requirements: *id006
|
112
|
-
- !ruby/object:Gem::Dependency
|
113
|
-
name: rake
|
114
119
|
prerelease: false
|
115
|
-
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
116
121
|
none: false
|
117
|
-
requirements:
|
118
|
-
- -
|
119
|
-
- !ruby/object:Gem::Version
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: rake
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
124
134
|
type: :development
|
125
|
-
version_requirements: *id007
|
126
|
-
- !ruby/object:Gem::Dependency
|
127
|
-
name: rr
|
128
135
|
prerelease: false
|
129
|
-
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
130
137
|
none: false
|
131
|
-
requirements:
|
132
|
-
- -
|
133
|
-
- !ruby/object:Gem::Version
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: rr
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
138
150
|
type: :development
|
139
|
-
version_requirements: *id008
|
140
|
-
- !ruby/object:Gem::Dependency
|
141
|
-
name: rspec
|
142
151
|
prerelease: false
|
143
|
-
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
144
153
|
none: false
|
145
|
-
requirements:
|
146
|
-
- -
|
147
|
-
- !ruby/object:Gem::Version
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
- !ruby/object:Gem::Dependency
|
159
|
+
name: rspec
|
160
|
+
requirement: !ruby/object:Gem::Requirement
|
161
|
+
none: false
|
162
|
+
requirements:
|
163
|
+
- - ! '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
152
166
|
type: :development
|
153
|
-
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
none: false
|
170
|
+
requirements:
|
171
|
+
- - ! '>='
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
154
174
|
description: Client library and command-line tool to interact with the Mortar service.
|
155
175
|
email: support@mortardata.com
|
156
|
-
executables:
|
176
|
+
executables:
|
157
177
|
- mortar
|
158
178
|
extensions: []
|
159
|
-
|
160
179
|
extra_rdoc_files: []
|
161
|
-
|
162
|
-
files:
|
180
|
+
files:
|
163
181
|
- README.md
|
164
182
|
- bin/mortar
|
165
183
|
- lib/mortar.rb
|
@@ -170,6 +188,7 @@ files:
|
|
170
188
|
- lib/mortar/command/base.rb
|
171
189
|
- lib/mortar/command/clusters.rb
|
172
190
|
- lib/mortar/command/describe.rb
|
191
|
+
- lib/mortar/command/fixtures.rb
|
173
192
|
- lib/mortar/command/generate.rb
|
174
193
|
- lib/mortar/command/help.rb
|
175
194
|
- lib/mortar/command/illustrate.rb
|
@@ -192,6 +211,7 @@ files:
|
|
192
211
|
- lib/mortar/templates/pigscript/pigscript.pig
|
193
212
|
- lib/mortar/templates/pigscript/python_udf.py
|
194
213
|
- lib/mortar/templates/project/README.md
|
214
|
+
- lib/mortar/templates/project/fixtures/gitkeep
|
195
215
|
- lib/mortar/templates/project/gitignore
|
196
216
|
- lib/mortar/templates/project/macros/gitkeep
|
197
217
|
- lib/mortar/templates/project/pigscripts/pigscript.pig
|
@@ -205,6 +225,7 @@ files:
|
|
205
225
|
- spec/mortar/command/base_spec.rb
|
206
226
|
- spec/mortar/command/clusters_spec.rb
|
207
227
|
- spec/mortar/command/describe_spec.rb
|
228
|
+
- spec/mortar/command/fixtures_spec.rb
|
208
229
|
- spec/mortar/command/generate_spec.rb
|
209
230
|
- spec/mortar/command/illustrate_spec.rb
|
210
231
|
- spec/mortar/command/jobs_spec.rb
|
@@ -221,38 +242,26 @@ files:
|
|
221
242
|
- spec/support/display_message_matcher.rb
|
222
243
|
homepage: http://mortardata.com/
|
223
244
|
licenses: []
|
224
|
-
|
225
245
|
post_install_message:
|
226
246
|
rdoc_options: []
|
227
|
-
|
228
|
-
require_paths:
|
247
|
+
require_paths:
|
229
248
|
- lib
|
230
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
249
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
231
250
|
none: false
|
232
|
-
requirements:
|
233
|
-
- -
|
234
|
-
- !ruby/object:Gem::Version
|
235
|
-
hash: 57
|
236
|
-
segments:
|
237
|
-
- 1
|
238
|
-
- 8
|
239
|
-
- 7
|
251
|
+
requirements:
|
252
|
+
- - ! '>='
|
253
|
+
- !ruby/object:Gem::Version
|
240
254
|
version: 1.8.7
|
241
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
255
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
242
256
|
none: false
|
243
|
-
requirements:
|
244
|
-
- -
|
245
|
-
- !ruby/object:Gem::Version
|
246
|
-
|
247
|
-
segments:
|
248
|
-
- 0
|
249
|
-
version: "0"
|
257
|
+
requirements:
|
258
|
+
- - ! '>='
|
259
|
+
- !ruby/object:Gem::Version
|
260
|
+
version: '0'
|
250
261
|
requirements: []
|
251
|
-
|
252
262
|
rubyforge_project:
|
253
263
|
rubygems_version: 1.8.24
|
254
264
|
signing_key:
|
255
265
|
specification_version: 3
|
256
266
|
summary: Client library and CLI to interact with the Mortar service.
|
257
267
|
test_files: []
|
258
|
-
|