mortar 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/README.md +36 -0
  2. data/bin/mortar +13 -0
  3. data/lib/mortar.rb +23 -0
  4. data/lib/mortar/auth.rb +312 -0
  5. data/lib/mortar/cli.rb +54 -0
  6. data/lib/mortar/command.rb +267 -0
  7. data/lib/mortar/command/auth.rb +96 -0
  8. data/lib/mortar/command/base.rb +319 -0
  9. data/lib/mortar/command/clusters.rb +41 -0
  10. data/lib/mortar/command/describe.rb +97 -0
  11. data/lib/mortar/command/generate.rb +121 -0
  12. data/lib/mortar/command/help.rb +166 -0
  13. data/lib/mortar/command/illustrate.rb +97 -0
  14. data/lib/mortar/command/jobs.rb +174 -0
  15. data/lib/mortar/command/pigscripts.rb +45 -0
  16. data/lib/mortar/command/projects.rb +128 -0
  17. data/lib/mortar/command/validate.rb +94 -0
  18. data/lib/mortar/command/version.rb +42 -0
  19. data/lib/mortar/errors.rb +24 -0
  20. data/lib/mortar/generators/generator_base.rb +107 -0
  21. data/lib/mortar/generators/macro_generator.rb +37 -0
  22. data/lib/mortar/generators/pigscript_generator.rb +40 -0
  23. data/lib/mortar/generators/project_generator.rb +67 -0
  24. data/lib/mortar/generators/udf_generator.rb +28 -0
  25. data/lib/mortar/git.rb +233 -0
  26. data/lib/mortar/helpers.rb +488 -0
  27. data/lib/mortar/project.rb +156 -0
  28. data/lib/mortar/snapshot.rb +39 -0
  29. data/lib/mortar/templates/macro/macro.pig +14 -0
  30. data/lib/mortar/templates/pigscript/pigscript.pig +38 -0
  31. data/lib/mortar/templates/pigscript/python_udf.py +13 -0
  32. data/lib/mortar/templates/project/Gemfile +3 -0
  33. data/lib/mortar/templates/project/README.md +8 -0
  34. data/lib/mortar/templates/project/gitignore +4 -0
  35. data/lib/mortar/templates/project/macros/gitkeep +0 -0
  36. data/lib/mortar/templates/project/pigscripts/pigscript.pig +35 -0
  37. data/lib/mortar/templates/project/udfs/python/python_udf.py +13 -0
  38. data/lib/mortar/templates/udf/python_udf.py +13 -0
  39. data/lib/mortar/version.rb +20 -0
  40. data/lib/vendor/mortar/okjson.rb +598 -0
  41. data/lib/vendor/mortar/uuid.rb +312 -0
  42. data/spec/mortar/auth_spec.rb +156 -0
  43. data/spec/mortar/command/auth_spec.rb +46 -0
  44. data/spec/mortar/command/base_spec.rb +82 -0
  45. data/spec/mortar/command/clusters_spec.rb +61 -0
  46. data/spec/mortar/command/describe_spec.rb +135 -0
  47. data/spec/mortar/command/generate_spec.rb +139 -0
  48. data/spec/mortar/command/illustrate_spec.rb +140 -0
  49. data/spec/mortar/command/jobs_spec.rb +364 -0
  50. data/spec/mortar/command/pigscripts_spec.rb +70 -0
  51. data/spec/mortar/command/projects_spec.rb +165 -0
  52. data/spec/mortar/command/validate_spec.rb +119 -0
  53. data/spec/mortar/command_spec.rb +122 -0
  54. data/spec/mortar/git_spec.rb +278 -0
  55. data/spec/mortar/helpers_spec.rb +82 -0
  56. data/spec/mortar/project_spec.rb +76 -0
  57. data/spec/mortar/snapshot_spec.rb +46 -0
  58. data/spec/spec.opts +1 -0
  59. data/spec/spec_helper.rb +278 -0
  60. data/spec/support/display_message_matcher.rb +68 -0
  61. metadata +259 -0
@@ -0,0 +1,97 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "mortar/command/base"
18
+ require "mortar/snapshot"
19
+
20
+ # manage pig scripts
21
+ #
22
+ class Mortar::Command::Describe < Mortar::Command::Base
23
+
24
+ include Mortar::Snapshot
25
+
26
+ # describe [PIGSCRIPT] [ALIAS]
27
+ #
28
+ # Describe the schema of an alias and all of its ancestors.
29
+ #
30
+ # -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
31
+ # -f, --param-file PARAMFILE # Load pig parameter values from a file.
32
+ #
33
+ # Examples:
34
+ #
35
+ # $ mortar describe
36
+ #
37
+ # TBD
38
+ #
39
+ def index
40
+ pigscript_name = shift_argument
41
+ alias_name = shift_argument
42
+ unless pigscript_name && alias_name
43
+ error("Usage: mortar describe PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
44
+ end
45
+ validate_arguments!
46
+ validate_git_based_project!
47
+ pigscript = validate_pigscript!(pigscript_name)
48
+ git_ref = create_and_push_snapshot_branch(git, project)
49
+
50
+ describe_id = nil
51
+ action("Starting describe") do
52
+ describe_id = api.post_describe(project.name, pigscript.name, alias_name, git_ref, :parameters => pig_parameters).body["describe_id"]
53
+ end
54
+
55
+ describe_result = nil
56
+ display
57
+ ticking(polling_interval) do |ticks|
58
+ describe_result = api.get_describe(describe_id, :exclude_result => true).body
59
+ is_finished =
60
+ Mortar::API::Describe::STATUSES_COMPLETE.include?(describe_result["status_code"])
61
+
62
+ redisplay("Status: %s %s" % [
63
+ describe_result['status_description'] + (is_finished ? "" : "..."),
64
+ is_finished ? " " : spinner(ticks)],
65
+ is_finished) # only display newline on last message
66
+ if is_finished
67
+ display
68
+ break
69
+ end
70
+ end
71
+
72
+ case describe_result['status_code']
73
+ when Mortar::API::Describe::STATUS_FAILURE
74
+ error_message = "Describe failed with #{describe_result['error_type'] || 'error'}"
75
+ if line_number = describe_result["line_number"]
76
+ error_message += " at Line #{line_number}"
77
+ if column_number = describe_result["column_number"]
78
+ error_message += ", Column #{column_number}"
79
+ end
80
+ end
81
+ error_context = get_error_message_context(describe_result['error_message'])
82
+ error_message += ":\n\n#{describe_result['error_message']}\n\n#{error_context}"
83
+ error(error_message)
84
+ when Mortar::API::Describe::STATUS_KILLED
85
+ error("Describe killed by user.")
86
+ when Mortar::API::Describe::STATUS_SUCCESS
87
+ web_result_url = describe_result['web_result_url']
88
+ display("Results available at #{web_result_url}")
89
+ action("Opening web browser to show results") do
90
+ require "launchy"
91
+ Launchy.open(web_result_url).join
92
+ end
93
+ else
94
+ raise RuntimeError, "Unknown describe status: #{describe_result['status']} for describe_id: #{describe_id}"
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,121 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "mortar/generators/project_generator"
18
+ require "mortar/generators/udf_generator"
19
+ require "mortar/generators/pigscript_generator"
20
+ require "mortar/generators/macro_generator"
21
+ require "mortar/command/base"
22
+
23
+ # generate new projects and scaffolding
24
+ #
25
+ class Mortar::Command::Generate < Mortar::Command::Base
26
+
27
+ # generate:project
28
+ #
29
+ # generate new project
30
+ #
31
+ #
32
+ # Examples:
33
+ #
34
+ # $ mortar generate:project
35
+ #
36
+ # TBD
37
+ #
38
+ def _project
39
+ project_name = shift_argument
40
+ unless project_name
41
+ error("Usage: mortar new PROJECTNAME\nMust specify PROJECTNAME.")
42
+ end
43
+ pigscript_name = project_name
44
+ app_generator = Mortar::Generators::ProjectGenerator.new
45
+ app_generator.generate_project(project_name, options)
46
+ end
47
+ alias_command "new", "generate:_project"
48
+ alias_command "generate:project", "generate:_project"
49
+
50
+
51
+
52
+ # generate:python_udf
53
+ #
54
+ # generate new python user defined function
55
+ #
56
+ #
57
+ # Examples:
58
+ #
59
+ # $ mortar generate:python_udf UDFNAME
60
+ #
61
+ # TBD
62
+ #
63
+ def python_udf
64
+ udf_name = shift_argument
65
+ unless udf_name
66
+ error("Usage: mortar generate:python_udf UDFNAME\nMust specify UDFNAME.")
67
+ end
68
+ udf_generator = Mortar::Generators::UDFGenerator.new
69
+ udf_generator.generate_python_udf(udf_name, project, options)
70
+
71
+ end
72
+
73
+ # generate:pigscript
74
+ #
75
+ # generate new pig script
76
+ #
77
+ # --skip-udf # Create the pig script without a partnered python udf
78
+ #
79
+ # Examples:
80
+ #
81
+ # $ mortar generate:pigscript SCRIPTNAME
82
+ #
83
+ # TBD
84
+ #
85
+ def pigscript
86
+ script_name = shift_argument
87
+ unless script_name
88
+ error("Usage: mortar generate:pigscript SCRIPTNAME\nMust specify SCRIPTNAME.")
89
+ end
90
+ options[:skip_udf] ||= false
91
+
92
+ script_generator = Mortar::Generators::PigscriptGenerator.new
93
+ script_generator.generate_pigscript(script_name, project, options)
94
+
95
+ end
96
+
97
+ # generate:macro
98
+ #
99
+ # generate new macro
100
+ #
101
+ #
102
+ # Examples:
103
+ #
104
+ # $ mortar generate:macro MACRONAME
105
+ #
106
+ # TBD
107
+ #
108
+ def macro
109
+ macro_name = shift_argument
110
+ unless macro_name
111
+ error("Usage: mortar generate:macro MACRONAME\nMust specify MACRONAME.")
112
+ end
113
+
114
+ macro_generator = Mortar::Generators::MacroGenerator.new
115
+ macro_generator.generate_macro(macro_name, project, options)
116
+
117
+ end
118
+
119
+
120
+
121
+ end
@@ -0,0 +1,166 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+ # Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2012,
17
+ # used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
18
+ #
19
+
20
+ require "mortar/command/base"
21
+
22
+ # list commands and display help
23
+ #
24
+ class Mortar::Command::Help < Mortar::Command::Base
25
+
26
+ PRIMARY_NAMESPACES = []
27
+
28
+ # help [COMMAND]
29
+ #
30
+ # list available commands or display help for a specific command
31
+ #
32
+ #Examples:
33
+ #
34
+ # $ mortar help
35
+ # Usage: mortar COMMAND [command-specific-options]
36
+ #
37
+ # Primary help topics, type "mortar help TOPIC" for more details:
38
+ #
39
+ # ...
40
+ #
41
+ # Additional topics:
42
+ #
43
+ # ...
44
+ #
45
+ def index
46
+ if command = args.shift
47
+ help_for_command(command)
48
+ else
49
+ help_for_root
50
+ end
51
+ end
52
+
53
+ alias_command "-h", "help"
54
+ alias_command "--help", "help"
55
+
56
+ def self.usage_for_command(command)
57
+ command = new.send(:commands)[command]
58
+ "Usage: mortar #{command[:banner]}" if command
59
+ end
60
+
61
+ private
62
+
63
+ def commands_for_namespace(name)
64
+ Mortar::Command.commands.values.select do |command|
65
+ command[:namespace] == name && command[:command] != name
66
+ end
67
+ end
68
+
69
+ def namespaces
70
+ namespaces = Mortar::Command.namespaces
71
+ namespaces.delete("app")
72
+ namespaces
73
+ end
74
+
75
+ def commands
76
+ Mortar::Command.commands
77
+ end
78
+
79
+ def legacy_help_for_namespace(namespace)
80
+ instance = Mortar::Command::Help.groups.map do |group|
81
+ [ group.title, group.select { |c| c.first =~ /^#{namespace}/ }.length ]
82
+ end.sort_by { |l| l.last }.last
83
+ return nil unless instance
84
+ return nil if instance.last.zero?
85
+ instance.first
86
+ end
87
+
88
+ def legacy_help_for_command(command)
89
+ Mortar::Command::Help.groups.each do |group|
90
+ group.each do |cmd, description|
91
+ return description if cmd.split(" ").first == command
92
+ end
93
+ end
94
+ nil
95
+ end
96
+
97
+ def primary_namespaces
98
+ PRIMARY_NAMESPACES.map { |name| namespaces[name] }.compact
99
+ end
100
+
101
+ def additional_namespaces
102
+ (namespaces.values - primary_namespaces)
103
+ end
104
+
105
+ def summary_for_namespaces(namespaces)
106
+ size = longest(namespaces.map { |n| n[:name] })
107
+ namespaces.sort_by {|namespace| namespace[:name]}.each do |namespace|
108
+ name = namespace[:name]
109
+ namespace[:description] ||= legacy_help_for_namespace(name)
110
+ puts " %-#{size}s # %s" % [ name, namespace[:description] ]
111
+ end
112
+ end
113
+
114
+ def help_for_root
115
+ puts "Usage: mortar COMMAND [command-specific-options]"
116
+ puts
117
+ puts "Primary help topics, type \"mortar help TOPIC\" for more details:"
118
+ puts
119
+ summary_for_namespaces(primary_namespaces)
120
+ puts
121
+ puts "Additional topics:"
122
+ puts
123
+ summary_for_namespaces(additional_namespaces)
124
+ puts
125
+ end
126
+
127
+ def help_for_namespace(name)
128
+ namespace_commands = commands_for_namespace(name)
129
+
130
+ unless namespace_commands.empty?
131
+ size = longest(namespace_commands.map { |c| c[:banner] })
132
+ namespace_commands.sort_by { |c| c[:banner].to_s }.each do |command|
133
+ next if command[:help] =~ /DEPRECATED/
134
+ command[:summary] ||= legacy_help_for_command(command[:command])
135
+ puts " %-#{size}s # %s" % [ command[:banner], command[:summary] ]
136
+ end
137
+ end
138
+ end
139
+
140
+ def help_for_command(name)
141
+ if command_alias = Mortar::Command.command_aliases[name]
142
+ display("Alias: #{name} redirects to #{command_alias}")
143
+ name = command_alias
144
+ end
145
+ if command = commands[name]
146
+ puts "Usage: mortar #{command[:banner]}"
147
+
148
+ if command[:help].strip.length > 0
149
+ puts command[:help].split("\n")[1..-1].join("\n")
150
+ else
151
+ puts
152
+ puts " " + legacy_help_for_command(name).to_s
153
+ end
154
+ puts
155
+ end
156
+
157
+ if commands_for_namespace(name).size > 0
158
+ puts "Additional commands, type \"mortar help COMMAND\" for more details:"
159
+ puts
160
+ help_for_namespace(name)
161
+ puts
162
+ elsif command.nil?
163
+ error "#{name} is not a mortar command. See `mortar help`."
164
+ end
165
+ end
166
+ end
@@ -0,0 +1,97 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "mortar/command/base"
18
+ require "mortar/snapshot"
19
+
20
+ # manage pig scripts
21
+ #
22
+ class Mortar::Command::Illustrate < Mortar::Command::Base
23
+
24
+ include Mortar::Snapshot
25
+
26
+ # illustrate [PIGSCRIPT] [ALIAS]
27
+ #
28
+ # Illustrate the effects and output of a pigscript.
29
+ #
30
+ # -p, --parameter NAME=VALUE # Set a pig parameter value in your script.
31
+ # -f, --param-file PARAMFILE # Load pig parameter values from a file.
32
+ #
33
+ # Examples:
34
+ #
35
+ # $ mortar illustrate
36
+ #
37
+ # TBD
38
+ #
39
+ def index
40
+ pigscript_name = shift_argument
41
+ alias_name = shift_argument
42
+ unless pigscript_name && alias_name
43
+ error("Usage: mortar illustrate PIGSCRIPT ALIAS\nMust specify PIGSCRIPT and ALIAS.")
44
+ end
45
+ validate_arguments!
46
+ validate_git_based_project!
47
+ pigscript = validate_pigscript!(pigscript_name)
48
+ git_ref = create_and_push_snapshot_branch(git, project)
49
+
50
+ illustrate_id = nil
51
+ action("Starting illustrate") do
52
+ illustrate_id = api.post_illustrate(project.name, pigscript.name, alias_name, git_ref, :parameters => pig_parameters).body["illustrate_id"]
53
+ end
54
+
55
+ illustrate_result = nil
56
+ display
57
+ ticking(polling_interval) do |ticks|
58
+ illustrate_result = api.get_illustrate(illustrate_id, :exclude_result => true).body
59
+ is_finished =
60
+ Mortar::API::Illustrate::STATUSES_COMPLETE.include?(illustrate_result["status_code"])
61
+
62
+ redisplay("Status: %s %s" % [
63
+ illustrate_result['status_description'] + (is_finished ? "" : "..."),
64
+ is_finished ? " " : spinner(ticks)],
65
+ is_finished) # only display newline on last message
66
+ if is_finished
67
+ display
68
+ break
69
+ end
70
+ end
71
+
72
+ case illustrate_result['status_code']
73
+ when Mortar::API::Illustrate::STATUS_FAILURE
74
+ error_message = "Illustrate failed with #{illustrate_result['error_type'] || 'error'}"
75
+ if line_number = illustrate_result["line_number"]
76
+ error_message += " at Line #{line_number}"
77
+ if column_number = illustrate_result["column_number"]
78
+ error_message += ", Column #{column_number}"
79
+ end
80
+ end
81
+ error_context = get_error_message_context(illustrate_result['error_message'])
82
+ error_message += ":\n\n#{illustrate_result['error_message']}\n\n#{error_context}"
83
+ error(error_message)
84
+ when Mortar::API::Illustrate::STATUS_KILLED
85
+ error("Illustrate killed by user.")
86
+ when Mortar::API::Illustrate::STATUS_SUCCESS
87
+ web_result_url = illustrate_result['web_result_url']
88
+ display("Results available at #{web_result_url}")
89
+ action("Opening web browser to show results") do
90
+ require "launchy"
91
+ Launchy.open(web_result_url).join
92
+ end
93
+ else
94
+ raise RuntimeError, "Unknown illustrate status: #{illustrate_result['status_code']} for illustrate_id: #{illustrate_id}"
95
+ end
96
+ end
97
+ end