right_link 5.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. data/actors/agent_manager.rb +88 -0
  2. data/actors/instance_scheduler.rb +321 -0
  3. data/actors/instance_services.rb +64 -0
  4. data/actors/instance_setup.rb +567 -0
  5. data/bin/cloud +25 -0
  6. data/bin/cook_runner +44 -0
  7. data/bin/deploy +120 -0
  8. data/bin/enroll +385 -0
  9. data/bin/rad +32 -0
  10. data/bin/rchk +29 -0
  11. data/bin/rnac +39 -0
  12. data/bin/rs_connect +33 -0
  13. data/bin/rs_log_level +31 -0
  14. data/bin/rs_ohai +28 -0
  15. data/bin/rs_reenroll +31 -0
  16. data/bin/rs_run_recipe +34 -0
  17. data/bin/rs_run_right_script +34 -0
  18. data/bin/rs_shutdown +33 -0
  19. data/bin/rs_tag +33 -0
  20. data/bin/rs_thunk +33 -0
  21. data/bin/rstat +31 -0
  22. data/bin/system +16 -0
  23. data/ext/Rakefile +18 -0
  24. data/init/config.yml +5 -0
  25. data/init/init.rb +79 -0
  26. data/lib/chef/ohai_setup.rb +51 -0
  27. data/lib/chef/plugins/cloud.rb +91 -0
  28. data/lib/chef/plugins/cloudstack.rb +23 -0
  29. data/lib/chef/plugins/ec2.rb +23 -0
  30. data/lib/chef/plugins/linux/block_device2.rb +24 -0
  31. data/lib/chef/plugins/rackspace.rb +23 -0
  32. data/lib/chef/plugins/rightscale.rb +125 -0
  33. data/lib/chef/plugins/windows/network.rb +114 -0
  34. data/lib/chef/plugins.rb +74 -0
  35. data/lib/chef/providers/dns_dnsmadeeasy_provider.rb +81 -0
  36. data/lib/chef/providers/dns_resource.rb +100 -0
  37. data/lib/chef/providers/executable_schedule_provider.rb +70 -0
  38. data/lib/chef/providers/executable_schedule_resource.rb +144 -0
  39. data/lib/chef/providers/remote_recipe_provider.rb +86 -0
  40. data/lib/chef/providers/remote_recipe_resource.rb +101 -0
  41. data/lib/chef/providers/right_link_tag_provider.rb +73 -0
  42. data/lib/chef/providers/right_link_tag_resource.rb +59 -0
  43. data/lib/chef/providers/right_script_provider.rb +190 -0
  44. data/lib/chef/providers/right_script_resource.rb +113 -0
  45. data/lib/chef/providers/rs_shutdown_provider.rb +75 -0
  46. data/lib/chef/providers/rs_shutdown_resource.rb +55 -0
  47. data/lib/chef/providers/server_collection_provider.rb +66 -0
  48. data/lib/chef/providers/server_collection_resource.rb +93 -0
  49. data/lib/chef/providers/windows/powershell_provider.rb +151 -0
  50. data/lib/chef/providers/windows/powershell_resource.rb +111 -0
  51. data/lib/chef/providers/windows/unsupported_provider.rb +51 -0
  52. data/lib/chef/right_providers.rb +55 -0
  53. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/ChefNodeCmdlet.csproj +104 -0
  54. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/ChefNodeCmdlet.dll-Help.xml +141 -0
  55. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/Exceptions.cs +182 -0
  56. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetChefNodeCommand.cs +58 -0
  57. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetChefNodeRequest.cs +46 -0
  58. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetChefNodeResponse.cs +45 -0
  59. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetCurrentResourceCommand.cs +58 -0
  60. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetCurrentResourceRequest.cs +46 -0
  61. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetCurrentResourceResponse.cs +45 -0
  62. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNewResourceCommand.cs +58 -0
  63. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNewResourceRequest.cs +46 -0
  64. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNewResourceResponse.cs +45 -0
  65. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNextActionCommand.cs +178 -0
  66. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNextActionRequest.cs +67 -0
  67. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNextActionResponse.cs +58 -0
  68. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNodeValueCommandBase.cs +142 -0
  69. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNodeValueRequestBase.cs +64 -0
  70. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/GetNodeValueResponseBase.cs +69 -0
  71. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/JsonTransport.cs +110 -0
  72. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/PipeClient.cs +158 -0
  73. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/PipeServer.cs +142 -0
  74. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/Properties/AssemblyInfo.cs +16 -0
  75. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/ProtocolConstants.cs +55 -0
  76. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/ProtocolUtilities.cs +77 -0
  77. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/ReadMe.txt +53 -0
  78. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetChefNodeCommand.cs +59 -0
  79. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetChefNodeRequest.cs +46 -0
  80. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetChefNodeResponse.cs +58 -0
  81. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetCurrentResourceCommand.cs +59 -0
  82. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetCurrentResourceRequest.cs +46 -0
  83. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetCurrentResourceResponse.cs +40 -0
  84. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetNewResourceCommand.cs +59 -0
  85. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetNewResourceRequest.cs +46 -0
  86. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetNewResourceResponse.cs +40 -0
  87. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetNodeValueCommandBase.cs +293 -0
  88. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetNodeValueRequestBase.cs +75 -0
  89. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/SetNodeValueResponseBase.cs +45 -0
  90. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet/Transport.cs +91 -0
  91. data/lib/chef/windows/ChefNodeCmdlet/ChefNodeCmdlet.sln +35 -0
  92. data/lib/chef/windows/ChefNodeCmdlet/TestChefNodeCmdlet/Program.cs +374 -0
  93. data/lib/chef/windows/ChefNodeCmdlet/TestChefNodeCmdlet/Properties/AssemblyInfo.cs +16 -0
  94. data/lib/chef/windows/ChefNodeCmdlet/TestChefNodeCmdlet/TestChefNodeCmdlet.csproj +65 -0
  95. data/lib/chef/windows/ChefNodeCmdlet/TestNextActionCmdlet/Program.cs +136 -0
  96. data/lib/chef/windows/ChefNodeCmdlet/TestNextActionCmdlet/Properties/AssemblyInfo.cs +36 -0
  97. data/lib/chef/windows/ChefNodeCmdlet/TestNextActionCmdlet/ReadMe.txt +46 -0
  98. data/lib/chef/windows/ChefNodeCmdlet/TestNextActionCmdlet/TestNextActionCmdlet.csproj +68 -0
  99. data/lib/chef/windows/bin/Newtonsoft.Json.dll +0 -0
  100. data/lib/chef/windows/chef_node_server.rb +463 -0
  101. data/lib/chef/windows/dynamic_powershell_provider.rb +296 -0
  102. data/lib/chef/windows/pipe_server.rb +283 -0
  103. data/lib/chef/windows/powershell_host.rb +285 -0
  104. data/lib/chef/windows/powershell_pipe_server.rb +136 -0
  105. data/lib/chef/windows/powershell_provider_base.rb +92 -0
  106. data/lib/chef/windows/scripts/run_loop.ps1 +105 -0
  107. data/lib/clouds/cloud.rb +557 -0
  108. data/lib/clouds/cloud_factory.rb +250 -0
  109. data/lib/clouds/cloud_utilities.rb +244 -0
  110. data/lib/clouds/clouds/azure.rb +106 -0
  111. data/lib/clouds/clouds/cloudstack.rb +114 -0
  112. data/lib/clouds/clouds/ec2.rb +113 -0
  113. data/lib/clouds/clouds/eucalyptus.rb +46 -0
  114. data/lib/clouds/clouds/google.rb +102 -0
  115. data/lib/clouds/clouds/none.rb +76 -0
  116. data/lib/clouds/clouds/openstack.rb +30 -0
  117. data/lib/clouds/clouds/rackspace-ng.rb +54 -0
  118. data/lib/clouds/clouds/rackspace.rb +78 -0
  119. data/lib/clouds/clouds/softlayer.rb +91 -0
  120. data/lib/clouds/metadata_formatter.rb +108 -0
  121. data/lib/clouds/metadata_provider.rb +128 -0
  122. data/lib/clouds/metadata_source.rb +87 -0
  123. data/lib/clouds/metadata_sources/certificate_metadata_source.rb +207 -0
  124. data/lib/clouds/metadata_sources/config_drive_metadata_source.rb +129 -0
  125. data/lib/clouds/metadata_sources/file_metadata_source.rb +74 -0
  126. data/lib/clouds/metadata_sources/http_metadata_source.rb +277 -0
  127. data/lib/clouds/metadata_sources/selective_metadata_source.rb +122 -0
  128. data/lib/clouds/metadata_tree_climber.rb +144 -0
  129. data/lib/clouds/metadata_writer.rb +155 -0
  130. data/lib/clouds/metadata_writers/dictionary_metadata_writer.rb +72 -0
  131. data/lib/clouds/metadata_writers/ruby_metadata_writer.rb +76 -0
  132. data/lib/clouds/metadata_writers/shell_metadata_writer.rb +121 -0
  133. data/lib/clouds/register_clouds.rb +34 -0
  134. data/lib/clouds.rb +32 -0
  135. data/lib/gem_dependencies.rb +83 -0
  136. data/lib/git_hooks/commit-msg.rb +7 -0
  137. data/lib/instance/agent_config.rb +168 -0
  138. data/lib/instance/agent_watcher.rb +233 -0
  139. data/lib/instance/audit_cook_stub.rb +104 -0
  140. data/lib/instance/audit_proxy.rb +247 -0
  141. data/lib/instance/bundle_queue.rb +104 -0
  142. data/lib/instance/cook/agent_connection.rb +109 -0
  143. data/lib/instance/cook/audit_logger.rb +165 -0
  144. data/lib/instance/cook/audit_stub.rb +142 -0
  145. data/lib/instance/cook/ca-bundle.crt +2794 -0
  146. data/lib/instance/cook/chef_state.rb +211 -0
  147. data/lib/instance/cook/cook.rb +306 -0
  148. data/lib/instance/cook/cook_state.rb +298 -0
  149. data/lib/instance/cook/cookbook_path_mapping.rb +66 -0
  150. data/lib/instance/cook/cookbook_repo_retriever.rb +190 -0
  151. data/lib/instance/cook/executable_sequence.rb +765 -0
  152. data/lib/instance/cook/external_parameter_gatherer.rb +190 -0
  153. data/lib/instance/cook/repose_downloader.rb +349 -0
  154. data/lib/instance/cook/shutdown_request_proxy.rb +121 -0
  155. data/lib/instance/cook.rb +41 -0
  156. data/lib/instance/downloader.rb +208 -0
  157. data/lib/instance/duplicable.rb +67 -0
  158. data/lib/instance/exceptions.rb +49 -0
  159. data/lib/instance/executable_sequence_proxy.rb +278 -0
  160. data/lib/instance/instance_commands.rb +577 -0
  161. data/lib/instance/instance_state.rb +633 -0
  162. data/lib/instance/json_utilities.rb +102 -0
  163. data/lib/instance/login_manager.rb +533 -0
  164. data/lib/instance/login_user_manager.rb +522 -0
  165. data/lib/instance/message_encoder.rb +118 -0
  166. data/lib/instance/multi_thread_bundle_queue.rb +232 -0
  167. data/lib/instance/operation_context.rb +60 -0
  168. data/lib/instance/options_bag.rb +65 -0
  169. data/lib/instance/payload_formatter.rb +46 -0
  170. data/lib/instance/policy.rb +53 -0
  171. data/lib/instance/policy_audit.rb +100 -0
  172. data/lib/instance/policy_manager.rb +146 -0
  173. data/lib/instance/reenroll_manager.rb +104 -0
  174. data/lib/instance/right_scripts_cookbook.rb +181 -0
  175. data/lib/instance/shutdown_request.rb +221 -0
  176. data/lib/instance/single_thread_bundle_queue.rb +189 -0
  177. data/lib/instance/volume_management.rb +450 -0
  178. data/lib/instance.rb +50 -0
  179. data/lib/repo_conf_generators/apt_conf_generators.rb +106 -0
  180. data/lib/repo_conf_generators/gem_conf_generators.rb +80 -0
  181. data/lib/repo_conf_generators/rightscale_conf_generators.rb +254 -0
  182. data/lib/repo_conf_generators/rightscale_key.pub +17 -0
  183. data/lib/repo_conf_generators/yum_conf_generators.rb +225 -0
  184. data/lib/repo_conf_generators.rb +30 -0
  185. data/lib/run_shell.rb +28 -0
  186. data/scripts/agent_checker.rb +571 -0
  187. data/scripts/agent_controller.rb +247 -0
  188. data/scripts/agent_deployer.rb +148 -0
  189. data/scripts/bundle_runner.rb +336 -0
  190. data/scripts/cloud_controller.rb +176 -0
  191. data/scripts/log_level_manager.rb +142 -0
  192. data/scripts/ohai_runner.rb +33 -0
  193. data/scripts/reenroller.rb +193 -0
  194. data/scripts/server_importer.rb +293 -0
  195. data/scripts/shutdown_client.rb +183 -0
  196. data/scripts/system_configurator.rb +367 -0
  197. data/scripts/tagger.rb +381 -0
  198. data/scripts/thunker.rb +356 -0
  199. metadata +418 -0
@@ -0,0 +1,190 @@
1
+ #
2
+ # Copyright (c) 2011 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+
23
+ require 'singleton'
24
+
25
+ module RightScale
26
+
27
+ # Provides access to RightLink agent audit methods
28
+ class ExternalParameterGatherer
29
+ include EM::Deferrable
30
+
31
+ # Failure title and message if any
32
+ attr_reader :failure_title, :failure_message
33
+
34
+ # Initialize parameter gatherer
35
+ #
36
+ # === Parameters
37
+ # bundle<RightScale::ExecutableBundle>:: the bundle for which to gather inputs
38
+ # options[:listen_port]:: Command server listen port
39
+ # options[:cookie]:: Command protocol cookie
40
+ #
41
+ # === Return
42
+ # true:: Always return true
43
+ def initialize(bundle, options)
44
+ @serializer = Serializer.new
45
+ @audit = AuditStub.instance
46
+ @cookie = options[:cookie]
47
+ @listen_port = options[:listen_port]
48
+ @thread_name = options[:thread_name]
49
+ @executables_inputs = {}
50
+
51
+ bundle.executables.each do |exe|
52
+ externals = exe.external_inputs
53
+ next if externals.nil? || externals.empty?
54
+ @executables_inputs[exe] = externals.dup
55
+ end
56
+ end
57
+
58
+ #TODO docs
59
+ def run
60
+ if done?
61
+ #we might not have ANY external parameters!
62
+ report_success
63
+ return true
64
+ end
65
+
66
+ @audit.create_new_section('Retrieving credentials')
67
+
68
+ #Preflight to check validity of cred objects
69
+ ok = true
70
+ @executables_inputs.each_pair do |exe, inputs|
71
+ inputs.each_pair do |name, location|
72
+ next if location.is_a?(RightScale::SecureDocumentLocation)
73
+ msg = "The provided credential (#{location.class.name}) is incompatible with this version of RightLink"
74
+ report_failure('Cannot process external input', msg)
75
+ ok = false
76
+ end
77
+ end
78
+
79
+ return false unless ok
80
+
81
+ @executables_inputs.each_pair do |exe, inputs|
82
+ inputs.each_pair do |name, location|
83
+ payload = {
84
+ :ticket => location.ticket,
85
+ :namespace => location.namespace,
86
+ :names => [location.name]
87
+ }
88
+ options = {
89
+ :targets => location.targets
90
+ }
91
+ self.send_idempotent_request('/vault/read_documents', payload, options) do |data|
92
+ handle_response(exe, name, location, data)
93
+ end
94
+ end
95
+ end
96
+ rescue Exception => e
97
+ report_failure('Credential gathering failed', "The following execption occured while gathering credentials", e)
98
+ end
99
+
100
+ protected
101
+
102
+ # Handle a RightNet response to our idempotent request. Could be success, failure or unexpected.
103
+ def handle_response(exe, name, location, response)
104
+ result = @serializer.load(response)
105
+
106
+ if result.success?
107
+ #Since we only ask for one credential at a time, we can do this...
108
+ secure_document = result.content.first
109
+ if secure_document.envelope_mime_type.nil?
110
+ @executables_inputs[exe][name] = secure_document
111
+ @audit.append_info("Got #{name} of '#{exe.nickname}'; #{count_remaining} remain.")
112
+ if done?
113
+ @audit.append_info("All credential values have been retrieved and processed.")
114
+ report_success
115
+ end
116
+ else
117
+ # The call succeeded but we can't process the credential value
118
+ msg = "The #{name} input of '#{exe.nickname}' was retrieved from the external source, but its type " +
119
+ "(#{secure_document.envelope_mime_type}) is incompatible with this version of RightLink."
120
+ report_failure('Cannot process credential', msg)
121
+ end
122
+ else # We got a result, but it was a failure...
123
+ msg = "Could not retrieve the value of the #{name} input of '#{exe.nickname}' " +
124
+ "from the external source. Reason for failure: #{result.content}."
125
+ report_failure('Failed to retrieve credential', msg)
126
+ end
127
+ rescue Exception => e
128
+ msg = "An unexpected error occurred while retrieving the value of the #{name} input of '#{exe.nickname}.'"
129
+ report_failure('Unexpected error while retrieving credentials', msg, e)
130
+ end
131
+
132
+ # Return the number of credentials remaining to be gathered
133
+ def count_remaining
134
+ count = @executables_inputs.values.map { |a| a.values.count { |p| not p.is_a?(RightScale::SecureDocument) } }
135
+ return count.inject { |sum,x| sum + x } || 0
136
+ end
137
+
138
+ # Sugar for count_remaining == 0
139
+ def done?
140
+ count_remaining == 0
141
+ end
142
+
143
+ # Do the actual substitution of credential values into the bundle
144
+ def substitute_parameters
145
+ @executables_inputs.each_pair do |exe, inputs|
146
+ inputs.each_pair do |name, value|
147
+ case exe
148
+ when RightScale::RecipeInstantiation
149
+ exe.attributes[name] = value.content
150
+ when RightScale::RightScriptInstantiation
151
+ exe.parameters[name] = value.content
152
+ end
153
+ end
154
+ end
155
+ end
156
+
157
+ # Report the completion of a successful run by updating our Deferrable disposition.
158
+ def report_success
159
+ substitute_parameters
160
+ EM.next_tick { succeed }
161
+ end
162
+
163
+ # Report a failure by setting some attributes that our caller will query, then updating our Deferrable
164
+ # disposition so our caller gets notified via errback.
165
+ def report_failure(title, message, exception=nil)
166
+ if exception
167
+ Log.error("ExternalParameterGatherer failed due to " +
168
+ "#{exception.class.name}: #{exception.message} (#{exception.backtrace.first})")
169
+ end
170
+
171
+ @failure_title = title
172
+ @failure_message = message
173
+ EM.next_tick { fail }
174
+ end
175
+
176
+ # Use the command protocol to send an idempotent request. This class cannot reuse Cook's
177
+ # implementation of the command-proto request wrappers because we must gather credentials
178
+ # concurrently for performance reasons. The easiest way to do this is simply to open a
179
+ # new command proto socket for every distinct request we make.
180
+ def send_idempotent_request(operation, payload, options={}, &callback)
181
+ connection = EM.connect('127.0.0.1', @listen_port, AgentConnection, @cookie, @thread_name, callback)
182
+ EM.next_tick do
183
+ connection.send_command(:name => :send_idempotent_request, :type => operation,
184
+ :payload => payload, :options => options)
185
+ end
186
+ end
187
+
188
+ end
189
+
190
+ end
@@ -0,0 +1,349 @@
1
+ #
2
+ # Copyright (c) 2009-2013 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+
23
+ require 'right_support/net/http_client'
24
+ require 'uri'
25
+
26
+ module RightScale
27
+
28
+ # Abstract download capabilities
29
+ class ReposeDownloader
30
+
31
+ # Environment variables to examine for proxy settings, in order.
32
+ PROXY_ENVIRONMENT_VARIABLES = ['HTTPS_PROXY', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY']
33
+
34
+ # Class names of exceptions to be re-raised as a ConnectionException
35
+ CONNECTION_EXCEPTIONS = ['Errno::ECONNREFUSED', 'Errno::ETIMEDOUT', 'SocketError',
36
+ 'RestClient::InternalServerError', 'RestClient::RequestTimeout']
37
+
38
+ # max timeout 8 (2**3) minutes for each retry
39
+ RETRY_BACKOFF_MAX = 3
40
+
41
+ # retry 5 times maximum
42
+ RETRY_MAX_ATTEMPTS = 5
43
+
44
+ class ConnectionException < Exception; end
45
+ class DownloadException < Exception; end
46
+
47
+ include RightSupport::Log::Mixin
48
+
49
+ # (Integer) Size in bytes of last successful download (nil if none)
50
+ attr_reader :size
51
+
52
+ # (Integer) Speed in bytes/seconds of last successful download (nil if none)
53
+ attr_reader :speed
54
+
55
+ # (String) Last resource downloaded
56
+ attr_reader :sanitized_resource
57
+
58
+ # Hash of IP Address => Hostname
59
+ attr_reader :ips
60
+
61
+ # Initializes a Downloader with a list of hostnames
62
+ #
63
+ # The purpose of this method is to instantiate a Downloader.
64
+ # It will perform DNS resolution on the hostnames provided
65
+ # and will configure a proxy if necessary
66
+ #
67
+ # === Parameters
68
+ # @param <[String]> Hostnames to resolve
69
+ #
70
+ # === Return
71
+ # @return [Downloader]
72
+ #
73
+ def initialize(hostnames)
74
+ raise ArgumentError, "At least one hostname must be provided" if hostnames.empty?
75
+ hostnames = [hostnames] unless hostnames.respond_to?(:each)
76
+ @ips = resolve(hostnames)
77
+ @hostnames = hostnames
78
+
79
+ proxy_var = PROXY_ENVIRONMENT_VARIABLES.detect { |v| ENV.has_key?(v) }
80
+ @proxy = ENV[proxy_var].match(/^[[:alpha:]]+:\/\//) ? URI.parse(ENV[proxy_var]) : URI.parse("http://" + ENV[proxy_var]) if proxy_var
81
+ end
82
+
83
+ # Downloads an attachment from Repose
84
+ #
85
+ # The purpose of this method is to download the specified attachment from Repose
86
+ # If a failure is encountered it will provide proper feedback regarding the nature
87
+ # of the failure
88
+ #
89
+ # === Parameters
90
+ # @param [String] Resource URI to parse and fetch
91
+ #
92
+ # === Block
93
+ # @yield [] A block is mandatory
94
+ # @yieldreturn [String] The stream that is being fetched
95
+ #
96
+ def download(resource)
97
+ client = get_http_client
98
+ @size = 0
99
+ @speed = 0
100
+ @sanitized_resource = sanitize_resource(resource)
101
+ resource = parse_resource(resource)
102
+ attempts = 0
103
+
104
+ begin
105
+ balancer.request do |endpoint|
106
+ RightSupport::Net::SSL.with_expected_hostname(ips[endpoint]) do
107
+ logger.info("Requesting '#{sanitized_resource}' from '#{endpoint}'")
108
+
109
+ attempts += 1
110
+ t0 = Time.now
111
+
112
+ # Previously we accessed RestClient directly and used it's wrapper method to instantiate
113
+ # a RestClient::Request object. This wrapper was not passing all options down the stack
114
+ # so now we invoke the RestClient::Request object directly, passing it our desired options
115
+ client.execute(:method => :get, :url => "https://#{endpoint}:443#{resource}", :timeout => calculate_timeout(attempts), :verify_ssl => OpenSSL::SSL::VERIFY_PEER, :ssl_ca_file => get_ca_file, :headers => {:user_agent => "RightLink v#{AgentConfig.protocol_version}"}) do |response, request, result|
116
+ if result.kind_of?(Net::HTTPSuccess)
117
+ @size = result.content_length
118
+ @speed = @size / (Time.now - t0)
119
+ yield response
120
+ else
121
+ response.return!(request, result)
122
+ end
123
+ end
124
+ end
125
+ end
126
+ rescue Exception => e
127
+ list = parse_exception_message(e)
128
+ message = list.join(", ")
129
+ logger.error("Request '#{sanitized_resource}' failed - #{message}")
130
+ raise ConnectionException, message unless (list & CONNECTION_EXCEPTIONS).empty?
131
+ raise DownloadException, message
132
+ end
133
+ end
134
+
135
+ # Message summarizing last successful download details
136
+ #
137
+ # === Return
138
+ # @return [String] Message with last downloaded resource, download size and speed
139
+ #
140
+ def details
141
+ "Downloaded '#{@sanitized_resource}' (#{ scale(size.to_i).join(' ') }) at #{ scale(speed.to_i).join(' ') }/s"
142
+ end
143
+
144
+ protected
145
+
146
+ # Resolve a list of hostnames to a hash of Hostname => IP Addresses
147
+ #
148
+ # The purpose of this method is to lookup all IP addresses per hostname and
149
+ # build a lookup hash that maps IP addresses back to their original hostname
150
+ # so we can perform TLS hostname verification.
151
+ #
152
+ # === Parameters
153
+ # @param <[String]> Hostnames to resolve
154
+ #
155
+ # === Return
156
+ # @return [Hash]
157
+ # * :key [<String>] a key (IP Address) that accepts a hostname string as it's value
158
+ #
159
+ def resolve(hostnames)
160
+ ips = {}
161
+ hostnames.each do |hostname|
162
+ infos = nil
163
+ attempts = RETRY_MAX_ATTEMPTS
164
+ begin
165
+ infos = Socket.getaddrinfo(hostname, 443, Socket::AF_INET, Socket::SOCK_STREAM, Socket::IPPROTO_TCP)
166
+ rescue Exception => e
167
+ if attempts > 0
168
+ attempts -= 1
169
+ retry
170
+ else
171
+ logger.error "Failed to resolve hostnames: #{e.class.name}: #{e.message}"
172
+ raise e
173
+ end
174
+ end
175
+
176
+ # Randomly permute the addrinfos of each hostname to help spread load.
177
+ infos.shuffle.each do |info|
178
+ ip = info[3]
179
+ ips[ip] = hostname
180
+ end
181
+ end
182
+ ips
183
+ end
184
+
185
+ # Parses a resource into a Repose-appropriate format
186
+ #
187
+ # The purpose of this method is to parse the resource given into the proper resource
188
+ # format that the ReposeDownloader class is expecting
189
+ #
190
+ # === Parameters
191
+ # @param [String] Resource URI to parse
192
+ #
193
+ # === Block
194
+ # @return [String] The parsed URI
195
+ #
196
+ def parse_resource(resource)
197
+ resource = URI::parse(resource)
198
+ raise ArgumentError, "Invalid resource provided. Resource must be a fully qualified URL" unless resource
199
+ "#{resource.path}?#{resource.query}"
200
+ end
201
+
202
+ # Parse Exception message and return it
203
+ #
204
+ # The purpose of this method is to parse the message portion of RequestBalancer
205
+ # Exceptions to determine the actual Exceptions that resulted in all endpoints
206
+ # failing to return a non-Exception.
207
+ #
208
+ # === Parameters
209
+ # @param [Exception] Exception to parse
210
+ #
211
+ # === Return
212
+ # @return [Array] List of exception class names
213
+
214
+ def parse_exception_message(e)
215
+ if e.kind_of?(RightSupport::Net::NoResult)
216
+ # Expected format of exception message: "... endpoints: ('<ip address>' => <exception class name array>, ...)""
217
+ i = 0
218
+ e.message.split(/\[|\]/).select {((i += 1) % 2) == 0 }.map { |s| s.split(/,\s*/) }.flatten
219
+ else
220
+ [e.class.name]
221
+ end
222
+ end
223
+
224
+ # Orders ips by hostnames
225
+ #
226
+ # The purpose of this method is to sort ips of hostnames so it tries all IPs of hostname 1,
227
+ # then all IPs of hostname 2, etc
228
+ #
229
+ # == Return
230
+ # @return [Array] array of ips ordered by hostnames
231
+ #
232
+ def hostnames_ips
233
+ @hostnames.map do |hostname|
234
+ # TODO change to Hash#select once we switch to 1.9
235
+ ips.reject { |ip, host| host != hostname }.keys
236
+ end.flatten
237
+ end
238
+
239
+ # Create and return a RequestBalancer instance
240
+ #
241
+ # The purpose of this method is to create a RequestBalancer that will be used
242
+ # to service all 'download' requests. Once a valid endpoint is found, the
243
+ # balancer will 'stick' with it. It will consider a response of '408: RequestTimeout' and
244
+ # '500: InternalServerError' as retryable exceptions and all other HTTP error codes to
245
+ # indicate a fatal exception that should abort the load-balanced request
246
+ #
247
+ # === Return
248
+ # @return [RightSupport::Net::RequestBalancer]
249
+ #
250
+ def balancer
251
+ @balancer ||= RightSupport::Net::RequestBalancer.new(
252
+ hostnames_ips,
253
+ :policy => RightSupport::Net::LB::Sticky,
254
+ :retry => RETRY_MAX_ATTEMPTS,
255
+ :fatal => lambda do |e|
256
+ if RightSupport::Net::RequestBalancer::DEFAULT_FATAL_EXCEPTIONS.any? { |c| e.is_a?(c) }
257
+ true
258
+ elsif e.respond_to?(:http_code) && (e.http_code != nil)
259
+ (e.http_code >= 400 && e.http_code < 500) && (e.http_code != 408 && e.http_code != 500 )
260
+ else
261
+ false
262
+ end
263
+ end
264
+ )
265
+ end
266
+
267
+ # Exponential incremental timeout algorithm. Returns the amount of
268
+ # of time to wait for the next iteration
269
+ #
270
+ # === Parameters
271
+ # @param [String] Number of attempts
272
+ #
273
+ # === Return
274
+ # @return [Integer] Timeout to use for next iteration
275
+ #
276
+ def calculate_timeout(attempts)
277
+ timeout_exponent = [attempts, RETRY_BACKOFF_MAX].min
278
+ (2 ** timeout_exponent) * 60
279
+ end
280
+
281
+ # Returns a path to a CA file
282
+ #
283
+ # The CA bundle is a basically static collection of trusted certs of top-level CAs.
284
+ # It should be provided by the OS, but because of our cross-platform nature and
285
+ # the lib we're using, we need to supply our own. We stole curl's.
286
+ #
287
+ # === Return
288
+ # @return [String] Path to a CA file
289
+ #
290
+ def get_ca_file
291
+ ca_file = File.normalize_path(File.join(File.dirname(__FILE__), 'ca-bundle.crt'))
292
+ end
293
+
294
+ # Instantiates an HTTP Client
295
+ #
296
+ # The purpose of this method is to create an HTTP Client that will be used to
297
+ # make requests in the download method
298
+ #
299
+ # === Return
300
+ # @return [RestClient]
301
+ #
302
+ def get_http_client
303
+ RestClient.proxy = @proxy.to_s if @proxy
304
+ RestClient
305
+ RestClient::Request
306
+ end
307
+
308
+ # Return a sanitized value from given argument
309
+ #
310
+ # The purpose of this method is to return a value that can be securely
311
+ # displayed in logs and audits
312
+ #
313
+ # === Parameters
314
+ # @param [String] 'Resource' to parse
315
+ #
316
+ # === Return
317
+ # @return [String] 'Resource' portion of resource provided
318
+ #
319
+ def sanitize_resource(resource)
320
+ URI::split(resource)[5].split("/").last
321
+ end
322
+
323
+ # Return scale and scaled value from given argument
324
+ #
325
+ # The purpose of this method is to convert bytes to a nicer format for display
326
+ # Scale can be B, KB, MB or GB
327
+ #
328
+ # === Parameters
329
+ # @param [Integer] Value in bytes
330
+ #
331
+ # === Return
332
+ # @return <[Integer], [String]> First element is scaled value, second element is scale
333
+ #
334
+ def scale(value)
335
+ case value
336
+ when 0..1023
337
+ [value, 'B']
338
+ when 1024..1024**2 - 1
339
+ [value / 1024, 'KB']
340
+ when 1024^2..1024**3 - 1
341
+ [value / 1024**2, 'MB']
342
+ else
343
+ [value / 1024**3, 'GB']
344
+ end
345
+ end
346
+
347
+ end
348
+
349
+ end
@@ -0,0 +1,121 @@
1
+ #
2
+ # Copyright (c) 2011 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+
23
+ module RightScale
24
+
25
+ # Proxy for a remote shutdown request state.
26
+ class ShutdownRequestProxy
27
+
28
+ include ShutdownRequestMixin
29
+
30
+ # exceptions.
31
+ class ShutdownQueryFailed < Exception; end
32
+
33
+ # Class initializer.
34
+ #
35
+ # === Return
36
+ # always true
37
+ def self.init(command_client)
38
+ @@command_client = command_client
39
+ true
40
+ end
41
+
42
+ # Factory method
43
+ #
44
+ # === Return
45
+ # (ShutdownRequestProxy):: the proxy instance for this class
46
+ def self.instance
47
+ # note that we never want the proxy to use a cached instance of the
48
+ # shutdown state as RightScripts and command-line actions can change the
49
+ # state without directly notifying the proxy.
50
+ result = send_shutdown_request(:name => :get_shutdown_request)
51
+ raise ShutdownQueryFailed.new("Unable to retrieve state of shutdown request from parent process.") unless result
52
+ return result
53
+ end
54
+
55
+ # Submits a new shutdown request state which may be superceded by a
56
+ # previous, higher-priority shutdown level.
57
+ #
58
+ # === Parameters
59
+ # request[:level](String):: shutdown level
60
+ # request[:immediately](Boolean):: shutdown immediacy or nil
61
+ #
62
+ # === Returns
63
+ # result(ShutdownRequestProxy):: the updated shutdown request state
64
+ def self.submit(request)
65
+ level = request[:kind] || request[:level]
66
+ immediately = !!request[:immediately]
67
+ result = send_shutdown_request(:name => :set_shutdown_request, :level => level, :immediately => immediately)
68
+ raise ShutdownQueryFailed.new("Unable to set state of shutdown request on parent process.") unless result
69
+ return result
70
+ end
71
+
72
+ protected
73
+
74
+ # Sends a get/set shutdown request using the command client.
75
+ #
76
+ # === Parameters
77
+ # payload(Hash):: parameters to send
78
+ #
79
+ # === Return
80
+ # result(ShutdownRequestProxy):: current shutdown request state or nil
81
+ def self.send_shutdown_request(payload)
82
+ # check initialization.
83
+ raise NotInitialized.new("ShutdownRequestProxy.init has not been called") unless defined?(@@command_client)
84
+
85
+ # use a queue to block and wait for response.
86
+ result_queue = Queue.new
87
+ @@command_client.send_command(payload) { |response| enqueue_result(result_queue, response) }
88
+ result = result_queue.shift
89
+ raise ShutdownQueryFailed.new("Unable to retrieve state of shutdown request from parent process.") unless result
90
+ return result
91
+ end
92
+
93
+ # Pushes a valid shutdown request on the response queue or else nil for
94
+ # synchronization purposes.
95
+ #
96
+ # === Parameters
97
+ # result_queue(Queue):: queue for push
98
+ # response(Hash):: response payload
99
+ #
100
+ # === Return
101
+ # always true
102
+ def self.enqueue_result(result_queue, response)
103
+ result = nil
104
+ begin
105
+ if response[:error]
106
+ Log.error("Failed exchanging shutdown state: #{response[:error]}")
107
+ else
108
+ result = ShutdownRequestProxy.new
109
+ result.level = response[:level]
110
+ result.immediately! if response[:immediately]
111
+ end
112
+ rescue Exception => e
113
+ Log.error("Failed exchanging shutdown state", e, :trace)
114
+ end
115
+ result_queue << result
116
+ true
117
+ end
118
+
119
+ end # ShutdownRequestProxy
120
+
121
+ end # RightScale
@@ -0,0 +1,41 @@
1
+ #
2
+ # Copyright (c) 2009-2011 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+
23
+ # Load files required by then runner process
24
+ # This process is responsible for running Chef
25
+ # It's a short lived process that runs one Chef converge then dies
26
+ # It talks back to the RightLink agent using the command protocol
27
+
28
+ COOK_BASE_DIR = File.join(File.dirname(__FILE__), 'cook')
29
+
30
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'cook.rb'))
31
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'agent_connection.rb'))
32
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'audit_stub.rb'))
33
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'audit_logger.rb'))
34
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'cook_state.rb'))
35
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'chef_state.rb'))
36
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'external_parameter_gatherer'))
37
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'cookbook_path_mapping.rb'))
38
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'cookbook_repo_retriever.rb'))
39
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'executable_sequence.rb'))
40
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'repose_downloader.rb'))
41
+ require File.normalize_path(File.join(COOK_BASE_DIR, 'shutdown_request_proxy.rb'))