clc-fork-chef-metal 0.11.beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +106 -0
  3. data/LICENSE +201 -0
  4. data/README.md +201 -0
  5. data/Rakefile +6 -0
  6. data/bin/metal +276 -0
  7. data/lib/chef/provider/machine.rb +147 -0
  8. data/lib/chef/provider/machine_batch.rb +130 -0
  9. data/lib/chef/provider/machine_execute.rb +30 -0
  10. data/lib/chef/provider/machine_file.rb +49 -0
  11. data/lib/chef/resource/machine.rb +95 -0
  12. data/lib/chef/resource/machine_batch.rb +20 -0
  13. data/lib/chef/resource/machine_execute.rb +22 -0
  14. data/lib/chef/resource/machine_file.rb +28 -0
  15. data/lib/chef_metal.rb +62 -0
  16. data/lib/chef_metal/action_handler.rb +63 -0
  17. data/lib/chef_metal/add_prefix_action_handler.rb +29 -0
  18. data/lib/chef_metal/chef_machine_spec.rb +64 -0
  19. data/lib/chef_metal/chef_provider_action_handler.rb +72 -0
  20. data/lib/chef_metal/chef_run_data.rb +80 -0
  21. data/lib/chef_metal/convergence_strategy.rb +26 -0
  22. data/lib/chef_metal/convergence_strategy/install_cached.rb +157 -0
  23. data/lib/chef_metal/convergence_strategy/install_msi.rb +56 -0
  24. data/lib/chef_metal/convergence_strategy/install_sh.rb +51 -0
  25. data/lib/chef_metal/convergence_strategy/no_converge.rb +38 -0
  26. data/lib/chef_metal/convergence_strategy/precreate_chef_objects.rb +180 -0
  27. data/lib/chef_metal/driver.rb +267 -0
  28. data/lib/chef_metal/machine.rb +110 -0
  29. data/lib/chef_metal/machine/basic_machine.rb +82 -0
  30. data/lib/chef_metal/machine/unix_machine.rb +276 -0
  31. data/lib/chef_metal/machine/windows_machine.rb +102 -0
  32. data/lib/chef_metal/machine_spec.rb +78 -0
  33. data/lib/chef_metal/recipe_dsl.rb +84 -0
  34. data/lib/chef_metal/transport.rb +87 -0
  35. data/lib/chef_metal/transport/ssh.rb +235 -0
  36. data/lib/chef_metal/transport/winrm.rb +109 -0
  37. data/lib/chef_metal/version.rb +3 -0
  38. metadata +223 -0
@@ -0,0 +1,102 @@
1
+ require 'chef_metal/machine/basic_machine'
2
+
3
+ module ChefMetal
4
+ class Machine
5
+ class WindowsMachine < BasicMachine
6
+ def initialize(machine_spec, transport, convergence_strategy)
7
+ super
8
+ end
9
+
10
+ # Options include:
11
+ #
12
+ # command_prefix - prefix to put in front of any command, e.g. sudo
13
+ attr_reader :options
14
+
15
+ # Delete file
16
+ def delete_file(action_handler, path)
17
+ if file_exists?(path)
18
+ action_handler.perform_action "delete file #{escape(path)} on #{machine_spec.name}" do
19
+ transport.execute("Remove-Item #{escape(path)}").error!
20
+ end
21
+ end
22
+ end
23
+
24
+ def is_directory?(path)
25
+ parse_boolean(transport.execute("Test-Path #{escape(path)} -pathtype container", :read_only => true).stdout)
26
+ end
27
+
28
+ # Return true or false depending on whether file exists
29
+ def file_exists?(path)
30
+ parse_boolean(transport.execute("Test-Path #{escape(path)}", :read_only => true).stdout)
31
+ end
32
+
33
+ def files_different?(path, local_path, content=nil)
34
+ if !file_exists?(path) || (local_path && !File.exists?(local_path))
35
+ return true
36
+ end
37
+
38
+ # Get remote checksum of file (from http://stackoverflow.com/a/13926809)
39
+ result = transport.execute(<<-EOM, :read_only => true)
40
+ $md5 = [System.Security.Cryptography.MD5]::Create("MD5")
41
+ $fd = [System.IO.File]::OpenRead(#{path.inspect})
42
+ $buf = new-object byte[] (1024*1024*8) # 8mb buffer
43
+ while (($read_len = $fd.Read($buf,0,$buf.length)) -eq $buf.length){
44
+ $total += $buf.length
45
+ $md5.TransformBlock($buf,$offset,$buf.length,$buf,$offset)
46
+ }
47
+ # finalize the last read
48
+ $md5.TransformFinalBlock($buf,0,$read_len)
49
+ $hash = $md5.Hash
50
+ # convert hash bytes to hex formatted string
51
+ $hash | foreach { $hash_txt += $_.ToString("x2") }
52
+ $hash_txt
53
+ EOM
54
+ result.error!
55
+ remote_sum = result.stdout.split(' ')[0]
56
+ digest = Digest::SHA256.new
57
+ if content
58
+ digest.update(content)
59
+ else
60
+ File.open(local_path, 'rb') do |io|
61
+ while (buf = io.read(4096)) && buf.length > 0
62
+ digest.update(buf)
63
+ end
64
+ end
65
+ end
66
+ remote_sum != digest.hexdigest
67
+ end
68
+
69
+ def create_dir(action_handler, path)
70
+ if !file_exists?(path)
71
+ action_handler.perform_action "create directory #{path} on #{machine_spec.name}" do
72
+ transport.execute("New-Item #{escape(path)} -Type directory")
73
+ end
74
+ end
75
+ end
76
+
77
+ # Set file attributes { :owner, :group, :rights }
78
+ # def set_attributes(action_handler, path, attributes)
79
+ # end
80
+
81
+ # Get file attributes { :owner, :group, :rights }
82
+ # def get_attributes(path)
83
+ # end
84
+
85
+ def dirname_on_machine(path)
86
+ path.split(/[\\\/]/)[0..-2].join('\\')
87
+ end
88
+
89
+ def escape(string)
90
+ transport.escape(string)
91
+ end
92
+
93
+ def parse_boolean(string)
94
+ if string =~ /^\s*true\s*$/mi
95
+ true
96
+ else
97
+ false
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,78 @@
1
+ module ChefMetal
2
+ #
3
+ # Specification for a machine. Sufficient information to find and contact it
4
+ # after it has been set up.
5
+ #
6
+ class MachineSpec
7
+ def initialize(node)
8
+ @node = node
9
+ end
10
+
11
+ attr_reader :node
12
+
13
+ #
14
+ # Globally unique identifier for this machine. Does not depend on the machine's
15
+ # location or existence.
16
+ #
17
+ def id
18
+ raise "id unimplemented"
19
+ end
20
+
21
+ #
22
+ # Name of the machine. Corresponds to the name in "machine 'name' do" ...
23
+ #
24
+ def name
25
+ node['name']
26
+ end
27
+
28
+ #
29
+ # Location of this machine. This should be a freeform hash, with enough
30
+ # information for the driver to look it up and create a Machine object to
31
+ # access it.
32
+ #
33
+ # This MUST include a 'driver_url' attribute with the driver's URL in it.
34
+ #
35
+ # chef-metal will do its darnedest to not lose this information.
36
+ #
37
+ def location
38
+ metal_attr('location')
39
+ end
40
+
41
+ #
42
+ # Set the location for this machine.
43
+ #
44
+ def location=(value)
45
+ set_metal_attr('location', value)
46
+ end
47
+
48
+ # URL to the driver. Convenience for location['driver_url']
49
+ def driver_url
50
+ location ? location['driver_url'] : nil
51
+ end
52
+
53
+ #
54
+ # Save this node to the server. If you have significant information that
55
+ # could be lost, you should do this as quickly as possible. Data will be
56
+ # saved automatically for you after allocate_machine and ready_machine.
57
+ #
58
+ def save(action_handler)
59
+ raise "save unimplemented"
60
+ end
61
+
62
+ protected
63
+
64
+ def metal_attr(attr)
65
+ if node['normal'] && node['normal']['metal']
66
+ node['normal']['metal'][attr]
67
+ else
68
+ nil
69
+ end
70
+ end
71
+
72
+ def set_metal_attr(attr, value)
73
+ node['normal'] ||= {}
74
+ node['normal']['metal'] ||= {}
75
+ node['normal']['metal'][attr] = value
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,84 @@
1
+ require 'chef_metal/chef_run_data'
2
+ require 'chef/resource_collection'
3
+
4
+ require 'chef/resource/machine'
5
+ require 'chef/provider/machine'
6
+ require 'chef/resource/machine_batch'
7
+ require 'chef/provider/machine_batch'
8
+ require 'chef/resource/machine_file'
9
+ require 'chef/provider/machine_file'
10
+ require 'chef/resource/machine_execute'
11
+ require 'chef/provider/machine_execute'
12
+
13
+ class Chef
14
+ module DSL
15
+ module Recipe
16
+ def with_driver(driver, &block)
17
+ if driver.is_a?(String)
18
+ run_context.chef_metal.with_driver(driver, &block)
19
+ elsif driver.is_a?(ChefMetal::Driver)
20
+ run_context.chef_metal.with_driver(run_context.chef_metal.driver_for_url(driver), &block)
21
+ else
22
+ raise "with_driver accepts either a driver URL string or a ChefMetal::Driver instance. You tried passing a #{driver.class}."
23
+ end
24
+ end
25
+
26
+ def with_machine_options(machine_options, &block)
27
+ run_context.chef_metal.with_machine_options(machine_options, &block)
28
+ end
29
+
30
+ def with_machine_batch(the_machine_batch, options = {}, &block)
31
+ if the_machine_batch.is_a?(String)
32
+ the_machine_batch = machine_batch the_machine_batch do
33
+ if options[:action]
34
+ action options[:action]
35
+ end
36
+ if options[:max_simultaneous]
37
+ max_simultaneous options[:max_simultaneous]
38
+ end
39
+ end
40
+ end
41
+ run_context.chef_metal.with_machine_batch(the_machine_batch, &block)
42
+ end
43
+
44
+ def current_machine_options
45
+ run_context.chef_metal.current_machine_options
46
+ end
47
+
48
+ def add_machine_options(options, &block)
49
+ run_context.chef_metal.add_machine_options(options, &block)
50
+ end
51
+
52
+ # When the machine resource is first declared, create a machine_batch (if there
53
+ # isn't one already)
54
+ def machine(name, &block)
55
+ if !run_context.chef_metal.current_machine_batch
56
+ run_context.chef_metal.with_machine_batch declare_resource(:machine_batch, 'default', caller[0])
57
+ end
58
+ declare_resource(:machine, name, caller[0], &block)
59
+ end
60
+ end
61
+ end
62
+
63
+ class Config
64
+ default(:driver) { ENV['CHEF_DRIVER'] }
65
+ # config_context :drivers do
66
+ # # each key is a driver_url, and each value can have driver, driver_options and machine_options
67
+ # config_strict_mode false
68
+ # end
69
+ # config_context :driver_options do
70
+ # # open ended for whatever the driver wants
71
+ # config_strict_mode false
72
+ # end
73
+ # config_context :machine_options do
74
+ # # open ended for whatever the driver wants
75
+ # config_strict_mode false
76
+ # end
77
+ end
78
+
79
+ class RunContext
80
+ def chef_metal
81
+ @chef_metal ||= ChefMetal::ChefRunData.new(config)
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,87 @@
1
+ require 'timeout'
2
+
3
+ module ChefMetal
4
+ class Transport
5
+ DEFAULT_TIMEOUT = 15*60
6
+
7
+ # Execute a program on the remote host.
8
+ #
9
+ # == Arguments
10
+ # command: command to run. May be a shell-escaped string or a pre-split array containing [PROGRAM, ARG1, ARG2, ...].
11
+ # options: hash of options, including but not limited to:
12
+ # :timeout => NUM_SECONDS - time to wait before program finishes (throws an exception otherwise). Set to nil or 0 to run with no timeout. Defaults to 15 minutes.
13
+ # :stream => BOOLEAN - true to stream stdout and stderr to the console.
14
+ # :stream => BLOCK - block to stream stdout and stderr to (block.call(stdout_chunk, stderr_chunk))
15
+ # :stream_stdout => FD - FD to stream stdout to (defaults to IO.stdout)
16
+ # :stream_stderr => FD - FD to stream stderr to (defaults to IO.stderr)
17
+ # :read_only => BOOLEAN - true if command is guaranteed not to change system state (useful for Docker)
18
+ def execute(command, options = {})
19
+ raise "execute not overridden on #{self.class}"
20
+ end
21
+
22
+ def read_file(path)
23
+ raise "read_file not overridden on #{self.class}"
24
+ end
25
+
26
+ def write_file(path, content)
27
+ raise "write_file not overridden on #{self.class}"
28
+ end
29
+
30
+ def download_file(path, local_path)
31
+ IO.write(local_path, read_file(path))
32
+ end
33
+
34
+ def upload_file(local_path, path)
35
+ write_file(path, IO.read(local_path))
36
+ end
37
+
38
+ def make_url_available_to_remote(local_url)
39
+ raise "make_url_available_to_remote not overridden on #{self.class}"
40
+ end
41
+
42
+ def disconnect
43
+ raise "disconnect not overridden on #{self.class}"
44
+ end
45
+
46
+ def available?
47
+ raise "available? not overridden on #{self.class}"
48
+ end
49
+
50
+ # Config hash, including :log_level and :logger as keys
51
+ def config
52
+ raise "config not overridden on #{self.class}"
53
+ end
54
+
55
+ protected
56
+
57
+ # Helper to implement stdout/stderr streaming in execute
58
+ def stream_chunk(options, stdout_chunk, stderr_chunk)
59
+ if options[:stream].is_a?(Proc)
60
+ options[:stream].call(stdout_chunk, stderr_chunk)
61
+ else
62
+ if stdout_chunk
63
+ if options[:stream_stdout]
64
+ options[:stream_stdout].print stdout_chunk
65
+ elsif options[:stream] || config[:log_level] == :debug
66
+ STDOUT.print stdout_chunk
67
+ end
68
+ end
69
+ if stderr_chunk
70
+ if options[:stream_stderr]
71
+ options[:stream_stderr].print stderr_chunk
72
+ elsif options[:stream] || config[:log_level] == :debug
73
+ STDERR.print stderr_chunk
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+ def with_execute_timeout(options, &block)
80
+ Timeout::timeout(execute_timeout(options), &block)
81
+ end
82
+
83
+ def execute_timeout(options)
84
+ options.has_key?(:timeout) ? options[:timeout] : DEFAULT_TIMEOUT
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,235 @@
1
+ require 'chef_metal/transport'
2
+ require 'uri'
3
+ require 'socket'
4
+ require 'timeout'
5
+ require 'net/ssh'
6
+ require 'net/scp'
7
+ require 'net/ssh/gateway'
8
+
9
+ module ChefMetal
10
+ class Transport
11
+ class SSH < ChefMetal::Transport
12
+ def initialize(host, username, ssh_options, options, global_config)
13
+ @host = host
14
+ @username = username
15
+ @ssh_options = ssh_options
16
+ @options = options
17
+ @config = global_config
18
+ end
19
+
20
+ attr_reader :host
21
+ attr_reader :username
22
+ attr_reader :ssh_options
23
+ attr_reader :options
24
+ attr_reader :config
25
+
26
+ def execute(command, execute_options = {})
27
+ Chef::Log.info("Executing #{options[:prefix]}#{command} on #{username}@#{host}")
28
+ stdout = ''
29
+ stderr = ''
30
+ exitstatus = nil
31
+ session # grab session outside timeout, it has its own timeout
32
+ with_execute_timeout(execute_options) do
33
+ channel = session.open_channel do |channel|
34
+ # Enable PTY unless otherwise specified, some instances require this
35
+ unless options[:ssh_pty_enable] == false
36
+ channel.request_pty do |chan, success|
37
+ raise "could not get pty" if !success && options[:ssh_pty_enable]
38
+ end
39
+ end
40
+
41
+ channel.exec("#{options[:prefix]}#{command}") do |ch, success|
42
+ raise "could not execute command: #{command.inspect}" unless success
43
+
44
+ channel.on_data do |ch2, data|
45
+ stdout << data
46
+ stream_chunk(execute_options, data, nil)
47
+ end
48
+
49
+ channel.on_extended_data do |ch2, type, data|
50
+ stderr << data
51
+ stream_chunk(execute_options, nil, data)
52
+ end
53
+
54
+ channel.on_request "exit-status" do |ch, data|
55
+ exitstatus = data.read_long
56
+ end
57
+ end
58
+ end
59
+
60
+ channel.wait
61
+ end
62
+
63
+ Chef::Log.info("Completed #{command} on #{username}@#{host}: exit status #{exitstatus}")
64
+ Chef::Log.debug("Stdout was:\n#{stdout}") if stdout != '' && !options[:stream] && !options[:stream_stdout] && config[:log_level] != :debug
65
+ Chef::Log.info("Stderr was:\n#{stderr}") if stderr != '' && !options[:stream] && !options[:stream_stderr] && config[:log_level] != :debug
66
+ SSHResult.new(command, execute_options, stdout, stderr, exitstatus)
67
+ end
68
+
69
+ def read_file(path)
70
+ Chef::Log.debug("Reading file #{path} from #{username}@#{host}")
71
+ result = StringIO.new
72
+ download(path, result)
73
+ result.string
74
+ end
75
+
76
+ def download_file(path, local_path)
77
+ Chef::Log.debug("Downloading file #{path} from #{username}@#{host} to local #{local_path}")
78
+ download(path, local_path)
79
+ end
80
+
81
+ def write_file(path, content)
82
+ execute("mkdir -p #{File.dirname(path)}").error!
83
+ if options[:prefix]
84
+ # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
85
+ remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
86
+ Chef::Log.debug("Writing #{content.length} bytes to #{remote_tempfile} on #{username}@#{host}")
87
+ Net::SCP.new(session).upload!(StringIO.new(content), remote_tempfile)
88
+ execute("mv #{remote_tempfile} #{path}").error!
89
+ else
90
+ Chef::Log.debug("Writing #{content.length} bytes to #{path} on #{username}@#{host}")
91
+ Net::SCP.new(session).upload!(StringIO.new(content), path)
92
+ end
93
+ end
94
+
95
+ def upload_file(local_path, path)
96
+ execute("mkdir -p #{File.dirname(path)}").error!
97
+ if options[:prefix]
98
+ # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
99
+ remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
100
+ Chef::Log.debug("Uploading #{local_path} to #{remote_tempfile} on #{username}@#{host}")
101
+ Net::SCP.new(session).upload!(local_path, remote_tempfile)
102
+ execute("mv #{remote_tempfile} #{path}").error!
103
+ else
104
+ Chef::Log.debug("Uploading #{local_path} to #{path} on #{username}@#{host}")
105
+ Net::SCP.new(session).upload!(local_path, path)
106
+ end
107
+ end
108
+
109
+ def make_url_available_to_remote(local_url)
110
+ uri = URI(local_url)
111
+ host = Socket.getaddrinfo(uri.host, uri.scheme, nil, :STREAM)[0][3]
112
+ if host == '127.0.0.1' || host == '[::1]'
113
+ unless session.forward.active_remotes.any? { |port, bind| port == uri.port && bind == '127.0.0.1' }
114
+ # TODO IPv6
115
+ Chef::Log.debug("Forwarding local server 127.0.0.1:#{uri.port} to port #{uri.port} on #{username}@#{host}")
116
+ session.forward.remote(uri.port, '127.0.0.1', uri.port)
117
+ end
118
+ end
119
+ local_url
120
+ end
121
+
122
+ def disconnect
123
+ if @session
124
+ begin
125
+ Chef::Log.debug("Closing SSH session on #{username}@#{host}")
126
+ @session.close
127
+ rescue
128
+ end
129
+ @session = nil
130
+ end
131
+ end
132
+
133
+ def available?
134
+ # If you can't pwd within 10 seconds, you can't pwd
135
+ execute('pwd', :timeout => 10)
136
+ true
137
+ rescue Timeout::Error, Errno::EHOSTUNREACH, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ECONNRESET, Net::SSH::Disconnect
138
+ Chef::Log.debug("#{username}@#{host} unavailable: network connection failed or broke: #{$!.inspect}")
139
+ false
140
+ rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch
141
+ Chef::Log.debug("#{username}@#{host} unavailable: SSH authentication error: #{$!.inspect} ")
142
+ false
143
+ end
144
+
145
+ protected
146
+
147
+ def gateway?
148
+ options.key?(:ssh_gateway) and ! options[:ssh_gateway].nil?
149
+ end
150
+
151
+ def gateway
152
+ @gateway ||= begin
153
+ gw_host, gw_user = options[:ssh_gateway].split('@').reverse
154
+ gw_host, gw_port = gw_host.split(':')
155
+ gw_user = ssh_options[:ssh_username] unless gw_user
156
+
157
+ ssh_start_opts = { timeout:10 }.merge(ssh_options)
158
+ ssh_start_opts[:port] = gw_port || 22
159
+
160
+ Chef::Log.debug("Opening SSH gateway to #{gw_user}@#{gw_host} with options #{ssh_start_opts.inspect}")
161
+ begin
162
+ Net::SSH::Gateway.new(gw_host, gw_user, ssh_start_opts)
163
+ rescue Errno::ETIMEDOUT
164
+ Chef::Log.debug("Timed out connecting to gateway: #{$!}")
165
+ raise InitialConnectTimeout.new($!)
166
+ end
167
+ end
168
+ end
169
+
170
+ def session
171
+ @session ||= begin
172
+ ssh_start_opts = { timeout:10 }.merge(ssh_options)
173
+ Chef::Log.debug("Opening SSH connection to #{username}@#{host} with options #{ssh_start_opts.inspect}")
174
+ # Small initial connection timeout (10s) to help us fail faster when server is just dead
175
+ begin
176
+ if gateway? then gateway.ssh(host, username, ssh_start_opts)
177
+ else Net::SSH.start(host, username, ssh_start_opts)
178
+ end
179
+ rescue Timeout::Error
180
+ Chef::Log.debug("Timed out connecting to SSH: #{$!}")
181
+ raise InitialConnectTimeout.new($!)
182
+ end
183
+ end
184
+ end
185
+
186
+ def download(path, local_path)
187
+ channel = Net::SCP.new(session).download(path, local_path)
188
+ begin
189
+ channel.wait
190
+ rescue Net::SCP::Error => e
191
+ # TODO we need a way to distinguish between "directory of file does not exist" and "SCP did not finish successfully"
192
+ nil
193
+ # ensure the channel is closed when a rescue happens above
194
+ ensure
195
+ channel.close
196
+ channel.wait
197
+ end
198
+ nil
199
+ end
200
+
201
+ class SSHResult
202
+ def initialize(command, options, stdout, stderr, exitstatus)
203
+ @command = command
204
+ @options = options
205
+ @stdout = stdout
206
+ @stderr = stderr
207
+ @exitstatus = exitstatus
208
+ end
209
+
210
+ attr_reader :command
211
+ attr_reader :options
212
+ attr_reader :stdout
213
+ attr_reader :stderr
214
+ attr_reader :exitstatus
215
+
216
+ def error!
217
+ if exitstatus != 0
218
+ # TODO stdout/stderr is already printed at info/debug level. Let's not print it twice, it's a lot.
219
+ msg = "Error: command '#{command}' exited with code #{exitstatus}.\n"
220
+ raise msg
221
+ end
222
+ end
223
+ end
224
+
225
+ class InitialConnectTimeout < Timeout::Error
226
+ def initialize(original_error)
227
+ super(original_error.message)
228
+ @original_error = original_error
229
+ end
230
+
231
+ attr_reader :original_error
232
+ end
233
+ end
234
+ end
235
+ end