chef-workflow 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,100 @@
1
+ require 'chef-workflow/support/ip'
2
+ require 'chef-workflow/support/knife'
3
+ require 'chef-workflow/support/debug'
4
+ require 'net/ssh'
5
+
6
+ module ChefWorkflow
7
+ #
8
+ # Helper for performing SSH on groups of servers. Intended to be mixed into
9
+ # test case classes.
10
+ #
11
+ module SSHHelper
12
+ include ChefWorkflow::DebugSupport
13
+
14
+ #
15
+ # run a command against a group of servers. These commands are run in
16
+ # parallel, but the command itself does not complete until all the threads
17
+ # have finished running.
18
+ #
19
+ def ssh_role_command(role, command)
20
+ t = []
21
+ ChefWorkflow::IPSupport.get_role_ips(role).each do |ip|
22
+ t.push(
23
+ Thread.new do
24
+ ssh_command(ip, command)
25
+ end
26
+ )
27
+ end
28
+ t.each(&:join)
29
+ end
30
+
31
+ #
32
+ # takes a block which it uses inside of the open_channel block that Net::SSH
33
+ # uses. Intended to provide a consistent way of setting up Net::SSH Makes
34
+ # heavy use of KnifeSupport to determine how to drive the command.
35
+ #
36
+ def configure_ssh_command(ip, command)
37
+ command = "#{ChefWorkflow::KnifeSupport.use_sudo ? 'sudo ': ''}#{command}"
38
+
39
+ options = { }
40
+
41
+ options[:password] = ChefWorkflow::KnifeSupport.ssh_password if ChefWorkflow::KnifeSupport.ssh_password
42
+ options[:keys] = [ChefWorkflow::KnifeSupport.ssh_identity_file] if ChefWorkflow::KnifeSupport.ssh_identity_file
43
+
44
+ Net::SSH.start(ip, ChefWorkflow::KnifeSupport.ssh_user, options) do |ssh|
45
+ ssh.open_channel do |ch|
46
+ ch.on_open_failed do |ch, code, desc|
47
+ raise "Connection Error to #{ip}: #{desc}"
48
+ end
49
+
50
+ ch.exec(command) do |ch, success|
51
+ yield ch, success
52
+ end
53
+ end
54
+
55
+ ssh.loop
56
+ end
57
+ end
58
+
59
+ #
60
+ # Run a command against a single IP. Returns the exit status.
61
+ #
62
+ #
63
+ def ssh_command(ip, command)
64
+ configure_ssh_command(ip, command) do |ch, success|
65
+ return 1 unless success
66
+
67
+ if_debug(2) do
68
+ ch.on_data do |ch, data|
69
+ $stderr.puts data
70
+ end
71
+ end
72
+
73
+ ch.on_request("exit-status") do |ch, data|
74
+ return data.read_long
75
+ end
76
+ end
77
+ end
78
+
79
+ #
80
+ # run a command, and instead of capturing the exit status, return the data
81
+ # captured during the command run.
82
+ #
83
+ def ssh_capture(ip, command)
84
+ retval = ""
85
+ configure_ssh_command(ip, command) do |ch, success|
86
+ return "" unless success
87
+
88
+ ch.on_data do |ch, data|
89
+ retval << data
90
+ end
91
+
92
+ ch.on_request("exit-status") do |ch, data|
93
+ return retval
94
+ end
95
+ end
96
+
97
+ return retval
98
+ end
99
+ end
100
+ end
@@ -1,41 +1,45 @@
1
1
  require 'fileutils'
2
2
  require 'chef-workflow/support/generic'
3
3
 
4
- #
5
- # Vagrant configuration settings. Uses `GenericSupport`.
6
- #
7
- class VagrantSupport
8
- # The default vagrant box we use for provisioning.
9
- DEFAULT_VAGRANT_BOX = "http://files.vagrantup.com/precise32.box"
10
-
11
- # the calculated box, currently taken from the box_url. Expect this to change.
12
- attr_reader :box
13
-
14
- # FIXME: support non-url boxes and ram configurations
15
- def initialize(box_url=DEFAULT_VAGRANT_BOX)
16
- self.box_url = box_url
17
- end
18
-
4
+ module ChefWorkflow
19
5
  #
20
- # Set or retrieve the box_url. See #box_url=.
6
+ # Vagrant configuration settings. Uses `GenericSupport`.
21
7
  #
22
- def box_url(arg=nil)
23
- if arg
24
- self.box_url = arg
8
+ class VagrantSupport
9
+ include ChefWorkflow::GenericSupport
10
+
11
+ # The default vagrant box we use for provisioning.
12
+ DEFAULT_VAGRANT_BOX = "http://files.vagrantup.com/precise32.box"
13
+
14
+ # the calculated box, currently taken from the box_url. Expect this to change.
15
+ attr_reader :box
16
+
17
+ #--
18
+ # FIXME: support non-url boxes and ram configurations
19
+ #++
20
+ def initialize(box_url=DEFAULT_VAGRANT_BOX)
21
+ self.box_url = box_url
25
22
  end
26
23
 
27
- @box_url
28
- end
24
+ #
25
+ # Set or retrieve the box_url. See #box_url=.
26
+ #
27
+ def box_url(arg=nil)
28
+ if arg
29
+ self.box_url = arg
30
+ end
29
31
 
30
- #
31
- # Set the box_url. The box name is derived from the url currently.
32
- #
33
- def box_url=(url)
34
- @box_url = url
35
- @box = File.basename(url).gsub('\.box', '')
36
- end
32
+ @box_url
33
+ end
37
34
 
38
- include GenericSupport
35
+ #
36
+ # Set the box_url. The box name is derived from the url currently.
37
+ #
38
+ def box_url=(url)
39
+ @box_url = url
40
+ @box = File.basename(url).gsub(/\.box$/, '')
41
+ end
42
+ end
39
43
  end
40
44
 
41
- VagrantSupport.configure
45
+ ChefWorkflow::VagrantSupport.configure
@@ -3,69 +3,40 @@ require 'fileutils'
3
3
  require 'chef-workflow/support/general'
4
4
  require 'chef-workflow/support/attr'
5
5
  require 'chef-workflow/support/debug'
6
+ require 'chef-workflow/support/db/group'
7
+ require 'chef-workflow/support/db/basic'
6
8
 
7
9
  #--
8
10
  # XXX see the dynamic require at the bottom
9
11
  #++
10
12
 
11
- #
12
- # This class mainly exists to track the run state of the Scheduler, and is kept
13
- # simple so that the contents can be marshalled and restored from a file.
14
- #
15
- class VM
16
- class << self
17
- extend AttrSupport
18
- fancy_attr :vm_file
19
- end
20
-
21
- include DebugSupport
22
- extend AttrSupport
23
-
13
+ module ChefWorkflow
24
14
  #
25
- # If a file exists that contains a VM object, load it. Use VM.vm_file to
26
- # control the location of this file.
15
+ # This class mainly exists to track the run state of the Scheduler, and is kept
16
+ # simple so that the contents can be marshalled and restored from a file.
27
17
  #
28
- def self.load_from_file
29
- vm_file = GeneralSupport.singleton.vm_file
30
-
31
- if File.file?(vm_file)
32
- return Marshal.load(File.binread(vm_file || DEFAULT_VM_FILE))
18
+ class VM
19
+ include ChefWorkflow::DebugSupport
20
+
21
+ # the vm groups and their provisioning lists.
22
+ attr_reader :groups
23
+ # the dependencies that each vm group depends on
24
+ attr_reader :dependencies
25
+ # the set of provisioned (solved) groups
26
+ attr_reader :provisioned
27
+ # the set of provisioning (working) groups
28
+ attr_reader :working
29
+
30
+ def initialize
31
+ @groups = ChefWorkflow::DatabaseSupport::VMGroup.new('vm_groups', false)
32
+ @dependencies = ChefWorkflow::DatabaseSupport::VMGroup.new('vm_dependencies', true)
33
+ @provisioned = ChefWorkflow::DatabaseSupport::Set.new('vm_scheduler', 'provisioned')
34
+ @working = ChefWorkflow::DatabaseSupport::Set.new('vm_scheduler', 'working')
33
35
  end
34
-
35
- return nil
36
- end
37
-
38
- #
39
- # Save the marshalled representation to a file. Use VM.vm_file to control the
40
- # location of this file.
41
- #
42
- def save_to_file
43
- vm_file = GeneralSupport.singleton.vm_file
44
- marshalled = Marshal.dump(self)
45
- FileUtils.mkdir_p(File.dirname(vm_file))
46
- File.binwrite(vm_file, marshalled)
47
36
  end
48
37
 
49
- # the vm groups and their provisioning lists.
50
- attr_reader :groups
51
- # the dependencies that each vm group depends on
52
- attr_reader :dependencies
53
- # the set of provisioned (solved) groups
54
- attr_reader :provisioned
55
- # the set of provisioning (working) groups
56
- attr_reader :working
57
-
58
- def clean
59
- @groups = { }
60
- @dependencies = { }
61
- @provisioned = Set.new
62
- @working = Set.new
38
+ # XXX require all the provisioners -- marshal will blow up unless this is done.
39
+ Dir[File.join(File.expand_path(File.dirname(__FILE__)), 'vm', '*')].each do |x|
40
+ require x if File.file?(x)
63
41
  end
64
-
65
- alias initialize clean
66
- end
67
-
68
- # XXX require all the provisioners -- marshal will blow up unless this is done.
69
- Dir[File.join(File.expand_path(File.dirname(__FILE__)), 'vm', '*')].each do |x|
70
- require x
71
42
  end
@@ -1,31 +1,40 @@
1
- require 'chef-workflow/support/knife'
1
+ require 'chef-workflow/support/debug'
2
2
  require 'chef-workflow/support/knife-plugin'
3
- require 'chef/knife/server_bootstrap_standalone'
4
3
 
5
- class VM
6
- class ChefServerProvisioner
7
- include DebugSupport
8
- include KnifePluginSupport
4
+ module ChefWorkflow
5
+ class VM
6
+ class ChefServerProvisioner
7
+ include ChefWorkflow::DebugSupport
8
+ include ChefWorkflow::KnifePluginSupport
9
9
 
10
- attr_accessor :name
10
+ attr_accessor :name
11
11
 
12
- def startup(*args)
13
- ip = args.first.first #arg
12
+ def startup(*args)
13
+ require 'chef-workflow/support/knife'
14
+ require 'chef/knife/ssh' # required for chef 10.12
15
+ require 'chef/knife/server_bootstrap_standalone'
14
16
 
15
- raise "No IP to use for the chef server" unless ip
17
+ ip = args.first.first #arg
16
18
 
17
- args = %W[--node-name test-chef-server --host #{ip}]
19
+ raise "No IP to use for the chef server" unless ip
18
20
 
19
- args += %W[--ssh-user #{KnifeSupport.singleton.ssh_user}] if KnifeSupport.singleton.ssh_user
20
- args += %W[--ssh-password #{KnifeSupport.singleton.ssh_password}] if KnifeSupport.singleton.ssh_password
21
- args += %W[--identity-file #{KnifeSupport.singleton.ssh_identity_file}] if KnifeSupport.singleton.ssh_identity_file
21
+ args = %W[--node-name test-chef-server --host #{ip}]
22
22
 
23
- init_knife_plugin(Chef::Knife::ServerBootstrapStandalone, args).run
24
- true
25
- end
23
+ args += %W[--ssh-user #{ChefWorkflow::KnifeSupport.ssh_user}] if ChefWorkflow::KnifeSupport.ssh_user
24
+ args += %W[--ssh-password #{ChefWorkflow::KnifeSupport.ssh_password}] if ChefWorkflow::KnifeSupport.ssh_password
25
+ args += %W[--identity-file #{ChefWorkflow::KnifeSupport.ssh_identity_file}] if ChefWorkflow::KnifeSupport.ssh_identity_file
26
+
27
+ init_knife_plugin(Chef::Knife::ServerBootstrapStandalone, args).run
28
+ true
29
+ end
30
+
31
+ def shutdown
32
+ true
33
+ end
26
34
 
27
- def shutdown
28
- true
35
+ def report
36
+ [name]
37
+ end
29
38
  end
30
39
  end
31
40
  end
@@ -1,146 +1,175 @@
1
- require 'chef-workflow/support/ec2'
2
- require 'chef-workflow/support/ip'
3
1
  require 'chef-workflow/support/debug'
4
- require 'net/ssh'
5
- require 'timeout'
6
2
 
7
- class VM
8
- class EC2Provisioner
9
- include DebugSupport
3
+ module ChefWorkflow
4
+ class VM
5
+ class EC2Provisioner
6
+ include ChefWorkflow::DebugSupport
10
7
 
11
- attr_accessor :name
8
+ attr_accessor :name
12
9
 
13
- def initialize(name, number_of_servers)
14
- @name = name
15
- @number_of_servers = number_of_servers
16
- @instance_ids = []
17
- end
10
+ def initialize(name, number_of_servers)
11
+ require 'chef-workflow/support/ec2'
12
+ require 'chef-workflow/support/ip'
13
+ require 'net/ssh'
14
+ require 'timeout'
18
15
 
19
- def ssh_connection_check(ip)
20
- Net::SSH.start(ip, KnifeSupport.singleton.ssh_user, { :keys => [KnifeSupport.singleton.ssh_identity_file] }) do |ssh|
21
- ssh.open_channel do |ch|
22
- ch.on_open_failed do
23
- return false
24
- end
16
+ @name = name
17
+ @number_of_servers = number_of_servers
18
+ init_instance_ids
19
+ end
20
+
21
+ def init_instance_ids
22
+ @instance_ids = ChefWorkflow::DatabaseSupport::Set.new("vm_ec2_instances", name)
23
+ end
25
24
 
26
- ch.exec("exit 0") do
27
- return true
25
+ def ssh_connection_check(ip)
26
+ Net::SSH.start(ip, ChefWorkflow::KnifeSupport.ssh_user, { :keys => [ChefWorkflow::KnifeSupport.ssh_identity_file] }) do |ssh|
27
+ ssh.open_channel do |ch|
28
+ ch.on_open_failed do
29
+ return false
30
+ end
31
+
32
+ ch.exec("exit 0") do
33
+ return true
34
+ end
28
35
  end
36
+ ssh.loop
29
37
  end
38
+ rescue Exception => e
39
+ return false
30
40
  end
31
- rescue
32
- return false
33
- end
34
41
 
35
- def ec2
36
- EC2Support.singleton
37
- end
42
+ def ec2
43
+ ChefWorkflow::EC2Support
44
+ end
38
45
 
39
- def startup(*args)
40
- aws_ec2 = ec2.ec2_obj
46
+ def startup(*args)
47
+ aws_ec2 = ec2.ec2_obj
41
48
 
42
- ec2.assert_security_groups
49
+ ec2.assert_security_groups
43
50
 
44
- instances = aws_ec2.instances.create(
45
- :count => @number_of_servers,
46
- :image_id => ec2.ami,
47
- :security_groups => ec2.security_groups,
48
- :key_name => ec2.ssh_key,
49
- :instance_type => ec2.instance_type
50
- )
51
+ instances = aws_ec2.instances.create(
52
+ :count => @number_of_servers,
53
+ :image_id => ec2.ami,
54
+ :security_groups => ec2.security_groups,
55
+ :key_name => ec2.ssh_key,
56
+ :instance_type => ec2.instance_type
57
+ )
51
58
 
52
- #
53
- # instances isn't actually an array above -- see this url:
54
- #
55
- # https://github.com/aws/aws-sdk-ruby/issues/100
56
- #
57
- # Actually make it a real array here so it's useful.
58
- #
59
+ #
60
+ # instances isn't actually an array above -- see this url:
61
+ #
62
+ # https://github.com/aws/aws-sdk-ruby/issues/100
63
+ #
64
+ # Actually make it a real array here so it's useful.
65
+ #
59
66
 
60
- if instances.kind_of?(Array)
61
- new_instances = []
67
+ if instances.kind_of?(Array)
68
+ new_instances = []
62
69
 
63
- instances.each do |instance|
64
- new_instances.push(instance)
70
+ instances.each do |instance|
71
+ new_instances.push(instance)
72
+ end
73
+
74
+ instances = new_instances
75
+ else
76
+ instances = [instances]
65
77
  end
66
78
 
67
- instances = new_instances
68
- else
69
- instances = [instances]
70
- end
79
+ #
80
+ # There are instances where AWS won't acknowledge a created instance
81
+ # right away. Let's make sure the API server knows they all exist before
82
+ # moving forward.
83
+ #
71
84
 
72
- #
73
- # There are instances where AWS won't acknowledge a created instance
74
- # right away. Let's make sure the API server knows they all exist before
75
- # moving forward.
76
- #
85
+ unresolved_instances = instances.dup
77
86
 
78
- unresolved_instances = instances.dup
87
+ until unresolved_instances.empty?
88
+ instance = unresolved_instances.shift
79
89
 
80
- until unresolved_instances.empty?
81
- instance = unresolved_instances.shift
90
+ unless (instance.status rescue nil)
91
+ if_debug(3) do
92
+ $stderr.puts "API server doesn't think #{instance.id} exists yet."
93
+ end
82
94
 
83
- unless (instance.status rescue nil)
84
- if_debug(3) do
85
- $stderr.puts "API server doesn't think #{instance.id} exists yet."
95
+ sleep 0.3
96
+ unresolved_instances.push(instance)
86
97
  end
87
-
88
- sleep 0.3
89
- unresolved_instances.push(instance)
90
98
  end
91
- end
92
-
93
- ip_addresses = []
94
-
95
- instances.each do |instance|
96
- @instance_ids.push(instance.id)
97
- end
98
99
 
99
- begin
100
- Timeout.timeout(ec2.provision_wait) do
101
- until instances.empty?
102
- instance = instances.shift
100
+ ip_addresses = []
103
101
 
104
- ready = false
102
+ instances.each do |instance|
103
+ @instance_ids.add(instance.id)
104
+ end
105
105
 
106
- if instance.status == :running
107
- ready = ssh_connection_check(instance.ip_address)
108
- unless ready
109
- if_debug(3) do
110
- $stderr.puts "Instance #{instance.id} running, but ssh isn't up yet."
106
+ ipaddress_mutex = Mutex.new
107
+ debug_mutex = Mutex.new
108
+
109
+ begin
110
+ Timeout.timeout(ec2.provision_wait) do
111
+ instances.map do |instance|
112
+ Thread.new do
113
+ loop do
114
+ ready = false
115
+
116
+ if instance.status == :running
117
+ ready = ssh_connection_check(instance.ip_address)
118
+ unless ready
119
+ if_debug(3) do
120
+ debug_mutex.synchronize do
121
+ $stderr.puts "Instance #{instance.id} running, but ssh isn't up yet."
122
+ end
123
+ end
124
+ end
125
+ else
126
+ if_debug(3) do
127
+ debug_mutex.synchronize do
128
+ $stderr.puts "#{instance.id} isn't running yet -- scheduling for re-check"
129
+ end
130
+ end
131
+ end
132
+
133
+ if ready
134
+ ipaddress_mutex.synchronize do
135
+ ip_addresses.push(instance.ip_address)
136
+ end
137
+ break
138
+ end
139
+
140
+ sleep 2
111
141
  end
112
142
  end
113
- else
114
- if_debug(3) do
115
- $stderr.puts "#{instance.id} isn't running yet -- scheduling for re-check"
116
- end
117
- end
118
-
119
- if ready
120
- ip_addresses.push(instance.ip_address)
121
- IPSupport.singleton.assign_role_ip(name, instance.ip_address)
122
- else
123
- sleep 0.3
124
- instances.push(instance)
125
- end
143
+ end.each(&:join)
126
144
  end
145
+ rescue TimeoutError
146
+ raise "instances timed out waiting for ec2"
147
+ end
148
+
149
+ ip_addresses.each do |ipaddr|
150
+ ChefWorkflow::IPSupport.assign_role_ip(name, ipaddr)
127
151
  end
128
- rescue TimeoutError
129
- raise "instances timed out waiting for ec2"
152
+
153
+ return ip_addresses
130
154
  end
131
155
 
132
- return ip_addresses
133
- end
156
+ def shutdown
157
+ aws_ec2 = ec2.ec2_obj
158
+
159
+ @instance_ids.each do |instance_id|
160
+ aws_ec2.instances[instance_id].terminate
161
+ end
134
162
 
135
- def shutdown
136
- aws_ec2 = ec2.ec2_obj
163
+ @instance_ids.clear
137
164
 
138
- @instance_ids.each do |instance_id|
139
- aws_ec2.instances[instance_id].terminate
165
+ ChefWorkflow::IPSupport.delete_role(name)
166
+ return true
140
167
  end
141
168
 
142
- IPSupport.singleton.delete_role(name)
143
- return true
169
+ def report
170
+ init_instance_ids
171
+ ["#{@number_of_servers} servers; instance ids: #{@instance_ids.to_a.join(" ")}"]
172
+ end
144
173
  end
145
174
  end
146
175
  end