slugforge 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +316 -0
  3. data/bin/slugforge +9 -0
  4. data/lib/slugforge.rb +19 -0
  5. data/lib/slugforge/build.rb +4 -0
  6. data/lib/slugforge/build/build_project.rb +31 -0
  7. data/lib/slugforge/build/export_upstart.rb +85 -0
  8. data/lib/slugforge/build/package.rb +63 -0
  9. data/lib/slugforge/cli.rb +125 -0
  10. data/lib/slugforge/commands.rb +130 -0
  11. data/lib/slugforge/commands/build.rb +20 -0
  12. data/lib/slugforge/commands/config.rb +24 -0
  13. data/lib/slugforge/commands/deploy.rb +383 -0
  14. data/lib/slugforge/commands/project.rb +21 -0
  15. data/lib/slugforge/commands/tag.rb +148 -0
  16. data/lib/slugforge/commands/wrangler.rb +142 -0
  17. data/lib/slugforge/configuration.rb +125 -0
  18. data/lib/slugforge/helper.rb +186 -0
  19. data/lib/slugforge/helper/build.rb +46 -0
  20. data/lib/slugforge/helper/config.rb +37 -0
  21. data/lib/slugforge/helper/enumerable.rb +46 -0
  22. data/lib/slugforge/helper/fog.rb +90 -0
  23. data/lib/slugforge/helper/git.rb +89 -0
  24. data/lib/slugforge/helper/path.rb +76 -0
  25. data/lib/slugforge/helper/project.rb +86 -0
  26. data/lib/slugforge/models/host.rb +233 -0
  27. data/lib/slugforge/models/host/fog_host.rb +33 -0
  28. data/lib/slugforge/models/host/hostname_host.rb +9 -0
  29. data/lib/slugforge/models/host/ip_address_host.rb +9 -0
  30. data/lib/slugforge/models/host_group.rb +65 -0
  31. data/lib/slugforge/models/host_group/aws_tag_group.rb +22 -0
  32. data/lib/slugforge/models/host_group/ec2_instance_group.rb +21 -0
  33. data/lib/slugforge/models/host_group/hostname_group.rb +16 -0
  34. data/lib/slugforge/models/host_group/ip_address_group.rb +16 -0
  35. data/lib/slugforge/models/host_group/security_group_group.rb +20 -0
  36. data/lib/slugforge/models/logger.rb +36 -0
  37. data/lib/slugforge/models/tag_manager.rb +125 -0
  38. data/lib/slugforge/slugins.rb +125 -0
  39. data/lib/slugforge/version.rb +9 -0
  40. data/scripts/post-install.sh +143 -0
  41. data/scripts/unicorn-shepherd.sh +305 -0
  42. data/spec/fixtures/array.yaml +3 -0
  43. data/spec/fixtures/fog_credentials.yaml +4 -0
  44. data/spec/fixtures/invalid_syntax.yaml +1 -0
  45. data/spec/fixtures/one.yaml +3 -0
  46. data/spec/fixtures/two.yaml +3 -0
  47. data/spec/fixtures/valid.yaml +4 -0
  48. data/spec/slugforge/commands/deploy_spec.rb +72 -0
  49. data/spec/slugforge/commands_spec.rb +33 -0
  50. data/spec/slugforge/configuration_spec.rb +200 -0
  51. data/spec/slugforge/helper/fog_spec.rb +81 -0
  52. data/spec/slugforge/helper/git_spec.rb +152 -0
  53. data/spec/slugforge/models/host_group/aws_tag_group_spec.rb +54 -0
  54. data/spec/slugforge/models/host_group/ec2_instance_group_spec.rb +51 -0
  55. data/spec/slugforge/models/host_group/hostname_group_spec.rb +20 -0
  56. data/spec/slugforge/models/host_group/ip_address_group_spec.rb +54 -0
  57. data/spec/slugforge/models/host_group/security_group_group_spec.rb +52 -0
  58. data/spec/slugforge/models/tag_manager_spec.rb +75 -0
  59. data/spec/spec_helper.rb +37 -0
  60. data/spec/support/env.rb +3 -0
  61. data/spec/support/example_groups/configuration_writer.rb +24 -0
  62. data/spec/support/example_groups/helper_provider.rb +10 -0
  63. data/spec/support/factories.rb +18 -0
  64. data/spec/support/fog.rb +15 -0
  65. data/spec/support/helpers.rb +18 -0
  66. data/spec/support/mock_logger.rb +6 -0
  67. data/spec/support/ssh.rb +8 -0
  68. data/spec/support/streams.rb +13 -0
  69. data/templates/foreman/master.conf.erb +21 -0
  70. data/templates/foreman/process-master.conf.erb +2 -0
  71. data/templates/foreman/process.conf.erb +19 -0
  72. metadata +344 -0
@@ -0,0 +1,233 @@
1
+ require 'net/ssh'
2
+ require 'net/scp'
3
+
4
+ module Slugforge
5
+ class Host
6
+ attr_reader :pattern, :server, :slug_name, :status
7
+
8
+ def initialize(pattern, server=nil)
9
+ @pattern = pattern
10
+ @server = server
11
+ @deploy_results = []
12
+ @timeline = []
13
+ @start_time = Time.now
14
+ @actions = []
15
+ self
16
+ end
17
+
18
+ def name
19
+ "name:#{@pattern}"
20
+ end
21
+
22
+ def ip
23
+ @pattern
24
+ end
25
+
26
+ def ssh_host
27
+ @pattern
28
+ end
29
+
30
+ def id
31
+ nil
32
+ end
33
+
34
+ def is_autoscaled?
35
+ false
36
+ end
37
+
38
+ def to_status
39
+ {
40
+ :name => name,
41
+ :ip => ip,
42
+ :pattern => @pattern,
43
+ :slug_name => @slug_name,
44
+ :status => @status.to_s,
45
+ :output => @deploy_results,
46
+ :start_time => @start_time,
47
+ :timeline => timeline,
48
+ }
49
+ end
50
+
51
+ def elapsed_time
52
+ Time.at(Time.now - @start_time).strftime('%M:%S')
53
+ end
54
+
55
+ def record_event(status)
56
+ @status = status
57
+ @timeline << {:status => status, :elapsed_time => elapsed_time}
58
+ end
59
+
60
+ def timeline
61
+ @timeline.map { |event| "#{event[:status]} @ #{event[:elapsed_time]}" }.join ', '
62
+ end
63
+
64
+ def complete?
65
+ success? || failed?
66
+ end
67
+
68
+ def success?
69
+ [:deployed, :terminated].include?(@status) && output.empty?
70
+ end
71
+
72
+ def failed?
73
+ @status == :failed?
74
+ end
75
+
76
+ def add_action(action)
77
+ @actions << action
78
+ end
79
+
80
+ def has_action?(action)
81
+ @actions.include?(action)
82
+ end
83
+
84
+ def remove_action(action)
85
+ @actions.delete(action)
86
+ end
87
+
88
+ def stage?
89
+ @actions.include?(:stage)
90
+ end
91
+
92
+ def install?
93
+ @actions.include?(:install)
94
+ end
95
+
96
+ def effective_action
97
+ return "installing" if @actions.include?(:install)
98
+ "staging"
99
+ end
100
+
101
+ def terminated?
102
+ @status == :terminated?
103
+ end
104
+
105
+ def output
106
+ @deploy_results.map { |result| result[:output] unless result[:exit_code] == 0 }.compact
107
+ end
108
+
109
+ def deploy(slug_name, logger, opts)
110
+ begin
111
+ record_event :started
112
+ if opts[:pretend]
113
+ logger.log "not actually #{effective_action} slug (#{name})", {:color => :yellow, :status => :pretend}
114
+ else
115
+ logger.say_status :deploy, "#{effective_action} to host #{name} as #{username(opts)}", :green
116
+ Net::SSH.start(ssh_host, username(opts), ssh_opts(opts)) do |ssh|
117
+ host_slug = detect_slug(ssh, slug_name, logger) unless opts[:force]
118
+ host_slug ||= copy_slug(ssh, slug_name, logger, opts)
119
+ explode_slug(ssh, host_slug, logger, opts) if stage?
120
+ install_slug(ssh, host_slug, logger, opts) if install?
121
+ end
122
+ end
123
+ record_event :deployed
124
+ rescue => e
125
+ record_event :failed
126
+ message = "#{e.class.to_s}: #{e.to_s}"
127
+ logger.log "#{message} (#{ip}: #{name})", {:color => :red, :status => :fail}
128
+ @deploy_results << {:output => message}
129
+ end
130
+ logger.say_status :deploy, "#{effective_action} complete for host: #{name}", success? ? :green : :red
131
+ @deploy_results
132
+ end
133
+
134
+ private
135
+ def ssh_opts(opts = {})
136
+ ssh_opts = { :forward_agent => true }
137
+ if opts[:identity]
138
+ ssh_opts[:key_data] = File.read(opts[:identity])
139
+ ssh_opts[:keys_only] = true
140
+ end
141
+ ssh_opts
142
+ end
143
+
144
+ def detect_slug(ssh, slug_name, logger)
145
+ found_path = ['/tmp', '/mnt'].select do |path|
146
+ file_count(ssh, path, slug_name) > 0
147
+ end.compact
148
+ return nil if found_path.empty?
149
+ slug_name_with_path = "#{found_path.first}/#{slug_name}"
150
+ logger.log "found existing slug (#{slug_name_with_path}) on host (#{name}); use --force to overwrite slug", {:color => :yellow, :status => :detect, :log_level => :vervose}
151
+ record_event :detected
152
+ slug_name_with_path
153
+ end
154
+
155
+ def copy_slug(ssh, slug_name, logger, opts)
156
+ slug_name_with_path = "/mnt/#{slug_name}"
157
+ case opts[:copy_type]
158
+ when :ssh
159
+ logger.log "interactive mode enabled (be sure to set slug_name)", {:color => :yellow, :status => :copy, :log_level => :verbose}
160
+ binding.pry
161
+ when :scp
162
+ logger.log "copying slug to host via SCP (#{name})", {:color => :green, :status => :copy, :log_level => :verbose}
163
+ scp_upload ip, username(opts), opts[:filename], "#{slug_name}", logger, :ssh => ssh_opts
164
+ logger.log "moving slug from ~ to /mnt as root", {:color => :green, :status => :copy, :log_level => :verbose}
165
+ @deploy_results << ssh_command(ssh, "sudo mv #{slug_name} #{slug_name_with_path}", logger)
166
+ else # AWS S3 command line by default
167
+ logger.log "copying slug to host via aws s3 command (#{name})", {:color => :green, :status => :copy, :log_level => :verbose}
168
+ @deploy_results << ssh_command(ssh, "sudo -- sh -c 'export AWS_ACCESS_KEY_ID=#{opts[:aws_session][:aws_access_key_id]}; export AWS_SECRET_ACCESS_KEY=#{opts[:aws_session][:aws_secret_access_key]}; export AWS_SECURITY_TOKEN=#{opts[:aws_session][:aws_session_token]}; export AWS_DEFAULT_REGION=#{opts[:aws_session][:aws_region]}; aws s3 cp #{opts[:s3_url]} #{slug_name_with_path}'; echo \"SSH_COMMAND_EXIT_CODE=$?\"", logger)
169
+ end
170
+ record_event :copied
171
+ slug_name_with_path
172
+ end
173
+
174
+ def username(opts)
175
+ opts[:username] || Net::SSH.configuration_for(ip)[:user] || `whoami`.chomp
176
+ end
177
+
178
+ def explode_slug(ssh, slug_name_with_path, logger, opts)
179
+ logger.log "exploding package as root #{"for user " + opts[:owner] if opts[:owner]}", {:color => :green, :status => :stage, :log_level => :verbose}
180
+ @deploy_results << ssh_command(ssh, slug_install_command(slug_name_with_path, opts[:deploy_dir], {:unpack => true, :owner => opts[:owner], :env => opts[:env]}), logger)
181
+ @slug_name = slug_name
182
+ end
183
+
184
+ def install_slug(ssh, slug_name_with_path, logger, opts)
185
+ logger.log "installing package as root #{"for user " + opts[:owner] if opts[:owner]}", {:color => :green, :status => :install, :log_level => :verbose}
186
+ @deploy_results << ssh_command(ssh, slug_install_command(slug_name_with_path, opts[:deploy_dir], {:owner => opts[:owner], :env => opts[:env], :force => opts[:force]}), logger)
187
+ @slug_name = slug_name
188
+ record_event :installed
189
+ end
190
+
191
+ def file_count(ssh, path, file)
192
+ ssh.exec!("find #{path} -maxdepth 1 -name '#{file}' -type f -size +0 | wc -l").to_i
193
+ end
194
+
195
+ def scp_upload(host, user, source, dest, logger, opts)
196
+ logger.log "SCP: #{source} to #{host}:#{dest}"
197
+ Net::SCP.upload!(host, user, source, dest, opts) do | ch, name, sent, total |
198
+ logger.log "\r#{name}: #{(sent * 100.0 / total).to_i}% "
199
+ end
200
+ logger.log
201
+ end
202
+
203
+ def ssh_command(ssh, command, logger)
204
+ output = ssh.exec!("#{command}")
205
+ exit_code_matches = /^SSH_COMMAND_EXIT_CODE=(\d+)$/.match(output)
206
+ exit_code = exit_code_matches ? exit_code_matches[1].to_i : 0
207
+ logger_opts = if exit_code == 0
208
+ {:color => :green, :log_level => :verbose}
209
+ else
210
+ {:color => :red}
211
+ end.merge({:status => :command})
212
+ logger.log "#{command}", logger_opts
213
+ logger.log "Output:\n#{output}", logger_opts
214
+ {:exit_code => exit_code, :output => output, :command => command, :username => ssh.options[:user]}
215
+ end
216
+
217
+ def slug_install_command(slug_name_with_path, deploy_dir, opts = {})
218
+ [ opts[:prefix],
219
+ "TERM=dumb sudo bash -l -c 'date >> /var/log/slug_deploy.log ; ",
220
+ "chmod +x #{slug_name_with_path} ",
221
+ "&& #{opts[:env]} #{slug_name_with_path} ",
222
+ '-y ', #always clobber existing installs
223
+ "-i #{deploy_dir} ",
224
+ opts[:owner] ? "-o #{opts[:owner]} " : '',
225
+ opts[:force] ? '-f ' : '',
226
+ opts[:unpack] ? '-u ' : '',
227
+ '-v ', #verbose
228
+ '| tee -a /var/log/slug_deploy.log ',
229
+ "; echo \"SSH_COMMAND_EXIT_CODE=$?\"'"
230
+ ].join('')
231
+ end
232
+ end
233
+ end
@@ -0,0 +1,33 @@
1
+ require 'slugforge/models/host'
2
+
3
+ module Slugforge
4
+ class FogHost < Host
5
+ def name
6
+ "instance:#{@server.id}, private_name:#{@server.private_dns_name}, public_name:#{@server.dns_name}, ip:#{@server.public_ip_address}"
7
+ end
8
+
9
+ def ip
10
+ @server.public_ip_address
11
+ end
12
+
13
+ def ssh_host
14
+ @server.dns_name
15
+ end
16
+
17
+ def id
18
+ @server.id
19
+ end
20
+
21
+ def is_autoscaled?
22
+ !@server.tags["aws:autoscaling:groupName"].nil?
23
+ end
24
+
25
+ def to_status
26
+ super.merge({
27
+ :instance_id => @server.id,
28
+ :private_name => @server.private_dns_name,
29
+ :public_name => @server.dns_name,
30
+ })
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,9 @@
1
+ require 'slugforge/models/host'
2
+
3
+ module Slugforge
4
+ class HostnameHost < Host
5
+ def name
6
+ "hostname:#{@pattern}"
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,9 @@
1
+ require 'slugforge/models/host'
2
+
3
+ module Slugforge
4
+ class IpAddressHost < Host
5
+ def name
6
+ "ip:#{@pattern}"
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,65 @@
1
+ require 'slugforge/models/host_group/ip_address_group'
2
+ require 'slugforge/models/host_group/ec2_instance_group'
3
+ require 'slugforge/models/host_group/hostname_group'
4
+ require 'slugforge/models/host_group/aws_tag_group'
5
+ require 'slugforge/models/host_group/security_group_group'
6
+
7
+ module Slugforge
8
+ class HostGroup
9
+ attr_reader :name, :hosts
10
+
11
+ def self.discover(patterns, compute)
12
+ patterns.map do |pattern|
13
+ IpAddressGroup.detect(pattern, compute) ||
14
+ Ec2InstanceGroup.detect(pattern, compute) ||
15
+ HostnameGroup.detect(pattern, compute) ||
16
+ AwsTagGroup.detect(pattern, compute) ||
17
+ SecurityGroupGroup.detect(pattern, compute) ||
18
+ # If nothing detected, return a "null" group
19
+ HostGroup.new(pattern, compute)
20
+ end
21
+ end
22
+
23
+ def initialize(pattern, compute)
24
+ @name = pattern
25
+ end
26
+
27
+ def install_all
28
+ return if @hosts.nil?
29
+ @hosts.each { |host| host.add_action(:install) }
30
+ end
31
+
32
+ def install_percent_of_hosts(value)
33
+ return if @hosts.nil?
34
+ count = (@hosts.count * value / 100.0).ceil
35
+ sorted_hosts[0...count].each { |host| host.add_action(:install) }
36
+ end
37
+
38
+ def install_number_of_hosts(value)
39
+ return if @hosts.nil?
40
+ count = [@hosts.count, value].min
41
+ sorted_hosts[0...count].each { |host| host.add_action(:install) }
42
+ end
43
+
44
+ def sorted_hosts
45
+ # We sort the hosts by IP to make the order deterministic before we filter
46
+ # by number or percent. That way when we move from 5% to 10% we end up at
47
+ # 10% of the hosts, not some value between 10% and 15%.
48
+ @hosts.sort_by { |host| host.ip }
49
+ end
50
+
51
+ def success?
52
+ @hosts.all?(&:success?)
53
+ end
54
+
55
+ def hosts_for_action(action)
56
+ @hosts.select { |host| host.has_action?(action) }
57
+ end
58
+
59
+ def self.detect(pattern, compute)
60
+ return nil unless pattern =~ self.matcher
61
+ group = self.new(pattern, compute)
62
+ group.hosts.empty? ? nil : group
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,22 @@
1
+ require 'slugforge/models/host/fog_host'
2
+
3
+ module Slugforge
4
+ class HostGroup ; end
5
+
6
+ class AwsTagGroup < HostGroup
7
+ def self.matcher
8
+ /^(\w+)=(\w+)$/
9
+ end
10
+
11
+ def initialize(pattern, compute)
12
+ matches = self.class.matcher.match(pattern)
13
+ return nil unless matches
14
+ @hosts = compute.servers.select do |server|
15
+ server.tags[matches[1]] == matches[2] && !server.public_ip_address.nil?
16
+ end.map do |server|
17
+ FogHost.new(pattern, server)
18
+ end
19
+ super
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,21 @@
1
+ require 'slugforge/models/host/fog_host'
2
+
3
+ module Slugforge
4
+ class HostGroup ; end
5
+
6
+ class Ec2InstanceGroup < HostGroup
7
+ def self.matcher
8
+ /^i-[0-9a-f]{8}$/i
9
+ end
10
+
11
+ def initialize(pattern, compute)
12
+ server = compute.servers.get(pattern)
13
+ @hosts = if server.nil? || server.public_ip_address.nil?
14
+ []
15
+ else
16
+ [ FogHost.new(pattern, server) ]
17
+ end
18
+ super
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,16 @@
1
+ require 'slugforge/models/host/hostname_host'
2
+
3
+ module Slugforge
4
+ class HostGroup ; end
5
+
6
+ class HostnameGroup < HostGroup
7
+ def self.matcher
8
+ /^[^.]+\./
9
+ end
10
+
11
+ def initialize(pattern, compute)
12
+ @hosts = [ HostnameHost.new(pattern) ]
13
+ super
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,16 @@
1
+ require 'slugforge/models/host/ip_address_host'
2
+
3
+ module Slugforge
4
+ class HostGroup ; end
5
+
6
+ class IpAddressGroup < HostGroup
7
+ def self.matcher
8
+ /^(\d{1,3}\.){3}(\d{1,3})$/
9
+ end
10
+
11
+ def initialize(pattern, compute)
12
+ @hosts = [ IpAddressHost.new(pattern) ]
13
+ super
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,20 @@
1
+ require 'slugforge/models/host/fog_host'
2
+
3
+ module Slugforge
4
+ class HostGroup ; end
5
+
6
+ class SecurityGroupGroup < HostGroup
7
+ def self.matcher
8
+ /\w+/
9
+ end
10
+
11
+ def initialize(pattern, compute)
12
+ @hosts = compute.servers.select do |server|
13
+ server.groups.include?(pattern) && !server.public_ip_address.nil?
14
+ end.map do |server|
15
+ FogHost.new(pattern, server)
16
+ end
17
+ super
18
+ end
19
+ end
20
+ end