beaker 1.13.1 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -143,26 +143,32 @@ module Beaker
143
143
  # @param [Hash{Symbol=>String}] opts The options
144
144
  # @option opts [String] :pe_ver_win Default PE version to install or upgrade to on Windows hosts
145
145
  # (Othersie uses individual Windows hosts pe_ver)
146
- # @option opts [String :pe_ver Default PE version to install or upgrade to
146
+ # @option opts [String] :pe_ver Default PE version to install or upgrade to
147
147
  # (Otherwise uses individual hosts pe_ver)
148
+ # @option opts [Boolean] :pe_debug (false) Should we run the installer in debug mode?
148
149
  # @example
149
150
  # on host, "#{installer_cmd(host, opts)} -a #{host['working_dir']}/answers"
150
151
  # @api private
151
152
  def installer_cmd(host, opts)
152
153
  version = host['pe_ver'] || opts[:pe_ver]
153
154
  if host['platform'] =~ /windows/
154
- version = opts[:pe_ver_win] || host['pe_ver']
155
- "cd #{host['working_dir']} && cmd /C 'start /w msiexec.exe /qn /i puppet-enterprise-#{version}.msi PUPPET_MASTER_SERVER=#{master} PUPPET_AGENT_CERTNAME=#{host}'"
155
+ version = host[:pe_ver] || opts['pe_ver_win']
156
+ log_file = "#{File.basename(host['working_dir'])}.log"
157
+ pe_debug = host[:pe_debug] || opts[:pe_debug] ? " && cat #{log_file}" : ''
158
+ "cd #{host['working_dir']} && cmd /C 'start /w msiexec.exe /qn /L*V #{log_file} /i puppet-enterprise-#{version}.msi PUPPET_MASTER_SERVER=#{master} PUPPET_AGENT_CERTNAME=#{host}'#{pe_debug}"
156
159
  elsif host['platform'] =~ /osx/
157
160
  version = host['pe_ver'] || opts[:pe_ver]
158
- "cd #{host['working_dir']} && hdiutil attach #{host['dist']}.dmg && installer -pkg /Volumes/puppet-enterprise-#{version}/puppet-enterprise-installer-#{version}.pkg -target /"
161
+ pe_debug = host[:pe_debug] || opts[:pe_debug] ? ' -verboseR' : ''
162
+ "cd #{host['working_dir']} && hdiutil attach #{host['dist']}.dmg && installer#{pe_debug} -pkg /Volumes/puppet-enterprise-#{version}/puppet-enterprise-installer-#{version}.pkg -target /"
159
163
 
160
164
  # Frictionless install didn't exist pre-3.2.0, so in that case we fall
161
165
  # through and do a regular install.
162
166
  elsif host['roles'].include? 'frictionless' and ! version_is_less(version, '3.2.0')
163
- "cd #{host['working_dir']} && curl -kO https://#{master}:8140/packages/#{version}/install.bash && bash install.bash"
167
+ pe_debug = host[:pe_debug] || opts[:pe_debug] ? ' -x' : ''
168
+ "cd #{host['working_dir']} && curl -kO https://#{master}:8140/packages/#{version}/install.bash && bash#{pe_debug} install.bash"
164
169
  else
165
- "cd #{host['working_dir']}/#{host['dist']} && ./#{host['pe_installer']} -a #{host['working_dir']}/answers"
170
+ pe_debug = host[:pe_debug] || opts[:pe_debug] ? ' -D' : ''
171
+ "cd #{host['working_dir']}/#{host['dist']} && ./#{host['pe_installer']}#{pe_debug} -a #{host['working_dir']}/answers"
166
172
  end
167
173
  end
168
174
 
@@ -184,6 +190,74 @@ module Beaker
184
190
  end
185
191
  end
186
192
 
193
+ # Fetch file_name from the given base_url into dst_dir.
194
+ #
195
+ # @param [String] base_url The base url from which to recursively download
196
+ # files.
197
+ # @param [String] file_name The trailing name compnent of both the source url
198
+ # and the destination file.
199
+ # @param [String] dst_dir The local destination directory.
200
+ #
201
+ # @return [String] dst The name of the newly-created file.
202
+ #
203
+ # @!visibility private
204
+ def fetch_http_file(base_url, file_name, dst_dir)
205
+ FileUtils.makedirs(dst_dir)
206
+ src = "#{base_url}/#{file_name}"
207
+ dst = File.join(dst_dir, file_name)
208
+ if File.exists?(dst)
209
+ logger.notify "Already fetched #{dst}"
210
+ else
211
+ logger.notify "Fetching: #{src}"
212
+ logger.notify " and saving to #{dst}"
213
+ open(src) do |remote|
214
+ File.open(dst, "w") do |file|
215
+ FileUtils.copy_stream(remote, file)
216
+ end
217
+ end
218
+ end
219
+ return dst
220
+ end
221
+
222
+ # Recursively fetch the contents of the given http url, ignoring
223
+ # `index.html` and `*.gif` files.
224
+ #
225
+ # @param [String] url The base http url from which to recursively download
226
+ # files.
227
+ # @param [String] dst_dir The local destination directory.
228
+ #
229
+ # @return [String] dst The name of the newly-created subdirectory of
230
+ # dst_dir.
231
+ #
232
+ # @!visibility private
233
+ def fetch_http_dir(url, dst_dir)
234
+ logger.notify "fetch_http_dir (url: #{url}, dst_dir #{dst_dir})"
235
+ if url[-1, 1] !~ /\//
236
+ url += '/'
237
+ end
238
+ url = URI.parse(url)
239
+ chunks = url.path.split('/')
240
+ dst = File.join(dst_dir, chunks.last)
241
+ #determine directory structure to cut
242
+ #only want to keep the last directory, thus cut total number of dirs - 2 (hostname + last dir name)
243
+ cut = chunks.length - 2
244
+ wget_command = "wget -nv -P #{dst_dir} --reject \"index.html*\",\"*.gif\" --cut-dirs=#{cut} -np -nH --no-check-certificate -r #{url}"
245
+
246
+ logger.notify "Fetching remote directory: #{url}"
247
+ logger.notify " and saving to #{dst}"
248
+ logger.notify " using command: #{wget_command}"
249
+
250
+ #in ruby 1.9+ we can upgrade this to popen3 to gain access to the subprocess pid
251
+ result = `#{wget_command} 2>&1`
252
+ result.each_line do |line|
253
+ logger.debug(line)
254
+ end
255
+ if $?.to_i != 0
256
+ raise "Failed to fetch_remote_dir '#{url}' (exit code #{$?}"
257
+ end
258
+ dst
259
+ end
260
+
187
261
  #Determine the PE package to download/upload on a mac host, download/upload that package onto the host.
188
262
  # Assumed file name format: puppet-enterprise-3.3.0-rc1-559-g97f0833-osx-10.9-x86_64.dmg.
189
263
  # @param [Host] host The mac host to download/upload and unpack PE onto
@@ -545,8 +619,12 @@ module Beaker
545
619
  on host, "apt-get install -y hiera=#{opts[:hiera_version]}-1puppetlabs1"
546
620
  end
547
621
 
548
- puppet_pkg = opts[:version] ? "puppet=#{opts[:version]}-1puppetlabs1" : 'puppet'
549
- on host, "apt-get install -y #{puppet_pkg}"
622
+ if opts[:version]
623
+ on host, "apt-get install -y puppet-common=#{opts[:version]}-1puppetlabs1"
624
+ on host, "apt-get install -y puppet=#{opts[:version]}-1puppetlabs1"
625
+ else
626
+ on host, 'apt-get install -y puppet'
627
+ end
550
628
  end
551
629
 
552
630
  # Installs Puppet and dependencies from msi
@@ -559,7 +637,7 @@ module Beaker
559
637
  # @api private
560
638
  def install_puppet_from_msi( host, opts )
561
639
  on host, "curl -O http://downloads.puppetlabs.com/windows/puppet-#{opts[:version]}.msi"
562
- on host, "msiexec /qn /i puppet-#{relver}.msi"
640
+ on host, "msiexec /qn /i puppet-#{opts[:version]}.msi"
563
641
 
564
642
  #Because the msi installer doesn't add Puppet to the environment path
565
643
  if fact_on(host, 'architecture').eql?('x86_64')
@@ -678,6 +756,148 @@ module Beaker
678
756
  do_install(sorted_hosts, options.merge({:type => :upgrade}))
679
757
  options['upgrade'] = true
680
758
  end
759
+
760
+ # Install official puppetlabs release repository configuration on host.
761
+ #
762
+ # @param [Host] host An object implementing {Beaker::Hosts}'s
763
+ # interface.
764
+ #
765
+ # @note This method only works on redhat-like and debian-like hosts.
766
+ #
767
+ def install_puppetlabs_release_repo ( host )
768
+ variant, version, arch, codename = host['platform'].to_array
769
+
770
+ case variant
771
+ when /^(fedora|el|centos)$/
772
+ variant = (($1 == 'centos') ? 'el' : $1)
773
+
774
+ rpm = options[:release_yum_repo_url] +
775
+ "/puppetlabs-release-%s-%s.noarch.rpm" % [variant, version]
776
+
777
+ on host, "rpm -ivh --force #{rpm}"
778
+
779
+ when /^(debian|ubuntu)$/
780
+ deb = options[:release_apt_repo_url] + "puppetlabs-release-%s.deb" % codename
781
+
782
+ on host, "wget -O /tmp/puppet.deb #{deb}"
783
+ on host, "dpkg -i --force-all /tmp/puppet.deb"
784
+ on host, "apt-get update"
785
+ else
786
+ raise "No repository installation step for #{variant} yet..."
787
+ end
788
+ end
789
+
790
+ # Install development repository on the given host. This method pushes all
791
+ # repository information including package files for the specified
792
+ # package_name to the host and modifies the repository configuration file
793
+ # to point at the new repository. This is particularly useful for
794
+ # installing development packages on hosts that can't access the builds
795
+ # server.
796
+ #
797
+ # @param [Host] host An object implementing {Beaker::Hosts}'s
798
+ # interface.
799
+ # @param [String] package_name The name of the package whose repository is
800
+ # being installed.
801
+ # @param [String] build_version A string identifying the output of a
802
+ # packaging job for use in looking up
803
+ # repository directory information
804
+ # @param [String] repo_configs_dir A local directory where repository files will be
805
+ # stored as an intermediate step before
806
+ # pushing them to the given host.
807
+ #
808
+ # @note This method only works on redhat-like and debian-like hosts.
809
+ #
810
+ def install_puppetlabs_dev_repo ( host, package_name, build_version,
811
+ repo_configs_dir = 'tmp/repo_configs' )
812
+ variant, version, arch, codename = host['platform'].to_array
813
+ platform_configs_dir = File.join(repo_configs_dir, variant)
814
+
815
+ # some of the uses of dev_builds_url below can't include protocol info,
816
+ # pluse this opens up possibility of switching the behavior on provided
817
+ # url type
818
+ _, protocol, hostname = options[:dev_builds_url].partition /.*:\/\//
819
+ dev_builds_url = protocol + hostname
820
+
821
+ case variant
822
+ when /^(fedora|el|centos)$/
823
+ variant = (($1 == 'centos') ? 'el' : $1)
824
+ fedora_prefix = ((variant == 'fedora') ? 'f' : '')
825
+
826
+ if host.is_pe?
827
+ pattern = "pl-%s-%s-repos-pe-%s-%s%s-%s.repo"
828
+ else
829
+ pattern = "pl-%s-%s-%s-%s%s-%s.repo"
830
+ end
831
+
832
+ repo_filename = pattern % [
833
+ package_name,
834
+ build_version,
835
+ variant,
836
+ fedora_prefix,
837
+ version,
838
+ arch
839
+ ]
840
+
841
+ repo = fetch_http_file( "%s/%s/%s/repo_configs/rpm/" %
842
+ [ dev_builds_url, package_name, build_version ],
843
+ repo_filename,
844
+ platform_configs_dir)
845
+
846
+ link = "%s/%s/%s/repos/%s/%s%s/products/%s/" %
847
+ [ dev_builds_url, package_name, build_version, variant,
848
+ fedora_prefix, version, arch ]
849
+
850
+ if not link_exists?( link )
851
+ link = "%s/%s/%s/repos/%s/%s%s/devel/%s/" %
852
+ [ dev_builds_url, package_name, build_version, variant,
853
+ fedora_prefix, version, arch ]
854
+ end
855
+
856
+ if not link_exists?( link )
857
+ raise "Unable to reach a repo directory at #{link}"
858
+ end
859
+
860
+ repo_dir = fetch_http_dir( link, platform_configs_dir )
861
+
862
+ config_dir = '/etc/yum.repos.d/'
863
+ scp_to host, repo, config_dir
864
+ scp_to host, repo_dir, '/root'
865
+
866
+ search = "baseurl\\s*=\\s*http:\\/\\/#{hostname}.*$"
867
+ replace = "baseurl=file:\\/\\/\\/root\\/#{arch}"
868
+ sed_command = "sed -i 's/#{search}/#{replace}/'"
869
+ find_and_sed = "find #{config_dir} -name \"*.repo\" -exec #{sed_command} {} \\;"
870
+
871
+ on host, find_and_sed
872
+
873
+ when /^(debian|ubuntu)$/
874
+ list = fetch_http_file( "%s/%s/%s/repo_configs/deb/" %
875
+ [ dev_builds_url, package_name, build_version ],
876
+ "pl-%s-%s-%s.list" %
877
+ [ package_name, build_version, codename ],
878
+ platform_configs_dir )
879
+
880
+ repo_dir = fetch_http_dir( "%s/%s/%s/repos/apt/%s" %
881
+ [ dev_builds_url, package_name,
882
+ build_version, codename ],
883
+ platform_configs_dir )
884
+
885
+ config_dir = '/etc/apt/sources.list.d'
886
+ scp_to host, list, config_dir
887
+ scp_to host, repo_dir, '/root'
888
+
889
+ search = "'deb\\s\\+http:\\/\\/#{hostname}.*$"
890
+ replace = "'deb file:\\/\\/\\/root\\/#{codename} #{codename} main'"
891
+ sed_command = "sed -i 's/#{search}/#{replace}/'"
892
+ find_and_sed = "find #{config_dir} -name \"*.list\" -exec #{sed_command} {} \\;"
893
+
894
+ on host, find_and_sed
895
+ on host, "apt-get update"
896
+
897
+ else
898
+ raise "No repository installation step for #{variant} yet..."
899
+ end
900
+ end
681
901
  end
682
902
  end
683
903
  end
@@ -19,8 +19,8 @@ module Unix
19
19
  h.merge({
20
20
  'user' => 'root',
21
21
  'group' => 'pe-puppet',
22
- 'service-wait' => false,
23
22
  'service-prefix'=> '/etc/init.d/',
23
+ 'master-start-curl-retries' => 120,
24
24
  'puppetservice' => 'pe-httpd',
25
25
  'puppetpath' => '/etc/puppetlabs/puppet',
26
26
  'puppetbin' => '/opt/puppet/bin/puppet',
@@ -39,7 +39,7 @@ module Unix
39
39
  h.merge({
40
40
  'user' => 'root',
41
41
  'group' => 'puppet',
42
- 'service-wait' => false,
42
+ 'master-start-curl-retries' => 120,
43
43
  'puppetpath' => '/etc/puppet',
44
44
  'puppetvardir' => '/var/lib/puppet',
45
45
  'puppetbin' => '/usr/bin/puppet',
@@ -49,17 +49,23 @@ module Unix::Pkg
49
49
  end
50
50
  end
51
51
 
52
- def install_package(name, cmdline_args = '')
52
+ def install_package(name, cmdline_args = '', version = nil)
53
53
  case self['platform']
54
54
  when /sles-/
55
55
  execute("zypper --non-interactive in #{name}")
56
56
  when /el-4/
57
57
  @logger.debug("Package installation not supported on rhel4")
58
58
  when /fedora|centos|el-/
59
+ if version
60
+ name = "#{name}-#{version}"
61
+ end
59
62
  execute("yum -y #{cmdline_args} install #{name}")
60
63
  when /ubuntu|debian/
64
+ if version
65
+ name = "#{name}=#{version}"
66
+ end
61
67
  update_apt_if_needed
62
- execute("apt-get install #{cmdline_args} -y #{name}")
68
+ execute("apt-get install --force-yes #{cmdline_args} -y #{name}")
63
69
  when /solaris-11/
64
70
  execute("pkg #{cmdline_args} install #{name}")
65
71
  when /solaris-10/
@@ -19,7 +19,6 @@ module Windows
19
19
  h.merge({
20
20
  'user' => 'Administrator',
21
21
  'group' => 'Administrators',
22
- 'service-wait' => false,
23
22
  'puppetservice' => 'pe-httpd',
24
23
  'puppetpath' => '`cygpath -smF 35`/PuppetLabs/puppet/etc',
25
24
  'puppetvardir' => '`cygpath -smF 35`/PuppetLabs/puppet/var',
@@ -91,7 +91,7 @@ module Beaker
91
91
  host.install_package pkg
92
92
  end
93
93
  end
94
- when host['platform'] !~ /aix|solaris|windows|sles-/
94
+ when host['platform'] !~ /aix|solaris|windows|sles-|osx-/
95
95
  UNIX_PACKAGES.each do |pkg|
96
96
  if not host.check_for_package pkg
97
97
  host.install_package pkg
@@ -103,48 +103,6 @@ module Beaker
103
103
  report_and_raise(logger, e, "validate")
104
104
  end
105
105
 
106
- #Update /etc/hosts on the master node to include a rule for lookup of the master by name/ip.
107
- # @param [Host, Array<Host>, String, Symbol] hosts One or more hosts to act upon
108
- # @param [Hash{Symbol=>String}] opts Options to alter execution.
109
- # @option opts [Beaker::Logger] :logger A {Beaker::Logger} object
110
- def add_master_entry hosts, opts
111
- logger = opts[:logger]
112
- master = only_host_with_role(hosts, :master)
113
- logger.notify "Add Master entry to /etc/hosts on #{master.name}"
114
- if master["hypervisor"] and master["hypervisor"] =~ /docker/
115
- # skip on docker because as of 0.8.1 /etc/hosts isn't modifiable
116
- # https://github.com/dotcloud/docker/issues/2267
117
- logger.debug "Don't update master entry on docker masters"
118
- return
119
- end
120
- if master["hypervisor"] and master["hypervisor"] =~ /vagrant/
121
- logger.debug "Don't update master entry on vagrant masters"
122
- return
123
- end
124
- logger.debug "Get ip address of Master #{master}"
125
- if master['platform'].include? 'solaris'
126
- stdout = master.exec(Command.new("ifconfig -a inet| awk '/broadcast/ {print $2}' | cut -d/ -f1 | head -1")).stdout
127
- else
128
- stdout = master.exec(Command.new("ip a|awk '/global/{print$2}' | cut -d/ -f1 | head -1")).stdout
129
- end
130
- ip=stdout.chomp
131
-
132
- path = ETC_HOSTS_PATH
133
- if master['platform'].include? 'solaris'
134
- path = ETC_HOSTS_PATH_SOLARIS
135
- end
136
-
137
- logger.debug "Update %s on #{master}" % path
138
- # Preserve the mode the easy way...
139
- master.exec(Command.new("cp %s %s.old" % [path, path]))
140
- master.exec(Command.new("cp %s %s.new" % [path, path]))
141
- master.exec(Command.new("grep -v '#{ip} #{master}' %s > %s.new" % [path, path]))
142
- master.exec(Command.new("echo '#{ip} #{master}' >> %s.new" % path))
143
- master.exec(Command.new("mv %s.new %s" % [path, path]))
144
- rescue => e
145
- report_and_raise(logger, e, "add_master_entry")
146
- end
147
-
148
106
  #Install a set of authorized keys using {HostPrebuiltSteps::ROOT_KEYS_SCRIPT}. This is a
149
107
  #convenience method to allow for easy login to hosts after they have been provisioned with
150
108
  #Beaker.
@@ -88,9 +88,6 @@ module Beaker
88
88
  if @options[:add_el_extras]
89
89
  add_el_extras(@hosts, @options)
90
90
  end
91
- if @options[:add_master_entry]
92
- add_master_entry(@hosts, @options)
93
- end
94
91
  if @options[:package_proxy]
95
92
  package_proxy(@hosts, @options)
96
93
  end
@@ -35,7 +35,12 @@ module Beaker
35
35
  container.start({"PublishAllPorts" => true, "Privileged" => true})
36
36
 
37
37
  # Find out where the ssh port is from the container
38
- ip = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostIp"]
38
+ if ENV['DOCKER_HOST']
39
+ ip = URI.parse(ENV['DOCKER_HOST']).host
40
+ @logger.info("Using docker server at #{ip}")
41
+ else
42
+ ip = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostIp"]
43
+ end
39
44
  port = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostPort"]
40
45
 
41
46
  # Update host metadata
data/lib/beaker/logger.rb CHANGED
@@ -133,6 +133,25 @@ module Beaker
133
133
  LOG_LEVELS[@log_level] >= LOG_LEVELS[:notify]
134
134
  end
135
135
 
136
+ # Remove invalid UTF-8 codes from provided string(s)
137
+ # @param [String, Array<String>] string The string(s) to remove invalid codes from
138
+ def convert string
139
+ if string.kind_of?(Array)
140
+ string.map do |s|
141
+ convert s
142
+ end
143
+ else
144
+ if string.respond_to?( :encode )
145
+ # We're running in >= 1.9 and we'll need to convert
146
+ # Remove invalide and undefined UTF-8 character encodings
147
+ string.encode('UTF-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')
148
+ else
149
+ # We're running 1.8, do nothing
150
+ string
151
+ end
152
+ end
153
+ end
154
+
136
155
  # Custom reporting for messages generated by host SUTs.
137
156
  # Will not print unless we are at {LOG_LEVELS} 'verbose' or higher.
138
157
  # Strips any color codes already in the provided messages, then adds logger color codes before reporting
@@ -196,8 +215,8 @@ module Beaker
196
215
  # @param [String] lines A single or array of lines to removed color codes from
197
216
  # @return [Array<String>] An array of strings that do not have color codes
198
217
  def strip_colors_from lines
199
- Array(lines).map do |line|
200
- line.gsub /\e\[(\d+;)?\d+m/, ''
218
+ Array( lines ).map do |line|
219
+ convert(line).gsub(/(\e|\^\[)\[(\d*;)*\d*m/, '')
201
220
  end
202
221
  end
203
222
 
@@ -209,7 +228,7 @@ module Beaker
209
228
  print_statement = add_newline ? :puts : :print
210
229
  @destinations.each do |to|
211
230
  to.print color_code if @color
212
- to.send print_statement, msg
231
+ to.send print_statement, convert( msg )
213
232
  to.print NORMAL if @color
214
233
  end
215
234
  end