vmfloaty 0.9.1 → 1.0.0

data/lib/vmfloaty/service.rb

@@ -7,11 +7,13 @@ require 'vmfloaty/ssh'
 
 class Service
   attr_reader :config
+  attr_accessor :silent
 
   def initialize(options, config_hash = {})
     options ||= Commander::Command::Options.new
     @config = Utils.get_service_config config_hash, options
     @service_object = Utils.get_service_object @config['type']
+    @silent = false
   end
 
   def method_missing(method_name, *args, &block)
@@ -36,7 +38,7 @@ class Service
 
   def user
     unless @config['user']
-      puts "Enter your #{@config['url']} service username:"
+      FloatyLogger.info "Enter your #{@config['url']} service username:"
       @config['user'] = STDIN.gets.chomp
     end
     @config['user']
@@ -44,7 +46,7 @@ class Service
 
   def token
     unless @config['token']
-      puts 'No token found. Retrieving a token...'
+      FloatyLogger.info 'No token found. Retrieving a token...'
       @config['token'] = get_new_token(nil)
     end
     @config['token']
@@ -75,10 +77,14 @@ class Service
     @service_object.list_active verbose, url, token, user
   end
 
-  def retrieve(verbose, os_types, use_token = true)
-    puts 'Requesting a vm without a token...' unless use_token
+  def retrieve(verbose, os_types, use_token = true, ondemand = nil)
+    FloatyLogger.info 'Requesting a vm without a token...' unless use_token
     token_value = use_token ? token : nil
-    @service_object.retrieve verbose, os_types, token_value, url, user, @config
+    @service_object.retrieve verbose, os_types, token_value, url, user, @config, ondemand
+  end
+
+  def wait_for_request(verbose, requestid)
+    @service_object.wait_for_request verbose, requestid, url
   end
 
   def ssh(verbose, host_os, use_token = true)
@@ -87,20 +93,11 @@ class Service
       begin
         token_value = token || get_new_token(verbose)
       rescue TokenError => e
-        STDERR.puts e
-        STDERR.puts 'Could not get token... requesting vm without a token anyway...'
+        FloatyLogger.error e
+        FloatyLogger.info 'Could not get token... requesting vm without a token anyway...'
       end
     end
-    Ssh.ssh(verbose, host_os, token_value, url)
-  end
-
-  def pretty_print_running(verbose, hostnames = [])
-    if hostnames.empty?
-      puts 'You have no running VMs.'
-    else
-      puts 'Running VMs:'
-      @service_object.pretty_print_hosts(verbose, hostnames, url)
-    end
+    Ssh.ssh(verbose, self, host_os, token_value)
   end
 
   def query(verbose, hostname)
@@ -108,6 +105,7 @@ class Service
   end
 
   def modify(verbose, hostname, modify_hash)
+    maybe_use_vmpooler
     @service_object.modify verbose, url, hostname, token, modify_hash
   end
 
@@ -120,18 +118,35 @@ class Service
   end
 
   def summary(verbose)
+    maybe_use_vmpooler
     @service_object.summary verbose, url
   end
 
   def snapshot(verbose, hostname)
+    maybe_use_vmpooler
     @service_object.snapshot verbose, url, hostname, token
   end
 
   def revert(verbose, hostname, snapshot_sha)
+    maybe_use_vmpooler
     @service_object.revert verbose, url, hostname, token, snapshot_sha
   end
 
   def disk(verbose, hostname, disk)
+    maybe_use_vmpooler
     @service_object.disk(verbose, url, hostname, token, disk)
   end
+
+  # some methods do not exist for ABS, and if possible should target the Pooler service
+  def maybe_use_vmpooler
+    if @service_object.is_a?(ABS.class)
+      if !self.silent
+        FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using fallback_vmpooler config from ~/.vmfloaty.yml"
+        self.silent = true
+      end
+
+      @config = Utils.get_vmpooler_service_config(@config['vmpooler_fallback'])
+      @service_object = Pooler
+    end
+  end
 end

data/lib/vmfloaty/ssh.rb

@@ -14,21 +14,27 @@ class Ssh
     nil
   end
 
-  def self.ssh(verbose, host_os, token, url)
+  def self.command_string(verbose, service, host_os, use_token)
     ssh_path = which('ssh')
     raise 'Could not determine path to ssh' unless ssh_path
 
     os_types = {}
     os_types[host_os] = 1
 
-    response = Pooler.retrieve(verbose, os_types, token, url)
-    raise "Could not get vm from vmpooler:\n #{response}" unless response['ok']
+    response = service.retrieve(verbose, os_types, use_token)
+    raise "Could not get vm from #{service.type}:\n #{response}" unless response['ok']
 
     user = /win/.match?(host_os) ? 'Administrator' : 'root'
 
-    hostname = "#{response[host_os]['hostname']}.#{response['domain']}"
-    cmd = "#{ssh_path} #{user}@#{hostname}"
+    hostname = response[host_os]['hostname']
+    hostname = response[host_os]['hostname'][0] if response[host_os]['hostname'].is_a?(Array)
+    hostname = "#{hostname}.#{response['domain']}" unless hostname.end_with?('puppetlabs.net')
 
+    "#{ssh_path} #{user}@#{hostname}"
+  end
+
+  def self.ssh(verbose, service, host_os, use_token)
+    cmd = command_string(verbose, service, host_os, use_token)
     # TODO: Should this respect more ssh settings? Can it be configured
     #       by users ssh config and does this respect those settings?
     Kernel.exec(cmd)

data/lib/vmfloaty/utils.rb

@@ -3,6 +3,7 @@
 require 'vmfloaty/abs'
 require 'vmfloaty/nonstandard_pooler'
 require 'vmfloaty/pooler'
+require 'vmfloaty/conf'
 
 class Utils
   # TODO: Takes the json response body from an HTTP GET
@@ -45,7 +46,12 @@ class Utils
 
     result = {}
 
-    response_body.each do |os, value|
+    # ABS has a job_id associated with hosts so pass that along
+    abs_job_id = response_body.delete('job_id')
+    result['job_id'] = abs_job_id unless abs_job_id.nil?
+
+    filtered_response_body = response_body.reject { |key, _| key == 'request_id' || key == 'ready' }
+    filtered_response_body.each do |os, value|
       hostnames = Array(value['hostname'])
       hostnames.map! { |host| "#{host}.#{domain}" } if domain
       result[os] = hostnames
@@ -56,7 +62,8 @@ class Utils
 
   def self.format_host_output(hosts)
     hosts.flat_map do |os, names|
-      names.map { |name| "- #{name} (#{os})" }
+      # Assume hosts are stored in Arrays and ignore everything else
+      names.map { |name| "- #{name} (#{os})" } if names.is_a? Array
     end.join("\n")
   end
 
@@ -77,41 +84,96 @@ class Utils
     os_types
   end
 
-  def self.pretty_print_hosts(verbose, service, hostnames = [])
+  def self.print_fqdn_for_host(service, hostname, host_data)
+    case service.type
+    when 'ABS'
+      abs_hostnames = []
+
+      host_data['allocated_resources'].each do |vm_name, _i|
+        abs_hostnames << vm_name['hostname']
+      end
+
+      puts abs_hostnames.join("\n")
+    when 'Pooler'
+      puts "#{hostname}.#{host_data['domain']}"
+    when 'NonstandardPooler'
+      puts host_data['fqdn']
+    else
+      raise "Invalid service type #{service.type}"
+    end
+  end
+
+  def self.pretty_print_hosts(verbose, service, hostnames = [], print_to_stderr = false, indent = 0)
+    output_target = print_to_stderr ? $stderr : $stdout
+
+    fetched_data = self.get_host_data(verbose, service, hostnames)
+    fetched_data.each do |hostname, host_data|
+      case service.type
+      when 'ABS'
+        # For ABS, 'hostname' variable is the jobID
+        #
+        # Create a vmpooler service to query each hostname there so as to get the metadata too
+
+        output_target.puts "- [JobID:#{host_data['request']['job']['id']}] <#{host_data['state']}>"
+        host_data['allocated_resources'].each do |allocated_resources, _i|
+          if allocated_resources['engine'] == "vmpooler"
+            vmpooler_service = service.clone
+            vmpooler_service.silent = true
+            vmpooler_service.maybe_use_vmpooler
+            self.pretty_print_hosts(verbose, vmpooler_service, allocated_resources['hostname'].split('.')[0], print_to_stderr, indent+2)
+          else
+            #TODO we could add more specific metadata for the other services, nspooler and aws
+            output_target.puts "  - #{allocated_resources['hostname']} (#{allocated_resources['type']})"
+          end
+        end
+      when 'Pooler'
+        tag_pairs = []
+        tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil?
+        duration = "#{host_data['running']}/#{host_data['lifetime']} hours"
+        metadata = [host_data['template'], duration, *tag_pairs]
+        output_target.puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})".gsub(/^/, ' ' * indent)
+      when 'NonstandardPooler'
+        line = "- #{host_data['fqdn']} (#{host_data['os_triple']}"
+        line += ", #{host_data['hours_left_on_reservation']}h remaining"
+        line += ", reason: #{host_data['reserved_for_reason']}" unless host_data['reserved_for_reason'].empty?
+        line += ')'
+        output_target.puts line
+      else
+        raise "Invalid service type #{service.type}"
+      end
+    end
+  end
+
+  def self.get_host_data(verbose, service, hostnames = [])
+    result = {}
     hostnames = [hostnames] unless hostnames.is_a? Array
     hostnames.each do |hostname|
       begin
         response = service.query(verbose, hostname)
         host_data = response[hostname]
-
-        case service.type
-        when 'ABS'
-          # For ABS, 'hostname' variable is the jobID
-          if host_data['state'] == 'allocated' || host_data['state'] == 'filled'
-            host_data['allocated_resources'].each do |vm_name, _i|
-              puts "- [JobID:#{host_data['request']['job']['id']}] #{vm_name['hostname']} (#{vm_name['type']}) <#{host_data['state']}>"
+        if block_given?
+          yield host_data result
+        else
+          case service.type
+          when 'ABS'
+            # For ABS, 'hostname' variable is the jobID
+            if host_data['state'] == 'allocated' || host_data['state'] == 'filled'
+              result[hostname] = host_data
             end
+          when 'Pooler'
+            result[hostname] = host_data
+          when 'NonstandardPooler'
+            result[hostname] = host_data
+          else
+            raise "Invalid service type #{service.type}"
           end
-        when 'Pooler'
-          tag_pairs = []
-          tag_pairs = host_data['tags'].map { |key, value| "#{key}: #{value}" } unless host_data['tags'].nil?
-          duration = "#{host_data['running']}/#{host_data['lifetime']} hours"
-          metadata = [host_data['template'], duration, *tag_pairs]
-          puts "- #{hostname}.#{host_data['domain']} (#{metadata.join(', ')})"
-        when 'NonstandardPooler'
-          line = "- #{host_data['fqdn']} (#{host_data['os_triple']}"
-          line += ", #{host_data['hours_left_on_reservation']}h remaining"
-          line += ", reason: #{host_data['reserved_for_reason']}" unless host_data['reserved_for_reason'].empty?
-          line += ')'
-          puts line
-        else
-          raise "Invalid service type #{service.type}"
         end
       rescue StandardError => e
-        STDERR.puts("Something went wrong while trying to gather information on #{hostname}:")
-        STDERR.puts(e)
+        FloatyLogger.error("Something went wrong while trying to gather information on #{hostname}:")
+        FloatyLogger.error(e)
       end
     end
+    result
   end
 
   def self.pretty_print_status(verbose, service)
@@ -133,7 +195,7 @@ class Utils
           char = 'o'
           puts "#{name.ljust(width)} #{(char * ready).green}#{(char * pending).yellow}#{(char * missing).red}"
         rescue StandardError => e
-          puts "#{name.ljust(width)} #{e.red}"
+          FloatyLogger.error "#{name.ljust(width)} #{e.red}"
         end
       end
       puts message.colorize(status_response['status']['ok'] ? :default : :red)
@@ -152,11 +214,11 @@ class Utils
           char = 'o'
           puts "#{name.ljust(width)} #{(char * ready).green}#{(char * pending).yellow}#{(char * missing).red}"
         rescue StandardError => e
-          puts "#{name.ljust(width)} #{e.red}"
+          FloatyLogger.error "#{name.ljust(width)} #{e.red}"
         end
       end
     when 'ABS'
-      puts 'ABS Not OK'.red unless status_response
+      FloatyLogger.error 'ABS Not OK' unless status_response
       puts 'ABS is OK'.green if status_response
     else
       raise "Invalid service type #{service.type}"
@@ -172,12 +234,15 @@ class Utils
   end
 
   def self.get_service_object(type = '')
-    nspooler_strings = %w[ns nspooler nonstandard nonstandard_pooler]
     abs_strings = %w[abs alwaysbescheduling always_be_scheduling]
-    if nspooler_strings.include? type.downcase
-      NonstandardPooler
-    elsif abs_strings.include? type.downcase
+    nspooler_strings = %w[ns nspooler nonstandard nonstandard_pooler]
+    vmpooler_strings = %w[vmpooler]
+    if abs_strings.include? type.downcase
       ABS
+    elsif nspooler_strings.include? type.downcase
+      NonstandardPooler
+    elsif vmpooler_strings.include? type.downcase
+      Pooler
     else
       Pooler
     end
@@ -205,6 +270,9 @@ class Utils
         # If the service is configured but some values are missing, use the top-level defaults to fill them in
         service_config.merge! config['services'][options.service]
       end
+    # No config file but service is declared on command line
+    elsif !config['services'] && options.service
+      service_config['type'] = options.service
     end
 
     # Prioritize an explicitly specified url, user, or token if the user provided one
@@ -215,4 +283,30 @@ class Utils
 
     service_config
   end
+
+  # This method gets the vmpooler service configured in ~/.vmfloaty
+  def self.get_vmpooler_service_config(vmpooler_fallback)
+    config = Conf.read_config
+    # The top-level url, user, and token values in the config file are treated as defaults
+    service_config = {
+        'url'   => config['url'],
+        'user'  => config['user'],
+        'token' => config['token'],
+        'type'  => 'vmpooler',
+    }
+
+    # at a minimum, the url needs to be configured
+    if config['services'] && config['services'][vmpooler_fallback] && config['services'][vmpooler_fallback]['url']
+      # If the service is configured but some values are missing, use the top-level defaults to fill them in
+      service_config.merge! config['services'][vmpooler_fallback]
+    else
+      if vmpooler_fallback.nil?
+        raise ArgumentError, "The abs service should have a key named 'vmpooler_fallback' in ~/.vmfloaty.yml with a value that points to a vmpooler service name use this format:\nservices:\n  myabs:\n    url: 'http://abs.com'\n    user: 'superman'\n    token: 'kryptonite'\n    vmpooler_fallback: 'myvmpooler'\n  myvmpooler:\n    url: 'http://vmpooler.com'\n    user: 'superman'\n    token: 'kryptonite'"
+      else
+        raise ArgumentError, "Could not find a configured service named '#{vmpooler_fallback}' in ~/.vmfloaty.yml use this format:\nservices:\n  #{vmpooler_fallback}:\n    url: 'http://vmpooler.com'\n    user: 'superman'\n    token: 'kryptonite'"
+      end
+    end
+
+    service_config
+  end
 end

data/lib/vmfloaty/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 class Vmfloaty
-  VERSION = '0.9.1'
+  VERSION = '1.0.0'
 end
+

data/spec/spec_helper.rb

@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+require 'simplecov'
+require 'coveralls'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+])
+SimpleCov.start do
+  add_filter %r{^/spec/}
+end
+
 require 'vmfloaty'
 require 'webmock/rspec'
 

data/spec/vmfloaty/abs_spec.rb

@@ -9,15 +9,60 @@ describe ABS do
   before :each do
   end
 
+  describe '#list' do
+    it 'skips empty platforms and lists aws' do
+      stub_request(:get, "http://foo/status/platforms/vmpooler").
+          to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "http://foo/status/platforms/ondemand_vmpooler").
+          to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "http://foo/status/platforms/nspooler").
+          to_return(:status => 200, :body => "", :headers => {})
+      body = '{
+                "aws_platforms": [
+                  "amazon-6-x86_64",
+                  "amazon-7-x86_64",
+                  "amazon-7-arm64",
+                  "centos-7-x86-64-west",
+                  "redhat-8-arm64"
+                ]
+              }'
+      stub_request(:get, "http://foo/status/platforms/aws").
+          to_return(:status => 200, :body => body, :headers => {})
+
+
+      results = ABS.list(false, "http://foo")
+
+      expect(results).to include("amazon-6-x86_64", "amazon-7-x86_64", "amazon-7-arm64", "centos-7-x86-64-west", "redhat-8-arm64")
+    end
+    it 'legacy JSON string, prior to PR 306' do
+      stub_request(:get, "http://foo/status/platforms/vmpooler").
+          to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "http://foo/status/platforms/ondemand_vmpooler").
+          to_return(:status => 200, :body => "", :headers => {})
+      stub_request(:get, "http://foo/status/platforms/nspooler").
+          to_return(:status => 200, :body => "", :headers => {})
+      body = '{
+          "aws_platforms": "[\"amazon-6-x86_64\",\"amazon-7-x86_64\",\"amazon-7-arm64\",\"centos-7-x86-64-west\",\"redhat-8-arm64\"]"
+      }'
+      stub_request(:get, "http://foo/status/platforms/aws").
+          to_return(:status => 200, :body => body, :headers => {})
+
+      results = ABS.list(false, "http://foo")
+
+      expect(results).to include("amazon-6-x86_64", "amazon-7-x86_64", "amazon-7-arm64", "centos-7-x86-64-west", "redhat-8-arm64")
+    end
+  end
+
   describe '#format' do
-    it 'returns an hash formatted like a vmpooler return' do
+    it 'returns an hash formatted like a vmpooler return, plus the job_id' do
+      job_id = "generated_by_floaty_12345"
       abs_formatted_response = [
         { 'hostname' => 'aaaaaaaaaaaaaaa.delivery.puppetlabs.net', 'type' => 'centos-7.2-x86_64', 'engine' => 'vmpooler' },
         { 'hostname' => 'aaaaaaaaaaaaaab.delivery.puppetlabs.net', 'type' => 'centos-7.2-x86_64', 'engine' => 'vmpooler' },
         { 'hostname' => 'aaaaaaaaaaaaaac.delivery.puppetlabs.net', 'type' => 'ubuntu-7.2-x86_64', 'engine' => 'vmpooler' },
       ]
 
-      vmpooler_formatted_response = ABS.translated(abs_formatted_response)
+      vmpooler_formatted_response = ABS.translated(abs_formatted_response, job_id)
 
       vmpooler_formatted_compare = {
         'centos-7.2-x86_64' => {},
@@ -29,6 +74,8 @@ describe ABS do
 
       vmpooler_formatted_compare['ok'] = true
 
+      vmpooler_formatted_compare['job_id'] = job_id
+
       expect(vmpooler_formatted_response).to eq(vmpooler_formatted_compare)
       vmpooler_formatted_response.delete('ok')
       vmpooler_formatted_compare.delete('ok')
@@ -65,14 +112,14 @@ describe ABS do
 
     describe '#test_abs_status_queue_endpoint' do
       before :each do
-        # rubocop:disable Metrics/LineLength
+        # rubocop:disable Layout/LineLength
         @active_requests_response = '
         [
-          "{ \"state\":\"allocated\",\"last_processed\":\"2019-12-16 23:00:34 +0000\",\"allocated_resources\":[{\"hostname\":\"take-this.delivery.puppetlabs.net\",\"type\":\"win-2012r2-x86_64\",\"engine\":\"vmpooler\"}],\"audit_log\":{\"2019-12-13 16:45:29 +0000\":\"Allocated take-this.delivery.puppetlabs.net for job 1576255517241\"},\"request\":{\"resources\":{\"win-2012r2-x86_64\":1},\"job\":{\"id\":\"1576255517241\",\"tags\":{\"user\":\"test-user\"},\"user\":\"test-user\",\"time-received\":1576255519},\"priority\":1}}",
+          { "state":"allocated","last_processed":"2019-12-16 23:00:34 +0000","allocated_resources":[{"hostname":"take-this.delivery.puppetlabs.net","type":"win-2012r2-x86_64","engine":"vmpooler"}],"audit_log":{"2019-12-13 16:45:29 +0000":"Allocated take-this.delivery.puppetlabs.net for job 1576255517241"},"request":{"resources":{"win-2012r2-x86_64":1},"job":{"id":"1576255517241","tags":{"user":"test-user"},"user":"test-user","time-received":1576255519},"priority":1}},
           "null",
-          "{\"state\":\"allocated\",\"last_processed\":\"2019-12-16 23:00:34 +0000\",\"allocated_resources\":[{\"hostname\":\"not-this.delivery.puppetlabs.net\",\"type\":\"win-2012r2-x86_64\",\"engine\":\"vmpooler\"}],\"audit_log\":{\"2019-12-13 16:46:14 +0000\":\"Allocated not-this.delivery.puppetlabs.net for job 1576255565159\"},\"request\":{\"resources\":{\"win-2012r2-x86_64\":1},\"job\":{\"id\":\"1576255565159\",\"tags\":{\"user\":\"not-test-user\"},\"user\":\"not-test-user\",\"time-received\":1576255566},\"priority\":1}}"
+          {"state":"allocated","last_processed":"2019-12-16 23:00:34 +0000","allocated_resources":[{"hostname":"not-this.delivery.puppetlabs.net","type":"win-2012r2-x86_64","engine":"vmpooler"}],"audit_log":{"2019-12-13 16:46:14 +0000":"Allocated not-this.delivery.puppetlabs.net for job 1576255565159"},"request":{"resources":{"win-2012r2-x86_64":1},"job":{"id":"1576255565159","tags":{"user":"not-test-user"},"user":"not-test-user","time-received":1576255566},"priority":1}}
         ]'
-        # rubocop:enable Metrics/LineLength
+        # rubocop:enable Layout/LineLength
         @token = 'utpg2i2xswor6h8ttjhu3d47z53yy47y'
         @test_user = 'test-user'
       end
@@ -92,5 +139,35 @@ describe ABS do
         )
       end
     end
+
+    describe '#test_abs_delete_jobid' do
+      before :each do
+        # rubocop:disable Layout/LineLength
+        @active_requests_response = '
+        [
+          { "state":"allocated", "last_processed":"2020-01-17 22:29:13 +0000", "allocated_resources":[{"hostname":"craggy-chord.delivery.puppetlabs.net", "type":"centos-7-x86_64", "engine":"vmpooler"}, {"hostname":"visible-revival.delivery.puppetlabs.net", "type":"centos-7-x86_64", "engine":"vmpooler"}], "audit_log":{"2020-01-17 22:28:45 +0000":"Allocated craggy-chord.delivery.puppetlabs.net, visible-revival.delivery.puppetlabs.net for job 1579300120799"}, "request":{"resources":{"centos-7-x86_64":2}, "job":{"id":"1579300120799", "tags":{"user":"test-user"}, "user":"test-user", "time-received":1579300120}, "priority":3}}
+        ]'
+        @return_request = { '{"job_id":"1579300120799","hosts":{"hostname":"craggy-chord.delivery.puppetlabs.net","type":"centos-7-x86_64","engine":"vmpooler"},{"hostname":"visible-revival.delivery.puppetlabs.net","type":"centos-7-x86_64","engine":"vmpooler"}}'=>true }
+        # rubocop:enable Layout/LineLength
+        @token = 'utpg2i2xswor6h8ttjhu3d47z53yy47y'
+        @test_user = 'test-user'
+        # Job ID
+        @hosts = ['1579300120799']
+      end
+
+      it 'will delete the whole job' do
+        stub_request(:get, 'https://abs.example.com/status/queue')
+          .to_return(:status => 200, :body => @active_requests_response, :headers => {})
+        stub_request(:post, 'https://abs.example.com/return')
+          .with(:body => @return_request)
+          .to_return(:status => 200, :body => 'OK', :headers => {})
+
+        ret = ABS.delete(false, @abs_url, @hosts, @token, @test_user)
+
+        expect(ret).to include(
+          'craggy-chord.delivery.puppetlabs.net' => { 'ok'=>true }, 'visible-revival.delivery.puppetlabs.net' => { 'ok'=>true },
+        )
+      end
+    end
   end
 end