beaker-google 0.1.0 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZmUwMDUwOWJjOWQ0ZDA4NjA4OTExYTdiNzM0ODRjN2M5NDQ2OWQ3OA==
-  data.tar.gz: !binary |-
-    NDkzYzcyNDQ3OWVhZGY2NzdjOGNmZmRkYTY4NzE2NmQxZmY5YjM0Ng==
+SHA1:
+  metadata.gz: 6e55acfef7e248d670044f6b0c4b4c074a519995
+  data.tar.gz: 8eb716c6f275f48c984dac7a6333801ffa59084c
 SHA512:
-  metadata.gz: !binary |-
-    YmQ5MDFiZGJhMWY0OWIyZjgyNTc3OTM0NDliOGIzYmJiMjVhYzhmNjJlYzc4
-    YzQ4ZmFiZWI4NzQ0ZTFkYjEyZjlhY2ZlZDk4ZWZhMjRhN2FlOTY0NGFkODE2
-    NDE0ZDc2MGMwMDcyNDM0ZGIwNmVlZWUxMjNhMGY3MzM2MTAwNTY=
-  data.tar.gz: !binary |-
-    MzMwNzc3MGY2NzNmZjQyMmY5N2ViZTk3NWUzZTAxN2JiMzBlM2MxNjAwZmE5
-    NWVlNGJiODY0ZWM3YzdkYTJjZWNiMGUwMjMyM2RjN2YxZjU2NzcyZjZlODMz
-    NDczYWY5OGE1YWQ3NmNlYTI4MjcyZWFkNWE1OGNjOWVkOWQyNDY=
+  metadata.gz: 9ac89dd3dec4ba387d25d7e0405dfa46033ac5fc95b68105ba09951ef724797166db9a1160e391e19db7b59180a3a8a1e99520220b58f8d78dcab74a9761bd3e
+  data.tar.gz: 4e05347be2fe493f0c77d2fc3a9591af27e5d09b73c1c3f07bd68793872011ca82aa5a21cdfacf33bb87f2e7af9a75c9f3a33e3ba9966b8479f90f070ac2fc21

data/README.md

@@ -1,21 +1,39 @@
 # beaker-google
 
-Beaker library to use google hypervisor
+Beaker library to use the Google hypervisor
 
 # How to use this wizardry
 
-This is a gem that allows you to use hosts with [google compute](google_compute_engine.md) hypervisor with [beaker](https://github.com/puppetlabs/beaker). 
+This is a gem that allows you to use hosts with [google compute](google_compute_engine.md) hypervisor with [Beaker](https://github.com/puppetlabs/beaker).
 
-### Right Now? (beaker 3.x)
+See the [documentation](docs/manual.md) for the full manual.
 
-This gem is already included as [beaker dependency](https://github.com/puppetlabs/beaker/blob/master/beaker.gemspec) for you, so you don't need to do anything special to use this gem's functionality with beaker.
+Beaker will automatically load the appropriate hypervisors for any given hosts file, so as long as your project dependencies are satisfied there's nothing else to do. No need to `require` this library in your tests.
 
-### In beaker's Next Major Version? (beaker 4.x)
+## With Beaker 3.x
+This gem is already included as [beaker dependency](https://github.com/puppetlabs/beaker/blob/master/beaker.gemspec)
+for you, so you don't need to do anything special to use this gem's
+functionality with Beaker.
 
-In beaker's next major version, the requirement for beaker-google will be pulled
-from that repo. When that happens, then the usage pattern will change. In order
-to use this then, you'll need to include beaker-google as a dependency right
-next to beaker itself.
+This library is included as a dependency of Beaker 3.x versions, so there's nothing to do.
+
+## With Beaker 4.x
+
+As of Beaker 4.0, all hypervisor and DSL extension libraries have been removed and are no longer dependencies. In order to use a specific hypervisor or DSL extension library in your project, you will need to include them alongside Beaker in your Gemfile or project.gemspec. E.g.
+
+~~~ruby
+# Gemfile
+gem 'beaker', '~>4.0'
+gem 'beaker-aws'
+# project.gemspec
+s.add_runtime_dependency 'beaker', '~>4.0'
+s.add_runtime_dependency 'beaker-aws'
+~~~
+
+In Beaker's next major version, the requirement for `beaker-google` will be
+pulled from that repo. When that happens, then the usage pattern will change.
+In order to use this then, you'll need to include `beaker-google` as a dependency
+right next to beaker itself.
 
 # Contributing
 

data/beaker-google.gemspec

@@ -20,7 +20,12 @@ Gem::Specification.new do |s|
   # Testing dependencies
   s.add_development_dependency 'rspec', '~> 3.0'
   s.add_development_dependency 'rspec-its'
-  s.add_development_dependency 'fakefs', '~> 0.6'
+  # pin fakefs for Ruby < 2.3
+  if RUBY_VERSION < "2.3"
+    s.add_development_dependency 'fakefs', '~> 0.6', '< 0.14'
+  else
+    s.add_development_dependency 'fakefs', '~> 0.6'
+  end
   s.add_development_dependency 'rake', '~> 10.1'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'pry', '~> 0.10'
@@ -32,7 +37,7 @@ Gem::Specification.new do |s|
 
   # Run time dependencies
   s.add_runtime_dependency 'stringify-hash', '~> 0.0.0'
-  s.add_runtime_dependency 'google-api-client', '~> 0.9'
+  s.add_runtime_dependency 'google-api-client', '~> 0.8.0'
 
 end
 

data/docs/manual.md

@@ -0,0 +1,122 @@
+# Using the Google Compute Engine Provisioner
+
+This [Beaker](https://github.com/puppetlabs/beaker) provisioner manages instances from pre-existing [Google Compute Engine](https://cloud.google.com/compute) images.
+
+Currently supported [GCE images](https://cloud.google.com/compute/docs/images#os-compute-support) include:
+  * Debian
+  * CentOS
+  * Red Hat Enterprise Linux
+
+## Prerequisites
+
+  * A [Google Compute Engine project](https://cloud.google.com/resource-manager/docs/creating-managing-projects)
+  * An active [service account](https://cloud.google.com/compute/docs/access/service-accounts)
+    to your Google Compute Engine project, along with the following
+    information:
+    * The service account **private key file** (named xxx-privatekey.p12).
+    * The service account **email address** (named xxx@developer.gserviceaccount.com).
+    * The service account **password**.
+    * A **ssh keypair**, [set up for GCE](https://developers.google.com/compute/docs/console#sshkeys)
+      * Load the private key into your `ssh-agent` prior to running Beaker (or
+        leave off a password, but this is not recommended)
+      * Name the pair `google_compute_engine`
+        * You can tell Beaker to use a different public key by setting `BEAKER_gce_ssh_public_key` to the path of your public key file
+      * Place the [public key](https://git-scm.com/book/gr/v2/Git-on-the-Server-Generating-Your-SSH-Public-Key) in your Google Compute Engine [project metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata)
+        * `key`: `sshKeys`
+        * `value` is the contents of your `google_compute.pub` with `google_compute:` _prepended_, eg: `google_compute:ssh-rsaAAA(...) user@machine.local`
+
+## Setting Up Beaker
+
+You are now ready to set up the Beaker [nodeset](https://github.com/puppetlabs/beaker-rspec#typical-workflow) for your GCE instances
+
+### Example GCE Hosts File
+
+  ```yaml
+  HOSTS:
+    debian-7-master:
+      roles:
+        - server
+        - master
+        - agent
+      platform: debian-7-x86_64
+      hypervisor: google
+      # The GCE image name to use
+      image: debian-7-XXXX
+
+    centos-6-agent:
+      roles:
+        - agent
+      platform: el-6-x86_64
+      hypervisor: google
+      # The GCE image name to use
+      image: centos-6-XXXX
+      # The GCE Machine type to use
+      gce_machine_type: n1-highmem-2
+
+    redhat-7-agent:
+      roles:
+        - agent
+      platform: el-7-x86_64
+      hypervisor: google
+      # The GCE image name to use
+      image: redhat-7-XXXX
+
+  CONFIG:
+    type: aio
+
+    # Optional: The number of seconds to attempt execution and then quit
+    timeout: 10
+
+    # You don't have to specify this if you set the environment variable
+    # BEAKER_gce_project
+    gce_project: google-compute-project-name
+
+    # Service account key
+    #
+    # If not defined, this key will try to be found at
+    #  $HOME/.beaker/gce/project-name.p12
+    #
+    # You can also specify the key using the environment variable
+    # BEAKER_gce_keyfile
+    gce_keyfile: /path/to/*****-privatekey.p12
+
+    # This is almost always the same!
+    #
+    # You don't have to specify this unless you have a custom 'gce_password'
+    #
+    # You can also specify this by setting the environment variable
+    # BEAKER_gce_password
+    gce_password: notasecret
+
+    # Service account email
+    #
+    # You don't have to specify this if you set the environment variable
+    # BEAKER_gce_email
+    gce_email: *********@developer.gserviceaccount.com
+  ```
+
+Google Compute cloud instances and disks are deleted after test runs, but it is
+up to the owner of the Google Compute Engine project to ensure that any zombie
+instances/disks are properly removed should Beaker fail during cleanup.
+
+**NOTE:** If zombie resources are left in GCE the code helps you identify runs
+that have not been cleaned up by prefixing all of the assets with the **same**
+identifier that follows the pattern of `beaker-TIMESTAMP`.
+
+### Tips and Tricks
+
+To most easily use this plugin with GCE projects, you may want to do the following:
+
+1. `mkdir -p $HOME/.beaker/gce`
+2. `cd $HOME/.beaker/gce`
+3. Add the following to a file named `config` with your values in place
+
+   ```bash
+   export BEAKER_gce_ssh_public_key="$HOME/.beaker/gce/rsa.pub"
+   export BEAKER_gce_project='my-beaker-tests'
+   export BEAKER_gce_keyfile="$HOME/.beaker/Beaker_Testing.p12"
+   export BEAKER_gce_email='myuser@my-beaker-tests.iam.gserviceaccount.com'
+   ```
+
+4. Copy your testing account `p12` file into `$HOME/.beaker/gce/Beaker_Testing.p12`
+5. Run your test after running `source $HOME/.beaker/gce/config`

data/lib/beaker-google/version.rb

@@ -1,3 +1,3 @@
 module BeakerGoogle
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/beaker/hypervisor/google.rb

@@ -1,4 +1,4 @@
 require 'beaker/hypervisor/google_compute'
 
-class Google < GoogleCompute
+class Beaker::Google < Beaker::GoogleCompute
 end

data/lib/beaker/hypervisor/google_compute.rb

@@ -1,33 +1,65 @@
 require 'time'
 
 module Beaker
-  #Beaker support for the Google Compute Engine.
+
+  # Beaker support for the Google Compute Engine.
   class GoogleCompute < Beaker::Hypervisor
 
     SLEEPWAIT = 5
-    #number of hours before an instance is considered a zombie
+
+    # Hours before an instance is considered a zombie
     ZOMBIE = 3
 
-    #Create the array of metaData, each member being a hash with a :key and a :value.  Sets
-    #:department, :project and :jenkins_build_url.
+    # Do some reasonable sleuthing on the SSH public key for GCE
+    def find_google_ssh_public_key
+      keyfile = ENV.fetch('BEAKER_gce_ssh_public_key', File.join(ENV['HOME'], '.ssh', 'google_compute_engine.pub'))
+
+      if @options[:gce_ssh_public_key] && !File.exist?(keyfile)
+        keyfile = @options[:gce_ssh_public_key]
+      end
+
+      raise("Could not find GCE Public SSH Key at '#{keyfile}'") unless File.exist?(keyfile)
+
+      return keyfile
+    end
+
+    # Create the array of metaData, each member being a hash with a :key and a
+    # :value.  Sets :department, :project and :jenkins_build_url.
     def format_metadata
       [ {:key => :department, :value => @options[:department]},
         {:key => :project, :value => @options[:project]},
-        {:key => :jenkins_build_url, :value => @options[:jenkins_build_url]} ].delete_if { |member| member[:value].nil? or member[:value].empty?}
+        {:key => :jenkins_build_url, :value => @options[:jenkins_build_url]},
+        {:key => :sshKeys, :value => "google_compute:#{File.read(find_google_ssh_public_key).strip}" }
+      ].delete_if { |member| member[:value].nil? or member[:value].empty?}
     end
 
-    #Create a new instance of the Google Compute Engine hypervisor object
-    #@param [<Host>] google_hosts The array of google hosts to provision, may ONLY be of platforms /centos-6-.*/ and
-    #                             /debian-7-.*/.  We currently only support the Google Compute provided templates.
-    #@param [Hash{Symbol=>String}] options The options hash containing configuration values
-    #@option options [String] :gce_project The Google Compute Project name to connect to
-    #@option options [String] :gce_keyfile The location of the Google Compute service account keyfile
-    #@option options [String] :gce_password The password for the Google Compute service account key
-    #@option options [String] :gce_email The email address for the Google Compute service account
-    #@option options [String] :gce_machine_type A Google Compute machine type used to create instances, defaults to n1-highmem-2
-    #@option options [Integer] :timeout The amount of time to attempt execution before quiting and exiting with failure
+    # Create a new instance of the Google Compute Engine hypervisor object
+    #
+    # @param [<Host>] google_hosts The Array of google hosts to provision, may
+    # ONLY be of platforms /centos-*/, /debian-*/, /rhel-*/, /suse-*/. Only
+    # supports the Google Compute provided templates.
+    #
+    # @param [Hash{Symbol=>String}] options The options hash containing
+    # configuration values @option options [String] :gce_project The Google
+    # Compute Project name to connect to
+    #
+    # @option options [String] :gce_keyfile The location of the Google Compute
+    # service account keyfile
+    #
+    # @option options [String] :gce_password The password for the Google Compute
+    # service account key
+    #
+    # @option options [String] :gce_email The email address for the Google
+    # Compute service account
+    #
+    # @option options [String] :gce_machine_type A Google Compute machine type
+    # used to create instances, defaults to n1-highmem-2
+    #
+    # @option options [Integer] :timeout The amount of time to attempt execution
+    # before quiting and exiting with failure
     def initialize(google_hosts, options)
       require 'beaker/hypervisor/google_compute_helper'
+
       @options = options
       @logger = options[:logger]
       @hosts = google_hosts
@@ -35,66 +67,77 @@ module Beaker
       @gce_helper = GoogleComputeHelper.new(options)
     end
 
-    #Create and configure virtual machines in the Google Compute Engine, including their associated disks and firewall rules
-    #Currently ONLY supports Google Compute provided templates of CENTOS-6 and DEBIAN-7
+    # Create and configure virtual machines in the Google Compute Engine,
+    # including their associated disks and firewall rules
     def provision
-      try = 1
       attempts = @options[:timeout].to_i / SLEEPWAIT
       start = Time.now
 
-      #get machineType resource, used by all instances
+      test_group_identifier = "beaker-#{start.to_i}-"
+
+      # get machineType resource, used by all instances
       machineType = @gce_helper.get_machineType(start, attempts)
 
-      #set firewall to open pe ports
+      # set firewall to open pe ports
       network = @gce_helper.get_network(start, attempts)
-      @firewall = generate_host_name
+      @firewall = test_group_identifier + generate_host_name
       @gce_helper.create_firewall(@firewall, network, start, attempts)
 
       @logger.debug("Created Google Compute firewall #{@firewall}")
 
 
       @hosts.each do |host|
-        gplatform = Platform.new(host[:image] || host[:platform])
+        if host[:image]
+          gplatform = host[:image]
+        elsif host[:platform]
+          gplatform = Platform.new(host[:platform])
+        else
+          raise('You must specify either :image or :platform, or both as necessary')
+        end
+
         img = @gce_helper.get_latest_image(gplatform, start, attempts)
-        host['diskname'] = generate_host_name
+
+        unique_host_id = test_group_identifier + generate_host_name
+
+        host['diskname'] = unique_host_id
         disk = @gce_helper.create_disk(host['diskname'], img, start, attempts)
         @logger.debug("Created Google Compute disk for #{host.name}: #{host['diskname']}")
 
-        #create new host name
-        host['vmhostname'] = generate_host_name
+        # create new host name
+        host['vmhostname'] = unique_host_id
         #add a new instance of the image
         instance = @gce_helper.create_instance(host['vmhostname'], img, machineType, disk, start, attempts)
         @logger.debug("Created Google Compute instance for #{host.name}: #{host['vmhostname']}")
 
-        #add metadata to instance, if there is any to set
+        # add metadata to instance, if there is any to set
         mdata = format_metadata
-        if not mdata.empty?
+        unless mdata.empty?
           @gce_helper.setMetadata_on_instance(host['vmhostname'], instance['metadata']['fingerprint'],
                                               mdata,
                                               start, attempts)
           @logger.debug("Added tags to Google Compute instance #{host.name}: #{host['vmhostname']}")
         end
 
-        #get ip for this host
+        # get ip for this host
         host['ip'] = instance['networkInterfaces'][0]['accessConfigs'][0]['natIP']
 
-        #configure ssh
+        # configure ssh
         default_user = host['user']
         host['user'] = 'google_compute'
 
-        disable_se_linux(host, @options)
         copy_ssh_to_root(host, @options)
         enable_root_login(host, @options)
         host['user'] = default_user
 
-        #shut down connection, will reconnect on next exec
+        # shut down connection, will reconnect on next exec
         host.close
 
         @logger.debug("Instance ready: #{host['vmhostname']} for #{host.name}}")
       end
     end
 
-    #Shutdown and destroy virtual machines in the Google Compute Engine, including their associated disks and firewall rules
+    # Shutdown and destroy virtual machines in the Google Compute Engine,
+    # including their associated disks and firewall rules
     def cleanup()
       attempts = @options[:timeout].to_i / SLEEPWAIT
       start = Time.now
@@ -110,18 +153,18 @@ module Beaker
 
     end
 
-    #Shutdown and destroy Google Compute instances (including their associated disks and firewall rules)
-    #that have been alive longer than ZOMBIE hours.
+    # Shutdown and destroy Google Compute instances (including their associated
+    # disks and firewall rules) that have been alive longer than ZOMBIE hours.
     def kill_zombies(max_age = ZOMBIE)
       now = start = Time.now
       attempts = @options[:timeout].to_i / SLEEPWAIT
 
-      #get rid of old instances
+      # get rid of old instances
       instances = @gce_helper.list_instances(start, attempts)
       if instances
         instances.each do |instance|
           created = Time.parse(instance['creationTimestamp'])
-          alive = (now - created ) /60 /60
+          alive = (now - created )/60/60
           if alive >= max_age
             #kill it with fire!
             @logger.debug("Deleting zombie instance #{instance['name']}")
@@ -131,14 +174,16 @@ module Beaker
       else
         @logger.debug("No zombie instances found")
       end
-      #get rid of old disks
+
+      # get rid of old disks
       disks = @gce_helper.list_disks(start, attempts)
       if disks
         disks.each do |disk|
           created = Time.parse(disk['creationTimestamp'])
-          alive = (now - created ) /60 /60
+          alive = (now - created )/60/60
           if alive >= max_age
-            #kill it with fire!
+
+            # kill it with fire!
             @logger.debug("Deleting zombie disk #{disk['name']}")
             @gce_helper.delete_disk( disk['name'], start, attempts )
           end
@@ -146,10 +191,11 @@ module Beaker
       else
         @logger.debug("No zombie disks found")
       end
-      #get rid of non-default firewalls
+
+      # get rid of non-default firewalls
       firewalls = @gce_helper.list_firewalls( start, attempts)
 
-      if firewalls and not firewalls.empty?
+      if firewalls && !firewalls.empty?
         firewalls.each do |firewall|
           @logger.debug("Deleting non-default firewall #{firewall['name']}")
           @gce_helper.delete_firewall( firewall['name'], start, attempts )
@@ -157,8 +203,6 @@ module Beaker
       else
         @logger.debug("No zombie firewalls found")
       end
-
     end
-
   end
 end