chef-provisioning-google 0.1.0

data/lib/chef/provisioning/google_driver/client/projects.rb

@@ -0,0 +1,39 @@
+require_relative "google_base"
+require_relative "metadata"
+
+class Chef
+  module Provisioning
+    module GoogleDriver
+      module Client
+        # Wraps a Projects service of the GCE API.
+        class Projects < GoogleBase
+
+          def get
+            # The default arguments are already enough for this call.
+            response = make_request(compute.projects.get)
+            raise_if_error(response)
+            Metadata.new(response)
+          end
+
+          # Takes a metadata object retrieved via #get and updates the metadata on GCE
+          # using the projects.set_common_instance_metadata API call.
+          # This fails if the metadata on GCE has been updated since the passed
+          # metadata object has been retrieved via #get.
+          # Note that this omits the API call if the metadata object hasn't changed
+          # locally since it was retrieved via #get.
+          def set_common_instance_metadata(metadata)
+            return nil unless metadata.changed?
+            response = make_request(
+              compute.projects.set_common_instance_metadata,
+              # Default paremeters are sufficient.
+              {},
+              { items: metadata.items, fingerprint: metadata.fingerprint }
+            )
+            operation_response(response)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/chef/provisioning/google_driver/client/zone_operations.rb

@@ -0,0 +1,18 @@
+require_relative "operations_base"
+
+class Chef
+  module Provisioning
+    module GoogleDriver
+      module Client
+        # Wraps a ZoneOperations service of the GCE API.
+        class ZoneOperations < OperationsBase
+
+          def operations_service
+            compute.zone_operations
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/chef/provisioning/google_driver/credentials.rb

@@ -0,0 +1,63 @@
+class Chef
+  module Provisioning
+    module GoogleDriver
+      # Access various forms of Google credentials
+      # TODO: load credentials from metadata server when provisioning from a GCE machine
+      class Credentials
+
+        def initialize
+          @credentials = {}
+        end
+
+        def [](name)
+          @credentials[name]
+        end
+
+        def []=(name, value)
+          @credentials[name] = value
+        end
+
+        def self.from_hash(h)
+          credentials = self.new
+          h.each do |k, v|
+            credentials[k] = v
+          end
+          credentials.validate!
+          credentials
+        end
+
+        # Validates whether the key settings are present in the credential object and keys are in the correct format.
+        # If no client_email is specified, method will try to load the client_email from the json key.
+        def validate!
+          unless self[:p12_key_path] || self[:json_key_path]
+            raise "Google key path is missing. Options provided: #{self.inspect}"
+          end
+          if self[:json_key_path]
+            json_key_hash = JSON.load(File.open(self[:json_key_path]))
+
+            unless self[:google_client_email]
+              # Try to load client_email from json if not present
+              if json_key_hash["client_email"]
+                self[:google_client_email] = json_key_hash["client_email"]
+              else
+                raise "google_client_email must be specified"
+              end
+            end
+
+            raise "Invalid Google JSON key, no private key" unless json_key_hash.include?("private_key")
+          elsif self[:p12_key_path]
+            raise "p12 key doesn't exist in the path specified" unless File.exist?(self[:p12_key_path])
+            raise "google_client_email must be specified" unless self[:google_client_email]
+          else
+            raise "json_key_path or p12_key_path is missing. Options provided: #{self}"
+          end
+        end
+
+        def load_defaults
+          raise NotImplementedError
+        end
+
+      end
+    end
+  end
+end

data/lib/chef/provisioning/google_driver/driver.rb

@@ -0,0 +1,313 @@
+require "chef/provisioning/driver"
+require "chef/provisioning/google_driver/credentials"
+require "chef/provisioning/google_driver/version"
+require "chef/mixin/deep_merge"
+require "chef/provisioning/convergence_strategy/install_cached"
+require "chef/provisioning/convergence_strategy/install_sh"
+require "chef/provisioning/convergence_strategy/install_msi"
+require "chef/provisioning/convergence_strategy/no_converge"
+require "chef/provisioning/transport/ssh"
+require "chef/provisioning/transport/winrm"
+require "chef/provisioning/machine/windows_machine"
+require "chef/provisioning/machine/unix_machine"
+require "chef/provisioning/machine_spec"
+
+require_relative "client/instances"
+require_relative "client/global_operations"
+require_relative "client/projects"
+require_relative "client/zone_operations"
+
+require "google/api_client"
+require "retryable"
+require "etc"
+
+class Chef
+  module Provisioning
+    module GoogleDriver
+      # Provisions machines using the Google SDK
+      # TODO look at the superclass comments for further explanation of the overridden methods in this class
+      class Driver < Chef::Provisioning::Driver
+
+        include Chef::Mixin::DeepMerge
+
+        attr_reader :google, :zone, :project, :instance_client, :global_operations_client, :zone_operations_client, :project_client
+        URL_REGEX = /^google:(.+?):(.+)$/
+
+        # URL scheme:
+        # google:zone
+        def self.from_url(driver_url, config)
+          self.new(driver_url, config)
+        end
+
+        def initialize(driver_url, config)
+          super
+
+          m = URL_REGEX.match(driver_url)
+          if m.nil?
+            raise "Driver URL [#{driver_url}] must match #{URL_REGEX.inspect}"
+          end
+          # TODO: Move zone into bootstrap_options
+          @zone = m[1]
+          @project = m[2]
+
+          @google = Google::APIClient.new(
+            :application_name => "chef-provisioning-google",
+            :application_version => Chef::Provisioning::GoogleDriver::VERSION
+          )
+          if google_credentials[:p12_key_path]
+            signing_key = Google::APIClient::KeyUtils.load_from_pkcs12(google_credentials[:p12_key_path], "notasecret")
+          elsif google_credentials[:json_key_path]
+            json_private_key = JSON.load(File.open(google_credentials[:json_key_path]))["private_key"]
+            signing_key = Google::APIClient::KeyUtils.load_from_pem(json_private_key, "notasecret")
+          end
+          google.authorization = Signet::OAuth2::Client.new(
+            :token_credential_uri => "https://accounts.google.com/o/oauth2/token",
+            :audience => "https://accounts.google.com/o/oauth2/token",
+            :scope => ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/compute.readonly"],
+            :issuer => google_credentials[:google_client_email],
+            :signing_key => signing_key
+          )
+          google.authorization.fetch_access_token!
+
+          @instance_client = Client::Instances.new(google, project, zone)
+          @project_client = Client::Projects.new(google, project, zone)
+          @global_operations_client =
+            Client::GlobalOperations.new(google, project, zone)
+          @zone_operations_client =
+            Client::ZoneOperations.new(google, project, zone)
+        end
+
+        def self.canonicalize_url(driver_url, config)
+          [ driver_url, config ]
+        end
+
+        def allocate_machine(action_handler, machine_spec, machine_options)
+          # TODO how do we handle running `allocate` and there is already a machine in GCE
+          #   but no node in chef?  We should just start tracking it.
+          if instance_for(machine_spec).nil?
+            name = machine_spec.name
+            operation = nil
+            action_handler.perform_action "creating instance named #{name} in zone #{zone}" do
+              default_options = instance_client.default_create_options(name)
+              options = hash_only_merge(default_options, machine_options[:insert_options])
+              operation = instance_client.create(options)
+            end
+            zone_operations_client.wait_for_done(action_handler, operation)
+            machine_spec.reference = {
+                "driver_version" => Chef::Provisioning::GoogleDriver::VERSION,
+                "allocated_at" => Time.now.utc.to_s,
+                "host_node" => action_handler.host_node,
+            }
+            machine_spec.driver_url = driver_url
+            # %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
+            %w{ssh_username sudo ssh_gateway key_name}.each do |key|
+              machine_spec.reference[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
+            end
+          end
+        end
+
+        def ready_machine(action_handler, machine_spec, machine_options)
+          name = machine_spec.name
+          instance = instance_for(machine_spec)
+
+          if instance.nil?
+            raise "Machine #{name} does not have an instance associated with it, or instance does not exist."
+          end
+
+          if !instance.running?
+            # could be PROVISIONING, STAGING, STOPPING, TERMINATED
+            if %w{STOPPING TERMINATED}.include?(instance.status)
+              action_handler.perform_action "instance named #{name} in zone #{zone} was stopped - starting it" do
+                instance_client.start(name)
+              end
+            end
+            instance_client.wait_for_status(action_handler, instance, "RUNNING")
+          end
+
+          # Refresh instance object so we get the new ip address and status
+          instance = instance_for(machine_spec)
+
+          wait_for_transport(action_handler, machine_spec, machine_options, instance)
+          machine_for(machine_spec, machine_options, instance)
+        end
+
+        def destroy_machine(action_handler, machine_spec, machine_options)
+          name = machine_spec.name
+          instance = instance_for(machine_spec)
+          # https://cloud.google.com/compute/docs/instances#checkmachinestatus
+          # TODO Shouldn't we also delete stopped machines?
+          if instance && !%w{STOPPING TERMINATED}.include?(instance.status)
+            operation = nil
+            action_handler.perform_action "destroying instance named #{name} in zone #{zone}" do
+              operation = instance_client.delete(name)
+            end
+            zone_operations_client.wait_for_done(action_handler, operation)
+          end
+
+          strategy = convergence_strategy_for(machine_spec, machine_options)
+          strategy.cleanup_convergence(action_handler, machine_spec)
+          # TODO clean up known_hosts entry
+        end
+
+        def stop_machine(action_handler, machine_spec, machine_options)
+          name = machine_spec.name
+          instance = instance_for(machine_spec)
+
+          if instance.nil?
+            raise "Machine #{name} does not have an instance associated with it, or instance does not exist."
+          end
+
+          unless instance.terminated?
+            unless instance.stopping?
+              action_handler.perform_action "stopping instance named #{name} in zone #{zone}" do
+                instance_client.stop(name)
+              end
+            end
+            instance_client.wait_for_status(action_handler, instance, "TERMINATED")
+          end
+
+          if instance.terminated?
+            Chef::Log.info "Instance #{instance.name} already stopped, nothing to do."
+          end
+        end
+
+        # TODO make these configurable and find a good place where to put them.
+        def tries
+          Client::GoogleBase::TRIES
+        end
+
+        def sleep
+          Client::GoogleBase::SLEEP_SECONDS
+        end
+
+        def wait_for_transport(action_handler, machine_spec, machine_options, instance)
+          transport = transport_for(machine_spec, machine_options, instance)
+          unless transport.available?
+            if action_handler.should_perform_actions
+              Retryable.retryable(:tries => tries, :sleep => sleep, :matching => /Not done/) do |retries, exception|
+                action_handler.report_progress("  waited #{retries * sleep}/#{tries * sleep}s for instance #{instance.name} to be connectable (transport up and running) ...")
+                raise "Not done" unless transport.available?
+              end
+              action_handler.report_progress "#{machine_spec.name} is now connectable"
+            end
+          end
+        end
+
+        def transport_for(machine_spec, machine_options, instance)
+          # TODO winrm
+          # if machine_spec.reference['is_windows']
+          #   create_winrm_transport(machine_spec, machine_options, instance)
+          # else
+          create_ssh_transport(machine_spec, machine_options, instance)
+          # end
+        end
+
+        def create_ssh_transport(machine_spec, machine_options, instance)
+          ssh_options = ssh_options_for(machine_spec, machine_options, instance)
+          username = machine_spec.reference["ssh_username"] || machine_options[:ssh_username] || Etc.getlogin
+          if machine_options.has_key?(:ssh_username) && machine_options[:ssh_username] != machine_spec.reference["ssh_username"]
+            Chef::Log.warn("Server #{machine_spec.name} was created with SSH username #{machine_spec.reference['ssh_username']} and machine_options specifies username #{machine_options[:ssh_username]}.  Using #{machine_spec.reference['ssh_username']}.  Please edit the node and change the chef_provisioning.reference.ssh_username attribute if you want to change it.")
+          end
+          options = {}
+          if machine_spec.reference[:sudo] || (!machine_spec.reference.has_key?(:sudo) && username != "root")
+            options[:prefix] = "sudo "
+          end
+
+          remote_host = instance.determine_remote_host
+
+          #Enable pty by default
+          options[:ssh_pty_enable] = true
+          options[:ssh_gateway] = machine_spec.reference["ssh_gateway"] if machine_spec.reference.has_key?("ssh_gateway")
+
+          Chef::Provisioning::Transport::SSH.new(remote_host, username, ssh_options, options, config)
+        end
+
+        def ssh_options_for(machine_spec, machine_options, instance)
+          result = {
+            :auth_methods => [ "publickey" ],
+            :keys_only => true,
+            :host_key_alias => "#{instance.id}.GOOGLE",
+          }.merge(machine_options[:ssh_options] || {})
+          # TODO right now we only allow keys created for the whole project and specified in the
+          # bootstrap options - look at AWS for other options
+          if machine_options[:key_name]
+            # TODO how do I add keys to config[:private_keys] ?
+            # result[:key_data] = [ get_private_key(machine_options[:key_name]) ]
+            # TODO: what to do if we find multiple valid keys in config[:private_key_paths] ?
+            config[:private_key_paths].each do |path|
+              result[:key_data] = IO.read("#{path}/#{machine_options[:key_name]}") if File.exist?("#{path}/#{machine_options[:key_name]}")
+            end
+            unless result[:key_data]
+              raise "#{machine_options[:key_name]} doesn't exist in private_key_paths:#{config[:private_key_paths]}"
+            end
+          else
+            raise "No key found to connect to #{machine_spec.name} (#{machine_spec.reference.inspect})!"
+          end
+          result
+        end
+
+        def machine_for(machine_spec, machine_options, instance = nil)
+          instance ||= instance_for(machine_spec)
+
+          unless instance
+            raise "Instance for node #{machine_spec.name} has not been created!"
+          end
+
+          # TODO winrm
+          # if machine_spec.reference['is_windows']
+          #   Chef::Provisioning::Machine::WindowsMachine.new(machine_spec, transport_for(machine_spec, machine_options, instance), convergence_strategy_for(machine_spec, machine_options))
+          # else
+          Chef::Provisioning::Machine::UnixMachine.new(machine_spec, transport_for(machine_spec, machine_options, instance), convergence_strategy_for(machine_spec, machine_options))
+          # end
+        end
+
+        def convergence_strategy_for(machine_spec, machine_options)
+          # Tell Ohai that this is an EC2 instance so that it runs the EC2 plugin
+          convergence_options = Cheffish::MergedConfig.new(
+            machine_options[:convergence_options] || {},
+            # TODO what is the right ohai hints file?
+            ohai_hints: { "google" => "" })
+
+          # Defaults
+          unless machine_spec.reference
+            return Chef::Provisioning::ConvergenceStrategy::NoConverge.new(convergence_options, config)
+          end
+
+          # TODO winrm
+          # if machine_spec.reference['is_windows']
+          #   Chef::Provisioning::ConvergenceStrategy::InstallMsi.new(convergence_options, config)
+          if machine_options[:cached_installer] == true
+            Chef::Provisioning::ConvergenceStrategy::InstallCached.new(convergence_options, config)
+          else
+            Chef::Provisioning::ConvergenceStrategy::InstallSh.new(convergence_options, config)
+          end
+        end
+
+        def instance_for(machine_spec)
+          if machine_spec.reference
+            if machine_spec.driver_url != driver_url
+              raise "Switching a machine's driver from #{machine_spec.driver_url} to #{driver_url} is not currently supported!  Use machine :destroy and then re-create the machine on the new driver."
+            end
+            instance_client.get(machine_spec.name)
+          end
+        end
+
+        private
+
+        # TODO load from file or env variables using common google method
+        # https://cloud.google.com/sdk/gcloud/#gcloud.auth
+        def google_credentials
+          # Grab the list of possible credentials
+          @google_credentials ||= if driver_options[:google_credentials]
+                                    Credentials.from_hash(driver_options[:google_credentials])
+                                  else
+                                    credentials = Credentials.new
+                                    credentials.load_defaults
+                                    credentials
+                                  end
+        end
+
+      end
+    end
+  end
+end

data/lib/chef/provisioning/google_driver/version.rb

@@ -0,0 +1,7 @@
+class Chef
+  module Provisioning
+    module GoogleDriver
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/chef/resource/google_key_pair.rb

@@ -0,0 +1,50 @@
+require "chef/resource/lwrp_base"
+
+class Chef::Resource::GoogleKeyPair < Chef::Resource::LWRPBase
+  self.resource_name = self.dsl_name
+
+  # TODO instance specific or project wide?
+  # Right now we only have project wide keys
+
+  provides :google_key_pair
+
+  actions :create, :destroy
+  default_action :create
+
+  # Private key to use as input (will be generated if it does not exist)
+  attribute :private_key_path, :kind_of => String
+  # Public key to use as input (will be generated if it does not exist)
+  attribute :public_key_path, :kind_of => String
+  # List of parameters to the private_key resource used for generation of the key
+  attribute :private_key_options, :kind_of => Hash
+
+  # This applies to both the local keys and the remote key
+  attribute :allow_overwrite, :kind_of => [TrueClass, FalseClass], :default => false
+
+  # TODO: add a `user` attribute which sets the user to login with the key
+  # GCE uses the key user to create a user on the instance, which may duplicate
+  # some logic the user is trying to do with their chef recipes
+
+  def after_created
+    # We default these here so load_current_resource can diff
+    if private_key_path.nil?
+      private_key_path ::File.join(driver.config[:private_key_write_path], "google_default")
+    elsif Pathname.new(private_key_path).relative?
+      private_key_path ::File.join(driver.config[:private_key_write_path], private_key_path)
+    end
+    # TODO you don't actually need to write the private key to disc if it isn't provided
+    # it can be read from the private key, but this code update needs testing
+    if public_key_path.nil?
+      public_key_path ::File.join(driver.config[:private_key_write_path], "google_default.pub")
+    elsif Pathname.new(public_key_path).relative?
+      public_key_path ::File.join(driver.config[:private_key_write_path], public_key_path)
+    end
+  end
+
+  # TODO introduce base class and add this as attribute, like AWS does
+  # Ideally, we won't be creating lots of copies of the same driver object, but it is okay
+  # if we do - they aren't singletons
+  def driver
+    run_context.chef_provisioning.driver_for(run_context.chef_provisioning.current_driver)
+  end
+end