knife-google 1.1.0 → 1.2.0

data/lib/chef/knife/google_project_list.rb

@@ -0,0 +1,178 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/google_base'
+require 'time'
+
+class Chef
+  class Knife
+    class GoogleProjectList < Knife
+
+      include Knife::GoogleBase
+
+      banner "knife google project list (options)"
+
+      option :limits,
+        :short => "-L",
+        :long => "--with-limits",
+        :description => "Additionally print the quota limit for each metric",
+        :required => false,
+        :boolean => true,
+        :default => false 
+
+      def run
+        $stdout.sync = true
+
+        project_list = [
+          ui.color("name", :bold),
+          ui.color('snapshots', :bold),
+          ui.color('networks', :bold),
+          ui.color('firewalls', :bold),
+          ui.color('images', :bold),
+          ui.color('routes', :bold),
+          ui.color('forwarding-rules', :bold),
+          ui.color('target-pools', :bold),
+          ui.color('health-checks', :bold)].flatten.compact
+
+        output_column_count = project_list.length
+
+        project = client.projects.project
+
+        project_list << project
+
+        snapshots_usage = "0"
+        snapshots_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "SNAPSHOTS"
+            snapshots_usage = "#{quota["usage"].to_i}"
+            snapshots_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          snapshots_quota = "#{snapshots_usage}/#{snapshots_limit}"
+        else
+          snapshots_quota = "#{snapshots_usage}"
+        end
+        project_list << snapshots_quota
+
+        networks_usage = "0"
+        networks_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "NETWORKS"
+            networks_usage = "#{quota["usage"].to_i}"
+            networks_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          networks_quota = "#{networks_usage}/#{networks_limit}" 
+        else
+          networks_quota = "#{networks_usage}"
+        end
+        project_list << networks_quota
+
+        firewalls_usage = "0"
+        firewalls_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "FIREWALLS"
+           firewalls_usage = "#{quota["usage"].to_i}"
+           firewalls_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          firewalls_quota = "#{firewalls_usage}/#{firewalls_limit}"
+        else
+          firewalls_quota = "#{firewalls_usage}"
+        end
+        project_list << firewalls_quota
+
+        images_usage = "0"
+        images_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "IMAGES"
+            images_usage = "#{quota["usage"].to_i}"
+            images_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          images_quota = "#{images_usage}/#{images_limit}"
+        else
+          images_quota = "#{images_usage}"
+        end
+        project_list << images_quota
+
+        routes_usage = "0"
+        routes_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "ROUTES"
+            routes_usage = "#{quota["usage"].to_i}"
+            routes_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          routes_quota = "#{routes_usage}/#{routes_limit}"
+        else
+          routes_quota = "#{routes_usage}"
+        end
+        project_list << routes_quota
+
+        forwarding_usage = "0"
+        forwarding_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "FORWARDING_RULES"
+            forwarding_usage = "#{quota["usage"].to_i}"
+            forwarding_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          forwarding_quota = "#{forwarding_usage}/#{forwarding_limit}"
+        else
+          forwarding_quota = "#{forwarding_usage}"
+        end
+        project_list << forwarding_quota
+
+        target_usage = "0"
+        target_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "TARGET_POOLS"
+            target_usage = "#{quota["usage"].to_i}"
+            target_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          target_quota = "#{target_usage}/#{target_limit}"
+        else
+          target_quota = "#{target_usage}"
+        end
+        project_list << target_quota
+
+        health_usage = "0"
+        health_limit = "0"
+        client.projects.get(project).quotas.each do |quota|
+          if quota["metric"] == "HEALTH_CHECKS"
+            health_usage = "#{quota["usage"].to_i}"
+            health_limit = "#{quota["limit"].to_i}"
+          end
+        end
+        if config[:limits] == true
+          health_quota = "#{health_usage}/#{health_limit}"
+        else
+          health_quota = "#{health_usage}"
+        end
+        project_list << health_quota
+
+        ui.info(ui.list(project_list, :uneven_columns_across, output_column_count))
+      end
+    end
+  end
+end

data/lib/chef/knife/google_region_list.rb

@@ -0,0 +1,125 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/google_base'
+require 'time'
+
+class Chef
+  class Knife
+    class GoogleRegionList < Knife
+
+      include Knife::GoogleBase
+
+      banner "knife google region list (options)"
+
+      option :limits,
+        :short => "-L",
+        :long => "--with-limits",
+        :description => "Additionally print the quota limit for each metric",
+        :required => false,
+        :boolean => true,
+        :default => false 
+
+      def run
+        $stdout.sync = true
+
+        region_list = [
+          ui.color("name", :bold),
+          ui.color('status', :bold),
+          ui.color('deprecation', :bold),
+          ui.color('cpus', :bold),
+          ui.color('disks-total-gb', :bold),
+          ui.color('in-use-addresses', :bold),
+          ui.color('static-addresses', :bold)].flatten.compact
+
+        output_column_count = region_list.length
+
+        client.regions.list.each do |region|
+          region_list << region.name
+          region_list << begin
+            status = region.status.downcase
+            case status
+            when 'up'
+              ui.color(status, :green)
+            else
+              ui.color(status, :red)
+            end
+          end
+          deprecation_state = "-"
+          if region.deprecated.respond_to?('state')
+            deprecation_state = region.deprecated.state
+          end
+          region_list << deprecation_state
+          cpu_usage = "0"
+          cpu_limit = "0"
+          region.quotas.each do |quota|
+            if quota["metric"] == "CPUS"
+              cpu_usage = "#{quota["usage"].to_i}"
+              cpu_limit = "#{quota["limit"].to_i}"
+            end
+          end
+          if config[:limits] == true
+            cpu_quota = "#{cpu_usage}/#{cpu_limit}"
+          else
+            cpu_quota = "#{cpu_usage}"
+          end
+          region_list << cpu_quota
+          disk_usage = "0"
+          disk_limit = "0"
+          region.quotas.each do |quota|
+            if quota["metric"] == "DISKS_TOTAL_GB"
+              disk_usage = "#{quota["usage"].to_i}"
+              disk_limit = "#{quota["limit"].to_i}"
+            end
+          end
+          if config[:limits] == true
+            disk_quota = "#{disk_usage}/#{disk_limit}" 
+          else
+            disk_quota = "#{disk_usage}"
+          end
+          region_list << disk_quota
+          inuse_usage = "0"
+          inuse_limit = "0"
+          region.quotas.each do |quota|
+            if quota["metric"] == "IN_USE_ADDRESSES"
+             inuse_usage = "#{quota["usage"].to_i}"
+             inuse_limit = "#{quota["limit"].to_i}"
+            end
+          end
+          if config[:limits] == true
+            inuse_quota = "#{inuse_usage}/#{inuse_limit}"
+          else
+            inuse_quota = "#{inuse_usage}"
+          end
+          region_list << inuse_quota
+          static_usage = "0"
+          static_limit = "0"
+          region.quotas.each do |quota|
+            if quota["metric"] == "STATIC_ADDRESSES"
+              static_usage = "#{quota["usage"].to_i}"
+              static_limit = "#{quota["limit"].to_i}"
+            end
+          end
+          if config[:limits] == true
+            static_quota = "#{static_usage}/#{static_limit}"
+          else
+            static_quota = "#{static_usage}"
+          end
+          region_list << static_quota
+        end
+        ui.info(ui.list(region_list, :uneven_columns_across, output_column_count))
+      end
+    end
+  end
+end

data/lib/chef/knife/google_server_create.rb

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'timeout'
 require 'chef/knife/google_base'
 
 class Chef
@@ -34,47 +35,89 @@ class Chef
 
       option :machine_type,
         :short => "-m MACHINE_TYPE",
-        :long => "--google-compute-machine MACHINE_TYPE",
+        :long => "--gce-machine-type MACHINE_TYPE",
         :description => "The machine type of server (n1-highcpu-2, n1-highcpu-2-d, etc)",
         :required => true
 
       option :image,
         :short => "-I IMAGE",
-        :long => "--google-compute-image IMAGE",
+        :long => "--gce-image IMAGE",
         :description => "The Image for the server",
         :required => true
 
       option :image_project_id,
-        :short => "-J IMAGE_PROJECT_ID",
-        :long => "--google-compute-image-project-id IMAGE_PROJECT_ID",
+        :long => "--gce-image-project-id IMAGE_PROJECT_ID",
         :description => "The project-id containing the Image (debian-cloud, centos-cloud, etc)",
         :default => "" 
 
       option :zone,
         :short => "-Z ZONE",
-        :long => "--google-compute-zone ZONE",
+        :long => "--gce-zone ZONE",
         :description => "The Zone for this server"
 
+      option :boot_disk_name,
+        :long => "--gce-boot-disk-name DISK",
+        :description => "Name of persistent boot disk; default is to use the server name",
+        :default => ""
+
+      option :boot_disk_size,
+        :long => "--gce-boot-disk-size SIZE",
+        :description => "Size of the persistent boot disk between 10 and 10000 GB, specified in GB; default is '10' GB",
+        :default => "10"
+
+      option :auto_restart,
+        :long => "--[no-]gce-auto-server-restart",
+        :description => "Compute Engine can automatically restart your VM instance if it is terminated for non-user-initiated reasons; enabled by default.",
+        :boolean => true,
+        :default => true
+
+      option :auto_migrate,
+        :long => "--[no-]gce-auto-server-migrate",
+        :description => "Compute Engine can migrate your VM instance to other hardware without downtime prior to periodic infrastructure maintenance, otherwise the server is terminated; enabled by default.",
+        :boolean => true,
+        :default => true
+
       option :network,
         :short => "-n NETWORK",
-        :long => "--google-compute-network NETWORK",
+        :long => "--gce-network NETWORK",
         :description => "The network for this server; default is 'default'",
         :default => "default"
 
       option :tags,
         :short => "-T TAG1,TAG2,TAG3",
-        :long => "--google-compute-tags TAG1,TAG2,TAG3",
+        :long => "--gce-tags TAG1,TAG2,TAG3",
         :description => "Tags for this server",
         :proc => Proc.new { |tags| tags.split(',') },
         :default => []
 
       option :metadata,
-        :short => "-M K=V[,K=V,...]",
-        :long => "--google-compute-metadata Key=Value[,Key=Value...]",
-        :description => "The metadata for this server",
+        :long => "--gce-metadata Key=Value[,Key=Value...]",
+        :description => "Additional metadata for this server",
         :proc => Proc.new { |metadata| metadata.split(',') },
         :default => []
 
+      option :service_account_scopes,
+        :long => "--gce-service-account-scopes SCOPE1,SCOPE2,SCOPE3",
+        :proc => Proc.new { |service_account_scopes| service_account_scopes.split(',') },
+        :description => "Service account scopes for this server",
+        :default => []
+
+      # GCE documentation uses the term 'service account name', the api uses the term 'email'
+      option :service_account_name,
+        :long => "--gce-service-account-name NAME",
+        :description => "Service account name for this server, typically in the form of '123845678986@project.gserviceaccount.com'; default is 'default'",
+        :default => "default"
+
+      option :instance_connect_ip,
+        :long => "--gce-server-connect-ip INTERFACE",
+        :description => "Whether to use PUBLIC or PRIVATE interface/address to connect; default is 'PUBLIC'",
+        :default => 'PUBLIC'
+
+      option :public_ip,
+        :long=> "--gce-public-ip IP_ADDRESS",
+        :description => "EPHEMERAL or static IP address or NONE; default is 'EPHEMERAL'",
+        :default => "EPHEMERAL"
+
       option :chef_node_name,
         :short => "-N NAME",
         :long => "--node-name NAME",
@@ -158,23 +201,6 @@ class Chef
            Chef::Config[:knife][:hints][name] = path ? JSON.parse(::File.read(path)) : Hash.new
         }
 
-      option :instance_connect_ip,
-        :long => "--google-compute-server-connect-ip PUBLIC",
-        :short => "-a PUBLIC",
-        :description => "Whether to use PUBLIC or PRIVATE address to connect; default is 'PUBLIC'",
-        :default => 'PUBLIC'
-
-      option :disks,
-        :long=> "--google-compute-disks DISK1,DISK2",
-        :proc => Proc.new { |metadata| metadata.split(',') },
-        :description => "Disks to be attached",
-        :default => []
-
-      option :public_ip,
-        :long=> "--google-compute-public-ip IP_ADDRESS",
-        :description => "EPHEMERAL or static IP address or NONE; default is 'EPHEMERAL'",
-        :default => "EPHEMERAL"
-
       def tcp_test_ssh(hostname, ssh_port)
         tcp_socket = TCPSocket.new(hostname, ssh_port)
         readable = IO.select([tcp_socket], nil, nil, 5)
@@ -200,7 +226,7 @@ class Chef
 
       def wait_for_tunnelled_sshd(hostname)
         print(".")
-        print(".") until tunnel_test_ssh(ssh_connect_host) {
+        print(".") until tunnel_test_ssh(hostname) {
           sleep @initial_sleep_delay ||= 40
           puts("done")
         }
@@ -237,7 +263,35 @@ class Chef
         end
       end
 
-      def bootstrap_for_node(instance,ssh_host)
+      def disk_exists(disk, zone)
+        # if client.disks.get errors with a Google::Compute::ResourceNotFound
+        # then the disk does not exist and can be created
+        client.disks.get(:disk => disk, :zone => selflink2name(zone))
+      rescue Google::Compute::ResourceNotFound
+        # disk does not exist
+        # continue provisioning
+        false
+      else
+        true
+      end
+
+      def wait_for_disk(disk, operation, zone)
+        Timeout::timeout(300) do
+          until disk.status == 'DONE'
+            ui.info(".")
+            sleep 1
+            disk = client.zoneOperations.get(:name => disk,
+                                             :operation => operation,
+                                             :zone => selflink2name(zone))
+          end
+          disk.target_link
+        end
+      rescue Timeout::Error
+        ui.error("Timeout exceeded with disk status: " + disk.status)
+        exit 1
+      end
+
+      def bootstrap_for_node(instance, ssh_host)
         bootstrap = Chef::Knife::Bootstrap.new
         bootstrap.name_args = [ssh_host]
         bootstrap.config[:run_list] = config[:run_list]
@@ -258,6 +312,7 @@ class Chef
         # Modify global configuration state to ensure hint gets set by
         # knife-bootstrap
         Chef::Config[:knife][:hints] ||= {}
+        Chef::Config[:knife][:hints]["gce"] ||= {}
         Chef::Config[:knife][:hints]["google"] ||= {}
         bootstrap
       end
@@ -265,14 +320,14 @@ class Chef
       def run
         $stdout.sync = true
         unless @name_args.size > 0
-          ui.error("Please provide the name of the new server")
+          ui.error("Please provide the name of the new server.")
           exit 1
         end
 
         begin
-          zone = client.zones.get(config[:zone] || Chef::Config[:knife][:google_compute_zone]).self_link
+          zone = client.zones.get(config[:zone] || Chef::Config[:knife][:gce_zone]).self_link
         rescue Google::Compute::ResourceNotFound
-          ui.error("Zone '#{config[:zone] || Chef::Config[:knife][:google_compute_zone]}' not found")
+          ui.error("Zone '#{config[:zone] || Chef::Config[:knife][:gce_zone]}' not found.")
           exit 1
         rescue Google::Compute::ParameterValidation
           ui.error("Must specify zone in knife config file or in command line as an option. Try --help.")
@@ -280,18 +335,33 @@ class Chef
         end
 
         begin
-          machine_type = client.machine_types.get(:name=>config[:machine_type], :zone=>selflink2name(zone)).self_link
+          machine_type = client.machine_types.get(:name => config[:machine_type],
+                                                  :zone => selflink2name(zone)).self_link
         rescue Google::Compute::ResourceNotFound
           ui.error("MachineType '#{config[:machine_type]}' not found")
           exit 1
         end
 
+        # this parameter is a string during the post and boolean otherwise
+        if config[:auto_restart] then
+          auto_restart = 'true'
+        else
+          auto_restart = 'false'
+        end
+
+        if config[:auto_migrate] then
+          auto_migrate = 'MIGRATE'
+        else
+          auto_migrate = 'TERMINATE'
+        end
+
         (checked_custom, checked_all) = false
         begin
           image_project = config[:image_project_id]
-          machine_type=~Regexp.new('/projects/(.*?)/')
+          # use zone url to determine project name
+          zone =~ Regexp.new('/projects/(.*?)/')
           project = $1
-          if image_project.empty?
+          if image_project.to_s.empty?
             unless checked_custom
               checked_custom = true
               ui.info("Looking for Image '#{config[:image]}' in Project '#{project}'")
@@ -323,6 +393,27 @@ class Chef
           end
         end
 
+        begin
+          boot_disk_size = config[:boot_disk_size].to_i
+          raise if !boot_disk_size.between?(10, 10000)
+        rescue
+          ui.error("Size of the persistent boot disk must be between 10 and 10000 GB.")
+          exit 1
+        end
+
+        if config[:boot_disk_name].to_s.empty? then
+          boot_disk_name = @name_args.first
+        else
+          boot_disk_name = config[:boot_disk_name]
+        end
+
+        ui.info("Waiting for the disk insert operation to complete")
+        boot_disk_insert = client.disks.insert(:sourceImage => image,
+                                               :zone => selflink2name(zone),
+                                               :name => boot_disk_name,
+                                               :sizeGb => boot_disk_size)
+        boot_disk_target_link = wait_for_disk(boot_disk_insert, boot_disk_insert.name, zone)
+
         begin
           network = client.networks.get(config[:network]).self_link
         rescue Google::Compute::ResourceNotFound
@@ -330,7 +421,6 @@ class Chef
           exit 1
         end
 
-        disks = config[:disks].collect{|disk| client.disks.get(:disk=>disk, :zone=>selflink2name(zone)).self_link}
         metadata = config[:metadata].collect{|pair| Hash[*pair.split('=')] }
         network_interface = {'network'=>network}
 
@@ -346,27 +436,65 @@ class Chef
           ui.error("Invalid public ip value : #{config[:public_ip]}")
           exit 1
         end
-        zone_operation = client.instances.create(:name=>@name_args.first, :zone=>selflink2name(zone),
-                                  :image=> image,
-                                  :machineType =>machine_type,
-                                  :disks=>disks,
-                                  :metadata=>{'items'=> metadata },
-                                  :networkInterfaces => [network_interface],
-                                  :tags=> config[:tags]
-                                )
 
         ui.info("Waiting for the create server operation to complete")
+        if !config[:service_account_scopes].any?
+          zone_operation = client.instances.create(:name => @name_args.first,
+                                                   :zone => selflink2name(zone),
+                                                   :machineType => machine_type,
+                                                   :disks => [{
+                                                     'boot' => true,
+                                                     'type' => 'PERSISTENT',
+                                                     'mode' => 'READ_WRITE',
+                                                     'deviceName' => selflink2name(boot_disk_target_link),
+                                                     'source' => boot_disk_target_link 
+                                                   }],
+                                                   :networkInterfaces => [network_interface],
+                                                   :scheduling => {
+                                                     'automaticRestart' => auto_restart,
+                                                     'onHostMaintenance' => auto_migrate
+                                                   },
+                                                   :metadata => { 'items' => metadata },
+                                                   :tags => { 'items' => config[:tags] }
+                                                  )
+        else
+          zone_operation = client.instances.create(:name => @name_args.first, 
+                                                   :zone=> selflink2name(zone),
+                                                   :machineType => machine_type,
+                                                   :disks => [{
+                                                     'boot' => true,
+                                                     'type' => 'PERSISTENT',
+                                                     'mode' => 'READ_WRITE',
+                                                     'deviceName' => selflink2name(boot_disk_target_link),
+                                                     'source' => boot_disk_target_link
+                                                   }],
+                                                   :networkInterfaces => [network_interface],
+                                                   :serviceAccounts => [{
+                                                     'kind' => 'compute#serviceAccount',
+                                                     'email' => config[:service_account_name],
+                                                     'scopes' => config[:service_account_scopes]
+                                                   }],
+                                                   :scheduling => {
+                                                     'automaticRestart' => auto_restart,
+                                                     'onHostMaintenance' => auto_migrate
+                                                   },
+                                                   :metadata => { 'items'=>metadata },
+                                                   :tags => { 'items' => config[:tags] }
+                                                  )
+        end
+
         until zone_operation.progress.to_i == 100
           ui.info(".")
           sleep 1
           zone_operation = client.zoneOperations.get(:name=>zone_operation, :operation=>zone_operation.name, :zone=>selflink2name(zone))
         end
+
         ui.info("Waiting for the servers to be in running state")
 
         @instance = client.instances.get(:name=>@name_args.first, :zone=>selflink2name(zone))
         msg_pair("Instance Name", @instance.name)
-        msg_pair("MachineType", selflink2name(@instance.machine_type))
-        msg_pair("Image", selflink2name(@instance.image))
+        msg_pair("Machine Type", selflink2name(@instance.machine_type))
+        msg_pair("Image", selflink2name(config[:image]))
         msg_pair("Zone", selflink2name(@instance.zone))
         msg_pair("Tags", @instance.tags.has_key?("items") ? @instance.tags["items"].join(",") : "None")
         until @instance.status == "RUNNING"