knife-ec-backup 2.0.0.beta.2 → 2.0.0.beta.3

data/lib/chef/knife/ec_base.rb

@@ -0,0 +1,148 @@
+#
+# Author:: Steven Danna (<steve@getchef.com>)
+# Copyright:: Copyright (c) 2014 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+
+class Chef
+  class Knife
+    module EcBase
+      class NoAdminFound < Exception; end
+
+      def self.included(includer)
+        includer.class_eval do
+
+          option :concurrency,
+            :long => '--concurrency THREADS',
+            :description => 'Maximum number of simultaneous requests to send (default: 10)'
+
+          option :webui_key,
+            :long => '--webui-key KEYPATH',
+            :description => 'Path to the WebUI Key (default: /etc/opscode/webui_priv.pem)',
+            :default => '/etc/opscode/webui_priv.pem'
+
+          option :skip_useracl,
+            :long => '--skip-useracl',
+            :boolean => true,
+            :default => false,
+            :description => "Skip downloading/restoring User ACLs.  This is required for EC 11.0.2 and lower"
+
+          option :skip_version,
+            :long => '--skip-version-check',
+            :boolean => true,
+            :default => false,
+            :description => "Skip Chef Server version check.  This will also skip any auto-configured options"
+
+          option :org,
+            :long => "--only-org ORG",
+            :description => "Only download/restore objects in the named organization (default: all orgs)"
+
+          option :sql_host,
+            :long => '--sql-host HOSTNAME',
+            :description => 'Postgresql database hostname (default: localhost)',
+            :default => "localhost"
+
+          option :sql_port,
+            :long => '--sql-port PORT',
+            :description => 'Postgresql database port (default: 5432)',
+            :default => 5432
+
+          option :sql_user,
+            :long => "--sql-user USERNAME",
+            :description => 'User used to connect to the postgresql database.'
+
+          option :sql_password,
+            :long => "--sql-password PASSWORD",
+            :description => 'Password used to connect to the postgresql database'
+
+          option :with_user_sql,
+            :long => '--with-user-sql',
+            :description => 'Try direct data base access for user export/import.  Required to properly handle passwords, keys, and USAGs'
+        end
+
+        attr_accessor :dest_dir
+
+        def configure_chef
+          super
+          Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
+          Chef::ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
+        end
+
+        def org_admin
+          rest = Chef::REST.new(Chef::Config.chef_server_url)
+          admin_users = rest.get_rest('groups/admins')['users']
+          org_members = rest.get_rest('users').map { |user| user['user']['username'] }
+          admin_users.delete_if { |user| !org_members.include?(user) || user == 'pivotal' }
+          if admin_users.empty?
+            raise Chef::Knife::EcBase::NoAdminFound
+          else
+            admin_users[0]
+          end
+        end
+      end
+
+      def server
+        @server ||= if Chef::Config.chef_server_root.nil?
+                      ui.warn("chef_server_root not found in knife configuration; using chef_server_url")
+                      Chef::Server.from_chef_server_url(Chef::Config.chef_server_url)
+                    else
+                      Chef::Server.new(Chef::Config.chef_server_root)
+                    end
+      end
+
+      def rest
+        @rest ||= Chef::REST.new(server.root_url)
+      end
+
+      def user_acl_rest
+        @user_acl_rest ||= if config[:skip_version]
+                             rest
+                           elsif server.supports_user_acls?
+                             rest
+                           elsif server.direct_account_access?
+                             Chef::REST.new("http://127.0.0.1:9465")
+                           end
+
+      end
+
+      def set_skip_user_acl!
+        config[:skip_useracl] ||= !(server.supports_user_acls? || server.direct_account_access?)
+      end
+
+      def set_client_config!
+        Chef::Config.custom_http_headers = (Chef::Config.custom_http_headers || {}).merge({'x-ops-request-source' => 'web'})
+        Chef::Config.node_name = 'pivotal'
+        Chef::Config.client_key = config[:webui_key]
+      end
+
+      def set_dest_dir_from_args!
+        if name_args.length <= 0
+          ui.error("Must specify backup directory as an argument.")
+          exit 1
+        end
+        @dest_dir = name_args[0]
+      end
+
+      def ensure_webui_key_exists!
+        if !File.exist?(config[:webui_key])
+          ui.error("Webui Key (#{config[:webui_key]}) does not exist.")
+          exit 1
+        end
+      end
+    end
+  end
+end

data/lib/chef/knife/ec_restore.rb

@@ -1,17 +1,13 @@
 require 'chef/knife'
+require 'chef/knife/ec_base'
 
 class Chef
   class Knife
     class EcRestore < Chef::Knife
-      banner "knife ec restore DIRECTORY"
 
-      option :concurrency,
-        :long => '--concurrency THREADS',
-        :description => 'Maximum number of simultaneous requests to send (default: 10)'
+      include Knife::EcBase
 
-      option :webui_key,
-        :long => '--webui-key KEYPATH',
-        :description => 'Used to set the path to the WebUI Key (default: /etc/opscode/webui_priv.pem)'
+      banner "knife ec restore DIRECTORY"
 
       option :overwrite_pivotal,
         :long => '--overwrite-pivotal',
@@ -19,48 +15,10 @@ class Chef
         :default => false,
         :description => "Whether to overwrite pivotal's key.  Once this is done, future requests will fail until you fix the private key."
 
-      option :skip_useracl,
-        :long => '--skip-useracl',
-        :boolean => true,
-        :default => false,
-        :description => "Whether to skip restoring User ACLs.  This is required for EC 11.0.2 and lower"
-
-      option :skip_version,
-        :long => '--skip-version-check',
-        :boolean => true,
-        :default => false,
-        :description => "Whether to skip checking the Chef Server version.  This will also skip any auto-configured options"
-
-      option :org,
-        :long => "--only-org ORG",
-        :description => "Only restore objects in the named organization (default: all orgs)"
-
       option :skip_users,
         :long => "--skip-users",
         :description => "Skip restoring users"
 
-      option :with_user_sql,
-        :long => "--with-user-sql",
-        :description => "Restore user id's, passwords, and keys from sql export"
-
-      option :sql_host,
-        :long => '--sql-host HOSTNAME',
-        :description => 'Postgresql database hostname (default: localhost)',
-        :default => "localhost"
-
-      option :sql_port,
-        :long => '--sql-port PORT',
-        :description => 'Postgresql database port (default: 5432)',
-        :default => 5432
-
-      option :sql_user,
-        :long => "--sql-user USERNAME",
-        :description => 'User used to connect to the postgresql database.'
-
-      option :sql_password,
-        :long => "--sql-password PASSWORD",
-        :description => 'Password used to connect to the postgresql database'
-
       deps do
         require 'chef/json_compat'
         require 'chef/chef_fs/config'
@@ -73,172 +31,104 @@ class Chef
         require 'securerandom'
         require 'chef/chef_fs/parallelizer'
         require 'chef/tsorter'
-      end
-
-      def configure_chef
-        super
-        Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
-        Chef::ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
+        require 'chef/server'
       end
 
       def run
-        #Check for destination directory argument
-        if name_args.length <= 0
-          ui.error("Must specify backup directory as an argument.")
-          exit 1
-        end
-        dest_dir = name_args[0]
-
-        #Check for pivotal user and key
-        node_name = Chef::Config.node_name
-        client_key = Chef::Config.client_key
-        if node_name != "pivotal"
-          if !File.exist?("/etc/opscode/pivotal.pem")
-            ui.error("Username not configured as pivotal and /etc/opscode/pivotal.pem does not exist.  It is recommended that you run this plugin from your Chef server.")
-            exit 1
-          end
-          Chef::Config.node_name = 'pivotal'
-          Chef::Config.client_key = '/etc/opscode/pivotal.pem'
+        set_dest_dir_from_args!
+        set_client_config!
+        ensure_webui_key_exists!
+        set_skip_user_acl!
+
+        restore_users unless config[:skip_users]
+        restore_user_sql if config[:with_user_sql]
+
+        for_each_organization do |orgname|
+          create_organization(orgname)
+          restore_open_invitations(orgname)
+          add_users_to_org(orgname)
+          upload_org_data(orgname)
         end
 
-        #Check for WebUI Key
-        if config[:webui_key] == nil
-          if !File.exist?("/etc/opscode/webui_priv.pem")
-            ui.error("WebUI not specified and /etc/opscode/webui_priv.pem does not exist.  It is recommended that you run this plugin from your Chef server.")
-            exit 1
-          end
-          ui.warn("WebUI not specified. Using /etc/opscode/webui_priv.pem")
-          webui_key = '/etc/opscode/webui_priv.pem'
+        if config[:skip_useracl]
+          ui.warn("Skipping user ACL update. To update user ACLs, remove --skip-useracl or upgrade your Enterprise Chef Server.")
         else
-          webui_key = config[:webui_key]
+          restore_user_acls
         end
+      end
 
-        #Set the server root
-        server_root = Chef::Config.chef_server_root
-        if server_root == nil
-          server_root = Chef::Config.chef_server_url.gsub(/\/organizations\/+[^\/]+\/*$/, '')
-          ui.warn("chef_server_root not found in knife configuration. Setting root to: #{server_root}")
-          Chef::Config.chef_server_root = server_root
+      def create_organization(orgname)
+        org = JSONCompat.from_json(File.read("#{dest_dir}/organizations/#{orgname}/org.json"))
+        rest.post_rest('organizations', org)
+      rescue Net::HTTPServerException => e
+        if e.response.code == "409"
+          rest.put_rest("organizations/#{orgname}", org)
+        else
+          raise
         end
+      end
 
-        if config[:skip_version] && config[:skip_useracl]
-          ui.warn("Skipping the Chef Server version check.  This will also skip any auto-configured options")
-          user_acl_rest = nil
-        elsif config[:skip_version] && !config[:skip_useracl]
-          ui.warn("Skipping the Chef Server version check.  This will also skip any auto-configured options")
-          user_acl_rest = rest
-        else # Grab Chef Server version number so that we can auto set options
-          uri = URI.parse("#{Chef::Config.chef_server_root}/version")
-          server_version = open(uri, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).each_line.first.split(' ').last
-          server_version_parts = server_version.split('.')
-
-          if server_version_parts.count == 3
-            puts "Detected Enterprise Chef Server version: #{server_version}"
-
-            # All versions of Chef Server below 11.0.X are unable to update user acls
-            if server_version_parts[0].to_i < 11 || (server_version_parts[0].to_i == 11 && server_version_parts[1].to_i == 0)
-              ui.warn("Your version of Enterprise Chef Server does not support the updating of User ACLs.  Setting skip-useracl to TRUE")
-              config[:skip_useracl] = true
-              user_acl_rest = nil
-            else
-              user_acl_rest = rest
+      def restore_open_invitations(orgname)
+        invitations = JSONCompat.from_json(File.read("#{dest_dir}/organizations/#{orgname}/invitations.json"))
+        invitations.each do |invitation|
+          begin
+            rest.post_rest("organizations/#{orgname}/association_requests", { 'user' => invitation['username'] })
+          rescue Net::HTTPServerException => e
+            if e.response.code != "409"
+              ui.error("Cannot create invitation #{invitation['id']}")
             end
-          else
-            ui.warn("Unable to detect Chef Server version.")
           end
         end
+      end
 
-        rest = Chef::REST.new(Chef::Config.chef_server_root)
-        # Restore users
-        restore_users(dest_dir, rest) unless config[:skip_users]
-        restore_user_sql(dest_dir) if config[:with_user_sql]
-
-        # Restore organizations
-        Dir.foreach("#{dest_dir}/organizations") do |name|
-          next if name == '..' || name == '.' || !File.directory?("#{dest_dir}/organizations/#{name}")
-          next unless (config[:org].nil? || config[:org] == name)
-          puts "Restoring org #{name} ..."
-
-          # Create organization
-          org = JSONCompat.from_json(IO.read("#{dest_dir}/organizations/#{name}/org.json"))
+      def add_users_to_org(orgname)
+        members = JSONCompat.from_json(File.read("#{dest_dir}/organizations/#{orgname}/members.json"))
+        members.each do |member|
+          username = member['user']['username']
           begin
-            rest.post_rest('organizations', org)
+            response = rest.post_rest("organizations/#{orgname}/association_requests", { 'user' => username })
+            association_id = response["uri"].split("/").last
+            rest.put_rest("users/#{username}/association_requests/#{association_id}", { 'response' => 'accept' })
           rescue Net::HTTPServerException => e
-            if e.response.code == "409"
-              rest.put_rest("organizations/#{name}", org)
-            else
+            if e.response.code != "409"
               raise
             end
           end
+        end
+      end
 
-          # Restore open invitations
-          invitations = JSONCompat.from_json(IO.read("#{dest_dir}/organizations/#{name}/invitations.json"))
-          invitations.each do |invitation|
-            begin
-              rest.post_rest("organizations/#{name}/association_requests", { 'user' => invitation['username'] })
-            rescue Net::HTTPServerException => e
-              if e.response.code != "409"
-                ui.error("Cannot create invitation #{invitation['id']}")
-              end
-            end
-          end
-
-          # Repopulate org members
-          members = JSONCompat.from_json(IO.read("#{dest_dir}/organizations/#{name}/members.json"))
-          members.each do |member|
-            username = member['user']['username']
-            begin
-              response = rest.post_rest("organizations/#{name}/association_requests", { 'user' => username })
-              association_id = response["uri"].split("/").last
-              rest.put_rest("users/#{username}/association_requests/#{association_id}", { 'response' => 'accept' })
-            rescue Net::HTTPServerException => e
-              if e.response.code != "409"
-                raise
-              end
-            end
-          end
-
-          # Upload org data
-          upload_org(dest_dir, webui_key, name)
+      def restore_user_acls
+        ui.msg "Restoring user ACLs ..."
+        for_each_user do |name|
+          user_acl = JSONCompat.from_json(File.read("#{dest_dir}/user_acls/#{name}.json"))
+          put_acl(user_acl_rest, "users/#{name}/_acl", user_acl)
         end
+      end
 
-        # Restore user ACLs
-        puts "Restoring user ACLs ..."
+      def for_each_user
         Dir.foreach("#{dest_dir}/users") do |filename|
           next if filename !~ /(.+)\.json/
           name = $1
-          if config[:skip_useracl]
-            ui.warn("Skipping user ACL update for #{name}. To update this ACL, remove --skip-useracl or upgrade your Enterprise Chef Server.")
-            next
-          end
           if name == 'pivotal' && !config[:overwrite_pivotal]
-            ui.warn("Skipping pivotal update.  To overwrite pivotal, pass --overwrite-pivotal.")
+            ui.warn("Skipping pivotal user.  To overwrite pivotal, pass --overwrite-pivotal.")
             next
           end
-
-          # Update user acl
-          user_acl = JSONCompat.from_json(IO.read("#{dest_dir}/user_acls/#{name}.json"))
-          put_acl(rest, "users/#{name}/_acl", user_acl)
+          yield name
         end
+      end
 
-
-        if @error
-          exit 1
+      def for_each_organization
+        Dir.foreach("#{dest_dir}/organizations") do |name|
+          next if name == '..' || name == '.' || !File.directory?("#{dest_dir}/organizations/#{name}")
+          next unless (config[:org].nil? || config[:org] == name)
+          yield name
         end
       end
 
-      def restore_users(dest_dir, rest)
-        puts "Restoring users ..."
-        Dir.foreach("#{dest_dir}/users") do |filename|
-          next if filename !~ /(.+)\.json/
-          name = $1
-          if name == 'pivotal' && !config[:overwrite_pivotal]
-            ui.warn("Skipping pivotal update.  To overwrite pivotal, pass --overwrite-pivotal.")
-            next
-          end
-
-          # Update user object
-          user = JSONCompat.from_json(IO.read("#{dest_dir}/users/#{name}.json"))
+      def restore_users
+        ui.msg "Restoring users ..."
+        for_each_user do |name|
+          user = JSONCompat.from_json(File.read("#{dest_dir}/users/#{name}.json"))
           begin
             # Supply password for new user
             user_with_password = user.dup
@@ -254,7 +144,7 @@ class Chef
         end
       end
 
-      def restore_user_sql(dest_dir)
+      def restore_user_sql
         require 'chef/knife/ec_key_import'
         k = Chef::Knife::EcKeyImport.new
         k.name_args = ["#{dest_dir}/key_dump.json"]
@@ -268,7 +158,7 @@ class Chef
       end
 
       PATHS = %w(chef_repo_path cookbook_path environment_path data_bag_path role_path node_path client_path acl_path group_path container_path)
-      def upload_org(dest_dir, webui_key, name)
+      def upload_org_data(name)
         old_config = Chef::Config.save
 
         begin
@@ -279,40 +169,42 @@ class Chef
 
           Chef::Config.chef_repo_path = "#{dest_dir}/organizations/#{name}"
           Chef::Config.versioned_cookbooks = true
-
-          Chef::Config.chef_server_url = "#{Chef::Config.chef_server_root}/organizations/#{name}"
+          Chef::Config.chef_server_url = "#{server.root_url}/organizations/#{name}"
 
           # Upload the admins group and billing-admins acls
-          puts "Restoring the org admin data"
+          ui.msg "Restoring the org admin data"
           chef_fs_config = Chef::ChefFS::Config.new
 
-          # Restore users w/o clients (which don't exist yet)
+          # Handle Admins and Billing Admins seperately
+          #
+          # admins: We need to upload admins first so that we
+          # can upload all of the other objects as a user in the org
+          # rather than as pivotal.  Because the clients, and groups, don't
+          # exist yet, we first upload the group with only the users.
+          #
+          # billing-admins: The default permissions on the
+          # billing-admin group only give update permissions to
+          # pivotal and members of the billing-admins group. Since we
+          # can't unsure that the admin we choose for uploading will
+          # be in the billing admins group, we have to upload this
+          # group as pivotal.  Thus, we upload its users and ACL here,
+          # and then update it again once all of the clients and
+          # groups are uploaded.
+          #
           ['admins', 'billing-admins'].each do |group|
             restore_group(chef_fs_config, group, :clients => false)
           end
 
           pattern = Chef::ChefFS::FilePattern.new('/acls/groups/billing-admins.json')
-          if Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.local_fs, chef_fs_config.chef_fs, nil, config, ui, proc { |entry| chef_fs_config.format_path(entry) })
-            @error = true
-          end
+          Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.local_fs,
+                                           chef_fs_config.chef_fs, nil, config, ui,
+                                           proc { |entry| chef_fs_config.format_path(entry)})
 
-          # Figure out who the admin is so we can spoof him and retrieve his stuff
-          rest = Chef::REST.new(Chef::Config.chef_server_url)
-          org_admins = rest.get_rest('groups/admins')['users']
-          org_members = rest.get_rest('users').map { |user| user['user']['username'] }
-          org_admins.delete_if { |user| !org_members.include?(user) || user == 'pivotal' }
-          if org_admins[0] != nil
-            # Using an org admin already on the destination server
-            Chef::Config.node_name = org_admins[0]
-            Chef::Config.client_key = webui_key
-          else
-            # No suitable org admins found, defaulting to pivotal
-            ui.warn("No suitable Organizational Admins found.  Defaulting to pivotal for org creation")
-          end
-          Chef::Config.custom_http_headers = (Chef::Config.custom_http_headers || {}).merge({'x-ops-request-source' => 'web'})
+          Chef::Config.node_name = org_admin
 
           # Restore the entire org skipping the admin data and restoring groups and acls last
-          puts "Restoring the rest of the org"
+          ui.msg "Restoring the rest of the org"
+          ui.debug "Using admin user: #{org_admin}"
           chef_fs_config = Chef::ChefFS::Config.new
           top_level_paths = chef_fs_config.local_fs.children.select { |entry| entry.name != 'acls' && entry.name != 'groups' }.map { |entry| entry.path }
 
@@ -326,10 +218,9 @@ class Chef
           (top_level_paths + group_paths + group_acl_paths + acl_paths).each do |path|
             Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new(path), chef_fs_config.local_fs, chef_fs_config.chef_fs, nil, config, ui, proc { |entry| chef_fs_config.format_path(entry) })
           end
-          # restore clients to groups, using the pivotal key again
-          Chef::Config[:node_name] = old_config[:node_name]
-          Chef::Config[:client_key] = old_config[:client_key]
-          Chef::Config.custom_http_headers = {}
+
+          # restore clients to groups, using the pivotal user again
+          Chef::Config[:node_name] = 'pivotal'
           ['admins', 'billing-admins'].each do |group|
             restore_group(Chef::ChefFS::Config.new, group)
           end
@@ -384,10 +275,6 @@ class Chef
         group.write(members.to_json)
       end
 
-      def parallelize(entries, options = {}, &block)
-        Chef::ChefFS::Parallelizer.parallelize(entries, options, &block)
-      end
-
       def put_acl(rest, url, acls)
         old_acls = rest.get_rest(url)
         old_acls = Chef::ChefFS::DataHandler::AclDataHandler.new.normalize(old_acls, nil)