knife-ec-backup 2.0.7 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b97714005f59fe60b36667b628c58f9de216f1e2
-  data.tar.gz: 8ba4fc5291580630844c140dea952c78d4a69db2
+  metadata.gz: 5c0eaeaca5c1a8707e66a26cafe08186a77ea6fa
+  data.tar.gz: 2b4675644bc5a526f3a26ad436dfd73a30ed1307
 SHA512:
-  metadata.gz: 49ea3afa7fa7ed54a318c828b235703fda5b38d28a171124ee4837efc43867ffecb4ff46b157c82aee9c7d93da65d39041f6aeb2f50f5792903e22d536e82f2c
-  data.tar.gz: 3443451e72baa28454cf805cfc8c20a241cc21bcf14ff286beb3d80e5caab8bc33d6c658f3191562d2ee0a59c5b8f482835be1b3028d15b272d7e2d433048af7
+  metadata.gz: ba31ebcf31a0aca8b4df975c9687436822c5fabdc81095b23e30c4e60e35f7ff88516334949ad53fa86c4d8fe4c6787c1ed5764fa9ddd0057e132ee73f78eab1
+  data.tar.gz: e751b01f0094618a3fb5a6249ef083a42d20e911135d69d07b9bba77a4438b4118fb1f7f80201ec86ec7cd392865f1d8241775c8b4f9e83305ad9fbf3d6871e8

data/README.md

@@ -84,23 +84,29 @@ It is recommended that you run this from a frontend Enterprise Chef Server, you
 
 # Subcommands
 
-## Common Option
+## Common Options
 
 The following options are supported across all subcommands:
 
-  * `--sql_host`:
+  * `--sql-host`:
     The hostname of the Chef Server's postgresql server. (default: localhost)
 
-  * `--sql_port`:
+  * `--sql-port`:
     The postgresql listening port on the Chef Server. (default: 5432)
 
-  * `--sql_user`:
+  * `--sql-user`:
     The username of postgresql user with access to the opscode_chef
     database. (default: autoconfigured from
     /etc/opscode/chef-server-running.json)
 
-  * `--sql_password`:
-    The password for the sql_user.  (default: autoconfigured from /etc/opscode/chef-server-running.json)
+  * `--sql-password`:
+    The password for the sql-user.  (default: autoconfigured from /etc/opscode/chef-server-running.json)
+
+  * `--purge`:
+    Whether to sync deletions from backup source to restore destination. (default: false)
+
+  * `--dry-run`:
+    Report what actions would be taken without performing any. (default: false)
 
 ## knife ec backup DEST_DIR (options)
 

data/lib/chef/knife/ec_backup.rb

@@ -24,6 +24,9 @@ class Chef
         set_skip_user_acl!
         ensure_webui_key_exists!
 
+        ensure_dir("#{dest_dir}/users")
+        ensure_dir("#{dest_dir}/user_acls") unless config[:skip_useracl]
+        ui.msg 'Downloading Users'
         for_each_user do |username, url|
           download_user(username, url)
           if config[:skip_useracl]
@@ -32,6 +35,7 @@ class Chef
             download_user_acl(username)
           end
         end
+        purge_users_on_backup
 
         if config[:with_user_sql] || config[:with_key_sql]
           export_from_sql
@@ -47,8 +51,30 @@ class Chef
         end
       end
 
+      def users_for_purge
+        purge_list = local_user_list - remote_user_list
+        # failsafe - don't delete pivotal
+        purge_list -= ['pivotal']
+        purge_list.each do |user|
+          yield user
+        end
+      end
+
+      def purge_users_on_backup
+        return unless config[:purge]
+        users_for_purge do |user|
+          ui.msg "Deleting user #{user} from local backup (purge is on)"
+          begin
+            ::File.delete("#{dest_dir}/users/#{user}.json")
+            ::File.delete("#{dest_dir}/user_acls/#{user}.json")
+          rescue Errno::ENOENT => e
+            ui.warn "Failed to find local #{user} data #{e}"
+          end
+        end
+      end
+
       def for_each_user
-        rest.get('/users').each_pair do |name, url|
+        remote_users.each_pair do |name, url|
           yield name, url
         end
       end
@@ -73,14 +99,12 @@ class Chef
       end
 
       def download_user(username, url)
-        ensure_dir("#{dest_dir}/users")
         File.open("#{dest_dir}/users/#{username}.json", 'w') do |file|
           file.write(Chef::JSONCompat.to_json_pretty(rest.get(url)))
         end
       end
 
       def download_user_acl(username)
-        ensure_dir("#{dest_dir}/user_acls")
         File.open("#{dest_dir}/user_acls/#{username}.json", 'w') do |file|
           file.write(Chef::JSONCompat.to_json_pretty(user_acl_rest.get("users/#{username}/_acl")))
         end
@@ -152,16 +176,30 @@ class Chef
           chef_fs_copy_pattern('/groups/public_key_read_access.json', chef_fs_config)
           chef_fs_copy_pattern('/groups/admins.json', chef_fs_config)
 
-          # Set Chef::Config to use an organization administrator
-          Chef::Config.node_name = org_admin
+          Chef::Config.node_name = if config[:skip_version]
+                                     org_admin
+                                   else
+                                     server.supports_defaulting_to_pivotal? ? 'pivotal' : org_admin
+                                   end
 
-          # Download the entire org skipping the billing-admins, public_key_read_access group ACLs and the groups themselves
           chef_fs_config = Chef::ChefFS::Config.new
           top_level_paths = chef_fs_config.chef_fs.children.select { |entry| entry.name != 'acls' && entry.name != 'groups' }.map { |entry| entry.path }
-          acl_paths       = chef_fs_paths('/acls/*', chef_fs_config, ['groups'])
-          group_acl_paths = chef_fs_paths('/acls/groups/*', chef_fs_config, ['billing-admins','public_key_read_access'])
-          group_paths     = chef_fs_paths('/groups/*', chef_fs_config, ['billing-admins','public_key_read_access'])
-          (top_level_paths + group_acl_paths + acl_paths + group_paths).each do |path|
+
+          # The top level acl object names end with .json extension
+          # Therefore we can use Chef::ChefFS::FilePattern matching for items
+          # such as /acls/organizations.json
+          #
+          # 2nd level leaf /acl/*/* objects as well as /groups/* objects do not end with .json
+          # therefore we use normalize_path_name to add the .json extension
+          # for example: /acls/environments/_default
+
+           # Skip the billing-admins, public_key_read_access group ACLs and the groups since they've already been copied
+          exclude_list = ['billing-admins','public_key_read_access']
+
+          top_level_acls  = chef_fs_paths('/acls/*.json', chef_fs_config, [])
+          acl_paths       = chef_fs_paths('/acls/*/*', chef_fs_config, exclude_list)
+          group_paths     = chef_fs_paths('/groups/*', chef_fs_config, exclude_list)
+          (top_level_paths + top_level_acls + acl_paths + group_paths).each do |path|
             chef_fs_copy_pattern(path, chef_fs_config)
           end
         ensure

data/lib/chef/knife/ec_base.rb

@@ -18,11 +18,13 @@
 
 require 'chef/knife'
 require 'chef/server_api'
+require 'veil'
 
 class Chef
   class Knife
     module EcBase
       class NoAdminFound < Exception; end
+      class UnImplemented < Exception; end
 
       def self.included(includer)
         includer.class_eval do
@@ -33,8 +35,12 @@ class Chef
 
           option :webui_key,
             :long => '--webui-key KEYPATH',
-            :description => 'Path to the WebUI Key (default: /etc/opscode/webui_priv.pem)',
-            :default => '/etc/opscode/webui_priv.pem'
+            :description => 'Path to the WebUI Key (default: Read from secrets store or /etc/opscode/webui_priv.pem)'
+
+          option :secrets_file_path,
+            :long => '--secrets-file PATH',
+            :description => 'Path to a valid private-chef-secrets.json file (default: /etc/opscode/private-chef-secrets.json)',
+            :default => '/etc/opscode/private-chef-secrets.json'
 
           option :skip_useracl,
             :long => '--skip-useracl',
@@ -78,6 +84,17 @@ class Chef
             :long => '--with-key-sql',
             :description => 'Try direct data base access for key table export/import.  Required to properly handle rotated keys.'
 
+          option :purge,
+            :long => '--purge',
+            :boolean => true | false,
+            :default => false,
+            :description => 'Syncs deletions from backup source to restore destination.'
+
+          option :dry_run,
+            :long => '--dry-run',
+            :boolean => true | false,
+            :default => false,
+            :description => 'Report what actions would be taken without performing any.'
         end
 
         attr_accessor :dest_dir
@@ -116,6 +133,23 @@ class Chef
         @rest ||= Chef::ServerAPI.new(server.root_url, {:api_version => "0"})
       end
 
+      def remote_users
+        @remote_users ||= rest.get('/users')
+      end
+
+      def remote_user_list
+        @remote_user_list ||= remote_users.keys
+      end
+
+      def local_user_list
+        @local_user_list ||= Dir.glob("#{dest_dir}/users/*\.json").map { |u| File.basename(u, '.json') }
+      end
+
+      def users_for_purge
+        # not itended to be called from ec_base
+        raise Chef::Knife::EcBase::UnImplemented
+      end
+
       def user_acl_rest
         @user_acl_rest ||= if config[:skip_version]
                              rest
@@ -134,7 +168,7 @@ class Chef
       def set_client_config!
         Chef::Config.custom_http_headers = (Chef::Config.custom_http_headers || {}).merge({'x-ops-request-source' => 'web'})
         Chef::Config.node_name = 'pivotal'
-        Chef::Config.client_key = config[:webui_key]
+        Chef::Config.client_key = webui_key
       end
 
       def set_dest_dir_from_args!
@@ -145,8 +179,39 @@ class Chef
         @dest_dir = name_args[0]
       end
 
+      def webui_key
+        if config[:webui_key]
+          config[:webui_key]
+        elsif veil.exist?("chef-server", "webui_key")
+          temporary_webui_key
+        else
+          '/etc/opscode/webui_priv.pem'
+        end
+      end
+
+      def veil_config
+        { provider: 'chef-secrets-file',
+          path: config[:secrets_file_path] }
+      end
+
+      def veil
+        Veil::CredentialCollection.from_config(veil_config)
+      end
+
+      def temporary_webui_key
+        @temp_webui_key ||= begin
+                              key_data = veil.get("chef-server", "webui_key")
+                              f = Tempfile.new("knife-ec-backup")
+                              f.write(key_data)
+                              f.flush
+                              f.close
+                              f
+                            end
+        @temp_webui_key.path
+      end
+
       def ensure_webui_key_exists!
-        if !File.exist?(config[:webui_key])
+        if !File.exist?(webui_key)
           ui.error("Webui Key (#{config[:webui_key]}) does not exist.")
           exit 1
         end

data/lib/chef/knife/ec_key_base.rb

@@ -48,6 +48,11 @@ class Chef
           :long => "--sql-password PASSWORD",
           :description => 'Password used to connect to the postgresql database'
 
+          option :secrets_file_path,
+          :long => '--secrets-file PATH',
+          :description => 'Path to a valid private-chef-secrets.json file (default: /etc/opscode/private-chef-secrets.json)',
+          :default => '/etc/opscode/private-chef-secrets.json'
+
           option :skip_keys_table,
           :long => "--skip-keys-table",
           :description => "Skip Chef 12-only keys table",
@@ -83,7 +88,26 @@ class Chef
                        'postgresql'
                      end
           config[:sql_user] ||= running_config['private_chef'][hash_key]['sql_user']
-          config[:sql_password] ||= running_config['private_chef'][hash_key]['sql_password']
+          config[:sql_password] ||= sql_password
+        end
+      end
+
+      def veil_config
+        { provider: 'chef-secrets-file',
+          path: config[:secrets_file_path] }
+      end
+
+      def veil
+        Veil::CredentialCollection.from_config(veil_config)
+      end
+
+      def sql_password
+        if config[:sql_password]
+          config[:sql_password]
+        elsif veil.exist?("opscode_erchef", "sql_password")
+          veil.get("opscode_erchef", "sql_password")
+        else veil.exist?("postgresql", "sql_password")
+          veil.get("postgresql", "sql_password")
         end
       end
     end

data/lib/chef/knife/ec_restore.rb

@@ -145,6 +145,28 @@ class Chef
             end
           end
         end
+        purge_users_on_restore
+      end
+
+      def users_for_purge
+        purge_list = remote_user_list - local_user_list
+        # failsafe - don't delete pivotal
+        purge_list -= ['pivotal']
+        purge_list.each do |user|
+          yield user
+        end
+      end
+
+      def purge_users_on_restore
+        return unless config[:purge]
+        users_for_purge do |user|
+          ui.msg "Deleting user #{user} from remote (purge is on)"
+          begin
+            rest.delete("/users/#{user}")
+          rescue Net::HTTPServerException => e
+            ui.warn "Failed deleting user #{user} from remote #{e}"
+          end
+        end
       end
 
       def ec_key_import
@@ -229,11 +251,14 @@ class Chef
                                            proc { |entry| chef_fs_config.format_path(entry)})
           end
 
-          Chef::Config.node_name = org_admin
+          Chef::Config.node_name = if config[:skip_version]
+                                     org_admin
+                                   else
+                                     server.supports_defaulting_to_pivotal? ? 'pivotal' : org_admin
+                                   end
 
           # Restore the entire org skipping the admin data and restoring groups and acls last
           ui.msg "Restoring the rest of the org"
-          Chef::Log.debug "Using admin user: #{org_admin}"
           chef_fs_config = Chef::ChefFS::Config.new
           top_level_paths = chef_fs_config.local_fs.children.select { |entry| entry.name != 'acls' && entry.name != 'groups' }.map { |entry| entry.path }
 

data/lib/chef/server.rb

@@ -34,5 +34,9 @@ class Chef
     rescue
       false
     end
+
+    def supports_defaulting_to_pivotal?
+      version >= Gem::Version.new('12.1.0')
+    end
   end
 end

data/lib/knife_ec_backup/version.rb

@@ -1,3 +1,3 @@
 module KnifeECBackup
-  VERSION = '2.0.7'
+  VERSION = '2.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-ec-backup
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.1.0
 platform: ruby
 authors:
 - John Keiser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-09 00:00:00.000000000 Z
+date: 2017-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '11.8'
+- !ruby/object:Gem::Dependency
+  name: veil
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Backup and Restore of Enterprise Chef
 email: jkeiser@opscode.com
 executables: []
@@ -102,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Backup and Restore of Enterprise Chef