knife-ec-backup 2.0.0.beta.2 → 2.0.0.beta.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a427844677521bea76fb22774f405ae3ed97af6
-  data.tar.gz: ac92e509e8ecee7a4e14852f99c5e43a72305306
+  metadata.gz: 2078c6252a1d006cfb49b5f6c3c2a7fc8e47ad30
+  data.tar.gz: b67b59762f716a2b66677d56beb86de993020682
 SHA512:
-  metadata.gz: 4f27a84674dc02eaee070700efc7f8bf4e898710342d1059b3881deb6c4bd9b75b4aef109708604364a0188cf65a7c67b665460b08c47e019c3abdf77ca0a8a3
-  data.tar.gz: ad983d25da97492db691b431ac6fef6f60f468d314ea2909f74132eec633053a8a26cd88b03890703d6fa86f8bd133b7f9ca77091fccda2fdb603406682ff817
+  metadata.gz: ad00c21eb5460532a51301a12d91c786aca6c8cd77b745024b639a4bcdf082b8fb25e416bc68eee84a4b1bcd942e0d3915dfd0e5dc2fba69a4971c372bb9451f
+  data.tar.gz: 091c6bf72543e5d197c361f906d998904d89dcc7e22a5935b88fc170565f5cc303e6cad38b25ef4a512a8889647bc1bfca739cd9772f6853c965cc1c3ecefcf6

data/README.md

@@ -2,7 +2,14 @@
 
 # Description
 
-This is an UNOFFICIAL and EXPERIMENTAL knife plugin intended to back up and restore an entire Enterprise Chef / Private Chef server, preserving the data in an intermediate, editable text format.
+knife-ec-backup can backup and restore the data in an Enterprise Chef
+Server installation, preserving the data in an intermediate, editable
+text format.  It is similar to the `knife download` and `knife upload`
+commands and uses the same underlying libraries, but also includes
+workarounds for objects not yet supported by those tools and various
+Server API deficiencies.  The long-run goal is to improve `knife
+donwload`, `knife upload` and the Chef Server API and deprecate this
+tool.
 
 # Requirements
 
@@ -20,6 +27,13 @@ This will install the plugin directly on the Chef Server:
 
     /opt/opscode/embedded/bin/gem install knife-ec-backup
 
+The latest versions of knife-ec-backup require gems with native
+extensions, thus you must install a standard build toolchain.  To
+install knife-ec-backup without installing libpq development headers
+on your system, try the following:
+
+   /opt/opscode/embedded/bin/gem install knife-ec-backup -- --with-pg-config=/opt/opscode/embedded/postgresql/9.2/bin/pg_config
+
 ## Build from source
 Clone the git repository and run the following from inside:
 
@@ -29,22 +43,61 @@ Clone the git repository and run the following from inside:
 # Configuration
 
 ## Permissions
+
 Note that most users in an EC installation lack the permissions to pull all of the data from all organizations and other users.
 This plugin **REQUIRES THE PIVOTAL KEY AND WEBUI KEY** from the Chef Server.
 It is recommended that you run this from a frontend Enterprise Chef Server, you can use --user and --key to pass the pivotal information along.
 
 # Subcommands
 
+## Common Option
+
+The following options are supported across all subcommands:
+
+  * `--sql_host`:
+    The hostname of the Chef Server's postgresql server. (default: localhost)
+
+  * `--sql_port`:
+    The postgresql listening port on the Chef Server. (default: 5432)
+
+  * `--sql_user`:
+    The username of postgresql user with access to the opscode_chef
+    database. (default: autoconfigured from
+    /etc/opscode/chef-server-running.json)
+
+  * `--sql_password`:
+    The password for the sql_user.  (default: autoconfigured from /etc/opscode/chef-server-running.json)
+
 ## knife ec backup DEST_DIR (options)
 
 *Options*
 
+  * `--concurrency THREAD_COUNT`:
+    The maximum number of concurrent requests to make to the Chef
+    Server. (default: 10)
+
   * `--webui-key`:
     Used to set the path to the WebUI Key (default: /etc/opscode/webui_priv.pem)
+    skip any auto-configured options (default: false)
+
+  * `--with-user-sql`:
+    Whether to backup/restore user data directly from the database.  This
+    requires access to the listening postgresql port on the Chef
+    Server.  This is required to correctly handle user passwords and
+    to ensure user-specific association groups are not duplicated.
+
   * `--skip-useracl`:
-    Whether to skip downloading User ACLs.  This is required for EC 11.0.0 and lower (default: false)
-  * `--skip-version-check'`:
-    Whether to skip checking the Chef Server version.  This will also skip any auto-configured options (default: false)
+    Skip download/restore of the user ACLs.  User ACLs are the
+    permissions that actors have *on other global users*.  These are
+    not the ACLs that control what permissions users have on various
+    Chef objects.
+
+  * `--skip-version-check`:
+    Skip Chef Server version check. This will also skip any auto-configured options (default: false)
+
+  * `--only-org ORG`:
+    Only donwload/restore objects in the named organization. Global
+    objects such as users will still be downloaded/restored.
 
 Creates a repository of an entire Enterprise Chef / Private Chef server.
 
@@ -100,24 +153,82 @@ This compares very closely with the "knife download /" from an OSC server:
 
 ## knife ec restore DEST_DIR (options)
 
+Restores all data from the specified DEST_DIR to an Enterprise Chef /
+Private Chef server. DEST_DIR should be a backup directory created by
+`knife ec restore`
+
 *Options*
 
   * `--webui-key`:
     Used to set the path to the WebUI Key (default: /etc/opscode/webui_priv.pem)
+
   * `--overwrite-pivotal`:
-    Whether to overwrite pivotal's key.  Once this is done, future requests will fail until you fix the private key (default: false)
+    Whether to overwrite pivotal's key.  Once this is done, future
+    requests will fail until you fix the private key (default: false)
+
+  * `--skip-users`:
+    Skip the restore of global users.  This may cause organization
+    uploading to fail if the necessary users do not exist on the Chef
+    Server.
+
+  * `--concurrency THREAD_COUNT`:
+    The maximum number of concurrent requests to make to the Chef
+    Server. (default: 10)
+
+  * `--skip-version-check`:
+    Skip Chef Server version check. This will
+    also skip any auto-configured options (default: false)
+
+  * `--with-user-sql`:
+    Whether to backup/restore user data directly from the database.  This
+    requires access to the listening postgresql port on the Chef
+    Server.  This is required to correctly handle user passwords and
+    to ensure user-specific association groups are not
+    duplicated. This option will only work on `restore` if it was also
+    used during the `backup`.
+
   * `--skip-useracl`:
-    Whether to skip downloading User ACLs.  This is required for EC 11.0.0 and lower (default: false)
-  * `--skip-version-check'`:
-    Whether to skip checking the Chef Server version.  This will also skip any auto-configured options (default: false)
+    Skip download/restore of the user ACLs.  User ACLs are the
+    permissions that actors have *on other global users*.  These are
+    not the ACLs that control what permissions users have on various
+    Chef objects.
+
+  * `--only-org ORG`:
+    Only donwload/restore objects in the named organization. Global
+    objects such as users will still be downloaded/restored.
+
+## knife ec key export [FILENAME]
+
+Create a json representation of the users table from the Chef Server
+database.  If no argument is given, the name of the backup is
+`key_dump.json`.
+
+Please note, most users should use `knife ec backup` with the
+`--with-user-sql` option rather than this command.
+
+## knife ec key import [FILENAME]
+
+Import a json representation of the users table from FILENAME to the
+the Chef Server database.  If no argument is given, the filename is
+assumed to be `key_dump.json`.
+
+Please note, most user should use `knife ec restore` with the
+`--with-user-sql` option rather than this command.
+
+# Known Bugs
+
+- `knife ec restore` can fail to restore cookbooks, failing with an
+  internal server error. A common cause of this problem is a
+  concurrency bug in Chef Server. Setting `--concurrency 1` can often
+  work around the issue.
 
-Restores all data from a repository to an Enterprise Chef / Private Chef server.
+- `knife ec restore` can fail if the pool of pre-created organizations
+  can not keep up with the newly created organizations.  This can
+  typically be resolved simply be restarting the restore.  To avoid
+  this error for backups with large number of organizations, try
+  setting (in /etc/opscode/private-chef.rb):
 
-# TODO
+        opscode_org_creator['ready_org_depth']
 
-* Ensure easy installation into embedded ruby gemset on Chef Server.
-* Remove requirement for Knife Essentials gem to be installed.
-* Single org backups.
-* This plugin does **NOT** currently backup user passwords.  **They will have to be reset after a restore.**
-* This plugin does **NOT** currently restore user public keys.  **Private keys will have to be reset after a restore.**
-* This plugin does **NOT** currently restore custom user ACLs.  **It will revert back to default ACLs on a restore.**
+  to the number of organizations in your backup and waiting for the
+  pool to fill before running `knife ec restore`

data/lib/chef/knife/ec_backup.rb

@@ -1,214 +1,133 @@
 require 'chef/knife'
+require 'chef/knife/ec_base'
 
 class Chef
   class Knife
     class EcBackup < Chef::Knife
-      banner "knife ec backup DIRECTORY"
-
-      option :concurrency,
-        :long => '--concurrency THREADS',
-        :description => 'Maximum number of simultaneous requests to send (default: 10)'
-
-      option :webui_key,
-        :long => '--webui-key KEYPATH',
-        :description => 'Used to set the path to the WebUI Key (default: /etc/opscode/webui_priv.pem)'
-
-      option :skip_useracl,
-        :long => '--skip-useracl',
-        :boolean => true,
-        :default => false,
-        :description => "Whether to skip downloading User ACLs.  This is required for EC 11.0.0 and lower"
-
-      option :skip_version,
-        :long => '--skip-version-check',
-        :boolean => true,
-        :default => false,
-        :description => "Whether to skip checking the Chef Server version.  This will also skip any auto-configured options"
-
-      option :with_user_sql,
-        :long => '--with-user-sql',
-        :description => 'Whether to try direct data base access for user export.  Required to properly handle passwords, keys, and USAGs'
-
-      option :org,
-        :long => '--only-org ORGNAME',
-        :description => "Only back up objects in the named organization (default: all orgs)"
-
-      option :sql_host,
-        :long => '--sql-host HOSTNAME',
-        :description => 'Postgresql database hostname (default: localhost)',
-        :default => "localhost"
 
-      option :sql_port,
-        :long => '--sql-port PORT',
-        :description => 'Postgresql database port (default: 5432)',
-        :default => 5432
-
-      option :sql_user,
-        :long => "--sql-user USERNAME",
-        :description => 'User used to connect to the postgresql database.'
-
-      option :sql_password,
-        :long => "--sql-password PASSWORD",
-        :description => 'Password used to connect to the postgresql database'
+      include Knife::EcBase
 
+      banner "knife ec backup DIRECTORY"
 
       deps do
         require 'chef/chef_fs/config'
         require 'chef/chef_fs/file_system'
         require 'chef/chef_fs/file_pattern'
         require 'chef/chef_fs/parallelizer'
-      end
-
-      def configure_chef
-        super
-        Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
-        Chef::ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
+        require 'chef/server'
+        require 'fileutils'
       end
 
       def run
-        #Check for destination directory argument
-        if name_args.length <= 0
-          ui.error("Must specify backup directory as an argument.")
-          exit 1
-        end
-        dest_dir = name_args[0]
-
+        set_dest_dir_from_args!
+        set_client_config!
+        set_skip_user_acl!
+        ensure_webui_key_exists!
 
-        #Check for pivotal user and key
-        node_name = Chef::Config.node_name
-        client_key = Chef::Config.client_key
-        if node_name != "pivotal"
-          if !File.exist?("/etc/opscode/pivotal.pem")
-            ui.error("Username not configured as pivotal and /etc/opscode/pivotal.pem does not exist.  It is recommended that you run this plugin from your Chef server.")
-            exit 1
+        for_each_user do |username, url|
+          download_user(username, url)
+          if config[:skip_useracl]
+            ui.warn("Skipping user ACL download for #{username}. To download this ACL, remove --skip-useracl or upgrade your Enterprise Chef Server.")
+          else
+            download_user_acl(username)
           end
-          Chef::Config.node_name = 'pivotal'
-          Chef::Config.client_key = '/etc/opscode/pivotal.pem'
         end
 
-        #Check for WebUI Key
-        if config[:webui_key] == nil
-          if !File.exist?("/etc/opscode/webui_priv.pem")
-            ui.error("WebUI not specified and /etc/opscode/webui_priv.pem does not exist.  It is recommended that you run this plugin from your Chef server.")
-            exit 1
-          end
-          ui.warn("WebUI not specified. Using /etc/opscode/webui_priv.pem")
-          webui_key = '/etc/opscode/webui_priv.pem'
-        else
-          webui_key = config[:webui_key]
+        if config[:with_user_sql]
+          export_users_from_sql
         end
 
-        #Set the server root
-        server_root = Chef::Config.chef_server_root
-        if server_root == nil
-          server_root = Chef::Config.chef_server_url.gsub(/\/organizations\/+[^\/]+\/*$/, '')
-          ui.warn("chef_server_root not found in knife configuration. Setting root to: #{server_root}")
-          Chef::Config.chef_server_root = server_root
+        ensure_dir("#{dest_dir}/organizations")
+        for_each_organization do |org_object|
+          name = org_object['name']
+          write_org_object_to_disk(org_object)
+          download_org_data(name)
+          download_org_members(name)
+          download_org_invitations(name)
         end
+      end
 
-        rest = Chef::REST.new(Chef::Config.chef_server_root)
-       
-        if config[:skip_version] && config[:skip_useracl]
-          ui.warn("Skipping the Chef Server version check.  This will also skip any auto-configured options")
-          user_acl_rest = nil
-        elsif config[:skip_version] && !config[:skip_useracl]
-          ui.warn("Skipping the Chef Server version check.  This will also skip any auto-configured options")
-          user_acl_rest = rest
-        else # Grab Chef Server version number so that we can auto set options
-          uri = URI.parse("#{Chef::Config.chef_server_root}/version")
-          server_version = open(uri, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).each_line.first.split(' ').last
-          server_version_parts = server_version.split('.')
-
-          if server_version_parts.count == 3
-            puts "Detected Enterprise Chef Server version: #{server_version}"
-
-            # All versions of Chef Server below 11.0.1 are missing the GET User ACL helper in nginx
-            if server_version_parts[0].to_i < 11 || (server_version_parts[0].to_i == 11 && server_version_parts[1].to_i == 0 && server_version_parts[0].to_i < 1)
-              #Check to see if Opscode-Account can be directly from the local machine  
-              begin
-                user_acl_rest.get('users')
-                ui.warn("Your version of Enterprise Chef Server does not support the downloading of User ACLs.  Using local connection to backup")
-                user_acl_rest = Chef::REST.new("http://127.0.0.1:9465")
-              rescue
-                ui.warn("Your version of Enterprise Chef Server does not support the downloading of User ACLs.  Setting skip-useracl to TRUE")
-                config[:skip_useracl] = true
-                user_acl_rest = nil
-              end
-            else
-              user_acl_rest = rest
-            end
+      def for_each_user
+        rest.get_rest('/users').each_pair do |name, url|
+          yield name, url
+        end
+      end
 
+      def for_each_organization
+        rest.get_rest('/organizations').each_pair do |name, url|
+          next unless (config[:org].nil? || config[:org] == name)
+          ui.msg "Downloading organization object for #{name} from #{url}"
+          org = rest.get_rest(url)
+          # Enterprise Chef 11 and below uses a pool of precreated
+          # organizations to account for slow organization creation
+          # using CouchDB. Thus, on server versions < 12 we want to
+          # skip any of these precreated organizations by checking if
+          # they have been assigned or not.  The Chef 12 API does not
+          # return an assigned_at field.
+          if org['assigned_at'] || server.version >= Gem::Version.new("12")
+            yield org
           else
-            ui.warn("Unable to detect Chef Server version.")
+            ui.msg "Skipping pre-created org #{name}"
           end
         end
+      end
 
-        # Grab users
-        puts "Grabbing users ..."
+      def download_user(username, url)
         ensure_dir("#{dest_dir}/users")
-        ensure_dir("#{dest_dir}/user_acls")
-
-        rest.get_rest('/users').each_pair do |name, url|
-          File.open("#{dest_dir}/users/#{name}.json", 'w') do |file|
-            file.write(Chef::JSONCompat.to_json_pretty(rest.get_rest(url)))
-          end
-
-          if config[:skip_useracl]
-            ui.warn("Skipping user ACL download for #{name}. To download this ACL, remove --skip-useracl or upgrade your Enterprise Chef Server.")
-            next
-          end
+        File.open("#{dest_dir}/users/#{username}.json", 'w') do |file|
+          file.write(Chef::JSONCompat.to_json_pretty(rest.get_rest(url)))
+        end
+      end
 
-          File.open("#{dest_dir}/user_acls/#{name}.json", 'w') do |file|
-            file.write(Chef::JSONCompat.to_json_pretty(user_acl_rest.get_rest("users/#{name}/_acl")))
-          end
+      def download_user_acl(username)
+        ensure_dir("#{dest_dir}/user_acls")
+        File.open("#{dest_dir}/user_acls/#{username}.json", 'w') do |file|
+          file.write(Chef::JSONCompat.to_json_pretty(user_acl_rest.get_rest("users/#{username}/_acl")))
         end
+      end
 
-        if config[:with_user_sql]
-          require 'chef/knife/ec_key_export'
-          Chef::Knife::EcKeyExport.deps
-          k = Chef::Knife::EcKeyExport.new
-          k.name_args = ["#{dest_dir}/key_dump.json"]
-          k.config[:sql_host] = config[:sql_host]
-          k.config[:sql_port] = config[:sql_port]
-          k.config[:sql_user] = config[:sql_user]
-          k.config[:sql_password] = config[:sql_password]
-          k.run
+      def export_users_from_sql
+        require 'chef/knife/ec_key_export'
+        Chef::Knife::EcKeyExport.deps
+        k = Chef::Knife::EcKeyExport.new
+        k.name_args = ["#{dest_dir}/key_dump.json"]
+        k.config[:sql_host] = config[:sql_host]
+        k.config[:sql_port] = config[:sql_port]
+        k.config[:sql_user] = config[:sql_user]
+        k.config[:sql_password] = config[:sql_password]
+        k.run
+      end
+
+      def write_org_object_to_disk(org_object)
+        name = org_object['name']
+        ensure_dir("#{dest_dir}/organizations/#{name}")
+        File.open("#{dest_dir}/organizations/#{name}/org.json", 'w') do |file|
+          file.write(Chef::JSONCompat.to_json_pretty(org_object))
         end
+      end
 
-        # Download organizations
-        ensure_dir("#{dest_dir}/organizations")
-        rest.get_rest('/organizations').each_pair do |name, url|
-          do_org = (config[:org].nil? || config[:org] == name)
-          org = rest.get_rest(url)
-          if org['assigned_at'] and do_org
-            puts "Grabbing organization #{name} ..."
-            ensure_dir("#{dest_dir}/organizations/#{name}")
-            download_org(dest_dir, webui_key, name)
-            File.open("#{dest_dir}/organizations/#{name}/org.json", 'w') do |file|
-              file.write(Chef::JSONCompat.to_json_pretty(org))
-            end
-            File.open("#{dest_dir}/organizations/#{name}/members.json", 'w') do |file|
-              file.write(Chef::JSONCompat.to_json_pretty(rest.get_rest("#{url}/users")))
-            end
-            File.open("#{dest_dir}/organizations/#{name}/invitations.json", 'w') do |file|
-              file.write(Chef::JSONCompat.to_json_pretty(rest.get_rest("#{url}/association_requests")))
-            end
-          end
+      def download_org_members(name)
+        ensure_dir("#{dest_dir}/organizations/#{name}")
+        File.open("#{dest_dir}/organizations/#{name}/members.json", 'w') do |file|
+          file.write(Chef::JSONCompat.to_json_pretty(rest.get_rest("/organizations/#{name}/users")))
         end
+      end
 
-        if @error
-          exit 1
+      def download_org_invitations(name)
+        ensure_dir("#{dest_dir}/organizations/#{name}")
+        File.open("#{dest_dir}/organizations/#{name}/invitations.json", 'w') do |file|
+          file.write(Chef::JSONCompat.to_json_pretty(rest.get_rest("/organizations/#{name}/association_requests")))
         end
       end
 
       def ensure_dir(dir)
         if !File.exist?(dir)
-          Dir.mkdir(dir)
+          FileUtils.mkdir_p(dir)
         end
       end
 
       PATHS = %w(chef_repo_path cookbook_path environment_path data_bag_path role_path node_path client_path acl_path group_path container_path)
-      def download_org(dest_dir, webui_key, name)
+      def download_org_data(name)
         old_config = Chef::Config.save
 
         begin
@@ -219,48 +138,47 @@ class Chef
           Chef::Config.chef_repo_path = "#{dest_dir}/organizations/#{name}"
           Chef::Config.versioned_cookbooks = true
 
-          Chef::Config.chef_server_url = "#{Chef::Config.chef_server_root}/organizations/#{name}"
+          Chef::Config.chef_server_url = "#{server.root_url}/organizations/#{name}"
 
           ensure_dir(Chef::Config.chef_repo_path)
 
           # Download the billing-admins ACL and group as pivotal
           chef_fs_config = Chef::ChefFS::Config.new
-          pattern = Chef::ChefFS::FilePattern.new('/acls/groups/billing-admins.json') 
-          if Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.chef_fs, chef_fs_config.local_fs, nil, config, ui, proc { |entry| chef_fs_config.format_path(entry) })
-            @error = true
-          end
-          pattern = Chef::ChefFS::FilePattern.new('/groups/billing-admins.json') 
-          if Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.chef_fs, chef_fs_config.local_fs, nil, config, ui, proc { |entry| chef_fs_config.format_path(entry) })
-            @error = true
-          end
-          pattern = Chef::ChefFS::FilePattern.new('/groups/admins.json') 
-          if Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.chef_fs, chef_fs_config.local_fs, nil, config, ui, proc { |entry| chef_fs_config.format_path(entry) })
-            @error = true
-          end
+          chef_fs_copy_pattern('/acls/groups/billing-admins.json', chef_fs_config)
+          chef_fs_copy_pattern('/groups/billing-admins.json', chef_fs_config)
+          chef_fs_copy_pattern('/groups/admins.json', chef_fs_config)
 
-          # Figure out who the admin is so we can spoof him and retrieve his stuff
-          rest = Chef::REST.new(Chef::Config.chef_server_url)
-          admin_users = rest.get_rest('groups/admins')['users']
-          org_members = rest.get_rest('users').map { |user| user['user']['username'] }
-          admin_users.delete_if { |user| !org_members.include?(user) }
-          Chef::Config.node_name = admin_users[0]
-          Chef::Config.client_key = webui_key
-          Chef::Config.custom_http_headers = (Chef::Config.custom_http_headers || {}).merge({'x-ops-request-source' => 'web'})
+          # Set Chef::Config to use an organization administrator
+          Chef::Config.node_name = org_admin
 
           # Download the entire org skipping the billing admins group ACL and the group itself
           chef_fs_config = Chef::ChefFS::Config.new
           top_level_paths = chef_fs_config.chef_fs.children.select { |entry| entry.name != 'acls' && entry.name != 'groups' }.map { |entry| entry.path }
-          acl_paths = Chef::ChefFS::FileSystem.list(chef_fs_config.chef_fs, Chef::ChefFS::FilePattern.new('/acls/*')).select { |entry| entry.name != 'groups' }.map { |entry| entry.path }
-          group_acl_paths = Chef::ChefFS::FileSystem.list(chef_fs_config.chef_fs, Chef::ChefFS::FilePattern.new('/acls/groups/*')).select { |entry| entry.name != 'billing-admins.json' }.map { |entry| entry.path }
-          group_paths = Chef::ChefFS::FileSystem.list(chef_fs_config.chef_fs, Chef::ChefFS::FilePattern.new('/groups/*')).select { |entry| entry.name != 'billing-admins.json' }.map { |entry| entry.path }
+          acl_paths       = chef_fs_paths('/acls/*', chef_fs_config, 'groups')
+          group_acl_paths = chef_fs_paths('/acls/groups/*', chef_fs_config, 'billing-admins.json')
+          group_paths     = chef_fs_paths('/groups/*', chef_fs_config, 'billing-admins.json')
           (top_level_paths + group_acl_paths + acl_paths + group_paths).each do |path|
-            Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new(path), chef_fs_config.chef_fs, chef_fs_config.local_fs, nil, config, ui, proc { |entry| chef_fs_config.format_path(entry) })
+            chef_fs_copy_pattern(path, chef_fs_config)
           end
-
         ensure
           Chef::Config.restore(old_config)
         end
       end
+
+      def chef_fs_paths(pattern_str, chef_fs_config, exclude=nil)
+        pattern = Chef::ChefFS::FilePattern.new(pattern_str)
+        list = Chef::ChefFS::FileSystem.list(chef_fs_config.chef_fs, pattern)
+        list = list.select { |entry| entry.name != exclude } if ! exclude.nil?
+        list.map {|entry| entry.path }
+      end
+
+      def chef_fs_copy_pattern(pattern_str, chef_fs_config)
+        pattern = Chef::ChefFS::FilePattern.new(pattern_str)
+        Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.chef_fs,
+                                         chef_fs_config.local_fs, nil,
+                                         config, ui,
+                                         proc { |entry| chef_fs_config.format_path(entry) })
+      end
     end
   end
 end