knife-tidy 0.7.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea6f3ab3ae6f2d01e70092c6df3e6ef4496b70bb
-  data.tar.gz: a5b9ba72970cbf016296c590a4177e23b07a4e82
+  metadata.gz: 1e5c5c4edef9b63996bbfa1caf5890a40f443ee0
+  data.tar.gz: 36eed111cf84de44150c8d39486ae05939cf37e1
 SHA512:
-  metadata.gz: 58fcda1b011c58c583535b929e56e7c50f37853b168544926886c71e373896dc3b985d454c85cd51e9035d41fdd472411cffc9d03876a74a1f0f3a9a972d0005
-  data.tar.gz: a87916d53272aa57ecaa8ef5a628cef0aaca192825bca03dab4a476660f2e803aec7e14f7fd486b6999ba2fb83d2cae068d453fc8828a06c949ee692dce5f23b
+  metadata.gz: edc4d0d480d39c99c0b59b54180003fa60c259c600d30a987bb4cd4e69ec67a54c6f6fc37b31a958d3d725f9f95f0bf79660cf6b554e664fcf764be00bda979b
+  data.tar.gz: 05b0a59b86a3f5a95ed136222555f984bbbe583fdd6acb2064398571b2c88c1ffa9abc7da7fd0917e984be5018c5d3a8c2ab25593ec1e8b0d831ea45012964c5

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Change Log
 
+## [1.0.0](https://github.com/chef-customers/knife-tidy/tree/1.0.0) (2017-12-04)
+[Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.7.0...1.0.0)
+
+**Merged pull requests:**
+
+- Enabled cookbook deletion [\#71](https://github.com/chef-customers/knife-tidy/pull/71) ([itmustbejj](https://github.com/itmustbejj))
+- Add option for backup path to server clean [\#70](https://github.com/chef-customers/knife-tidy/pull/70) ([TheLunaticScripter](https://github.com/TheLunaticScripter))
+- Warn the user if there are nodes created in the last hour that haven'… [\#67](https://github.com/chef-customers/knife-tidy/pull/67) ([itmustbejj](https://github.com/itmustbejj))
+- Add guard to skip generating org reports if the search index is not u… [\#66](https://github.com/chef-customers/knife-tidy/pull/66) ([itmustbejj](https://github.com/itmustbejj))
+- Enable server clean command and clarify confirmation dialogue [\#31](https://github.com/chef-customers/knife-tidy/pull/31) ([jonlives](https://github.com/jonlives))
+
 ## [0.7.0](https://github.com/chef-customers/knife-tidy/tree/0.7.0) (2017-11-29)
 [Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.6.1...0.7.0)
 
@@ -10,6 +21,7 @@
 
 **Merged pull requests:**
 
+- release 0.7.0 [\#65](https://github.com/chef-customers/knife-tidy/pull/65) ([jeremymv2](https://github.com/jeremymv2))
 - Add admins/users groups to the read acl for clients from \< CS 12.5 [\#64](https://github.com/chef-customers/knife-tidy/pull/64) ([itmustbejj](https://github.com/itmustbejj))
 - Restore acls for ::server-admins and org read access groups if they a… [\#61](https://github.com/chef-customers/knife-tidy/pull/61) ([itmustbejj](https://github.com/itmustbejj))
 - Filter email notifications on org\_list config option. [\#60](https://github.com/chef-customers/knife-tidy/pull/60) ([itmustbejj](https://github.com/itmustbejj))

data/README.md

@@ -60,10 +60,17 @@ org_unused_cookbooks.json | List of cookbooks and versions that do not appear to
 ## $ knife tidy server clean --help
 Remove stale nodes that haven't checked-in to the Chef Server as defined by the `--node-threshold NUM_DAYS` option when the reports were generated.. The associated client and ACLs are also removed.
 
-Future: remove unused cookbooks - currently this feature is disabled.
-
 ## Options
 
+  * `--backup-path /path/to/an-ec-backup`
+    The location to the last backup of the target Chef Server. It is not recommended to run the clean command without first taking a current backup using [knife-ec-backup](https://github.com/chef/knife-ec-backup)
+
+  * `--only-cookbooks`
+    Only deletes the unused cookbooks from the target Chef Server. NOTE: Cannot be specified if `--only-nodes` is already specified
+
+  * `--only-nodes`
+    Only deltes the stale nodes, associated clients, and ACLs from the target Chef Server. NOTE: Cannot be specified if `--only-cookbooks` is already specified
+
   * `--dry-run`
     Do not perform any actual deletion, only report on what would have been deleted.
 

data/lib/chef/knife/tidy_backup_clean.rb

@@ -36,7 +36,7 @@ class Chef
         FileUtils.rm_f(action_needed_file_path)
 
         if config[:gen_gsub]
-          Chef::TidySubstitutions.new().boiler_plate
+          Chef::TidySubstitutions.new(nil, tidy).boiler_plate
           exit
         end
 
@@ -67,18 +67,18 @@ class Chef
 
         completion_message
 
-        puts "\nWARNING: ** Unrepairable Items **\nPlease see #{action_needed_file_path}\n" if ::File.exist?(action_needed_file_path)
+        ui.stdout.puts "\nWARNING: ** Unrepairable Items **\nPlease see #{action_needed_file_path}\n" if ::File.exist?(action_needed_file_path)
       end
 
       def validate_user_emails
         emails_seen = []
         tidy.global_user_names.each do |user|
           email = ''
-          puts "INFO: Validating #{user}"
+          ui.stdout.puts "INFO: Validating #{user}"
           the_user = FFI_Yajl::Parser.parse(::File.read(::File.join(tidy.users_path, "#{user}.json")), symbolize_names: false)
           if the_user.has_key?('email') && the_user['email'].match(/\A[^@\s]+@[^@\s]+\z/)
             if emails_seen.include?(the_user['email'])
-              puts "REPAIRING: Already saw #{user}'s email, creating a unique one."
+              ui.stdout.puts "REPAIRING: Already saw #{user}'s email, creating a unique one."
               email = tidy.unique_email
               new_user = the_user.dup
               new_user['email'] = email
@@ -88,7 +88,7 @@ class Chef
               emails_seen.push(the_user['email'])
             end
           else
-            puts "REPAIRING: User #{user} does not have a valid email, creating a unique one."
+            ui.stdout.puts "REPAIRING: User #{user} does not have a valid email, creating a unique one."
             email = tidy.unique_email
             new_user = the_user.dup
             new_user['email'] = email
@@ -102,11 +102,11 @@ class Chef
       # The existence of anything else will cause a restore to fail
       # EC11 backups will contain org objects with 6 extra fields including org_type, billing_plan, assigned_at, etc
       def fix_org_object(org)
-        puts "INFO: Validating org object for #{org}"
+        ui.stdout.puts "INFO: Validating org object for #{org}"
         org_object = load_org_object(org)
 
         unless org_object.keys.count == 3  # cheapo, maybe expect the exact names?
-          puts "REPAIRING: org object for #{org} contains extra/missing fields. Fixing that for you"
+          ui.stdout.puts "REPAIRING: org object for #{org} contains extra/missing fields. Fixing that for you"
           # quick/dirty attempt at fixing any of the required fields in case they're nil
           good_name = org_object['name'] || org
           good_full_name = org_object['full_name'] || org
@@ -120,7 +120,7 @@ class Chef
       def load_org_object(org)
         JSON.parse(File.read(File.join(tidy.org_path(org), 'org.json')))
       rescue Errno::ENOENT, JSON::ParserError
-        puts "REPAIRING: org object for organization #{org} is missing or corrupt. Generating a new one"
+        ui.stdout.puts "REPAIRING: org object for organization #{org} is missing or corrupt. Generating a new one"
         return { name: org, full_name: org, guid: SecureRandom.uuid.gsub('-','') }
       end
 
@@ -129,7 +129,7 @@ class Chef
       end
 
       def add_cookbook_name_to_metadata(cookbook_name, rb_path)
-        puts "REPAIRING: Correcting `name` in #{rb_path}"
+        ui.stdout.puts "REPAIRING: Correcting `name` in #{rb_path}"
         content = IO.readlines(rb_path)
         new_content = content.reject { |line| line =~ /^name\s+/ }
         name_field = "name '#{cookbook_name}'\n"
@@ -150,7 +150,7 @@ class Chef
               metadata = FFI_Yajl::Parser.parse(::File.read(json_path), symbolize_names: false)
               if metadata['name'] != cookbook_name
                 metadata['name'] = cookbook_name
-                puts "REPAIRING: Correcting `name` in #{json_path}`"
+                ui.stdout.puts "REPAIRING: Correcting `name` in #{json_path}`"
                 ::File.open(json_path, 'w') do |f|
                   f.write(Chef::JSONCompat.to_json_pretty(metadata))
                 end
@@ -163,7 +163,7 @@ class Chef
       def load_cookbooks(org)
         cl = Chef::CookbookLoader.new(tidy.cookbooks_path(org))
         for_each_cookbook_basename(org) do |cookbook|
-          puts "INFO: Loading #{cookbook}"
+          ui.stdout.puts "INFO: Loading #{cookbook}"
           ret = cl.load_cookbook(cookbook)
           if ret.nil?
             action_needed("ACTION NEEDED: Something's wrong with the #{cookbook} cookbook in org #{org} - cannot load it! Moving to cookbooks.broken folder.")
@@ -192,8 +192,8 @@ class Chef
 
       def fix_chef_sugar_metadata
         Dir[::File.join(tidy.backup_path, 'organizations/*/cookbooks/chef-sugar*/metadata.rb')].each do |file|
-          puts 'INFO: Searching for known chef-sugar problems when uploading.'
-          s = Chef::TidySubstitutions.new
+          ui.stdout.puts 'INFO: Searching for known chef-sugar problems when uploading.'
+          s = Chef::TidySubstitutions.new(nil, tidy)
           version = s.cookbook_version_from_path(file)
           patterns = [
             {
@@ -216,10 +216,10 @@ class Chef
           name = tidy.cookbook_name_from_path(cookbook_path)
           md_path = ::File.join(cookbook_path, 'metadata.rb')
           unless ::File.exist?(md_path)
-            puts "INFO: No metadata.rb in #{cookbook_path} - skipping"
+            ui.stdout.puts "INFO: No metadata.rb in #{cookbook_path} - skipping"
             next
           end
-          Chef::TidySubstitutions.new.sub_in_file(
+          Chef::TidySubstitutions.new(nil, tidy).sub_in_file(
             ::File.join(cookbook_path, 'metadata.rb'),
             Regexp.new("^depends +['\"]#{name}['\"]"),
             "# depends '#{name}' # knife-tidy was here")
@@ -232,7 +232,7 @@ class Chef
         md = metadata.dup
         metadata.each_pair do |key, value|
           if value.nil?
-            puts "REPAIRING: Fixing null value for key #{key} in #{json_path}"
+            ui.stdout.puts "REPAIRING: Fixing null value for key #{key} in #{json_path}"
             md[key] = 'default value'
           end
         end
@@ -241,7 +241,7 @@ class Chef
             # platform key cannot contain comma delimited values
             md['platforms'].delete(key) if key =~ /,/
             if value.kind_of?(Array) && value.empty?
-              puts "REPAIRING: Fixing empty platform key for for key #{key} in #{json_path}"
+              ui.stdout.puts "REPAIRING: Fixing empty platform key for for key #{key} in #{json_path}"
               md['platforms'][key] = '>= 0.0.0'
             end
           end
@@ -258,10 +258,10 @@ class Chef
           create_minimal_metadata(path)
         end
         unless ::File.exist?(md_path)
-          puts "INFO: No metadata.rb in #{path} - skipping"
+          ui.stdout.puts "INFO: No metadata.rb in #{path} - skipping"
           return
         end
-        puts "INFO: Generating new metadata.json for #{path}"
+        ui.stdout.puts "INFO: Generating new metadata.json for #{path}"
         md = Chef::Cookbook::Metadata.new
         md.name(cookbook)
         md.from_file(md_path)
@@ -290,7 +290,7 @@ class Chef
         metadata['maintainer'] = 'the maintainer'
         metadata['maintainer_email'] = 'the maintainer email'
         rb_file = ::File.join(cookbook_path, 'metadata.rb')
-        puts "REPAIRING: no metadata files exist for #{cookbook_path}, creating #{rb_file}"
+        ui.stdout.puts "REPAIRING: no metadata files exist for #{cookbook_path}, creating #{rb_file}"
         ::File.open(rb_file, 'w') do |f|
           metadata.each_pair do |key, value|
             f.write("#{key} '#{value}'\n")
@@ -362,7 +362,7 @@ class Chef
             rl << item
             new_role['run_list'].push(item)
           rescue ArgumentError
-            puts "REPAIRING: Invalid Recipe Item: #{item} in run_list from #{role_path}"
+            ui.stdout.puts "REPAIRING: Invalid Recipe Item: #{item} in run_list from #{role_path}"
           end
         end
         if the_role.has_key?('env_run_lists')
@@ -373,7 +373,7 @@ class Chef
                 rl << item
                 new_role['env_run_lists'][key].push(item)
               rescue ArgumentError
-                puts "REPAIRING: Invalid Recipe Item: #{item} in env_run_lists #{key} from #{role_path}"
+                ui.stdout.puts "REPAIRING: Invalid Recipe Item: #{item} in env_run_lists #{key} from #{role_path}"
               end
             end
           end
@@ -384,7 +384,7 @@ class Chef
 
       def validate_roles(org)
         for_each_role(org) do |role_path|
-          puts "INFO: Validating Role at #{role_path}"
+          ui.stdout.puts "INFO: Validating Role at #{role_path}"
           begin
             Chef::Role.from_hash(FFI_Yajl::Parser.parse(::File.read(role_path), symbolize_names: false))
           rescue ArgumentError
@@ -395,12 +395,12 @@ class Chef
 
       def validate_invitations(org)
         invite_file = tidy.invitations_path(org)
-        puts "INFO: validating org #{org} invites in #{invite_file}"
+        ui.stdout.puts "INFO: validating org #{org} invites in #{invite_file}"
         invitations = FFI_Yajl::Parser.parse(::File.read(invite_file), symbolize_names: false)
         invitations_new = []
         invitations.each do |invite|
           if invite['username'].nil?
-            puts "REPAIRING: Dropping corrupt invitations for #{org} in file #{invite_file}"
+            ui.stdout.puts "REPAIRING: Dropping corrupt invitations for #{org} in file #{invite_file}"
           else
             invite_hash = {}
             invite_hash['id'] = invite['id']

data/lib/chef/knife/tidy_base.rb

@@ -58,7 +58,21 @@ class Chef
       end
 
       def completion_message
-        puts "#{ui.color("** Finished **", :magenta)}"
+        ui.stdout.puts "#{ui.color("** Finished **", :magenta)}"
+      end
+
+      def action_needed_file_path
+        ::File.expand_path('knife-tidy-actions-needed.txt')
+      end
+
+      def server_warnings_file_path
+        ::File.expand_path('reports/knife-tidy-server-warnings.txt')
+      end
+
+      def action_needed(msg, file_path=action_needed_file_path)
+        ::File.open(file_path, 'a') do |f|
+          f.write(msg + "\n")
+        end
       end
     end
   end

data/lib/chef/knife/tidy_server_clean.rb

@@ -3,7 +3,6 @@ require 'chef/knife/tidy_base'
 class Chef
   class Knife
     class TidyServerClean < Knife
-
       include Knife::TidyBase
 
       deps do
@@ -13,6 +12,10 @@ class Chef
 
       banner "knife tidy server clean (options)"
 
+      option :backup_path,
+        :long => '--backup-path path/to/backup',
+        :description => 'The path to the knife-ec-backup backup directory'
+
       option :concurrency,
         :long => '--concurrency THREADS',
         :default => 1,
@@ -42,6 +45,16 @@ class Chef
           exit 1
         end
 
+        while config[:backup_path].nil?
+          user_value = ui.ask_question("It is not recommended to run this command without specifying a current backup directory.\nPlease specify a backup directory:")
+          config[:backup_path] = user_value == '' ? nil : user_value
+        end
+
+        unless ::File.directory?(config[:backup_path])
+          ui.error 'Must specify valid --backup-path'
+          exit 1
+        end
+
         deletions = if config[:only_cookbooks]
                       "cookbooks"
                     elsif config[:only_nodes]
@@ -56,9 +69,13 @@ class Chef
                  all_orgs
                end
 
-        ui.warn "This operation will affect the following Orgs on #{server.root_url}\n\n#{orgs}\n\n"
-
-        ui.confirm("About to delete #{deletions} from the Chef Server identified in the #{tidy.reports_dir} directory! Are you sure you wish to continue") unless config[:unattended]
+        ui.warn "This operation will affect the following Orgs on #{server.root_url}: #{orgs}"
+        if ::File.exist?(server_warnings_file_path)
+          ::File.read(::File.expand_path('reports/knife-tidy-server-warnings.txt')).each_line do |line|
+            ui.warn(line)
+          end
+        end
+        ui.confirm("This command will delete #{deletions} identified by the knife-tidy reports in #{tidy.reports_dir} from the Chef Server specified in your knife configuration file. \n\n The Chef server to be used is currently #{server.root_url}.\n\n Please be sure this is the Chef server you wish to delete data from. \n\nWould you like to continue?") unless config[:unattended]
 
         orgs.each do |org|
           clean_cookbooks(org) unless config[:only_nodes]
@@ -69,12 +86,10 @@ class Chef
       end
 
       def clean_cookbooks(org)
-        ui.warn "Cleaning cookbooks is a feature not yet enabled."
-        return
         queue = Chef::Util::ThreadedJobQueue.new
         unused_cookbooks_file = ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json")
         return unless ::File.exist?(unused_cookbooks_file)
-        puts "INFO: Cleaning cookbooks for Org: #{org}, using #{unused_cookbooks_file}"
+        ui.stdout.puts "INFO: Cleaning cookbooks for Org: #{org}, using #{unused_cookbooks_file}"
         unused_cookbooks = FFI_Yajl::Parser.parse(::File.read(unused_cookbooks_file), symbolize_names: true)
         unused_cookbooks.keys.each do |cookbook|
           versions = unused_cookbooks[cookbook]
@@ -100,7 +115,7 @@ class Chef
         queue = Chef::Util::ThreadedJobQueue.new
         stale_nodes_file = ::File.join(tidy.reports_dir, "#{org}_stale_nodes.json")
         return unless ::File.exist?(stale_nodes_file)
-        puts "INFO: Cleaning stale nodes for Org: #{org}, using #{stale_nodes_file}"
+        ui.stdout.puts "INFO: Cleaning stale nodes for Org: #{org}, using #{stale_nodes_file}"
         stale_nodes = FFI_Yajl::Parser.parse(::File.read(stale_nodes_file), symbolize_names: true)
         stale_nodes[:list].each do |node|
           queue << lambda { delete_node_job(org, node) }

data/lib/chef/knife/tidy_server_report.rb

@@ -19,8 +19,9 @@ class Chef
 
       def run
         ensure_reports_dir!
+        FileUtils.rm_f(server_warnings_file_path)
 
-        ui.warn "Writing to #{tidy.reports_dir} directory"
+        ui.stdout.puts(ui.color("Writing to #{tidy.reports_dir} directory", :magenta))
         delete_existing_reports
 
         orgs = if config[:org_list]
@@ -29,20 +30,37 @@ class Chef
                  all_orgs
                end
 
-        pre_12_3_nodes = []
         stale_orgs = []
         node_threshold = config[:node_threshold].to_i
 
         orgs.each do |org|
+          pre_12_3_nodes = []
+          unconverged_recent_nodes = []
           ui.info "  Organization: #{org}"
           cb_list = cookbook_list(org)
           version_count = cookbook_count(cb_list).sort_by(&:last).reverse.to_h
           used_cookbooks = {}
           nodes = nodes_list(org)
+          db_nodes = rest.get("/organizations/#{org}/nodes")
+          unless nodes.length == db_nodes.length
+            ood_message = "Search index is out of date! No cleanup action will be taken for #{org}."
+            ui.error(ood_message)
+            action_needed(ood_message, server_warnings_file_path)
+            next
+          end
 
           nodes.each do |node|
-            chef_version = Chef::VersionString.new(node['chef_packages']['chef']['version'])
-            if chef_version < "12.3"
+            # If the node hasn't checked in.
+            if !node['chef_packages']
+              # If the node is under an hour old.
+              if (Time.now.to_i - node['ohai_time'].to_i) < 3600
+                unconverged_recent_nodes << node['name']
+              end
+              next
+            end
+            chef_version = Gem::Version.new(node['chef_packages']['chef']['version'])
+            # If the node has checked in within the node_threshold with a client older than 12.3
+            if chef_version < Gem::Version.new("12.3") && (Time.now.to_i - node['ohai_time'].to_i) <= node_threshold * 86400
               pre_12_3_nodes << node['name']
             end
           end
@@ -58,10 +76,8 @@ class Chef
             end
           end
 
-          Chef::Log.debug("Used cookbook list before checking environments: #{used_cookbooks}")
           pins = environment_constraints(org)
           used_cookbooks = check_environment_pins(used_cookbooks, pins, cb_list)
-          Chef::Log.debug("Used cookbook list after checking environments: #{used_cookbooks}")
 
           stale_nodes = []
           nodes.each do |n|
@@ -73,12 +89,19 @@ class Chef
           stale_nodes_hash = {'threshold_days': node_threshold, 'org_total_node_count': nodes.count, 'count': stale_nodes.count, 'list': stale_nodes}
           stale_orgs.push(org) if stale_nodes.count == nodes.count
 
-          tidy.write_new_file(unused_cookbooks(used_cookbooks, cb_list), ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json"))
-          tidy.write_new_file(version_count, ::File.join(tidy.reports_dir, "#{org}_cookbook_count.json"))
-          tidy.write_new_file(stale_nodes_hash, ::File.join(tidy.reports_dir, "#{org}_stale_nodes.json"))
+          tidy.write_new_file(unused_cookbooks(used_cookbooks, cb_list), ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json"), backup=false)
+          tidy.write_new_file(version_count, ::File.join(tidy.reports_dir, "#{org}_cookbook_count.json"), backup=false)
+          tidy.write_new_file(stale_nodes_hash, ::File.join(tidy.reports_dir, "#{org}_stale_nodes.json"), backup=false)
 
           if pre_12_3_nodes.length > 0
-            ui.warn "#{pre_12_3_nodes.length} nodes have been detected in the organization #{org} running chef-client versions prior to 12.3 - this means that the list of stale cookbooks for these nodes may not have been correctly calculated and your report may not be complete for this organization."
+            pre_12_3_message = "#{pre_12_3_nodes.length} nodes in organization #{org} have converged in the last #{node_threshold} days with a chef-client < 12.3. These nodes' cookbook versions WILL NOT be factored in the stale cookbooks versions report. Continuing with the server cleanup will delete cookbooks in-use by these nodes."
+            ui.warn(pre_12_3_message)
+            action_needed(pre_12_3_message, server_warnings_file_path)
+          end
+          if unconverged_recent_nodes.length > 0
+            unconverged_recent_message "#{unconverged_recent_nodes.length} nodes have been created in the last hour that have yet to converge in organization #{org}. These nodes WILL NOT be factored in the stale cookbook verisons report. Continuing with the server cleanup will delete cookbooks in-use by these nodes."
+            ui.warn(unconverged_recent_message)
+            action_needed(unconverged_recent_message, server_warnings_file_path)
           end
         end
 
@@ -141,10 +164,11 @@ class Chef
       def unused_cookbooks(used_list, cb_list)
         unused_list = {}
         cb_list.each do |name, versions|
+          versions.sort! {| a, b | Gem::Version.new(a) <=> Gem::Version.new(b) }
           if used_list[name].nil? # Not in the used list at all (Remove all versions)
             unused_list[name] = versions
           elsif used_list[name].sort != versions  # Is in the used cookbook list, but version arrays do not match (Find unused versions)
-            unused_list[name] = versions - used_list[name]
+            unused_list[name] = versions - used_list[name] - [versions.last]  # Don't delete the most recent version as it might not be in a run_list yet.
           end
         end
         unused_list
@@ -176,15 +200,18 @@ class Chef
       def check_cookbook_list(cb_list, cb, version)
         if cb_list[cb]
           cb_list[cb].each do |v|
+            versions_not_satisfied = []
             if Gem::Dependency.new('', version).match?('', v)
-              Chef::Log.debug("Pin of #{cb} can be satisfied by #{v}, adding to used list")
               return [v]
             else
-              Chef::Log.debug("Pin of #{cb} version #{version} not satisfied by #{v}")
+              versions_not_satisfied.push(v)
+            end
+            if v == cb_list[cb].last
+              ui.warn("Pin of #{cb} #{version} not satisfied by current versions of cookbook: [#{versions_not_satisfied.join(', ')}]")
             end
           end
         else
-          Chef::Log.info("Cookbook #{cb} version #{version} is pinned in an environment, but does not exist on the server in this org.")
+          ui.warn("Cookbook #{cb} #{version} is pinned in an environment, but does not exist on the server in this org.")
         end
         return nil
       end
@@ -192,20 +219,18 @@ class Chef
       def check_environment_pins(used_cookbooks, pins, cb_list)
         pins.each do |cb, versions|
           versions.each do |version|
+            next if version == "<= 0.0.0"
             if used_cookbooks[cb]
               # This pinned cookbook is in the used list, now check for a matching version.
               used_cookbooks[cb].each do |v|
                 if Gem::Dependency.new('', version).match?('', v)
-                  # This version in used_cookbooks satisfies the pin
-                  Chef::Log.debug("Pin of #{cb}: #{version} is satisfied by #{v}")
                   break
                 end
               end
               result = check_cookbook_list(cb_list, cb, version)
-              used_cookbooks[cb].push(result[0]) if result
+              used_cookbooks[cb].push(result[0]) if result && !used_cookbooks[cb].include?(result[0])
             else
               # No cookbook version for that pin, look through the full cookbook list for a match
-              Chef::Log.debug("No used cookbook #{cb}, checking the full cookbook list")
               result = check_cookbook_list(cb_list, cb, version)
               used_cookbooks[cb] = result if result
             end

data/lib/chef/tidy_acls.rb

@@ -18,26 +18,26 @@ class Chef
     end
 
     def load_users
-      puts "INFO: Loading users"
+      @tidy.ui.stdout.puts "INFO: Loading users"
       Dir[::File.join(@tidy.users_path, '*.json')].each do |user|
         @users.push(FFI_Yajl::Parser.parse(::File.read(user), symbolize_names: true))
       end
     end
 
     def load_members
-      puts "INFO: Loading members for #{@org}"
+      @tidy.ui.stdout.puts "INFO: Loading members for #{@org}"
       @members = FFI_Yajl::Parser.parse(::File.read(@tidy.members_path(@org)), symbolize_names: true)
     end
 
     def load_clients
-      puts "INFO: Loading clients for #{@org}"
+      @tidy.ui.stdout.puts "INFO: Loading clients for #{@org}"
       Dir[::File.join(@tidy.clients_path(@org), '*.json')].each do |client|
         @clients.push(FFI_Yajl::Parser.parse(::File.read(client), symbolize_names: true))
       end
     end
 
     def load_groups
-      puts "INFO: Loading groups for #{@org}"
+      @tidy.ui.stdout.puts "INFO: Loading groups for #{@org}"
       Dir[::File.join(@tidy.groups_path(@org), '*.json')].each do |group|
         @groups.push(FFI_Yajl::Parser.parse(::File.read(group), symbolize_names: true))
       end
@@ -48,7 +48,7 @@ class Chef
       load_members
       load_clients
       load_groups
-      puts "INFO: #{@org} Actors loaded!"
+      @tidy.ui.stdout.puts "INFO: #{@org} Actors loaded!"
     end
 
     def acl_ops
@@ -105,41 +105,34 @@ class Chef
     end
 
     def fix_ambiguous_actor(actor)
-      puts "REPAIRING: Ambiguous actor! #{actor} removing from #{@tidy.members_path(@org)}"
+      @tidy.ui.stdout.puts "REPAIRING: Ambiguous actor! #{actor} removing from #{@tidy.members_path(@org)}"
       remove_user_from_org(actor)
     end
 
     def add_client_to_org(actor)
       # TODO
-      puts "ACTION NEEDED: Client referenced in acl non-existant: #{actor}"
+      @tidy.ui.stdout.puts "ACTION NEEDED: Client referenced in acl non-existant: #{actor}"
     end
 
     def add_actor_to_members(actor)
-      puts "REPAIRING: Invalid actor: #{actor} adding to #{@tidy.members_path(@org)}"
+      @tidy.ui.stdout.puts "REPAIRING: Invalid actor: #{actor} adding to #{@tidy.members_path(@org)}"
       user = { user: { username: actor } }
       @members.push(user)
-      write_new_file(@members, @tidy.members_path(@org))
-    end
-
-    def write_new_file(contents, path)
-      FileUtils.cp(path, "#{path}.orig") unless ::File.exist?("#{path}.orig")
-      ::File.open(path, 'w+') do |f|
-         f.write(FFI_Yajl::Encoder.encode(contents, pretty: true))
-      end
+      @tidy.write_new_file(@members, @tidy.members_path(@org))
     end
 
     def remove_user_from_org(actor)
       @members.reject! { |user| user[:user][:username] == actor }
-      write_new_file(@members, @tidy.members_path(@org))
+      @tidy.write_new_file(@members, @tidy.members_path(@org))
     end
 
     def remove_group_from_acl(group, acl_file)
-      puts "REPAIRING: Removing invalid group: #{group} from #{acl_file}"
+      @tidy.ui.stdout.puts "REPAIRING: Removing invalid group: #{group} from #{acl_file}"
       acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
       acl_ops.each do |op|
         acl[op]['groups'].reject! { |the_group| the_group == group }
       end
-      write_new_file(acl, acl_file)
+      @tidy.write_new_file(acl, acl_file)
     end
 
     # Appends the proper acls for ::server-admins and the org's read access group if they are missing.
@@ -147,26 +140,26 @@ class Chef
       acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
       acl_ops.each do |op|
         unless acl[op]['groups'].include? '::server-admins'
-          puts "REPAIRING: Adding #{op} acl for ::server-admins in #{acl_file}"
+          @tidy.ui.stdout.puts "REPAIRING: Adding #{op} acl for ::server-admins in #{acl_file}"
           acl[op]['groups'].push('::server-admins')
         end
         if op == 'read' && !acl[op]['groups'].include?("::#{@org}_read_access_group")
-          puts "REPAIRING: Adding #{op} acl for ::#{@org}_read_access_group in #{acl_file}"
+          @tidy.ui.stdout.puts "REPAIRING: Adding #{op} acl for ::#{@org}_read_access_group in #{acl_file}"
           acl[op]['groups'].push("::#{@org}_read_access_group")
         end
       end
-      write_new_file(acl, acl_file)
+      @tidy.write_new_file(acl, acl_file)
     end
 
     def ensure_client_read_acls(acl_file)
       acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
       %w(users admins).each do | group |
         unless acl['read']['groups'].include? group
-          puts "REPAIRING: Adding read acl for #{group} in #{acl_file}"
+          @tidy.ui.stdout.puts "REPAIRING: Adding read acl for #{group} in #{acl_file}"
           acl['read']['groups'].push(group)
         end
       end
-      write_new_file(acl, acl_file)
+      @tidy.write_new_file(acl, acl_file)
     end
 
     def validate_acls
@@ -191,10 +184,32 @@ class Chef
       end
     end
 
+    def default_user_acl
+      return {:create=>{:actors=>["pivotal", client], :groups=>["::server-admins"]},
+              :read=>{:actors=>["pivotal", client], :groups=>["::server-admins", "::#{@org}_read_access_group"]},
+              :update=>{:actors=>["pivotal", client], :groups=>["::server-admins"]},
+              :delete=>{:actors=>["pivotal", client], :groups=>["::server-admins"]},
+              :grant=>{:actors=>["pivotal", client], :groups=>["::server-admins"]}}
+    end
+
+    def default_client_acl(client_name)
+      return {:create=>{:actors=>["pivotal", "#{@org}-validator", client_name], :groups=>["admins"]},
+              :read=>{:actors=>["pivotal", "#{@org}-validator", client_name], :groups=>["admins", "users"]},
+              :update=>{:actors=>["pivotal", client_name], :groups=>["admins"]},
+              :delete=>{:actors=>["pivotal", client_name], :groups=>["admins", "users"]},
+              :grant=>{:actors=>["pivotal", client_name], :groups=>["admins"]}}
+    end
+
     def validate_user_acls
       @members.each do |member|
         user_acl_path = ::File.join(@tidy.user_acls_path, "#{member[:user][:username]}.json")
-        user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+        begin
+          user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+        rescue Errno::ENOENT
+          @tidy.ui.stdout.puts "REPAIRING: Replacing missing user acl for #{member[:user][:username]}."
+          @tidy.write_new_file(default_user_acl, client_acl_path, backup=false)
+          user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+        end
         ensure_global_group_acls(user_acl_path)
         actors_groups = acl_actors_groups(user_acl)
         actors_groups[:groups].each do |group|
@@ -208,7 +223,13 @@ class Chef
     def validate_client_acls
       @clients.each do |client|
         client_acl_path = ::File.join(@tidy.org_acls_path(@org), 'clients', "#{client[:name]}.json")
-        client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+        begin
+          client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+        rescue Errno::ENOENT
+          @tidy.ui.stdout.puts "REPAIRING: Replacing missing client acl for #{client[:name]} in #{client_acl_path}."
+          @tidy.write_new_file(default_client_acl(client[:name]), client_acl_path, backup=false)
+          client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+        end
         ensure_client_read_acls(client_acl_path)
       end
     end

data/lib/chef/tidy_common.rb

@@ -1,5 +1,6 @@
 require 'ffi_yajl'
 require 'fileutils'
+require "chef/knife/core/ui"
 
 class Chef
   class TidyCommon
@@ -11,6 +12,10 @@ class Chef
       @backup_path = ::File.expand_path(backup_path)
     end
 
+    def ui
+      @ui ||= Chef::Knife::UI.new(STDOUT, STDERR, STDIN, {})
+    end
+
     def users_path
       @users_path ||= ::File.expand_path(::File.join(@backup_path, 'users'))
     end
@@ -62,8 +67,8 @@ class Chef
       end
     end
 
-    def write_new_file(contents, path)
-      if ::File.exist?(path)
+    def write_new_file(contents, path, backup=true)
+      if ::File.exist?(path) && backup
         FileUtils.cp(path, "#{path}.orig") unless ::File.exist?("#{path}.orig")
       end
       ::File.open(path, 'w+') do |f|

data/lib/chef/tidy_substitutions.rb

@@ -9,13 +9,14 @@ class Chef
 
     attr_accessor :file_path, :backup_path, :data
 
-    def initialize(file_path = nil, tidy_common = nil)
+    def initialize(file_path = nil, tidy_common)
       @file_path = file_path
-      @backup_path = tidy_common.backup_path if tidy_common
+      @tidy = tidy_common
+      @backup_path = tidy_common.backup_path
     end
 
     def load_data
-      puts "INFO: Loading substitutions from #{file_path}"
+      @tidy.ui.stdout.puts "INFO: Loading substitutions from #{file_path}"
       @data = FFI_Yajl::Parser.parse(::File.read(@file_path), symbolize_names: false)
     rescue Errno::ENOENT
       raise NoSubstitutionFile, file_path
@@ -23,7 +24,7 @@ class Chef
 
     def boiler_plate
       bp = ::File.join(File.dirname(__FILE__), '../../conf/substitutions.json.example')
-      puts "INFO: Creating boiler plate gsub file: 'substitutions.json'"
+      @tidy.ui.stdout.puts "INFO: Creating boiler plate gsub file: 'substitutions.json'"
       FileUtils.cp(bp, ::File.join(Dir.pwd, 'substitutions.json'))
     end
 
@@ -44,7 +45,7 @@ class Chef
           file.each_line do |line|
             if line.match(search)
               temp_file.puts replace
-              puts "INFO:  ++ #{path}"
+              @tidy.ui.stdout.puts "INFO:  ++ #{path}"
             else
               temp_file.puts line
             end
@@ -63,7 +64,7 @@ class Chef
       load_data
       @data.keys.each do |entry|
         @data[entry].keys.each do |glob|
-          puts "INFO: Running substitutions for #{entry} -> #{glob}"
+          @tidy.ui.stdout.puts "INFO: Running substitutions for #{entry} -> #{glob}"
           Dir[::File.join(@backup_path, glob)].each do |file|
             @data[entry][glob].each do |substitution|
               search = Regexp.new(substitution['pattern'])

data/lib/knife-tidy/version.rb

@@ -1,4 +1,4 @@
 module KnifeTidy
-  VERSION = '0.7.0'
+  VERSION = '1.0.0'
   MAJOR, MINOR, TINY = VERSION.split('.')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-tidy
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Jeremy Miller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-29 00:00:00.000000000 Z
+date: 2017-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake