knife-tidy 2.0.1 → 2.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23b1bd5cd9472c351eddcf5722b5883529ab52ff7d2823cd1f71ffd050ea382e
-  data.tar.gz: 47d38da1aaea0f9ea819a3fdac00ed8736196d9429c0629a26f2d8562a4f6549
+  metadata.gz: c7aaf34102e27291f03892bfb4eddb450aaed46c5e615818e48a59c174c91e42
+  data.tar.gz: f482bbfb1182b581532ef643900c6f4814dc800ff42e02ff0a29b8936faed77b
 SHA512:
-  metadata.gz: 564fbad7997899cc26a55b3c265edbd7a5e0be3aa518c86e4fb67b651e30a6e55311e0042ea2b0f7a009b1b1e0984894e54b6c23cd643d076097789c9a2e3c2a
-  data.tar.gz: 3ef4230e711258c422cf14b9a53b15bd9797e917afc79bf1dee2763f5ee748b84d4ee02190e20fe70ba27d65a8a448f0a243229090ef5ca4d8c710d1c521f82c
+  metadata.gz: a5192215cbd8341aef601e71bf6c42bb2eb5d19f652570734ac192151941d7b954da8d0fe5c504ea67b599772d09ea12da77d6d782f54e2c967c6c12fc9cef0d
+  data.tar.gz: 75b37b5663f0558791ad1a52b5281d84197e1def8ed2c15103896165a781b4a07f1ac4556f5d266e35fb662d76218e6be0ccf413d569e0b7b776075e5a5aca27

data/conf/substitutions.json.example

@@ -0,0 +1,10 @@
+{
+  "your-problem-descriptor":{
+    "organizations/*/cookbooks/*/metadata.rb":[
+      {
+        "pattern":"^version .*GO_PIPELINE_LABEL",
+        "replace":"version !COOKBOOK_VERSION!"
+      }
+    ]
+  }
+}

data/lib/chef/knife/tidy_backup_clean.rb

@@ -1,15 +1,15 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
     class TidyBackupClean < Knife
       deps do
-        require "chef/cookbook_loader"
+        require "chef/cookbook/cookbook_version_loader"
         require "chef/cookbook/metadata"
         require "chef/role"
         require "chef/run_list"
-        require "chef/tidy_substitutions"
-        require "chef/tidy_acls"
+        require_relative "../tidy_substitutions"
+        require_relative "../tidy_acls"
         require "ffi_yajl"
         require "fileutils"
         require "securerandom"
@@ -75,7 +75,7 @@ class Chef
         tidy.global_user_names.each do |user|
           email = ""
           ui.stdout.puts "INFO: Validating #{user}"
-          the_user = FFI_Yajl::Parser.parse(::File.read(::File.join(tidy.users_path, "#{user}.json")), symbolize_names: false)
+          the_user = tidy.json_file_to_hash(File.join(tidy.users_path, "#{user}.json"), symbolize_names: false)
           if the_user.key?("email") && the_user["email"].match(/\A[^@\s]+@[^@\s]+\z/)
             if emails_seen.include?(the_user["email"])
               ui.stdout.puts "REPAIRING: Already saw #{user}'s email, creating a unique one."
@@ -147,7 +147,7 @@ class Chef
             add_cookbook_name_to_metadata(cookbook_name, rb_path) if lines.empty?
           else
             if ::File.exist?(json_path)
-              metadata = FFI_Yajl::Parser.parse(::File.read(json_path), symbolize_names: false)
+              metadata = tidy.json_file_to_hash(json_path, symbolize_names: false)
               if metadata["name"] != cookbook_name
                 metadata["name"] = cookbook_name
                 ui.stdout.puts "REPAIRING: Correcting `name` in #{json_path}`"
@@ -161,26 +161,24 @@ class Chef
       end
 
       def load_cookbooks(org)
-        cl = Chef::CookbookLoader.new(tidy.cookbooks_path(org))
-        for_each_cookbook_basename(org) do |cookbook|
+        for_each_cookbook_path(org) do |cookbook|
+          cl = Chef::Cookbook::CookbookVersionLoader.new(cookbook)
           ui.stdout.puts "INFO: Loading #{cookbook}"
-          ret = cl.load_cookbook(cookbook)
+          ret = cl.load!
           if ret.nil?
             action_needed("ACTION NEEDED: Something's wrong with the #{cookbook} cookbook in org #{org} - cannot load it! Moving to cookbooks.broken folder.")
             broken_cookooks_add(org, cookbook)
           end
         end
-      rescue LoadError => e
+      rescue LoadError, Exceptions::MetadataNotValid => e
         ui.error e
         exit 1
       end
 
-      def broken_cookooks_add(org, cookbook)
+      def broken_cookooks_add(org, cookbook_path)
         broken_path = ::File.join(tidy.org_path(org), "cookbooks.broken")
         FileUtils.mkdir(broken_path) unless ::File.directory?(broken_path)
-        Dir[::File.join(tidy.cookbooks_path(org), "#{cookbook}*")].each do |cb|
-          FileUtils.mv(cb, broken_path, verbose: true, force: true)
-        end
+        FileUtils.mv(cookbook_path, broken_path, verbose: true, force: true)
       end
 
       def generate_new_metadata(org)
@@ -194,7 +192,7 @@ class Chef
         Dir[::File.join(tidy.backup_path, "organizations/*/cookbooks/chef-sugar*/metadata.rb")].each do |file|
           ui.stdout.puts "INFO: Searching for known chef-sugar problems when uploading."
           s = Chef::TidySubstitutions.new(nil, tidy)
-          version = s.cookbook_version_from_path(file)
+          version = tidy.cookbook_version_from_path(::File.dirname(file))
           patterns = [
             {
               search: "^require .*/lib/chef/sugar/version",
@@ -222,13 +220,14 @@ class Chef
           Chef::TidySubstitutions.new(nil, tidy).sub_in_file(
             ::File.join(cookbook_path, "metadata.rb"),
             Regexp.new("^depends +['\"]#{name}['\"]"),
-            "# depends '#{name}' # knife-tidy was here")
+            "# depends '#{name}' # knife-tidy was here"
+          )
         end
       end
 
       def fix_metadata_fields(cookbook_path)
         json_path = ::File.join(cookbook_path, "metadata.json")
-        metadata = FFI_Yajl::Parser.parse(::File.read(json_path), symbolize_names: false)
+        metadata = tidy.json_file_to_hash(json_path, symbolize_names: false)
         md = metadata.dup
         metadata.each_pair do |key, value|
           if value.nil?
@@ -279,9 +278,7 @@ class Chef
 
       def create_minimal_metadata(cookbook_path)
         name = tidy.cookbook_name_from_path(cookbook_path)
-        components = cookbook_path.split(File::SEPARATOR)
-        name_version = components[components.index("cookbooks") + 1]
-        version = name_version.match(/\d+\.\d+\.\d+/).to_s
+        version = tidy.cookbook_version_from_path(cookbook_path)
         metadata = {}
         metadata["name"] = name
         metadata["version"] = version
@@ -352,7 +349,7 @@ class Chef
       end
 
       def repair_role_run_lists(role_path)
-        the_role = FFI_Yajl::Parser.parse(::File.read(role_path), symbolize_names: false)
+        the_role = tidy.json_file_to_hash(role_path, symbolize_names: false)
         new_role = the_role.clone
         rl = Chef::RunList.new
         new_role["run_list"] = []
@@ -378,14 +375,14 @@ class Chef
           end
         end
         write_role(role_path, new_role)
-        # rubocop:enable MethodLength
+        # rubocop:enable Metrics/MethodLength
       end
 
       def validate_roles(org)
         for_each_role(org) do |role_path|
           ui.stdout.puts "INFO: Validating Role at #{role_path}"
           begin
-            Chef::Role.from_hash(FFI_Yajl::Parser.parse(::File.read(role_path), symbolize_names: false))
+            Chef::Role.from_hash(tidy.json_file_to_hash(role_path, symbolize_names: false))
           rescue ArgumentError
             repair_role_run_lists(role_path)
           end
@@ -395,10 +392,10 @@ class Chef
       def validate_clients_group(org)
         ui.stdout.puts "INFO: validating all clients for org #{org} exist in clients group"
         clients_group_path = ::File.join(tidy.groups_path(org), "clients.json")
-        existing_group_data = FFI_Yajl::Parser.parse(::File.read(clients_group_path), symbolize_names: false)
+        existing_group_data = tidy.json_file_to_hash(clients_group_path, symbolize_names: false)
         existing_group_data["clients"] = [] unless existing_group_data.key?("clients")
         if existing_group_data["clients"].length != tidy.client_names(org).length
-          ui.stdout.puts "REPAIRING: Adding #{(existing_group_data['clients'].length - tidy.client_names(org).length).abs} missing clients into #{org}'s client group file #{clients_group_path}"
+          ui.stdout.puts "REPAIRING: Adding #{(existing_group_data["clients"].length - tidy.client_names(org).length).abs} missing clients into #{org}'s client group file #{clients_group_path}"
           existing_group_data["clients"] = (existing_group_data["clients"] + tidy.client_names(org)).uniq
           ::File.open(clients_group_path, "w") do |f|
             f.write(Chef::JSONCompat.to_json_pretty(existing_group_data))
@@ -409,7 +406,7 @@ class Chef
       def validate_invitations(org)
         invite_file = tidy.invitations_path(org)
         ui.stdout.puts "INFO: validating org #{org} invites in #{invite_file}"
-        invitations = FFI_Yajl::Parser.parse(::File.read(invite_file), symbolize_names: false)
+        invitations = tidy.json_file_to_hash(invite_file, symbolize_names: false)
         invitations_new = []
         invitations.each do |invite|
           if invite["username"].nil?

data/lib/chef/knife/tidy_base.rb

@@ -24,8 +24,8 @@ class Chef
       def self.included(includer)
         includer.class_eval do
           deps do
-            require "chef/tidy_server"
-            require "chef/tidy_common"
+            require_relative "../tidy_server"
+            require_relative "../tidy_common"
           end
 
           option :org_list,

data/lib/chef/knife/tidy_notify.rb

@@ -1,4 +1,4 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
@@ -11,42 +11,42 @@ class Chef
       banner "knife tidy notify (options)"
 
       option :smtp_server,
-             short: "-s SERVER_NAME",
-             long: "--smtp_server SERVER_NAME",
-             default: "localhost",
-             description: "SMTP Server to be used for emailling reports to organization admins (defaults to localhost)"
+        short: "-s SERVER_NAME",
+        long: "--smtp_server SERVER_NAME",
+        default: "localhost",
+        description: "SMTP Server to be used for emailling reports to organization admins (defaults to localhost)"
 
       option :smtp_port,
-             short: "-p SMTP_PORT",
-             long: "--smtp_port SMTP_PORT",
-             default: 25,
-             description: "SMTP port to be used for emailling reports to organization admins (defaults to 25)"
+        short: "-p SMTP_PORT",
+        long: "--smtp_port SMTP_PORT",
+        default: 25,
+        description: "SMTP port to be used for emailling reports to organization admins (defaults to 25)"
 
       option :smtp_helo,
-             short: "-h SMTP_HELO",
-             long: "--smtp_helo SMTP_HELO",
-             default: "localhost",
-             description: "SMTP HELO to be used for emailling reports to organization admins (defaults to localhost)"
+        short: "-h SMTP_HELO",
+        long: "--smtp_helo SMTP_HELO",
+        default: "localhost",
+        description: "SMTP HELO to be used for emailling reports to organization admins (defaults to localhost)"
 
       option :smtp_username,
-             short: "-u SMTP_USERNAME",
-             long: "--smtp_username SMTP_USERNAME",
-             description: "SMTP Username to be used for emailling reports to organization admins"
+        short: "-u SMTP_USERNAME",
+        long: "--smtp_username SMTP_USERNAME",
+        description: "SMTP Username to be used for emailling reports to organization admins"
 
       option :smtp_password,
-             long: "--smtp_password SMTP_PASSWORD",
-             description: "SMTP Password to be used for emailling reports to organization admins"
+        long: "--smtp_password SMTP_PASSWORD",
+        description: "SMTP Password to be used for emailling reports to organization admins"
 
       option :smtp_from,
-             long: "--smtp_from SMTP_FROM",
-             description: "SMTP From address to be used for emailling reports to organization admins"
+        long: "--smtp_from SMTP_FROM",
+        description: "SMTP From address to be used for emailling reports to organization admins"
 
       option :smtp_use_tls,
-             long: "--smtp_use_tls",
-             short: "-t",
-             default: false,
-             boolean: true | false,
-             description: "Whether TLS should be used for emailling reports to organization admins (defaults to false if omitted)"
+        long: "--smtp_use_tls",
+        short: "-t",
+        default: false,
+        boolean: true | false,
+        description: "Whether TLS should be used for emailling reports to organization admins (defaults to false if omitted)"
 
       include Knife::TidyBase
 
@@ -54,13 +54,13 @@ class Chef
         reports_dir = tidy.reports_dir
         report_file_suffixes = ["_unused_cookbooks.json", "_cookbook_count.json", "_stale_nodes.json"]
         # Only grab the files matching the report_file_suffixes
-        report_files = Dir["#{reports_dir}/*{#{report_file_suffixes.join(',')}}"]
+        report_files = Dir["#{reports_dir}/*{#{report_file_suffixes.join(",")}}"]
 
         ui.info "Reading from #{tidy.reports_dir} directory"
 
         # Fetch list of organization names from reports directory
         begin
-          org_names = report_files.map { |r_file| r_file.match("#{reports_dir}\/(.*)(#{report_file_suffixes.join('|')})").captures.first }.uniq
+          org_names = report_files.map { |r_file| r_file.match("#{reports_dir}\/(.*)(#{report_file_suffixes.join("|")})").captures.first }.uniq
         rescue NoMethodError
           ui.stderr.puts "Failed to parse json reports files. Please ensure your reports are valid."
           return
@@ -88,7 +88,7 @@ class Chef
               file_name = "#{reports_dir}/#{org}#{report}"
               ui.info("  Parsing file #{file_name}")
               json_string = File.read(file_name)
-              reports[org][report] = FFI_Yajl::Parser.parse(json_string)
+              reports[org][report] = tidy.json_file_to_hash(json_string, symbolize_names: false)
             rescue Errno::ENOENT
               ui.info("    Skipping file #{file_name} - not found for organization #{org}")
               reports[org][report] = {}
@@ -120,7 +120,7 @@ class Chef
         mime_boundary = "==Multipart_Boundary_x#{srand}x"
         message = <<~MESSAGE_END
           From: Knife Tidy <#{config[:smtp_from]}>
-          To: #{recipients.map { |recipient| "#{recipient[:name]} <#{recipient[:email]}>" }.join(', ')}
+          To: #{recipients.map { |recipient| "#{recipient[:name]} <#{recipient[:email]}>" }.join(", ")}
           MIME-Version: 1.0
           Subject: Knife Tidy Cleanup Report for Organization "#{organization}"
           Content-Type: multipart/mixed; boundary="#{mime_boundary}";
@@ -172,13 +172,13 @@ class Chef
         table_body = if report_data[organization]["_unused_cookbooks.json"].empty?
                        "<tr><td colspan='2'>No unused cookbook versions</td></tr>"
                      else
-                       report_data[organization]["_unused_cookbooks.json"].map { |cookbook_name, cookbook_versions| "<tr><td>#{cookbook_name}</td><td>#{cookbook_versions.join('<br>')}</td></tr>" }.join("\n")
+                       report_data[organization]["_unused_cookbooks.json"].map { |cookbook_name, cookbook_versions| "<tr><td>#{cookbook_name}</td><td>#{cookbook_versions.join("<br>")}</td></tr>" }.join("\n")
                      end
         table_start + header_string + table_body + table_end
       end
 
       def generate_node_table(report_data, organization)
-        table_start = "<h2>Stale Nodes</h2><p>This table contains nodes that have not checked in to the Chef Server in #{report_data[organization]['_stale_nodes.json']['threshold_days']} days.<p><table border='1' cellpadding='1' cellspacing='0'>"
+        table_start = "<h2>Stale Nodes</h2><p>This table contains nodes that have not checked in to the Chef Server in #{report_data[organization]["_stale_nodes.json"]["threshold_days"]} days.<p><table border='1' cellpadding='1' cellspacing='0'>"
         table_end = "</table>"
         header_string = "<tr><th>Node Name</th></tr>"
         table_body = if report_data[organization]["_stale_nodes.json"].empty? || report_data[organization]["_stale_nodes.json"]["count"] == 0

data/lib/chef/knife/tidy_server_clean.rb

@@ -1,4 +1,4 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
@@ -89,8 +89,9 @@ class Chef
         queue = Chef::Util::ThreadedJobQueue.new
         unused_cookbooks_file = ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json")
         return unless ::File.exist?(unused_cookbooks_file)
+
         ui.stdout.puts "INFO: Cleaning cookbooks for Org: #{org}, using #{unused_cookbooks_file}"
-        unused_cookbooks = FFI_Yajl::Parser.parse(::File.read(unused_cookbooks_file), symbolize_names: true)
+        unused_cookbooks = tidy.json_file_to_hash(unused_cookbooks_file, symbolize_names: true)
         unused_cookbooks.keys.each do |cookbook|
           versions = unused_cookbooks[cookbook]
           versions.each do |version|
@@ -115,8 +116,9 @@ class Chef
         queue = Chef::Util::ThreadedJobQueue.new
         stale_nodes_file = ::File.join(tidy.reports_dir, "#{org}_stale_nodes.json")
         return unless ::File.exist?(stale_nodes_file)
+
         ui.stdout.puts "INFO: Cleaning stale nodes for Org: #{org}, using #{stale_nodes_file}"
-        stale_nodes = FFI_Yajl::Parser.parse(::File.read(stale_nodes_file), symbolize_names: true)
+        stale_nodes = tidy.json_file_to_hash(stale_nodes_file, symbolize_names: true)
         stale_nodes[:list].each do |node|
           queue << -> { delete_node_job(org, node) }
         end

data/lib/chef/knife/tidy_server_report.rb

@@ -1,4 +1,4 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
@@ -17,6 +17,11 @@ class Chef
         default: 30,
         description: "Maximum number of days since last checkin before node is considered stale (default: 30)"
 
+      option :keep_versions,
+        long: "--keep-versions MIN",
+        default: 0,
+        description: "Keep a minimum of this many versions of each cookbook (default: 0)"
+
       def run
         ensure_reports_dir!
         FileUtils.rm_f(server_warnings_file_path)
@@ -32,6 +37,7 @@ class Chef
 
         stale_orgs = []
         node_threshold = config[:node_threshold].to_i
+        keep_versions  = config[:keep_versions].to_i
 
         orgs.each do |org|
           pre_12_3_nodes = []
@@ -43,7 +49,7 @@ class Chef
           nodes = nodes_list(org)
           db_nodes = rest.get("/organizations/#{org}/nodes")
           unless nodes.length == db_nodes.length
-            ood_message = "Search index is out of date! No cleanup action will be taken for #{org}."
+            ood_message = "Search index is out of date (search returned #{nodes.length} nodes while the database indicates there are #{db_nodes.length} nodes! No action will be taken for #{org}. Perhaps a 'chef-server-ctl reindex' is in order?"
             ui.error(ood_message)
             action_needed(ood_message, server_warnings_file_path)
             next
@@ -51,7 +57,7 @@ class Chef
 
           nodes.each do |node|
             # If the node hasn't checked in.
-            if !node["chef_packages"]
+            unless node["chef_packages"]
               # If the node is under an hour old.
               if (Time.now.to_i - node["ohai_time"].to_i) < 3600
                 unconverged_recent_nodes << node["name"]
@@ -76,6 +82,9 @@ class Chef
             end
           end
 
+          used_cookbooks = keep_cookbook_versions(cb_list, keep_versions)
+
+          Chef::Log.debug("Used cookbook list before checking environments: #{used_cookbooks}")
           pins = environment_constraints(org)
           used_cookbooks = check_environment_pins(used_cookbooks, pins, cb_list)
 
@@ -153,6 +162,15 @@ class Chef
         cb_list
       end
 
+      def keep_cookbook_versions(cb_list, min)
+        retain = {}
+        cb_list.each do |name, versions|
+          keep = versions.sort { |a, b| Gem::Version.new(a) <=> Gem::Version.new(b) }.last(min)
+          retain[name] = keep
+        end
+        retain
+      end
+
       def cookbook_count(cb_list)
         cb_count_list = {}
         cb_list.each do |name, versions|
@@ -207,8 +225,9 @@ class Chef
             else
               versions_not_satisfied.push(v)
             end
+
             if v == cb_list[cb].last
-              ui.warn("Pin of #{cb} #{version} not satisfied by current versions of cookbook: [#{versions_not_satisfied.join(', ')}]")
+              ui.warn("Pin of #{cb} #{version} not satisfied by current versions of cookbook: [#{versions_not_satisfied.join(", ")}]")
             end
           end
         else
@@ -221,6 +240,7 @@ class Chef
         pins.each do |cb, versions|
           versions.each do |version|
             next if version == "<= 0.0.0"
+
             if used_cookbooks[cb]
               # This pinned cookbook is in the used list, now check for a matching version.
               used_cookbooks[cb].each do |v|

data/lib/chef/tidy_acls.rb

@@ -20,26 +20,26 @@ class Chef
     def load_users
       @tidy.ui.stdout.puts "INFO: Loading users"
       Dir[::File.join(@tidy.users_path, "*.json")].each do |user|
-        @users.push(FFI_Yajl::Parser.parse(::File.read(user), symbolize_names: true))
+        @users.push(@tidy.json_file_to_hash(user, symbolize_names: true))
       end
     end
 
     def load_members
       @tidy.ui.stdout.puts "INFO: Loading members for #{@org}"
-      @members = FFI_Yajl::Parser.parse(::File.read(@tidy.members_path(@org)), symbolize_names: true)
+      @members = @tidy.json_file_to_hash(@tidy.members_path(@org), symbolize_names: true)
     end
 
     def load_clients
       @tidy.ui.stdout.puts "INFO: Loading clients for #{@org}"
       Dir[::File.join(@tidy.clients_path(@org), "*.json")].each do |client|
-        @clients.push(FFI_Yajl::Parser.parse(::File.read(client), symbolize_names: true))
+        @clients.push(@tidy.json_file_to_hash(client, symbolize_names: true))
       end
     end
 
     def load_groups
       @tidy.ui.stdout.puts "INFO: Loading groups for #{@org}"
       Dir[::File.join(@tidy.groups_path(@org), "*.json")].each do |group|
-        @groups.push(FFI_Yajl::Parser.parse(::File.read(group), symbolize_names: true))
+        @groups.push(@tidy.json_file_to_hash(group, symbolize_names: true))
       end
     end
 
@@ -128,7 +128,7 @@ class Chef
 
     def remove_group_from_acl(group, acl_file)
       @tidy.ui.stdout.puts "REPAIRING: Removing invalid group: #{group} from #{acl_file}"
-      acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+      acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
       acl_ops.each do |op|
         acl[op]["groups"].reject! { |the_group| the_group == group }
       end
@@ -137,7 +137,7 @@ class Chef
 
     # Appends the proper acls for ::server-admins and the org's read access group if they are missing.
     def ensure_global_group_acls(acl_file)
-      acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+      acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
       acl_ops.each do |op|
         unless acl[op]["groups"].include? "::server-admins"
           @tidy.ui.stdout.puts "REPAIRING: Adding #{op} acl for ::server-admins in #{acl_file}"
@@ -152,7 +152,7 @@ class Chef
     end
 
     def ensure_client_read_acls(acl_file)
-      acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+      acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
       %w{users admins}.each do |group|
         unless acl["read"]["groups"].include? group
           @tidy.ui.stdout.puts "REPAIRING: Adding read acl for #{group} in #{acl_file}"
@@ -164,10 +164,11 @@ class Chef
 
     def validate_acls
       org_acls.each do |acl_file|
-        acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+        acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
         actors_groups = acl_actors_groups(acl)
         actors_groups[:actors].each do |actor|
           next if actor == "pivotal"
+
           if ambiguous_actor?(actor)
             fix_ambiguous_actor(actor)
           elsif missing_from_members?(actor)
@@ -202,11 +203,11 @@ class Chef
       @members.each do |member|
         user_acl_path = ::File.join(@tidy.user_acls_path, "#{member[:user][:username]}.json")
         begin
-          user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+          user_acl = @tidy.json_file_to_hash(user_acl_path, symbolize_names: false)
         rescue Errno::ENOENT
           @tidy.ui.stdout.puts "REPAIRING: Replacing missing user acl for #{member[:user][:username]}."
           @tidy.write_new_file(default_user_acl(member), user_acl_path, backup = false)
-          user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+          user_acl = @tidy.json_file_to_hash(user_acl_path, symbolize_names: false)
         end
         ensure_global_group_acls(user_acl_path)
         actors_groups = acl_actors_groups(user_acl)
@@ -220,11 +221,11 @@ class Chef
       @clients.each do |client|
         client_acl_path = ::File.join(@tidy.org_acls_path(@org), "clients", "#{client[:name]}.json")
         begin
-          client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+          client_acl = @tidy.json_file_to_hash(client_acl_path, symbolize_names: false)
         rescue Errno::ENOENT
           @tidy.ui.stdout.puts "REPAIRING: Replacing missing client acl for #{client[:name]} in #{client_acl_path}."
           @tidy.write_new_file(default_client_acl(client[:name]), client_acl_path, backup = false)
-          client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+          client_acl = @tidy.json_file_to_hash(client_acl_path, symbolize_names: false)
         end
         ensure_client_read_acls(client_acl_path)
       end

data/lib/chef/tidy_common.rb

@@ -12,61 +12,130 @@ class Chef
       @backup_path = ::File.expand_path(backup_path)
     end
 
+    #
+    # @return [Chef::Knife::UI]
+    #
     def ui
       @ui ||= Chef::Knife::UI.new(STDOUT, STDERR, STDIN, {})
     end
 
+    # The path to the users directory in the backup
+    #
+    # @return [String]
+    #
     def users_path
       @users_path ||= ::File.expand_path(::File.join(@backup_path, "users"))
     end
 
+    # The path to the members.json file in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def members_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "members.json"))
     end
 
+    # The path to the invitations.json file in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def invitations_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "invitations.json"))
     end
 
+    # The path to the clients directory in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def clients_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "clients"))
     end
 
+    # The paths to each of the client json files in the backup
+    #
+    # @param [String] org
+    #
+    # @return [Array]
+    #
     def client_names(org)
       Dir[::File.join(clients_path(org), "*")].map { |dir| ::File.basename(dir, ".json") }
     end
 
+    # The path to groups directory in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def groups_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "groups"))
     end
 
+    # The path to acls directory in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def org_acls_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "acls"))
     end
 
+    # The path to user_acls directory in the backup
+    #
+    # @return [String]
+    #
     def user_acls_path
       @user_acls_path ||= ::File.expand_path(::File.join(@backup_path, "user_acls"))
     end
 
+    # The path to cookbooks directory in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def cookbooks_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "cookbooks"))
     end
 
+    # The path to roles directory in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def roles_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org, "roles"))
     end
 
+    # The path to the org directory in the backup
+    #
+    # @param [String] org
+    #
+    # @return [String]
+    #
     def org_path(org)
       ::File.expand_path(::File.join(@backup_path, "organizations", org))
     end
 
+    # generate a bogus, but valid email
+    #
+    # @return [String]
+    #
     def unique_email
       (0...8).map { (65 + rand(26)).chr }.join.downcase +
         "@" + (0...8).map { (65 + rand(26)).chr }.join.downcase + ".com"
     end
 
     def save_user(user)
-      ::File.open(::File.join(users_path, "#{user['username']}.json"), "w+") do |f|
+      ::File.open(::File.join(users_path, "#{user["username"]}.json"), "w+") do |f|
         f.write(FFI_Yajl::Encoder.encode(user, pretty: true))
       end
     end
@@ -80,10 +149,62 @@ class Chef
       end
     end
 
+    # Read a json file and return a hash of parsed content with optional symbolized keys
+    #
+    # @param [String] path to file
+    # @param [double splat] options to pass FFI_Yajl::Parser.parse()
+    #
+    # @return [Hash] original json content as hash
+    #
+    # @example
+    # json_file_to_hash('/path/to/file.json', symbolize_names: true) => { foo: "bar" }
+    #
+    def json_file_to_hash(file_path, **options)
+      FFI_Yajl::Parser.parse(File.read(file_path), options)
+    rescue Errno::ENOENT, Errno::EACCES, FFI_Yajl::ParseError
+      puts "ERROR: unable to parse file: '#{file_path}'"
+      raise
+    end
+
+    #
+    # Determine the cookbook name from path
+    #
+    # @param [String] path The path of the cookbook.
+    #
+    # @return [String] The cookbook's name
+    #
+    # @example
+    # cookbook_version_from_path('/data/chef_backup/snapshots/20191008040001/organizations/myorg/cookbooks/chef-sugar-5.0.4') => 'chef-sugar'
+    #
     def cookbook_name_from_path(path)
       ::File.basename(path, "-*")
     end
 
+    #
+    # Determine the cookbook version from a path.
+    #
+    # @param [String] path The path of the cookbook.
+    #
+    # @return [String] The version of the cookbook.
+    #
+    # @example
+    # cookbook_version_from_path('/data/chef_backup/snapshots/20191008040001/organizations/myorg/cookbooks/chef-sugar-5.0.4') => '5.0.4'
+    # cookbook_version_from_path('/data/chef_backup/snapshots/20191008040001/organizations/myorg/cookbooks/chef-sugar-5.0.4/recipe/default.rb') => '5.0.4'
+    # cookbook_version_from_path('/data/chef_backup/snapshots/20191008040001/organizations/myorg/cookbooks/chef-sugar-5.0.4/files/cookbooks/default.rb') => '5.0.4'
+    #
+    def cookbook_version_from_path(path)
+      dirs = path.split(File::SEPARATOR)
+
+      until dirs.empty?
+        version_match = dirs[-1].match(/\d+\.\d+\.\d+/)
+        if dirs[-2] == "cookbooks" && version_match # we found the cookbook version not something that looks like one inside a cookbook path
+          return version_match.to_s
+        else
+          dirs.pop
+        end
+      end
+    end
+
     def global_user_names
       @global_user_names ||= Dir[::File.join(@backup_path, "users", "*")].map { |dir| ::File.basename(dir, ".json") }
     end

data/lib/chef/tidy_server.rb

@@ -7,7 +7,7 @@ class Chef
     end
 
     def self.from_chef_server_url(url)
-      url = url.gsub(/\/organizations\/+[^\/]+\/*$/, "")
+      url = url.gsub(%r{/organizations/+[^/]+/*$}, "")
       Chef::Server.new(url)
     end
   end

data/lib/chef/tidy_substitutions.rb

@@ -17,7 +17,7 @@ class Chef
 
     def load_data
       @tidy.ui.stdout.puts "INFO: Loading substitutions from #{file_path}"
-      @data = FFI_Yajl::Parser.parse(::File.read(@file_path), symbolize_names: false)
+      @data = @tidy.json_file_to_hash(@file_path, symbolize_names: false)
     rescue Errno::ENOENT
       raise NoSubstitutionFile, file_path
     end
@@ -28,14 +28,7 @@ class Chef
       FileUtils.cp(bp, ::File.join(Dir.pwd, "substitutions.json"))
     end
 
-    def cookbook_version_from_path(path)
-      components = path.split(File::SEPARATOR)
-      name_version = components[components.index("cookbooks") + 1]
-      name_version.match(/\d+\.\d+\.\d+/).to_s
-    end
-
-    def revert
-    end
+    def revert; end
 
     def sub_in_file(path, search, replace)
       temp_file = Tempfile.new("tidy")
@@ -68,7 +61,7 @@ class Chef
             @data[entry][glob].each do |substitution|
               search = Regexp.new(substitution["pattern"])
               replace = substitution["replace"].dup
-              replace.gsub!(/\!COOKBOOK_VERSION\!/) { |_m| "'" + cookbook_version_from_path(file) + "'" }
+              replace.gsub!(/\!COOKBOOK_VERSION\!/) { |_m| "'" + @tidy.cookbook_version_from_path(file) + "'" }
               sub_in_file(file, search, replace)
             end
           end

data/lib/knife-tidy/version.rb

@@ -1,4 +1,4 @@
 module KnifeTidy
-  VERSION = "2.0.1".freeze
+  VERSION = "2.0.15".freeze
   MAJOR, MINOR, TINY = VERSION.split(".")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-tidy
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.15
 platform: ruby
 authors:
 - Jeremy Miller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-07 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies: []
 description: Report on stale Chef Server nodes and cookbooks and clean up data integrity
   issues in a knife-ec-backup object based backup
@@ -19,6 +19,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
+- conf/substitutions.json.example
 - lib/chef/knife/tidy_backup_clean.rb
 - lib/chef/knife/tidy_base.rb
 - lib/chef/knife/tidy_notify.rb
@@ -48,8 +49,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Report on stale Chef Server nodes and cookbooks and clean up data integrity