knife-tidy 2.0.6 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 562e7dd8fc15d4b707b8b3d70c0bc54a3226f9d54d8bcc72ef4cf5436320fc1a
-  data.tar.gz: 61a42597e3340788e0f4040b5c251c5fce14622afadb077048886936e445815f
+  metadata.gz: a35c35e40e4f4c982f86ab15be05fa79192c55a8c748b4fd0a4dacb4e2332fee
+  data.tar.gz: 7c1a6ac3c38567e5015520c66fb2431c161f1edc2c8af0335870b8d5f1afd183
 SHA512:
-  metadata.gz: 3c475ece3e2a94d08a521f4262820e88e5103d8e1cc3cc7273dbd6c1ce639d122e2c00904a46fe71ddec0eefee87aa567f7de574fd965764cebe70135f65d7d0
-  data.tar.gz: 33c2841cfc1d1c51db87a06c7a61ffd26c630096962e0ce70aac4df6d9ac234818e82c0b7ab2894a0a440fd804cb5a15a05fdbe9d691dc84ca0f1b762157d5b7
+  metadata.gz: 40d92b7427aff1df3a7c4fcc7cc1df7742285cc5332b64e66213413b6dbc0a6e2b80d96554b2f76f21d3173499a53533d33366986f3b4446d09b5e42f312fc5a
+  data.tar.gz: 1a34a9e016c5fcc1eb81bea371f7fae6a2ffa5faf48b7e83771aa2b354517c3b951970929dc58de230e2ade19ea6e0d8b1c77b74b5ad7e1d9ed256b4f38f45a9

data/conf/substitutions.json.example

@@ -0,0 +1,10 @@
+{
+  "your-problem-descriptor":{
+    "organizations/*/cookbooks/*/metadata.rb":[
+      {
+        "pattern":"^version .*GO_PIPELINE_LABEL",
+        "replace":"version !COOKBOOK_VERSION!"
+      }
+    ]
+  }
+}

data/lib/chef/knife/tidy_backup_clean.rb

@@ -1,15 +1,15 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
     class TidyBackupClean < Knife
       deps do
-        require "chef/cookbook_loader"
+        require "chef/cookbook/cookbook_version_loader"
         require "chef/cookbook/metadata"
         require "chef/role"
         require "chef/run_list"
-        require "chef/tidy_substitutions"
-        require "chef/tidy_acls"
+        require_relative "../tidy_substitutions"
+        require_relative "../tidy_acls"
         require "ffi_yajl"
         require "fileutils"
         require "securerandom"
@@ -75,7 +75,7 @@ class Chef
         tidy.global_user_names.each do |user|
           email = ""
           ui.stdout.puts "INFO: Validating #{user}"
-          the_user = FFI_Yajl::Parser.parse(::File.read(::File.join(tidy.users_path, "#{user}.json")), symbolize_names: false)
+          the_user = tidy.json_file_to_hash(File.join(tidy.users_path, "#{user}.json"), symbolize_names: false)
           if the_user.key?("email") && the_user["email"].match(/\A[^@\s]+@[^@\s]+\z/)
             if emails_seen.include?(the_user["email"])
               ui.stdout.puts "REPAIRING: Already saw #{user}'s email, creating a unique one."
@@ -147,7 +147,7 @@ class Chef
             add_cookbook_name_to_metadata(cookbook_name, rb_path) if lines.empty?
           else
             if ::File.exist?(json_path)
-              metadata = FFI_Yajl::Parser.parse(::File.read(json_path), symbolize_names: false)
+              metadata = tidy.json_file_to_hash(json_path, symbolize_names: false)
               if metadata["name"] != cookbook_name
                 metadata["name"] = cookbook_name
                 ui.stdout.puts "REPAIRING: Correcting `name` in #{json_path}`"
@@ -161,26 +161,24 @@ class Chef
       end
 
       def load_cookbooks(org)
-        cl = Chef::CookbookLoader.new(tidy.cookbooks_path(org))
-        for_each_cookbook_basename(org) do |cookbook|
+        for_each_cookbook_path(org) do |cookbook|
+          cl = Chef::Cookbook::CookbookVersionLoader.new(cookbook)
           ui.stdout.puts "INFO: Loading #{cookbook}"
-          ret = cl.load_cookbook(cookbook)
+          ret = cl.load!
           if ret.nil?
             action_needed("ACTION NEEDED: Something's wrong with the #{cookbook} cookbook in org #{org} - cannot load it! Moving to cookbooks.broken folder.")
             broken_cookooks_add(org, cookbook)
           end
         end
-      rescue LoadError => e
+      rescue LoadError, Exceptions::MetadataNotValid => e
         ui.error e
         exit 1
       end
 
-      def broken_cookooks_add(org, cookbook)
+      def broken_cookooks_add(org, cookbook_path)
         broken_path = ::File.join(tidy.org_path(org), "cookbooks.broken")
         FileUtils.mkdir(broken_path) unless ::File.directory?(broken_path)
-        Dir[::File.join(tidy.cookbooks_path(org), "#{cookbook}*")].each do |cb|
-          FileUtils.mv(cb, broken_path, verbose: true, force: true)
-        end
+        FileUtils.mv(cookbook_path, broken_path, verbose: true, force: true)
       end
 
       def generate_new_metadata(org)
@@ -229,7 +227,7 @@ class Chef
 
       def fix_metadata_fields(cookbook_path)
         json_path = ::File.join(cookbook_path, "metadata.json")
-        metadata = FFI_Yajl::Parser.parse(::File.read(json_path), symbolize_names: false)
+        metadata = tidy.json_file_to_hash(json_path, symbolize_names: false)
         md = metadata.dup
         metadata.each_pair do |key, value|
           if value.nil?
@@ -351,7 +349,7 @@ class Chef
       end
 
       def repair_role_run_lists(role_path)
-        the_role = FFI_Yajl::Parser.parse(::File.read(role_path), symbolize_names: false)
+        the_role = tidy.json_file_to_hash(role_path, symbolize_names: false)
         new_role = the_role.clone
         rl = Chef::RunList.new
         new_role["run_list"] = []
@@ -377,14 +375,14 @@ class Chef
           end
         end
         write_role(role_path, new_role)
-        # rubocop:enable MethodLength
+        # rubocop:enable Metrics/MethodLength
       end
 
       def validate_roles(org)
         for_each_role(org) do |role_path|
           ui.stdout.puts "INFO: Validating Role at #{role_path}"
           begin
-            Chef::Role.from_hash(FFI_Yajl::Parser.parse(::File.read(role_path), symbolize_names: false))
+            Chef::Role.from_hash(tidy.json_file_to_hash(role_path, symbolize_names: false))
           rescue ArgumentError
             repair_role_run_lists(role_path)
           end
@@ -394,7 +392,7 @@ class Chef
       def validate_clients_group(org)
         ui.stdout.puts "INFO: validating all clients for org #{org} exist in clients group"
         clients_group_path = ::File.join(tidy.groups_path(org), "clients.json")
-        existing_group_data = FFI_Yajl::Parser.parse(::File.read(clients_group_path), symbolize_names: false)
+        existing_group_data = tidy.json_file_to_hash(clients_group_path, symbolize_names: false)
         existing_group_data["clients"] = [] unless existing_group_data.key?("clients")
         if existing_group_data["clients"].length != tidy.client_names(org).length
           ui.stdout.puts "REPAIRING: Adding #{(existing_group_data["clients"].length - tidy.client_names(org).length).abs} missing clients into #{org}'s client group file #{clients_group_path}"
@@ -408,7 +406,7 @@ class Chef
       def validate_invitations(org)
         invite_file = tidy.invitations_path(org)
         ui.stdout.puts "INFO: validating org #{org} invites in #{invite_file}"
-        invitations = FFI_Yajl::Parser.parse(::File.read(invite_file), symbolize_names: false)
+        invitations = tidy.json_file_to_hash(invite_file, symbolize_names: false)
         invitations_new = []
         invitations.each do |invite|
           if invite["username"].nil?

data/lib/chef/knife/tidy_base.rb

@@ -24,8 +24,8 @@ class Chef
       def self.included(includer)
         includer.class_eval do
           deps do
-            require "chef/tidy_server"
-            require "chef/tidy_common"
+            require_relative "../tidy_server"
+            require_relative "../tidy_common"
           end
 
           option :org_list,

data/lib/chef/knife/tidy_notify.rb

@@ -1,4 +1,4 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
@@ -88,7 +88,7 @@ class Chef
               file_name = "#{reports_dir}/#{org}#{report}"
               ui.info("  Parsing file #{file_name}")
               json_string = File.read(file_name)
-              reports[org][report] = FFI_Yajl::Parser.parse(json_string)
+              reports[org][report] = tidy.json_file_to_hash(json_string, symbolize_names: false)
             rescue Errno::ENOENT
               ui.info("    Skipping file #{file_name} - not found for organization #{org}")
               reports[org][report] = {}

data/lib/chef/knife/tidy_server_clean.rb

@@ -1,4 +1,4 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
@@ -91,7 +91,7 @@ class Chef
         return unless ::File.exist?(unused_cookbooks_file)
 
         ui.stdout.puts "INFO: Cleaning cookbooks for Org: #{org}, using #{unused_cookbooks_file}"
-        unused_cookbooks = FFI_Yajl::Parser.parse(::File.read(unused_cookbooks_file), symbolize_names: true)
+        unused_cookbooks = tidy.json_file_to_hash(unused_cookbooks_file, symbolize_names: true)
         unused_cookbooks.keys.each do |cookbook|
           versions = unused_cookbooks[cookbook]
           versions.each do |version|
@@ -118,7 +118,7 @@ class Chef
         return unless ::File.exist?(stale_nodes_file)
 
         ui.stdout.puts "INFO: Cleaning stale nodes for Org: #{org}, using #{stale_nodes_file}"
-        stale_nodes = FFI_Yajl::Parser.parse(::File.read(stale_nodes_file), symbolize_names: true)
+        stale_nodes = tidy.json_file_to_hash(stale_nodes_file, symbolize_names: true)
         stale_nodes[:list].each do |node|
           queue << -> { delete_node_job(org, node) }
         end

data/lib/chef/knife/tidy_server_report.rb

@@ -1,4 +1,4 @@
-require "chef/knife/tidy_base"
+require_relative "tidy_base"
 
 class Chef
   class Knife
@@ -17,6 +17,11 @@ class Chef
         default: 30,
         description: "Maximum number of days since last checkin before node is considered stale (default: 30)"
 
+      option :keep_versions,
+        long: "--keep-versions MIN",
+        default: 0,
+        description: "Keep a minimum of this many versions of each cookbook (default: 0)"
+
       def run
         ensure_reports_dir!
         FileUtils.rm_f(server_warnings_file_path)
@@ -32,6 +37,7 @@ class Chef
 
         stale_orgs = []
         node_threshold = config[:node_threshold].to_i
+        keep_versions  = config[:keep_versions].to_i
 
         orgs.each do |org|
           pre_12_3_nodes = []
@@ -76,6 +82,9 @@ class Chef
             end
           end
 
+          used_cookbooks = keep_cookbook_versions(cb_list, keep_versions)
+
+          Chef::Log.debug("Used cookbook list before checking environments: #{used_cookbooks}")
           pins = environment_constraints(org)
           used_cookbooks = check_environment_pins(used_cookbooks, pins, cb_list)
 
@@ -153,6 +162,15 @@ class Chef
         cb_list
       end
 
+      def keep_cookbook_versions(cb_list, min)
+        retain = {}
+        cb_list.each do |name, versions|
+          keep = versions.sort { |a, b| Gem::Version.new(a) <=> Gem::Version.new(b) }.last(min)
+          retain[name] = keep
+        end
+        retain
+      end
+
       def cookbook_count(cb_list)
         cb_count_list = {}
         cb_list.each do |name, versions|

data/lib/chef/tidy_acls.rb

@@ -20,26 +20,26 @@ class Chef
     def load_users
       @tidy.ui.stdout.puts "INFO: Loading users"
       Dir[::File.join(@tidy.users_path, "*.json")].each do |user|
-        @users.push(FFI_Yajl::Parser.parse(::File.read(user), symbolize_names: true))
+        @users.push(@tidy.json_file_to_hash(user, symbolize_names: true))
       end
     end
 
     def load_members
       @tidy.ui.stdout.puts "INFO: Loading members for #{@org}"
-      @members = FFI_Yajl::Parser.parse(::File.read(@tidy.members_path(@org)), symbolize_names: true)
+      @members = @tidy.json_file_to_hash(@tidy.members_path(@org), symbolize_names: true)
     end
 
     def load_clients
       @tidy.ui.stdout.puts "INFO: Loading clients for #{@org}"
       Dir[::File.join(@tidy.clients_path(@org), "*.json")].each do |client|
-        @clients.push(FFI_Yajl::Parser.parse(::File.read(client), symbolize_names: true))
+        @clients.push(@tidy.json_file_to_hash(client, symbolize_names: true))
       end
     end
 
     def load_groups
       @tidy.ui.stdout.puts "INFO: Loading groups for #{@org}"
       Dir[::File.join(@tidy.groups_path(@org), "*.json")].each do |group|
-        @groups.push(FFI_Yajl::Parser.parse(::File.read(group), symbolize_names: true))
+        @groups.push(@tidy.json_file_to_hash(group, symbolize_names: true))
       end
     end
 
@@ -128,7 +128,7 @@ class Chef
 
     def remove_group_from_acl(group, acl_file)
       @tidy.ui.stdout.puts "REPAIRING: Removing invalid group: #{group} from #{acl_file}"
-      acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+      acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
       acl_ops.each do |op|
         acl[op]["groups"].reject! { |the_group| the_group == group }
       end
@@ -137,7 +137,7 @@ class Chef
 
     # Appends the proper acls for ::server-admins and the org's read access group if they are missing.
     def ensure_global_group_acls(acl_file)
-      acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+      acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
       acl_ops.each do |op|
         unless acl[op]["groups"].include? "::server-admins"
           @tidy.ui.stdout.puts "REPAIRING: Adding #{op} acl for ::server-admins in #{acl_file}"
@@ -152,7 +152,7 @@ class Chef
     end
 
     def ensure_client_read_acls(acl_file)
-      acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+      acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
       %w{users admins}.each do |group|
         unless acl["read"]["groups"].include? group
           @tidy.ui.stdout.puts "REPAIRING: Adding read acl for #{group} in #{acl_file}"
@@ -164,7 +164,7 @@ class Chef
 
     def validate_acls
       org_acls.each do |acl_file|
-        acl = FFI_Yajl::Parser.parse(::File.read(acl_file), symbolize_names: false)
+        acl = @tidy.json_file_to_hash(acl_file, symbolize_names: false)
         actors_groups = acl_actors_groups(acl)
         actors_groups[:actors].each do |actor|
           next if actor == "pivotal"
@@ -203,11 +203,11 @@ class Chef
       @members.each do |member|
         user_acl_path = ::File.join(@tidy.user_acls_path, "#{member[:user][:username]}.json")
         begin
-          user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+          user_acl = @tidy.json_file_to_hash(user_acl_path, symbolize_names: false)
         rescue Errno::ENOENT
           @tidy.ui.stdout.puts "REPAIRING: Replacing missing user acl for #{member[:user][:username]}."
           @tidy.write_new_file(default_user_acl(member), user_acl_path, backup = false)
-          user_acl = FFI_Yajl::Parser.parse(::File.read(user_acl_path), symbolize_names: false)
+          user_acl = @tidy.json_file_to_hash(user_acl_path, symbolize_names: false)
         end
         ensure_global_group_acls(user_acl_path)
         actors_groups = acl_actors_groups(user_acl)
@@ -221,11 +221,11 @@ class Chef
       @clients.each do |client|
         client_acl_path = ::File.join(@tidy.org_acls_path(@org), "clients", "#{client[:name]}.json")
         begin
-          client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+          client_acl = @tidy.json_file_to_hash(client_acl_path, symbolize_names: false)
         rescue Errno::ENOENT
           @tidy.ui.stdout.puts "REPAIRING: Replacing missing client acl for #{client[:name]} in #{client_acl_path}."
           @tidy.write_new_file(default_client_acl(client[:name]), client_acl_path, backup = false)
-          client_acl = FFI_Yajl::Parser.parse(::File.read(client_acl_path), symbolize_names: false)
+          client_acl = @tidy.json_file_to_hash(client_acl_path, symbolize_names: false)
         end
         ensure_client_read_acls(client_acl_path)
       end

data/lib/chef/tidy_common.rb

@@ -149,6 +149,23 @@ class Chef
       end
     end
 
+    # Read a json file and return a hash of parsed content with optional symbolized keys
+    #
+    # @param [String] path to file
+    # @param [double splat] options to pass FFI_Yajl::Parser.parse()
+    #
+    # @return [Hash] original json content as hash
+    #
+    # @example
+    # json_file_to_hash('/path/to/file.json', symbolize_names: true) => { foo: "bar" }
+    #
+    def json_file_to_hash(file_path, **options)
+      FFI_Yajl::Parser.parse(File.read(file_path), options)
+    rescue Errno::ENOENT, Errno::EACCES, FFI_Yajl::ParseError
+      puts "ERROR: unable to parse file: '#{file_path}'"
+      raise
+    end
+
     #
     # Determine the cookbook name from path
     #

data/lib/chef/tidy_substitutions.rb

@@ -17,7 +17,7 @@ class Chef
 
     def load_data
       @tidy.ui.stdout.puts "INFO: Loading substitutions from #{file_path}"
-      @data = FFI_Yajl::Parser.parse(::File.read(@file_path), symbolize_names: false)
+      @data = @tidy.json_file_to_hash(@file_path, symbolize_names: false)
     rescue Errno::ENOENT
       raise NoSubstitutionFile, file_path
     end

data/lib/knife-tidy/version.rb

@@ -1,4 +1,4 @@
 module KnifeTidy
-  VERSION = "2.0.6".freeze
+  VERSION = "2.1.0".freeze
   MAJOR, MINOR, TINY = VERSION.split(".")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-tidy
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 2.1.0
 platform: ruby
 authors:
 - Jeremy Miller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-04 00:00:00.000000000 Z
+date: 2020-06-30 00:00:00.000000000 Z
 dependencies: []
 description: Report on stale Chef Server nodes and cookbooks and clean up data integrity
   issues in a knife-ec-backup object based backup
@@ -19,6 +19,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
+- conf/substitutions.json.example
 - lib/chef/knife/tidy_backup_clean.rb
 - lib/chef/knife/tidy_base.rb
 - lib/chef/knife/tidy_notify.rb