knife-tidy 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4617020ad74d1acc6de5f6c5e65c7f3506455689
+  data.tar.gz: be95ccdc41759a9949aa6ee3aa6afe221486c995
+SHA512:
+  metadata.gz: 4170963ff8992beec932a57db8715fa0edba110edee1c643c3f277d4d9b2568edb7ea846cee7033bfa397c4bcd699f5dc0a6aff1494b808c519c461c6269e810
+  data.tar.gz: acc9b53c4944f77bc0879805142be99b7143a891a2feb320a80767f2c89879c651d606f9fd56234ddc1801c13b59c17b61a86b3fc12e65140baa36bbc67ec212

data/.gitignore

@@ -0,0 +1,52 @@
+*.rbc
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+_Store
+reports/
+conf/*.json
+*~
+*#
+.#*
+\#*#
+.*.sw[a-z]
+*.un~
+*.tmp
+*.bk
+*.bkup
+
+# ruby/bundler files
+.ruby-version
+.ruby-gemset
+.rvmrc
+Gemfile.lock
+.bundle
+*.gem
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+.idea
+
+# chef stuff
+Berksfile.lock
+.kitchen
+.kitchen.local.yml
+vendor/
+.coverage/
+.zero-knife.rb
+Policyfile.lock.json
+.chef/
+
+# vagrant stuff
+.vagrant/
+.vagrant.d/
+.kitchen/

data/Gemfile

@@ -0,0 +1,38 @@
+source "https://rubygems.org"
+
+gemspec
+
+if vsn = ENV['TRAVIS_CHEF_VERSION']
+  if m = /branch:(?<branch>.*)$/.match(vsn)
+    gem 'chef', git: 'https://github.com/chef/chef', branch: m[:branch]
+  else
+    gem 'chef', vsn
+  end
+end
+
+group :development do
+  gem 'rspec'
+  gem 'rake'
+  gem 'simplecov'
+  gem 'fakefs'
+  gem "chefstyle", git: "https://github.com/chef/chefstyle.git"
+  gem "chef-zero"
+end
+
+group :changelog do
+  gem "github_changelog_generator", git: "https://github.com/chef/github-changelog-generator"
+end
+
+# This is here instead of gemspec so that we can
+# override which Chef gem to use when we do testing
+# Possibilities in the future include using environmental
+# variables, thus allowing us to to have Travis support
+
+# Examples you can use in Gemfile.local
+# gem 'chef', '~> 10.28'
+# gem 'chef' # latest
+# gem 'chef', git: 'git://github.com/mal/chef.git', branch: 'CHEF-3307'
+
+# If you want to load debugging tools into the bundle exec sandbox,
+# # add these additional dependencies into Gemfile.local
+eval(IO.read(__FILE__ + '.local'), binding) if File.exists?(__FILE__ + '.local')

data/LICENSE

@@ -0,0 +1,201 @@
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity authorized by
+   the copyright owner that is granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity. For the purposes of this definition,
+   "control" means (i) the power, direct or indirect, to cause the
+   direction or management of such entity, whether by contract or
+   otherwise, or (ii) ownership of fifty percent (50%) or more of the
+   outstanding shares, or (iii) beneficial ownership of such entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications,
+   including but not limited to software source code, documentation
+   source, and configuration files.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form, including but
+   not limited to compiled object code, generated documentation,
+   and conversions to other media types.
+
+   "Work" shall mean the work of authorship, whether in Source or
+   Object form, made available under the License, as indicated by a
+   copyright notice that is included in or attached to the work
+   (an example is provided in the Appendix below).
+
+   "Derivative Works" shall mean any work, whether in Source or Object
+   form, that is based on (or derived from) the Work and for which the
+   editorial revisions, annotations, elaborations, or other modifications
+   represent, as a whole, an original work of authorship. For the purposes
+   of this License, Derivative Works shall not include works that remain
+   separable from, or merely link (or bind by name) to the interfaces of,
+   the Work and Derivative Works thereof.
+
+   "Contribution" shall mean any work of authorship, including
+   the original version of the Work and any modifications or additions
+   to that Work or Derivative Works thereof, that is intentionally
+   submitted to Licensor for inclusion in the Work by the copyright owner
+   or by an individual or Legal Entity authorized to submit on behalf of
+   the copyright owner. For the purposes of this definition, "submitted"
+   means any form of electronic, verbal, or written communication sent
+   to the Licensor or its representatives, including but not limited to
+   communication on electronic mailing lists, source code control systems,
+   and issue tracking systems that are managed by, or on behalf of, the
+   Licensor for the purpose of discussing and improving the Work, but
+   excluding communication that is conspicuously marked or otherwise
+   designated in writing by the copyright owner as "Not a Contribution."
+
+   "Contributor" shall mean Licensor and any individual or Legal Entity
+   on behalf of whom a Contribution has been received by Licensor and
+   subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   copyright license to reproduce, prepare Derivative Works of,
+   publicly display, publicly perform, sublicense, and distribute the
+   Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   (except as stated in this section) patent license to make, have made,
+   use, offer to sell, sell, import, and otherwise transfer the Work,
+   where such license applies only to those patent claims licensable
+   by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s)
+   with the Work to which such Contribution(s) was submitted. If You
+   institute patent litigation against any entity (including a
+   cross-claim or counterclaim in a lawsuit) alleging that the Work
+   or a Contribution incorporated within the Work constitutes direct
+   or contributory patent infringement, then any patent licenses
+   granted to You under this License for that Work shall terminate
+   as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+   Work or Derivative Works thereof in any medium, with or without
+   modifications, and in Source or Object form, provided that You
+   meet the following conditions:
+
+   (a) You must give any other recipients of the Work or
+       Derivative Works a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices
+       stating that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works
+       that You distribute, all copyright, patent, trademark, and
+       attribution notices from the Source form of the Work,
+       excluding those notices that do not pertain to any part of
+       the Derivative Works; and
+
+   (d) If the Work includes a "NOTICE" text file as part of its
+       distribution, then any Derivative Works that You distribute must
+       include a readable copy of the attribution notices contained
+       within such NOTICE file, excluding those notices that do not
+       pertain to any part of the Derivative Works, in at least one
+       of the following places: within a NOTICE text file distributed
+       as part of the Derivative Works; within the Source form or
+       documentation, if provided along with the Derivative Works; or,
+       within a display generated by the Derivative Works, if and
+       wherever such third-party notices normally appear. The contents
+       of the NOTICE file are for informational purposes only and
+       do not modify the License. You may add Your own attribution
+       notices within Derivative Works that You distribute, alongside
+       or as an addendum to the NOTICE text from the Work, provided
+       that such additional attribution notices cannot be construed
+       as modifying the License.
+
+   You may add Your own copyright statement to Your modifications and
+   may provide additional or different license terms and conditions
+   for use, reproduction, or distribution of Your modifications, or
+   for any such Derivative Works as a whole, provided Your use,
+   reproduction, and distribution of the Work otherwise complies with
+   the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+   agreed to in writing, Licensor provides the Work (and each
+   Contributor provides its Contributions) on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied, including, without limitation, any warranties or conditions
+   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+   PARTICULAR PURPOSE. You are solely responsible for determining the
+   appropriateness of using or redistributing the Work and assume any
+   risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+   whether in tort (including negligence), contract, or otherwise,
+   unless required by applicable law (such as deliberate and grossly
+   negligent acts) or agreed to in writing, shall any Contributor be
+   liable to You for damages, including any direct, indirect, special,
+   incidental, or consequential damages of any character arising as a
+   result of this License or out of the use or inability to use the
+   Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses), even if such Contributor
+   has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+   the Work or Derivative Works thereof, You may choose to offer,
+   and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations and/or rights consistent with this
+   License. However, in accepting such obligations, You may act only
+   on Your own behalf and on Your sole responsibility, not on behalf
+   of any other Contributor, and only if You agree to indemnify,
+   defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason
+   of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+   To apply the Apache License to your work, attach the following
+   boilerplate notice, with the fields enclosed by brackets "[]"
+   replaced with your own identifying information. (Don't include
+   the brackets!)  The text should be enclosed in the appropriate
+   comment syntax for the file format. We also recommend that a
+   file or class name and description of purpose be included on the
+   same "printed page" as the copyright notice for easier
+   identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,65 @@
+# knife tidy
+
+# Summary
+
+This Chef Knife plugin has two primary purposes:
+ * Report on the state of Chef Server objects that can be tidied up (Future: clean up objects)
+ * Clean up data integrity issues from a object backup created by [knife-ec-backup](https://github.com/chef/knife-ec-backup)
+
+# Requirements
+
+A current Chef Client. Can easily be installed via [Chef DK](https://github.com/chef/chef-dk#installation)
+
+# Installation
+
+Via Gem
+```bash
+gem install knife-tidy
+```
+
+Via Source
+```bash
+git clone https://github.com/jeremymv2/knife-tidy.git
+cd knife-tidy
+gem build knife-tidy.gemspec && gem install knife-tidy-*.gem --no-ri --no-rdoc
+```
+
+## Common Options
+
+The following options are supported across all subcommands:
+
+  * `--orgs ORG1,ORG2`:
+    Only apply to objects in the named organizations (default: all orgs)
+
+# knife tidy server report (options)
+
+## Options
+
+  * `--node-threshold NUM_DAYS`
+    Maximum number of days since last checkin before node is considered stale (default: 30)
+
+## Notes
+  Generates json reports as such:
+
+File Name | Contents
+--- | ---
+<org>_<threshold_num>d_stale_nodes.json | Nodes in that org that have not checked in for the number of days specified.
+<org>_cookbook_count.json | Number of cookbook versions for each cookbook that that org.
+<org>_unused_cookbooks.json | List of cookbooks and versions that do not appear to be in-use for that org. This is determined by checking the versioned run list of each of the nodes in the org.
+
+# knife tidy backup clean (options)
+
+## Options
+
+  * `--repo-path /path/to/chef-repo`:
+    The Chef Repo to report on or change (such as one created from a
+    [knife-ec-backup](https://github.com/chef/knife-ec-backup)
+
+## Notes
+
+  * Items [Addressed](ITEMS_CLEANED.md)
+  * [To Do](TODO_LIST.md)
+
+# Credits
+
+  * Server Report was ported from Nolan Davidson's [chef-cleanup](https://github.com/nsdavidson/chef-cleanup)

data/Rakefile

@@ -0,0 +1,69 @@
+#!/usr/bin/env rake
+
+require_relative 'tasks/maintainers'
+
+# Style tests. cookstyle (rubocop) and Foodcritic
+namespace :style do
+  begin
+    require 'cookstyle'
+    require 'rubocop/rake_task'
+
+    desc 'Run Ruby style checks'
+    RuboCop::RakeTask.new(:ruby)
+  rescue LoadError => e
+    puts ">>> Gem load error: #{e}, omitting style:ruby" unless ENV['CI']
+  end
+
+  begin
+    require 'foodcritic'
+
+    desc 'Run Chef style checks'
+    FoodCritic::Rake::LintTask.new(:chef) do |t|
+      t.options = {
+        fail_tags: ['any'],
+        progress: true,
+      }
+    end
+  rescue LoadError
+    puts ">>> Gem load error: #{e}, omitting style:chef" unless ENV['CI']
+  end
+end
+
+desc 'Run all style checks'
+task style: ['style:chef', 'style:ruby']
+
+# ChefSpec
+begin
+  require 'rspec/core/rake_task'
+
+  desc 'Run ChefSpec examples'
+  RSpec::Core::RakeTask.new(:spec)
+rescue LoadError => e
+  puts ">>> Gem load error: #{e}, omitting spec" unless ENV['CI']
+end
+
+# Integration tests. Kitchen.ci
+namespace :integration do
+  begin
+    require 'kitchen/rake_tasks'
+
+    desc 'Run kitchen integration tests'
+    Kitchen::RakeTasks.new
+  rescue StandardError => e
+    puts ">>> Kitchen error: #{e}, omitting #{task.name}" unless ENV['CI']
+  end
+end
+
+namespace :supermarket do
+  begin
+    require 'stove/rake_task'
+
+    desc 'Publish cookbook to Supermarket with Stove'
+    Stove::RakeTask.new
+  rescue LoadError => e
+    puts ">>> Gem load error: #{e}, omitting #{task.name}" unless ENV['CI']
+  end
+end
+
+# Default
+task default: %w(style spec)

data/conf/substitutions.json.example

@@ -0,0 +1,14 @@
+{
+  "chef-sugar":{
+    "organizations/*/cookbooks/chef-sugar*/metadata.rb":[
+      {
+        "pattern":"require +File.expand_path('../lib/chef/sugar/version', __FILE__)",
+        "replace":"# require          File.expand_path('../lib/chef/sugar/version', __FILE__)"
+      },
+      {
+        "pattern":"version *Chef::Sugar::VERSION",
+        "replace":"# version          !COOKBOOK_VERSION!"
+      }
+    ]
+  }
+}

data/knife-tidy.gemspec

@@ -0,0 +1,32 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'knife-tidy/version'
+
+Gem::Specification.new do |s|
+  s.name             = "knife-tidy"
+  s.version          = KnifeTidy::VERSION
+  s.version          = "#{s.version}-pre#{ENV['TRAVIS_BUILD_NUMBER']}" if ENV["TRAVIS"]
+  s.has_rdoc         = true
+  s.authors          = ["Jeremy Miller"]
+  s.email            = ["jmiller@chef.io"]
+  s.summary          = "Report on stale Chef Server nodes and cookbooks and clean up data integrity issues in a knife-ec-backup object based backup"
+  s.description      = s.summary
+  s.homepage         = "https://github.com/jeremymv2/knife-tidy"
+  s.license          = "Apache License, v2.0"
+  s.files            = `git ls-files`.split("\n")
+  s.require_paths    = ["lib"]
+
+  s.required_ruby_version = ">= 2.2.0"
+
+  s.add_development_dependency "rake", "~> 11.0"
+  s.add_development_dependency "rspec", "~> 3.4"
+  s.add_development_dependency "aruba", "~> 0.6"
+  s.add_development_dependency "simplecov", "~> 0.9"
+  s.add_development_dependency "simplecov-console", "~> 0.2"
+  if ENV.key?("TRAVIS_BUILD") && RUBY_VERSION == "2.1.9"
+    # Test version of Chef with Chef Zero before
+    # /orgs/org/users/user/keys endpoint was added.
+    s.add_development_dependency "chef", "12.8.1"
+  else # Test most current version of Chef on 2.2.2
+    s.add_development_dependency :chef
+  end
+end

data/lib/chef/knife/tidy_backup_clean.rb

@@ -0,0 +1,111 @@
+require 'chef/knife/tidy_base'
+
+class Chef
+  class Knife
+    class TidyBackupClean < Knife
+
+      include Knife::TidyBase
+
+      deps do
+        require 'chef/cookbook_loader'
+        require 'chef/cookbook/metadata'
+        require 'chef/tidy_substitutions'
+      end
+
+      option :backup_path,
+        :long => '--backup-path path/to/backup',
+        :description => 'The path to the knife-ec-backup backup directory'
+
+      option :gsub_file,
+        :long => '--gsub-file path/to/gsub/file',
+        :description => 'The path to the file used for substitutions'
+
+      def run
+        unless config[:backup_path]
+          ui.error 'Must specify --backup-path'
+          exit 1
+        end
+
+        global_users.each do |user|
+          validate_user(user)
+        end
+
+        if config[:gsub_file]
+          if config[:gen_gsub_template]
+            Chef::TidySubstitutions.new(substitutions_file).boiler_plate
+          else
+            Chef::TidySubstitutions.new(substitutions_file, backup_path_expanded).run_substitutions
+          end
+        end
+
+        orgs.each do |org|
+          fix_self_dependencies(org)
+          load_cookbooks(org)
+          generate_new_metadata(org)
+        end
+      end
+
+      def load_cookbooks(org)
+        cl = Chef::CookbookLoader.new(cookbooks_path_expanded(org))
+        for_each_cookbook_basename(org) do |cookbook|
+          ui.info "Loading #{cookbook}"
+          ret = cl.load_cookbook(cookbook)
+          if ret.nil?
+            ui.error "Something's wrong with the #{cookbook} cookbook - cannot load it!"
+          end
+        end
+      rescue LoadError => e
+        ui.error e
+        ui.error 'Look at the cookbook above and determine what in the metadata.rb is causing the exception and rectify manually'
+        exit 1
+      end
+
+      def generate_new_metadata(org)
+        for_each_cookbook_path(org) do |cookbook_path|
+          generate_metadata_from_file(cookbook_name_from_path(cookbook_path), cookbook_path)
+        end
+      end
+
+      def fix_self_dependencies(org)
+        for_each_cookbook_path(org) do |cookbook_path|
+          name = cookbook_name_from_path(cookbook_path)
+          md_path = ::File.join(cookbook_path, 'metadata.rb')
+          unless ::File.exist?(md_path)
+            ui.warn "No metadata.rb in #{cookbook_path} - skipping"
+            next
+          end
+          Chef::TidySubstitutions.new.sub_in_file(
+            ::File.join(cookbook_path, 'metadata.rb'),
+            Regexp.new("^depends +['\"]#{name}['\"]"),
+            "# depends '#{name}' # knife-tidy was here")
+        end
+      end
+
+      def generate_metadata_from_file(cookbook, path)
+        md_path = ::File.join(path, 'metadata.rb')
+        unless ::File.exist?(md_path)
+          ui.warn "No metadata.rb in #{path} - skipping"
+          return
+        end
+        ui.info "   Generating new metadata.json for #{path}"
+        md = Chef::Cookbook::Metadata.new
+        md.name(cookbook)
+        md.from_file(md_path)
+        json_file = ::File.join(path, 'metadata.json')
+        ::File.open(json_file, 'w') do |f|
+          f.write(Chef::JSONCompat.to_json_pretty(md))
+        end
+      rescue Exceptions::ObsoleteDependencySyntax, Exceptions::InvalidVersionConstraint => e
+        ui.stderr.puts "ERROR: The cookbook '#{cookbook}' contains invalid or obsolete metadata syntax."
+        ui.stderr.puts "in #{file}:"
+        ui.stderr.puts
+        ui.stderr.puts e.message
+        exit 1
+      end
+
+      def validate_user(user)
+        ui.info "Validating user #{user}"
+      end
+    end
+  end
+end

data/lib/chef/knife/tidy_base.rb

@@ -0,0 +1,97 @@
+# Author:: Jeremy Miller (<jmiller@chef.io>)
+# Copyright:: Copyright (c) 2017 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+require 'chef/server_api'
+
+class Chef
+  class Knife
+    module TidyBase
+
+      def self.included(includer)
+        includer.class_eval do
+
+          deps do
+            require 'chef/tidy_server'
+          end
+
+          option :org_list,
+            :long => "--orgs ORG1,ORG2",
+            :description => "Only apply to objects in the named organizations"
+          end
+      end
+
+      def server
+        @server ||= if Chef::Config.chef_server_root.nil?
+                      ui.warn("chef_server_root not found in knife configuration; using chef_server_url")
+                      Chef::TidyServer.from_chef_server_url(Chef::Config.chef_server_url)
+                    else
+                      Chef::TidyServer.new(Chef::Config.chef_server_root)
+                    end
+      end
+
+      def rest
+        @rest ||= Chef::ServerAPI.new(server.root_url, {:api_version => "0"})
+      end
+
+      def backup_path_expanded
+        ::File.expand_path(config[:backup_path])
+      end
+
+      def cookbook_name_from_path(path)
+        ::File.basename(path, '-*')
+      end
+
+      def cookbooks_path_expanded(org)
+        ::File.expand_path(::File.join(backup_path_expanded, 'organizations', org, 'cookbooks'))
+      end
+
+      def substitutions_file
+        ::File.expand_path(config[:gsub_file])
+      end
+
+      def global_users
+        Dir[::File.join(backup_path_expanded, 'users', '*')].map { |dir| ::File.basename(dir, '.json') }
+      end
+
+      def orgs
+        if config[:org_list]
+          config[:org_list].split(',')
+        else
+          Dir[::File.join(backup_path_expanded, 'organizations', '*')].map { |dir| ::File.basename(dir) }
+        end
+      end
+
+      def for_each_cookbook_basename(org)
+        cookbooks_seen = []
+        Dir[::File.join(cookbooks_path_expanded(org), '**-**')].each do |cookbook|
+          name = cookbook_name_from_path(cookbook)
+          unless cookbooks_seen.include?(name)
+            cookbooks_seen.push(name)
+            yield name
+          end
+        end
+      end
+
+      def for_each_cookbook_path(org)
+        Dir[::File.join(cookbooks_path_expanded(org), '**')].each do |cookbook|
+          yield cookbook
+        end
+      end
+    end
+  end
+end

data/lib/chef/knife/tidy_server_report.rb

@@ -0,0 +1,132 @@
+require 'chef/knife/tidy_base'
+
+class Chef
+  class Knife
+    class TidyServerReport < Knife
+
+      include Knife::TidyBase
+
+      deps do
+        require 'ffi_yajl'
+      end
+
+      banner "knife tidy server report (OPTIONS)"
+
+      option :node_threshold,
+        :long => '--node-threshold NUM_DAYS',
+        :default => 30,
+        :description => 'Maximum number of days since last checkin before node is considered stale (default: 30)'
+
+      def run
+        ensure_reports_dir!
+
+        ui.warn "Writing to #{reports_dir} directory"
+
+        orgs = if config[:org_list]
+                 config[:org_list].split(',')
+               else
+                 all_orgs
+               end
+
+        stale_orgs = []
+        node_threshold = config[:node_threshold]
+
+        orgs.each do |org|
+          ui.info "  Organization: #{org}"
+          cb_list = cookbook_list(org)
+          version_count = cookbook_count(cb_list).sort_by(&:last).reverse.to_h
+          used_cookbooks = {}
+          nodes = nodes_list(org)[0]
+
+          nodes.select{|node| !node['cookbooks'].nil?}.each do |node|
+            node['cookbooks'].each do |name, version_hash|
+              version = Gem::Version.new(version_hash['version']).to_s
+              if used_cookbooks[name] && !used_cookbooks[name].include?(version)
+                used_cookbooks[name].push(version)
+              else
+                used_cookbooks[name] = [version]
+              end
+            end
+          end
+
+          stale_nodes = []
+          nodes.each do |n|
+            if (Time.now.to_i - n['ohai_time'].to_i) >= node_threshold * 86400
+              stale_nodes.push(n['name'])
+            end
+          end
+
+          stale_nodes_hash = {'threshold_days': node_threshold, 'count': stale_nodes.count, 'list': stale_nodes}
+          stale_orgs.push(org) if stale_nodes.count == nodes.count
+
+          report("#{org}_unused_cookbooks.json", unused_cookbooks(used_cookbooks, cb_list))
+          report("#{org}_unused_cookbooks.json", unused_cookbooks(used_cookbooks, cb_list))
+          report("#{org}_cookbook_count.json", version_count)
+          report("#{org}_#{node_threshold}d_stale_nodes.json", stale_nodes_hash)
+        end
+      end
+
+      def report(file_name, content)
+        ::File.write(::File.join(reports_dir, file_name), FFI_Yajl::Encoder.encode(content, pretty: true))
+      end
+
+      def reports_dir
+        ::File.join(Dir.pwd, 'reports')
+      end
+
+      def ensure_reports_dir!
+        Dir.mkdir(reports_dir) unless Dir.exist?(reports_dir)
+      end
+
+      def nodes_list(org)
+        Chef::Search::Query.new("#{server.root_url}/organizations/#{org}").search(
+          :node, '*:*',
+          :filter_result => {
+            'name' => ['name'],
+            'cookbooks' => ['cookbooks'],
+            'ohai_time' => ['ohai_time']
+          }
+        )
+      end
+
+      def cookbook_list(org)
+        cb_list = {}
+        rest.get("/organizations/#{org}/cookbooks?num_versions=all").each do |name, data|
+          data['versions'].each do |version_hash|
+            version = Gem::Version.new(version_hash['version']).to_s
+            if cb_list[name] && !cb_list[name].include?(version)
+              cb_list[name].push(version)
+            else
+              cb_list[name] = [version]
+            end
+          end
+        end
+        cb_list
+      end
+
+      def cookbook_count(cb_list)
+        cb_count_list = {}
+        cb_list.each do |name, versions|
+          cb_count_list[name] = versions.count
+        end
+        cb_count_list
+      end
+
+      def unused_cookbooks(used_list, cb_list)
+        unused_list = {}
+        cb_list.each do |name, versions|
+          if used_list[name].nil? # Not in the used list at all (Remove all versions)
+            unused_list[name] = versions
+          elsif used_list[name].sort != versions  # Is in the used cookbook list, but version arrays do not match (Find unused versions)
+            unused_list[name] = versions - used_list[name]
+          end
+        end
+        unused_list
+      end
+
+      def all_orgs
+        rest.get('organizations').keys
+      end
+    end
+  end
+end

data/lib/chef/tidy_server.rb

@@ -0,0 +1,14 @@
+class Chef
+  class TidyServer
+    attr_accessor :root_url
+
+    def initialize(root_url)
+      @root_url = root_url
+    end
+
+    def self.from_chef_server_url(url)
+      url = url.gsub(/\/organizations\/+[^\/]+\/*$/, '')
+      Chef::Server.new(url)
+    end
+  end
+end

data/lib/chef/tidy_substitutions.rb

@@ -0,0 +1,81 @@
+require 'ffi_yajl'
+require 'tempfile'
+require 'fileutils'
+require 'chef/log'
+
+class Chef
+  class TidySubstitutions
+    class NoSubstitutionFile < RuntimeError; end
+
+    attr_accessor :file_path, :backup_path, :data
+
+    def initialize(file_path = nil, backup_path = nil)
+      @file_path = file_path
+      @backup_path = backup_path
+    end
+
+    # Load the substitutions from disk
+    #
+    # @return [Hash]
+    #
+    def load_data
+      Chef::Log.info "Loading substitutions from #{file_path}"
+      @data = FFI_Yajl::Parser.parse(::File.read(@file_path), symbolize_names: false)
+    rescue Errno::ENOENT
+      raise NoSubstitutionFile, file_path
+    end
+
+    def boiler_plate
+
+    end
+
+    def cookbook_version_from_path(path)
+      components = path.split(File::SEPARATOR)
+      name_version = components[components.index('cookbooks')+1]
+      name_version.match(/\d+\.\d+\.\d+/).to_s
+    end
+
+    def revert
+
+    end
+
+    def sub_in_file(path, search, replace)
+      temp_file = Tempfile.new('tidy')
+      begin
+        File.open(path, 'r') do |file|
+          file.each_line do |line|
+            if line.match(search)
+              temp_file.puts replace
+              Chef::Log.info "  ++ #{path}"
+            else
+              temp_file.puts line
+            end
+          end
+        end
+        temp_file.close
+        FileUtils.mv(path, "#{path}.orig") unless ::File.exist?("#{path}.orig")
+        FileUtils.mv(temp_file.path, path)
+      ensure
+        temp_file.close
+        temp_file.unlink
+      end
+    end
+
+    def run_substitutions
+      load_data
+      @data.keys.each do |entry|
+        @data[entry].keys.each do |glob|
+          Chef::Log.info "Running substitutions for #{entry} -> #{glob}"
+          Dir[::File.join(backup_path, glob)].each do |file|
+            @data[entry][glob].each do |substitution|
+              search = Regexp.new(substitution['pattern'])
+              replace = substitution['replace'].dup
+              replace.gsub!(/\!COOKBOOK_VERSION\!/) { |m| "'" + cookbook_version_from_path(file) + "'" }
+              sub_in_file(file, search, replace)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/knife-tidy/version.rb

@@ -0,0 +1,4 @@
+module KnifeTidy
+  VERSION = '0.1.0'
+  MAJOR, MINOR, TINY = VERSION.split('.')
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: knife-tidy
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jeremy Miller
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Report on stale Chef Server nodes and cookbooks and clean up data integrity
+  issues in a knife-ec-backup object based backup
+email:
+- jmiller@chef.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- conf/substitutions.json.example
+- knife-tidy.gemspec
+- lib/chef/knife/tidy_backup_clean.rb
+- lib/chef/knife/tidy_base.rb
+- lib/chef/knife/tidy_server_report.rb
+- lib/chef/tidy_server.rb
+- lib/chef/tidy_substitutions.rb
+- lib/knife-tidy/version.rb
+homepage: https://github.com/jeremymv2/knife-tidy
+licenses:
+- Apache License, v2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Report on stale Chef Server nodes and cookbooks and clean up data integrity
+  issues in a knife-ec-backup object based backup
+test_files: []