knife-tidy 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bc910eba99a7cccb3e6ca423da6c116f1fd09af
-  data.tar.gz: fc67a3f6dd6f653e28336f6907e409b1dc6934c3
+  metadata.gz: 15b086bc7f2ebb5d3fe13ff649521b1e01d13df9
+  data.tar.gz: 1f32091c52ff24644a51df7f51474861446e5590
 SHA512:
-  metadata.gz: 133361ea06a8bc562de3d149e3b0e523627ab3bbdccbdac8d5a8e4205d0acaadfa249395403f9d2659de82b7cb34024e29affbc3d17cf63a523dc7a69027d560
-  data.tar.gz: a80bd192038fb73f4b375098f0958cdbd537c5d4321d0dc8f14251446788429145709016080fcb83edcc7ae2ed6363625cd975bbbe3c46e662d7094d6f172db9
+  metadata.gz: 0c88961a22947d751cab5e39bf8305076b2f89b53d6035d1f13bd5041ebe89b937fd4ac07a8a49488ee719cba91a642b5593a4ec2472fd9224ae1c6c0b2af01b
+  data.tar.gz: 52027a5cc0720febbee5c141e8db6b94245c8f5c07f73dca50a98adbedbea6f9d96bbb369ea205c197c6243b22d778df264efff304d016f2786770358b729fd0

data/.travis.yml

@@ -0,0 +1,27 @@
+language: ruby
+cache: bundler
+dist: trusty
+sudo: false
+
+# Early warning system to catch if Rubygems breaks something
+before_install:
+  - gem update --system
+  - gem uninstall bundler -a -x
+  - gem install bundler
+  - rm -f .bundle/config
+
+rvm:
+  - 2.2.7
+  - 2.3.4
+  - 2.4.1
+  - ruby-head
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+
+script: bundle exec rake spec
+
+branches:
+  only:
+  - master

data/CHANGELOG.md

@@ -1,22 +1,28 @@
 # Change Log
 
-## [0.1.0](https://github.com/jeremymv2/knife-tidy/tree/0.1.0) (2017-08-15)
-[Full Changelog](https://github.com/jeremymv2/knife-tidy/compare/0.2.0...0.1.0)
+
+## [0.2.1](https://github.com/chef-customers/knife-tidy/tree/0.2.1) (2017-09-01)
+[Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.2.0...0.2.1)
 
 **Merged pull requests:**
 
-- bump version to 0.2.0 [\#6](https://github.com/jeremymv2/knife-tidy/pull/6) ([jeremymv2](https://github.com/jeremymv2))
+- first round of tests [\#10](https://github.com/chef-customers/knife-tidy/pull/10) ([jeremymv2](https://github.com/jeremymv2))
+- Add environment checking for unused cookbook list [\#9](https://github.com/chef-customers/knife-tidy/pull/9) ([nsdavidson](https://github.com/nsdavidson))
+- disable [\#8](https://github.com/chef-customers/knife-tidy/pull/8) ([jeremymv2](https://github.com/jeremymv2))
+- server object deletion [\#7](https://github.com/chef-customers/knife-tidy/pull/7) ([jeremymv2](https://github.com/jeremymv2))
+- bump version to 0.2.0 [\#6](https://github.com/chef-customers/knife-tidy/pull/6) ([jeremymv2](https://github.com/jeremymv2))
 
-## [0.2.0](https://github.com/jeremymv2/knife-tidy/tree/0.2.0) (2017-08-16)
-[Full Changelog](https://github.com/jeremymv2/knife-tidy/compare/0.1.1...0.2.0)
+## [0.2.0](https://github.com/chef-customers/knife-tidy/tree/0.2.0) (2017-08-16)
+[Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.1.1...0.2.0)
 
 **Merged pull requests:**
 
-- moved all common functions to tidy\_common.rb [\#5](https://github.com/jeremymv2/knife-tidy/pull/5) ([jeremymv2](https://github.com/jeremymv2))
-- Jeremymv2/acl items [\#4](https://github.com/jeremymv2/knife-tidy/pull/4) ([jeremymv2](https://github.com/jeremymv2))
-- updated changelog [\#3](https://github.com/jeremymv2/knife-tidy/pull/3) ([jeremymv2](https://github.com/jeremymv2))
-- bump version to 0.1.1 [\#2](https://github.com/jeremymv2/knife-tidy/pull/2) ([jeremymv2](https://github.com/jeremymv2))
-
+- moved all common functions to tidy\_common.rb [\#5](https://github.com/chef-customers/knife-tidy/pull/5) ([jeremymv2](https://github.com/jeremymv2))
+- Jeremymv2/acl items [\#4](https://github.com/chef-customers/knife-tidy/pull/4) ([jeremymv2](https://github.com/jeremymv2))
+- updated changelog [\#3](https://github.com/chef-customers/knife-tidy/pull/3) ([jeremymv2](https://github.com/jeremymv2))
+- bump version to 0.1.1 [\#2](https://github.com/chef-customers/knife-tidy/pull/2) ([jeremymv2](https://github.com/jeremymv2))
 
+## [0.1.0](https://github.com/chef-customers/knife-tidy/tree/0.1.0) (2017-09-01)
+[Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.2.1...0.1.0)
 
-\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/README.md

@@ -55,10 +55,10 @@ org_unused_cookbooks.json | List of cookbooks and versions that do not appear to
 
 ## Options
 
-  * `--repo-path /path/to/chef-repo`:
+  * `--backup-path /path/to/an-ec-backup`:
     The Chef Repo to tidy up (such as one created from a [knife-ec-backup](https://github.com/chef/knife-ec-backup)
 
-  * `--gsub-file path/to/gsub/file`:
+  * `--gsub-file /path/to/gsub/file`:
     The path to the file used for substitutions. If non-existant, a boiler plate one will be created.
 
 ## Notes
@@ -78,7 +78,7 @@ org_unused_cookbooks.json | List of cookbooks and versions that do not appear to
       },
       {
         "pattern":"version *Chef::Sugar::VERSION",
-        "replace":"# version          !COOKBOOK_VERSION!"
+        "replace":"version          !COOKBOOK_VERSION!"
       }
     ]
   }

data/TODO.md

@@ -0,0 +1,5 @@
+## ec-backup
+ - add Metadata with last backup timestamp
+
+## tidy
+ - disallow server clean operation unless recent backup exists

data/knife-tidy.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.email            = ["jmiller@chef.io"]
   s.summary          = "Report on stale Chef Server nodes and cookbooks and clean up data integrity issues in a knife-ec-backup object based backup"
   s.description      = s.summary
-  s.homepage         = "https://github.com/jeremymv2/knife-tidy"
+  s.homepage         = "https://github.com/chef-customers/knife-tidy"
   s.license          = "Apache License, v2.0"
   s.files            = `git ls-files`.split("\n")
   s.require_paths    = ["lib"]

data/lib/chef/knife/tidy_backup_clean.rb

@@ -9,10 +9,11 @@ class Chef
         require 'chef/cookbook/metadata'
         require 'chef/tidy_substitutions'
         require 'chef/tidy_acls'
-        require 'chef/tidy_common'
         require 'ffi_yajl'
       end
 
+      banner "knife tidy backup clean (OPTIONS)"
+
       include Knife::TidyBase
 
       option :backup_path,
@@ -48,10 +49,8 @@ class Chef
           load_cookbooks(org)
           generate_new_metadata(org)
         end
-      end
 
-      def tidy
-        @tidy ||= Chef::TidyCommon.new(config[:backup_path])
+        completion_message
       end
 
       def validate_user_emails

data/lib/chef/knife/tidy_base.rb

@@ -27,6 +27,7 @@ class Chef
 
           deps do
             require 'chef/tidy_server'
+            require 'chef/tidy_common'
           end
 
           option :org_list,
@@ -45,7 +46,19 @@ class Chef
       end
 
       def rest
-        @rest ||= Chef::ServerAPI.new(server.root_url, {:api_version => "0"})
+        @rest ||= Chef::ServerAPI.new(server.root_url, keepalives: true)
+      end
+
+      def tidy
+        @tidy ||= if config[:backup_path].nil?
+                    Chef::TidyCommon.new
+                  else
+                    Chef::TidyCommon.new(config[:backup_path])
+                  end
+      end
+
+      def completion_message
+        puts "#{ui.color("** Finished **", :magenta)}"
       end
     end
   end

data/lib/chef/knife/tidy_server_clean.rb

@@ -0,0 +1,138 @@
+# not enabled
+return
+
+require 'chef/knife/tidy_base'
+
+class Chef
+  class Knife
+    class TidyServerClean < Knife
+
+      include Knife::TidyBase
+
+      deps do
+        require 'ffi_yajl'
+        require 'chef/util/threaded_job_queue'
+      end
+
+      banner "knife tidy server clean (options)"
+
+      option :concurrency,
+        :long => '--concurrency THREADS',
+        :default => 10,
+        :description => 'Maximum number of simultaneous requests to send (default: 10)'
+
+      option :only_cookbooks,
+        :long => '--only-cookbooks',
+        :description => 'Only delete unused cookbooks from Chef Server.'
+
+      option :only_nodes,
+        :long => '--only-nodes',
+        :description => 'Only delete stale nodes from Chef Server.'
+
+      def run
+        STDOUT.sync = true
+
+        ensure_reports_dir
+        ui.info "Reading from #{tidy.reports_dir} directory"
+
+        ui.info "Using thread concurrency #{config[:concurrency]}"
+        configure_chef
+
+        if config[:only_cookbooks] && config[:only_nodes]
+          ui.error 'Cannot use --only-cookbooks AND --only-nodes'
+          exit 1
+        end
+
+        ui.confirm('This operation will delete items on Chef Server, continue') unless config[:unattended]
+
+        orgs = if config[:org_list]
+                 config[:org_list].split(',')
+               else
+                 all_orgs
+               end
+
+        orgs.each do |org|
+          clean_cookbooks(org) unless config[:only_nodes]
+          clean_nodes(org) unless config[:only_cookbooks]
+        end
+
+        completion_message
+      end
+
+      def clean_cookbooks(org)
+        queue = Chef::Util::ThreadedJobQueue.new
+        unused_cookbooks_file = ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json")
+        return unless ::File.exist?(unused_cookbooks_file)
+        ui.info "Cleaning cookbooks for Org: #{org}, using #{unused_cookbooks_file}"
+        unused_cookbooks = FFI_Yajl::Parser.parse(::File.read(unused_cookbooks_file), symbolize_names: true)
+        unused_cookbooks.keys.each do |cookbook|
+          versions = unused_cookbooks[cookbook]
+          versions.each do |version|
+            queue << lambda { delete_cookbook_job(org, cookbook, version) }
+          end
+        end
+        queue.process(config[:concurrency].to_i)
+      end
+
+      def delete_cookbook_job(org, cookbook, version)
+        path = "/organizations/#{org}/cookbooks/#{cookbook}/#{version}"
+        rest.delete(path)
+        response = '200'
+      rescue Net::HTTPServerException
+        response = $!.response.code
+      ensure
+        formatted = response == '200' ?
+          ui.color(' Deleting  %-20s %-10s %10s', :green) :
+          ui.color(' Deleting  %-20s %-10s %10s', :red)
+        printf("#{formatted}\n", cookbook, version, response)
+      end
+
+      def clean_nodes(org)
+        queue = Chef::Util::ThreadedJobQueue.new
+        stale_nodes_file = ::File.join(tidy.reports_dir, "#{org}_stale_nodes.json")
+        return unless ::File.exist?(stale_nodes_file)
+        ui.info "Cleaning stale nodes for Org: #{org}, using #{stale_nodes_file}"
+        stale_nodes = FFI_Yajl::Parser.parse(::File.read(stale_nodes_file), symbolize_names: true)
+        stale_nodes[:list].each do |node|
+          queue << lambda { delete_node_job(org, node) }
+        end
+        queue.process(config[:concurrency].to_i)
+      end
+
+      def delete_node_job(org, node)
+        path = "/organizations/#{org}/nodes/#{node}"
+        rest.delete(path)
+        response = '200'
+      rescue Net::HTTPServerException
+        response = $!.response.code
+      ensure
+        formatted = response == '200' ?
+          ui.color(' Deleting  %-20s %10s', :green) :
+          ui.color(' Deleting  %-20s %10s', :red)
+        printf("#{formatted}\n", node, response)
+      end
+
+      def ensure_reports_dir
+        unless ::File.directory?(tidy.reports_dir)
+          ui.error "#{tidy.reports_dir} not found!"
+          exit 1
+        end
+      end
+
+      def report_files
+        Dir[::File.join(tidy.reports_dir, '**')]
+      end
+
+      def all_orgs
+        orgs = []
+        report_files.each do |file|
+          org = ::File.basename(file).match(/^(.*?)_/).captures[0]
+          if org
+            orgs.push(org) unless orgs.include?(org)
+          end
+        end
+        orgs
+      end
+    end
+  end
+end

data/lib/chef/knife/tidy_server_report.rb

@@ -10,7 +10,7 @@ class Chef
         require 'ffi_yajl'
       end
 
-      banner "knife tidy server report (OPTIONS)"
+      banner "knife tidy server report (options)"
 
       option :node_threshold,
         :long => '--node-threshold NUM_DAYS',
@@ -20,7 +20,8 @@ class Chef
       def run
         ensure_reports_dir!
 
-        ui.warn "Writing to #{reports_dir} directory"
+        ui.warn "Writing to #{tidy.reports_dir} directory"
+        delete_existing_reports
 
         orgs = if config[:org_list]
                  config[:org_list].split(',')
@@ -49,6 +50,11 @@ class Chef
             end
           end
 
+          Chef::Log.debug("Used cookbook list before checking environments: #{used_cookbooks}")
+          pins = environment_constraints(org)
+          used_cookbooks = check_environment_pins(used_cookbooks, pins, cb_list)
+          Chef::Log.debug("Used cookbook list after checking environments: #{used_cookbooks}")
+
           stale_nodes = []
           nodes.each do |n|
             if (Time.now.to_i - n['ohai_time'].to_i) >= node_threshold * 86400
@@ -59,23 +65,25 @@ class Chef
           stale_nodes_hash = {'threshold_days': node_threshold, 'count': stale_nodes.count, 'list': stale_nodes}
           stale_orgs.push(org) if stale_nodes.count == nodes.count
 
-          report("#{org}_unused_cookbooks.json", unused_cookbooks(used_cookbooks, cb_list))
-          report("#{org}_unused_cookbooks.json", unused_cookbooks(used_cookbooks, cb_list))
-          report("#{org}_cookbook_count.json", version_count)
-          report("#{org}_#{node_threshold}d_stale_nodes.json", stale_nodes_hash)
+          tidy.write_new_file(unused_cookbooks(used_cookbooks, cb_list), ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json"))
+          tidy.write_new_file(unused_cookbooks(used_cookbooks, cb_list), ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json"))
+          tidy.write_new_file(version_count, ::File.join(tidy.reports_dir, "#{org}_cookbook_count.json"))
+          tidy.write_new_file(stale_nodes_hash, ::File.join(tidy.reports_dir, "#{org}_stale_nodes.json"))
         end
-      end
 
-      def report(file_name, content)
-        ::File.write(::File.join(reports_dir, file_name), FFI_Yajl::Encoder.encode(content, pretty: true))
+        completion_message
       end
 
-      def reports_dir
-        ::File.join(Dir.pwd, 'reports')
+      def ensure_reports_dir!
+        Dir.mkdir(tidy.reports_dir) unless Dir.exist?(tidy.reports_dir)
       end
 
-      def ensure_reports_dir!
-        Dir.mkdir(reports_dir) unless Dir.exist?(reports_dir)
+      def delete_existing_reports
+        files = Dir[::File.join(tidy.reports_dir, '*.json')]
+        unless files.empty?
+          ui.confirm("You have existing reports in #{tidy.reports_dir}. Remove")
+          FileUtils.rm(files, :force => true)
+        end
       end
 
       def nodes_list(org)
@@ -127,6 +135,62 @@ class Chef
       def all_orgs
         rest.get('organizations').keys
       end
+
+      def all_environments(org)
+        rest.get("/organizations/#{org}/environments").values
+      end
+
+      def environment_constraints(org)
+        constraints = {}
+        all_environments(org).each do |env|
+          e = rest.get(env)
+          e['cookbook_versions'].each do |cb, version|
+            if constraints[cb]
+              constraints[cb].push(version) unless constraints[cb].include?(version)
+            else
+              constraints[cb] = [version]
+            end
+          end
+        end
+        constraints
+      end
+
+      def check_cookbook_list(cb_list, cb, version)
+        cb_list[cb].each do |v|
+          if Gem::Dependency.new('', version).match?('', v)
+            Chef::Log.debug("Pin of #{cb} can be satisfied by #{v}, adding to used list")
+            return [v]
+          else
+            Chef::Log.debug("Pin of #{cb} version #{version} not satisfied by #{v}")
+          end
+        end
+        return nil
+      end
+
+      def check_environment_pins(used_cookbooks, pins, cb_list)
+        pins.each do |cb, versions|
+          versions.each do |version|
+            if used_cookbooks[cb]
+              # This pinned cookbook is in the used list, now check for a matching version.
+              used_cookbooks[cb].each do |v|
+                if Gem::Dependency.new('', version).match?('', v)
+                  # This version in used_cookbooks satisfies the pin
+                  Chef::Log.debug("Pin of #{cb}: #{version} is satisfied by #{v}")
+                  break
+                end
+              end
+              result = check_cookbook_list(cb_list, cb, version)
+              used_cookbooks[cb].push(result[0]) if result
+            else
+              # No cookbook version for that pin, look through the full cookbook list for a match
+              Chef::Log.debug("No used cookbook #{cb}, checking the full cookbook list")
+              result = check_cookbook_list(cb_list, cb, version)
+              used_cookbooks[cb] = result if result
+            end
+          end
+        end
+        used_cookbooks
+      end
     end
   end
 end

data/lib/chef/tidy_common.rb

@@ -5,7 +5,7 @@ class Chef
   class TidyCommon
     attr_accessor :backup_path
 
-    def initialize(backup_path)
+    def initialize(backup_path = Dir.pwd)
       @backup_path = ::File.expand_path(backup_path)
     end
 
@@ -53,7 +53,9 @@ class Chef
     end
 
     def write_new_file(contents, path)
-      FileUtils.cp(path, "#{path}.orig") unless ::File.exist?("#{path}.orig")
+      if ::File.exist?(path)
+        FileUtils.cp(path, "#{path}.orig") unless ::File.exist?("#{path}.orig")
+      end
       ::File.open(path, 'w+') do |f|
          f.write(FFI_Yajl::Encoder.encode(contents, pretty: true))
       end
@@ -66,5 +68,9 @@ class Chef
     def global_user_names
       @global_user_names ||= Dir[::File.join(@backup_path, 'users', '*')].map { |dir| ::File.basename(dir, '.json') }
     end
+
+    def reports_dir
+      @reports_dir ||= ::File.join(Dir.pwd, 'reports')
+    end
   end
 end

data/lib/knife-tidy/version.rb

@@ -1,4 +1,4 @@
 module KnifeTidy
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
   MAJOR, MINOR, TINY = VERSION.split('.')
 end

data/spec/chef/knife/tidy_backup_clean_spec.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "spec_helper"))
+require 'chef/knife/tidy_backup_clean'
+require 'chef/knife'
+require 'chef/config'
+require 'stringio'
+
+class Tester < Chef::Knife
+  include Chef::Knife::TidyBackupClean
+end
+
+describe Chef::Knife::TidyBackupClean do
+  let(:t) { Tester.new }
+  before(:each) do
+    @rest = double('rest')
+    @stderr = StringIO.new
+    allow(t.ui).to receive(:stderr).and_return(@stderr)
+    allow(Chef::ServerAPI).to receive(:new).and_return(@rest)
+  end
+
+  context "completion_message" do
+    it "lets the user know we're Finished" do
+      expect{t.completion_message}.to output("** Finished **\n").to_stdout
+    end
+  end
+end

data/spec/chef/knife/tidy_base_spec.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "spec_helper"))
+require 'chef/knife/tidy_base'
+require 'chef/knife'
+require 'chef/config'
+require 'stringio'
+
+class Tester < Chef::Knife
+  include Chef::Knife::TidyBase
+end
+
+describe Chef::Knife::TidyBase do
+  let(:t) { Tester.new }
+  before(:each) do
+    @rest = double('rest')
+    @stderr = StringIO.new
+    allow(t.ui).to receive(:stderr).and_return(@stderr)
+    allow(Chef::ServerAPI).to receive(:new).and_return(@rest)
+  end
+
+  context "completion_message" do
+    it "lets the user know we're Finished" do
+      expect{t.completion_message}.to output("** Finished **\n").to_stdout
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+$:.unshift File.expand_path('../../lib', __FILE__)
+require 'chef'
+require 'chef/knife/tidy_base'
+require 'chef/knife/tidy_backup_clean'
+require 'chef/knife/tidy_server_report'
+require 'chef/knife/tidy_server_clean'
+
+# Clear config between each example
+# to avoid dependencies between examples
+RSpec.configure do |c|
+  c.raise_errors_for_deprecations!
+  c.filter_run_excluding :exclude => true
+  c.before(:each) do
+    Chef::Config.reset
+    Chef::Config[:knife] ={}
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-tidy
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Jeremy Miller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -103,22 +103,28 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
+- TODO.md
 - conf/substitutions.json.example
 - knife-tidy.gemspec
 - lib/chef/knife/tidy_backup_clean.rb
 - lib/chef/knife/tidy_base.rb
+- lib/chef/knife/tidy_server_clean.rb
 - lib/chef/knife/tidy_server_report.rb
 - lib/chef/tidy_acls.rb
 - lib/chef/tidy_common.rb
 - lib/chef/tidy_server.rb
 - lib/chef/tidy_substitutions.rb
 - lib/knife-tidy/version.rb
-homepage: https://github.com/jeremymv2/knife-tidy
+- spec/chef/knife/tidy_backup_clean_spec.rb
+- spec/chef/knife/tidy_base_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/chef-customers/knife-tidy
 licenses:
 - Apache License, v2.0
 metadata: {}