knife-tidy 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fdd8bded909178ab1e6055a4415dc0c675ee0441
-  data.tar.gz: 52b73bfc270082947f232fd333ecdd900bac2ac7
+  metadata.gz: e9766f389dc78f0c5c764b04ad056d0c74278751
+  data.tar.gz: 6df96008461814fa24005126d3df61a117288cd3
 SHA512:
-  metadata.gz: e843f9db36f6baf438c500ff2fd52b8e7e7151f06391597c3809dd52b4bf89124ba7a9cd2579d090c1eb9280b92145ff09a2738a799275cbb4bfb8de75b1865e
-  data.tar.gz: 7dcb30b9c176ccb3b087cbecdfdbe4ec03854c32643eb7de6b7592a24ae1d58a7b8009a3ce1ab0f38ddcab16114f9eae005ee6505958c4c4987a1f80133d5278
+  metadata.gz: 21bf46076b9e8dee58aea293820e13ea93d724ea240cb95c9b44e43942aab098a754aea3e31affbd9f481acacfc3eda39c9f9d1787210039aed5f20691ed00b5
+  data.tar.gz: ec2a71913df5a0e69949e3a0a6efe59814df404db9de70d29255a77a340430ca83836cdba78ad8d8927c8a183f2516ba027a54c07be1272681f5e33521e481a9

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Change Log
 
+## [0.5.0](https://github.com/chef-customers/knife-tidy/tree/0.5.0) (2017-10-06)
+[Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.4.1...0.5.0)
+
+**Closed issues:**
+
+- Supermarket.chef.io [\#37](https://github.com/chef-customers/knife-tidy/issues/37)
+
+**Merged pull requests:**
+
+- enabling deletion of stale nodes [\#44](https://github.com/chef-customers/knife-tidy/pull/44) ([jeremymv2](https://github.com/jeremymv2))
+- better warning message before confirmation [\#43](https://github.com/chef-customers/knife-tidy/pull/43) ([jeremymv2](https://github.com/jeremymv2))
+- Jeremymv2/chef sugar fix default [\#42](https://github.com/chef-customers/knife-tidy/pull/42) ([jeremymv2](https://github.com/jeremymv2))
+- deleting nodes also deletes client [\#41](https://github.com/chef-customers/knife-tidy/pull/41) ([jeremymv2](https://github.com/jeremymv2))
+- Change nodes\_list method to get all nodes [\#40](https://github.com/chef-customers/knife-tidy/pull/40) ([nsdavidson](https://github.com/nsdavidson))
+- bump version to 0.4.1 [\#39](https://github.com/chef-customers/knife-tidy/pull/39) ([jeremymv2](https://github.com/jeremymv2))
+
 ## [0.4.1](https://github.com/chef-customers/knife-tidy/tree/0.4.1) (2017-09-27)
 [Full Changelog](https://github.com/chef-customers/knife-tidy/compare/0.4.0...0.4.1)
 

data/README.md

@@ -2,9 +2,10 @@
 
 # Summary
 
-This Chef Knife plugin has two primary purposes:
- * Report on the state of Chef Server objects that can be tidied up (Future: clean up objects)
- * A companion tool to [knife-ec-backup](https://github.com/chef/knife-ec-backup) that will clean up data integrity issues in an object backup
+This Chef Knife plugin provides:
+ * Reports on the state of Chef Server objects that can be tidied up
+ * Removal of stale nodes (and associated clients and ACLs) identified by the above Reports
+ * A [knife-ec-backup](https://github.com/chef/knife-ec-backup) companion tool that will clean up data integrity issues in an object backup
 
 # Requirements
 
@@ -56,6 +57,21 @@ org_threshold_numdays_stale_nodes.json | Nodes in that org that have not checked
 org_cookbook_count.json | Number of cookbook versions for each cookbook that that org.
 org_unused_cookbooks.json | List of cookbooks and versions that do not appear to be in-use for that org. This is determined by checking the versioned run list of each of the nodes in the org.
 
+## $ knife tidy server clean --help
+Remove stale nodes that haven't checked-in to the Chef Server as defined by the `--node-threshold NUM_DAYS` option when the reports were generated.. The associated client and ACLs are also removed.
+
+Future: remove unused cookbooks - currently this feature is disabled.
+
+## Options
+
+  * `--dry-run`
+    Do not perform any actual deletion, only report on what would have been deleted.
+
+Example:
+```bash
+knife tidy server clean --orgs brewinc,acmeinc
+```
+
 ## $ knife tidy backup clean --help
 
 ## Options
@@ -80,18 +96,6 @@ knife tidy backup clean --backup-path backups/ --gsub-file substitutions.json
 
 ```json
 {
-  "chef-sugar":{
-    "organizations/*/cookbooks/chef-sugar*/metadata.rb":[
-      {
-        "pattern":"^require .*/lib/chef/sugar/version",
-        "replace":"# require          File.expand_path('../lib/chef/sugar/version', *__FILE__)"
-      },
-      {
-        "pattern":"version *Chef::Sugar::VERSION",
-        "replace":"version          !COOKBOOK_VERSION!"
-      }
-    ]
-  },
   "io-read-version-and-readme.md":{
     "organizations/*/cookbooks/*/metadata.rb":[
       {

data/conf/substitutions.json.example

@@ -1,25 +1,9 @@
 {
-  "chef-sugar":{
-    "organizations/*/cookbooks/chef-sugar*/metadata.rb":[
-      {
-        "pattern":"^require .*/lib/chef/sugar/version",
-        "replace":"# require          File.expand_path('../lib/chef/sugar/version', *__FILE__)"
-      },
-      {
-        "pattern":"version *Chef::Sugar::VERSION",
-        "replace":"version          !COOKBOOK_VERSION!"
-      }
-    ]
-  },
-  "io-read-version-and-readme.md":{
+  "your-problem-descriptor":{
     "organizations/*/cookbooks/*/metadata.rb":[
       {
-        "pattern":"^version +IO.read.* 'VERSION'.*",
+        "pattern":"^version .*GO_PIPELINE_LABEL",
         "replace":"version !COOKBOOK_VERSION!"
-      },
-      {
-        "pattern":"^long_description +IO.read.* 'README.md'.*",
-        "replace":"#long_description \"A Long Description..\""
       }
     ]
   }

data/lib/chef/knife/tidy_backup_clean.rb

@@ -45,6 +45,8 @@ class Chef
           exit 1
         end
 
+        fix_chef_sugar_metadata
+
         Chef::TidySubstitutions.new(substitutions_file, tidy).run_substitutions if config[:gsub_file]
 
         validate_user_emails
@@ -187,6 +189,27 @@ class Chef
         end
       end
 
+      def fix_chef_sugar_metadata
+        Dir[::File.join(tidy.backup_path, 'organizations/*/cookbooks/chef-sugar*/metadata.rb')].each do |file|
+          puts 'INFO: Searching for known chef-sugar problems when uploading.'
+          s = Chef::TidySubstitutions.new
+          version = s.cookbook_version_from_path(file)
+          patterns = [
+            {
+              search: '^require .*/lib/chef/sugar/version',
+              replace: "# require          File.expand_path('../lib/chef/sugar/version', *__FILE__)"
+            },
+            {
+              search: '^version *Chef::Sugar::VERSION',
+              replace: "version '#{version}'"
+            }
+          ]
+          patterns.each do |p|
+            s.sub_in_file(file, Regexp.new(p[:search]), p[:replace])
+          end
+        end
+      end
+
       def fix_self_dependencies(org)
         for_each_cookbook_path(org) do |cookbook_path|
           name = tidy.cookbook_name_from_path(cookbook_path)

data/lib/chef/knife/tidy_server_clean.rb

@@ -15,8 +15,8 @@ class Chef
 
       option :concurrency,
         :long => '--concurrency THREADS',
-        :default => 10,
-        :description => 'Maximum number of simultaneous requests to send (default: 10)'
+        :default => 1,
+        :description => 'Maximum number of simultaneous requests to send (default: 1)'
 
       option :only_cookbooks,
         :long => '--only-cookbooks',
@@ -24,23 +24,17 @@ class Chef
 
       option :only_nodes,
         :long => '--only-nodes',
-        :description => 'Only delete stale nodes from Chef Server.'
+        :description => 'Only delete stale nodes (and associated clients and ACLs) from Chef Server.'
 
       option :dry_run,
         :long => '--dry-run',
         :description => 'Do not perform any actual deletion, only report on what would have been deleted.'
 
       def run
-        # not enabled
-        ui.warn "This feature is not enabled"
-        exit
-
         STDOUT.sync = true
 
         ensure_reports_dir
-        puts "INFO: Reading from #{tidy.reports_dir} directory"
 
-        puts "INFO: Using thread concurrency #{config[:concurrency]}"
         configure_chef
 
         if config[:only_cookbooks] && config[:only_nodes]
@@ -48,7 +42,13 @@ class Chef
           exit 1
         end
 
-        ui.confirm('This operation will delete items on Chef Server, continue') unless config[:unattended]
+        deletions = if config[:only_cookbooks]
+                      "cookbooks"
+                    elsif config[:only_nodes]
+                      "nodes (and associated clients and ACLs)"
+                    else
+                      "cookbooks and nodes (and associated clients and ACLs)"
+                    end
 
         orgs = if config[:org_list]
                  config[:org_list].split(',')
@@ -56,6 +56,10 @@ class Chef
                  all_orgs
                end
 
+        ui.warn "This operation will affect the following Orgs on #{server.root_url}\n\n#{orgs}\n\n"
+
+        ui.confirm("About to delete #{deletions} from the Chef Server identified in the #{tidy.reports_dir} directory! Are you sure you wish to continue") unless config[:unattended]
+
         orgs.each do |org|
           clean_cookbooks(org) unless config[:only_nodes]
           clean_nodes(org) unless config[:only_cookbooks]
@@ -65,6 +69,8 @@ class Chef
       end
 
       def clean_cookbooks(org)
+        ui.warn "Cleaning cookbooks is a feature not yet enabled."
+        return
         queue = Chef::Util::ThreadedJobQueue.new
         unused_cookbooks_file = ::File.join(tidy.reports_dir, "#{org}_unused_cookbooks.json")
         return unless ::File.exist?(unused_cookbooks_file)
@@ -82,19 +88,12 @@ class Chef
       def delete_cookbook_job(org, cookbook, version)
         path = "/organizations/#{org}/cookbooks/#{cookbook}/#{version}"
         if config[:dry_run]
-          printf("INFO: Would have executed `rest.delete(#{path})`\n")
+          printf("DRYRUN: Would have executed `rest.delete(#{path})`\n")
           return
         end
+        printf("INFO: Deleting #{path}\n")
         rest.delete(path)
-        response = '200'
       rescue Net::HTTPServerException
-        response = $!.response.code
-      ensure
-        return if config[:dry_run]
-        formatted = response == '200' ?
-          ui.color(' Deleting  %-20s %-10s %10s', :green) :
-          ui.color(' Deleting  %-20s %-10s %10s', :red)
-        printf("#{formatted}\n", cookbook, version, response)
       end
 
       def clean_nodes(org)
@@ -110,21 +109,19 @@ class Chef
       end
 
       def delete_node_job(org, node)
-        path = "/organizations/#{org}/nodes/#{node}"
-        if config[:dry_run]
-          printf("INFO: Would have executed `rest.delete(#{path})`\n")
-          return
+        paths = ["/organizations/#{org}/nodes/#{node}", "/organizations/#{org}/clients/#{node}"]
+        paths.each do |path|
+          if config[:dry_run]
+            printf("DRYRUN: Would have executed `rest.delete(#{path})`\n")
+            next
+          else
+            begin
+              printf("INFO: Deleting #{path}\n")
+              rest.delete(path)
+            rescue Net::HTTPServerException
+            end
+          end
         end
-        rest.delete(path)
-        response = '200'
-      rescue Net::HTTPServerException
-        response = $!.response.code
-      ensure
-        return if config[:dry_run]
-        formatted = response == '200' ?
-          ui.color(' Deleting  %-20s %10s', :green) :
-          ui.color(' Deleting  %-20s %10s', :red)
-        printf("#{formatted}\n", node, response)
       end
 
       def ensure_reports_dir

data/lib/chef/knife/tidy_server_report.rb

@@ -37,7 +37,7 @@ class Chef
           cb_list = cookbook_list(org)
           version_count = cookbook_count(cb_list).sort_by(&:last).reverse.to_h
           used_cookbooks = {}
-          nodes = nodes_list(org)[0]
+          nodes = nodes_list(org)
 
           nodes.select{|node| !node['cookbooks'].nil?}.each do |node|
             node['cookbooks'].each do |name, version_hash|
@@ -86,7 +86,10 @@ class Chef
         end
       end
 
+      # Need the block here to get the search method to invoke multiple searches and 
+      # aggregate results for result sets over 1k.
       def nodes_list(org)
+        node_results = []
         Chef::Search::Query.new("#{server.root_url}/organizations/#{org}").search(
           :node, '*:*',
           :filter_result => {
@@ -94,7 +97,10 @@ class Chef
             'cookbooks' => ['cookbooks'],
             'ohai_time' => ['ohai_time']
           }
-        )
+        ) do |node|
+          node_results << node
+        end
+        node_results
       end
 
       def cookbook_list(org)

data/lib/knife-tidy/version.rb

@@ -1,4 +1,4 @@
 module KnifeTidy
-  VERSION = '0.4.1'
+  VERSION = '0.5.0'
   MAJOR, MINOR, TINY = VERSION.split('.')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: knife-tidy
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Jeremy Miller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-27 00:00:00.000000000 Z
+date: 2017-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake