knife-essentials 0.9.8 → 1.0.0.beta1

data/lib/chef_fs/file_system/chef_server_root_dir.rb

@@ -81,13 +81,7 @@ module ChefFS
             EnvironmentsDir.new(self),
             RestListDir.new("roles", self, nil, ChefFS::DataHandler::RoleDataHandler.new)
           ]
-          if repo_mode == 'everything'
-            result += [
-              RestListDir.new("clients", self, nil, ChefFS::DataHandler::ClientDataHandler.new),
-              NodesDir.new(self),
-              RestListDir.new("users", self, nil, ChefFS::DataHandler::UserDataHandler.new)
-            ]
-          elsif repo_mode == 'hosted_everything'
+          if repo_mode == 'hosted_everything'
             result += [
               AclsDir.new(self),
               RestListDir.new("clients", self, nil, ChefFS::DataHandler::ClientDataHandler.new),
@@ -95,6 +89,12 @@ module ChefFS
               RestListDir.new("groups", self, nil, ChefFS::DataHandler::GroupDataHandler.new),
               NodesDir.new(self)
             ]
+          elsif repo_mode != 'static'
+            result += [
+              RestListDir.new("clients", self, nil, ChefFS::DataHandler::ClientDataHandler.new),
+              NodesDir.new(self),
+              RestListDir.new("users", self, nil, ChefFS::DataHandler::UserDataHandler.new)
+            ]
           end
           result.sort_by { |child| child.name }
         end

data/lib/chef_fs/file_system/cookbook_dir.rb

@@ -150,7 +150,7 @@ module ChefFS
           return [ !exists?, nil, nil ]
         end
         are_same = true
-        ChefFS::CommandLine::diff_entries(self, other, nil, :name_only) do |type, old_entry, new_entry|
+        ChefFS::CommandLine::diff_entries(self, other, nil, :name_only).each do |type, old_entry, new_entry|
           if [ :directory_to_file, :file_to_directory, :deleted, :added, :modified ].include?(type)
             are_same = false
           end

data/lib/chef_fs/file_system/rest_list_entry.rb

@@ -105,6 +105,8 @@ module ChefFS
       end
 
       def compare_to(other)
+        # TODO this pair of reads can be parallelized
+
         # Grab the other value
         begin
           other_value_json = other.read

data/lib/chef_fs/knife.rb

@@ -20,6 +20,7 @@ require 'chef_fs/file_system/chef_server_root_dir'
 require 'chef_fs/file_system/chef_repository_file_system_root_dir'
 require 'chef_fs/file_pattern'
 require 'chef_fs/path_utils'
+require 'chef_fs/parallelizer'
 require 'chef/config'
 
 module ChefFS
@@ -27,16 +28,21 @@ module ChefFS
     def self.common_options
       option :repo_mode,
         :long => '--repo-mode MODE',
-        :description => "Specifies the local repository layout.  Values: default, everything, hosted_everything"
+        :description => "Specifies the local repository layout.  Values: static, everything, hosted_everything.  Default: everything/hosted_everything"
 
       option :chef_repo_path,
         :long => '--chef-repo-path PATH',
         :description => 'Overrides the location of chef repo. Default is specified by chef_repo_path in the config'
+
+      option :concurrency,
+        :long => '--concurrency THREADS',
+        :description => 'Maximum number of simultaneous requests to send (default: 10)'
     end
 
     def configure_chef
       super
       Chef::Config[:repo_mode] = config[:repo_mode] if config[:repo_mode]
+      Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
 
       # --chef-repo-path overrides all other paths
       path_variables = %w(acl_path client_path cookbook_path container_path data_bag_path environment_path group_path node_path role_path user_path)
@@ -65,6 +71,15 @@ module ChefFS
         Chef::Config[:role_path] = nil
       end
 
+      # Default to getting *everything* from the server.
+      if !Chef::Config[:repo_mode]
+        if Chef::Config[:chef_server_url] =~ /^[^\/]+\/+organizations\//
+          Chef::Config[:repo_mode] = 'hosted_everything'
+        else
+          Chef::Config[:repo_mode] = 'everything'
+        end
+      end
+
       # Infer any *_path variables that are not specified
       if Chef::Config[:chef_repo_path]
         path_variables.each do |variable_name|
@@ -76,6 +91,8 @@ module ChefFS
           end
         end
       end
+
+      ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
     end
 
     def chef_fs
@@ -91,12 +108,12 @@ module ChefFS
 
         result = {}
         case Chef::Config[:repo_mode]
-        when 'everything'
-          object_names = %w(clients cookbooks data_bags environments nodes roles users)
+        when 'static'
+          object_names = %w(cookbooks data_bags environments roles)
         when 'hosted_everything'
           object_names = %w(acls clients cookbooks containers data_bags environments groups nodes roles)
         else
-          object_names = %w(cookbooks data_bags environments roles)
+          object_names = %w(clients cookbooks data_bags environments nodes roles users)
         end
         object_names.each do |object_name|
           variable_name = "#{object_name[0..-2]}_path" # cookbooks -> cookbook_path
@@ -195,5 +212,8 @@ module ChefFS
       end
     end
 
+    def parallelize(inputs, options = {}, &block)
+      ChefFS::Parallelizer.parallelize(inputs, options, &block)
+    end
   end
 end

data/lib/chef_fs/parallelizer.rb

@@ -0,0 +1,127 @@
+module ChefFS
+  class Parallelizer
+    @@parallelizer = nil
+    @@threads = 0
+
+    def self.threads=(value)
+      if @@threads != value
+        @@threads = value
+        @@parallelizer = nil
+      end
+    end
+
+    def self.parallelize(enumerator, options = {}, &block)
+      @@parallelizer ||= Parallelizer.new(@@threads)
+      @@parallelizer.parallelize(enumerator, options, &block)
+    end
+
+    def initialize(threads)
+      @tasks_mutex = Mutex.new
+      @tasks = []
+      @threads = []
+      1.upto(threads) do
+        @threads << Thread.new { worker_loop }
+      end
+    end
+
+    def parallelize(enumerator, options = {}, &block)
+      task = ParallelizedResults.new(enumerator, options, &block)
+      @tasks_mutex.synchronize do
+        @tasks << task
+      end
+      task
+    end
+
+    class ParallelizedResults
+      include Enumerable
+
+      def initialize(enumerator, options, &block)
+        @inputs = enumerator.to_a
+        @options = options
+        @block = block
+
+        @mutex = Mutex.new
+        @outputs = []
+        @status = []
+      end
+
+      def each
+        next_index = 0
+        while true
+          # Report any results that already exist
+          while @status.length > next_index && ([:finished, :exception].include?(@status[next_index]))
+            if @status[next_index] == :finished
+              if @options[:flatten]
+                @outputs[next_index].each do |entry|
+                  yield entry
+                end
+              else
+                yield @outputs[next_index]
+              end
+            else
+              raise @outputs[next_index]
+            end
+            next_index = next_index + 1
+          end
+
+          # Pick up a result and process it, if there is one.  This ensures we
+          # move forward even if there are *zero* worker threads available.
+          if !process_input
+            # Exit if we're done.
+            if next_index >= @status.length
+              break
+            else
+              # Ruby 1.8 threading sucks.  Wait till we process more things.
+              sleep(0.05)
+            end
+          end
+        end
+      end
+
+      def process_input
+        # Grab the next one to process
+        index, input = @mutex.synchronize do
+          index = @status.length
+          if index >= @inputs.length
+            return nil
+          end
+          input = @inputs[index]
+          @status[index] = :started
+          [ index, input ]
+        end
+
+        begin
+          @outputs[index] = @block.call(input)
+          @status[index] = :finished
+        rescue
+          @outputs[index] = $!
+          @status[index] = :exception
+        end
+        index
+      end
+    end
+
+    private
+
+    def worker_loop
+      while true
+        begin
+          task = @tasks[0]
+          if task
+            if !task.process_input
+              @tasks_mutex.synchronize do
+                @tasks.delete(task)
+              end
+            end
+          else
+            # Ruby 1.8 threading sucks.  Wait a bit to see if another task comes in.
+            sleep(0.05)
+          end
+        rescue
+          puts "ERROR #{$!}"
+          puts $!.backtrace
+        end
+      end
+    end
+  end
+end

data/lib/chef_fs/raw_request.rb

@@ -23,10 +23,15 @@ module ChefFS
         response_body = chef_rest.decompress_body(response)
 
         if response.kind_of?(Net::HTTPSuccess)
-            response_body
+          response_body
         elsif redirect_location = redirected_to(response)
-          raise "Redirected to #{create_url(redirect_location)}"
-          follow_redirect {api_request(:GET, create_url(redirect_location))}
+          if [:GET, :HEAD].include?(method)
+            chef_rest.follow_redirect do
+              api_request(chef_rest, method, chef_rest.create_url(redirect_location))
+            end
+          else
+            raise Exceptions::InvalidRedirect, "#{method} request was redirected from #{url} to #{redirect_location}. Only GET and HEAD support redirects."
+          end
         else
           # have to decompress the body before making an exception for it. But the body could be nil.
           response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
@@ -57,7 +62,6 @@ module ChefFS
       response['location']
     end
 
-
     def self.build_headers(chef_rest, method, url, headers={}, json_body=false, raw=false)
       #        headers                 = @default_headers.merge(headers)
       #headers['Accept']       = "application/json" unless raw

data/lib/chef_fs/version.rb

@@ -1,3 +1,3 @@
 module ChefFS
-  VERSION = "0.9.8"
+  VERSION = "1.0.0.beta1"
 end

data/spec/integration/chef_repo_path_spec.rb

@@ -17,7 +17,7 @@ describe 'chef_repo_path tests' do
       file 'roles/role1.json', {}
       file 'users/user1.json', {}
 
-      file 'clients2/client1.json', {}
+      file 'clients2/client2.json', {}
       file 'cookbooks2/cookbook2/metadata.rb', ''
       file 'data_bags2/bag2/item2.json', {}
       file 'environments2/env2.json', {}
@@ -61,6 +61,8 @@ describe 'chef_repo_path tests' do
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client2.json
 cookbooks/
 cookbooks/cookbook2/
 cookbooks/cookbook2/metadata.rb
@@ -69,8 +71,12 @@ data_bags/bag2/
 data_bags/bag2/item2.json
 environments/
 environments/env2.json
+nodes/
+nodes/node2.json
 roles/
 roles/role2.json
+users/
+users/user2.json
 EOM
           end
         end
@@ -100,6 +106,8 @@ EOM
           cwd '.'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client2.json
 cookbooks/
 cookbooks/cookbook2/
 cookbooks/cookbook2/metadata.rb
@@ -108,8 +116,12 @@ data_bags/bag2/
 data_bags/bag2/item2.json
 environments/
 environments/env2.json
+nodes/
+nodes/node2.json
 roles/
 roles/role2.json
+users/
+users/user2.json
 EOM
           end
         end
@@ -165,6 +177,8 @@ EOM
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client3.json
 cookbooks/
 cookbooks/cookbook3/
 cookbooks/cookbook3/metadata.rb
@@ -173,8 +187,12 @@ data_bags/bag3/
 data_bags/bag3/item3.json
 environments/
 environments/env3.json
+nodes/
+nodes/node3.json
 roles/
 roles/role3.json
+users/
+users/user3.json
 EOM
           end
         end
@@ -351,6 +369,9 @@ EOM
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client1.json
+clients/client2.json
 cookbooks/
 cookbooks/cookbook1/
 cookbooks/cookbook1/metadata.rb
@@ -364,9 +385,15 @@ data_bags/bag2/item2.json
 environments/
 environments/env1.json
 environments/env2.json
+nodes/
+nodes/node1.json
+nodes/node2.json
 roles/
 roles/role1.json
 roles/role2.json
+users/
+users/user1.json
+users/user2.json
 EOM
           end
         end
@@ -399,6 +426,9 @@ EOM
           cwd '.'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client1.json
+clients/client3.json
 cookbooks/
 cookbooks/cookbook1/
 cookbooks/cookbook1/metadata.rb
@@ -412,9 +442,15 @@ data_bags/bag3/item3.json
 environments/
 environments/env1.json
 environments/env3.json
+nodes/
+nodes/node1.json
+nodes/node3.json
 roles/
 roles/role1.json
 roles/role3.json
+users/
+users/user1.json
+users/user3.json
 EOM
           end
         end
@@ -435,6 +471,9 @@ EOM
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client1.json
+clients/client3.json
 cookbooks/
 cookbooks/cookbook1/
 cookbooks/cookbook1/metadata.rb
@@ -448,9 +487,15 @@ data_bags/bag3/item3.json
 environments/
 environments/env1.json
 environments/env3.json
+nodes/
+nodes/node1.json
+nodes/node3.json
 roles/
 roles/role1.json
 roles/role3.json
+users/
+users/user1.json
+users/user3.json
 EOM
           end
         end
@@ -495,6 +540,8 @@ EOM
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client3.json
 cookbooks/
 cookbooks/cookbook3/
 cookbooks/cookbook3/metadata.rb
@@ -503,8 +550,12 @@ data_bags/bag3/
 data_bags/bag3/item3.json
 environments/
 environments/env3.json
+nodes/
+nodes/node3.json
 roles/
 roles/role3.json
+users/
+users/user3.json
 EOM
           end
         end
@@ -536,6 +587,9 @@ EOM
           cwd '.'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client1.json
+clients/client3.json
 cookbooks/
 cookbooks/cookbook1/
 cookbooks/cookbook1/metadata.rb
@@ -549,9 +603,15 @@ data_bags/bag3/item3.json
 environments/
 environments/env1.json
 environments/env3.json
+nodes/
+nodes/node1.json
+nodes/node3.json
 roles/
 roles/role1.json
 roles/role3.json
+users/
+users/user1.json
+users/user3.json
 EOM
           end
         end
@@ -572,6 +632,9 @@ EOM
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client1.json
+clients/client3.json
 cookbooks/
 cookbooks/cookbook1/
 cookbooks/cookbook1/metadata.rb
@@ -585,9 +648,15 @@ data_bags/bag3/item3.json
 environments/
 environments/env1.json
 environments/env3.json
+nodes/
+nodes/node1.json
+nodes/node3.json
 roles/
 roles/role1.json
 roles/role3.json
+users/
+users/user1.json
+users/user3.json
 EOM
           end
         end
@@ -635,6 +704,8 @@ EOM
           cwd 'chef_repo2'
           it 'knife list --local -Rfp lists everything' do
             knife('list --local -Rfp').should_succeed <<EOM
+clients/
+clients/client3.json
 cookbooks/
 cookbooks/cookbook3/
 cookbooks/cookbook3/metadata.rb
@@ -643,8 +714,12 @@ data_bags/bag/
 data_bags/bag/item.json
 environments/
 environments/env3.json
+nodes/
+nodes/node3.json
 roles/
 roles/role3.json
+users/
+users/user3.json
 EOM
           end
         end