knife-essentials 0.9.8 → 1.0.0.beta1

data/lib/chef/knife/delete_essentials.rb

@@ -38,7 +38,7 @@ class Chef
         error = false
         if config[:remote_only]
           pattern_args.each do |pattern|
-            ChefFS::FileSystem.list(chef_fs, pattern) do |result|
+            ChefFS::FileSystem.list(chef_fs, pattern).each do |result|
               if delete_result(result)
                 error = true
               end
@@ -46,7 +46,7 @@ class Chef
           end
         elsif config[:local_only]
           pattern_args.each do |pattern|
-            ChefFS::FileSystem.list(local_fs, pattern) do |result|
+            ChefFS::FileSystem.list(local_fs, pattern).each do |result|
               if delete_result(result)
                 error = true
               end
@@ -54,7 +54,7 @@ class Chef
           end
         else
           pattern_args.each do |pattern|
-            ChefFS::FileSystem.list_pairs(pattern, chef_fs, local_fs) do |chef_result, local_result|
+            ChefFS::FileSystem.list_pairs(pattern, chef_fs, local_fs).each do |chef_result, local_result|
               if delete_result(chef_result, local_result)
                 error = true
               end

data/lib/chef/knife/deps_essentials.rb

@@ -36,7 +36,7 @@ class Chef
         @root = config[:remote] ? chef_fs : local_fs
         dependencies = {}
         pattern_args.each do |pattern|
-          ChefFS::FileSystem.list(@root, pattern) do |entry|
+          ChefFS::FileSystem.list(@root, pattern).each do |entry|
             if config[:tree]
               print_dependencies_tree(entry, dependencies)
             else

data/lib/chef/knife/edit_essentials.rb

@@ -20,7 +20,7 @@ class Chef
         # Get the matches (recursively)
         error = false
         pattern_args.each do |pattern|
-          ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern) do |result|
+          ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern).each do |result|
             if result.dir?
               ui.error "#{format_path(result)}: is a directory" if pattern.exact_path
               error = true

data/lib/chef/knife/list_essentials.rb

@@ -43,21 +43,28 @@ class Chef
         patterns = name_args.length == 0 ? [""] : name_args
 
         # Get the matches (recursively)
-        results = []
-        dir_results = []
-        pattern_args_from(patterns).each do |pattern|
-          ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern) do |result|
-            if result.dir? && !config[:bare_directories]
-              dir_results += add_dir_result(result)
-            elsif result.exists?
-              results << result
-            elsif pattern.exact_path
-              ui.error "#{format_path(result)}: No such file or directory"
-              self.exit_code = 1
-            end
+        all_results = parallelize(pattern_args_from(patterns), :flatten => true) do |pattern|
+          pattern_results = ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern)
+          if pattern_results.first && !pattern_results.first.exists? && pattern.exact_path
+            ui.error "#{format_path(pattern_results.first)}: No such file or directory"
+            self.exit_code = 1
           end
+          pattern_results
         end
 
+        # Process directories
+        if !config[:bare_directories]
+          dir_results = parallelize(all_results.select { |result| result.dir? }, :flatten => true) do |result|
+            add_dir_result(result)
+          end.to_a
+        else
+          dir_results = []
+        end
+
+        # Process all other results
+        results = all_results.select { |result| result.exists? && (!result.dir? || config[:bare_directories]) }.to_a
+
+        # Flatten out directory results if necessary
         if config[:flat]
           dir_results.each do |result, children|
             results += children
@@ -65,9 +72,11 @@ class Chef
           dir_results = []
         end
 
+        # Sort by path for happy output
         results = results.sort_by { |result| result.path }
         dir_results = dir_results.sort_by { |result| result[0].path }
 
+        # Print!
         if results.length == 0 && dir_results.length == 1
           results = dir_results[0][1]
           dir_results = []
@@ -98,11 +107,8 @@ class Chef
 
         result = [ [ result, children ] ]
         if config[:recursive]
-          children.each do |child|
-            if child.dir?
-              result += add_dir_result(child)
-            end
-          end
+          child_dirs = children.select { |child| child.dir? }
+          result += parallelize(child_dirs, :flatten => true) { |child| add_dir_result(child) }.to_a
         end
         result
       end

data/lib/chef/knife/show_essentials.rb

@@ -19,26 +19,33 @@ class Chef
       def run
         # Get the matches (recursively)
         error = false
-        pattern_args.each do |pattern|
-          ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern) do |result|
-            if result.dir?
-              ui.error "#{format_path(result)}: is a directory" if pattern.exact_path
+        entry_values = parallelize(pattern_args, :flatten => true) do |pattern|
+          parallelize(ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern)) do |entry|
+            if entry.dir?
+              ui.error "#{format_path(entry)}: is a directory" if pattern.exact_path
               error = true
+              nil
             else
               begin
-                value = result.read
-                output "#{format_path(result)}:"
-                output(format_for_display(value))
+                [entry, entry.read]
               rescue ChefFS::FileSystem::OperationNotAllowedError => e
                 ui.error "#{format_path(e.entry)}: #{e.reason}."
                 error = true
+                nil
               rescue ChefFS::FileSystem::NotFoundError => e
                 ui.error "#{format_path(e.entry)}: No such file or directory"
                 error = true
+                nil
               end
             end
           end
         end
+        entry_values.each do |entry, value|
+          if entry
+            output "#{format_path(entry)}:"
+            output(format_for_display(value))
+          end
+        end
         if error
           exit 1
         end

data/lib/chef/knife/xargs_essentials.rb

@@ -74,7 +74,7 @@ class Chef
         # Get the matches (recursively)
         files = []
         pattern_args_from(get_patterns).each do |pattern|
-          ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern) do |result|
+          ChefFS::FileSystem.list(config[:local] ? local_fs : chef_fs, pattern).each do |result|
             if result.dir?
               # TODO option to include directories
               ui.warn "#{format_path(result)}: is a directory.  Will not run #{command} on it."

data/lib/chef_fs/command_line.rb

@@ -30,7 +30,7 @@ module ChefFS
 
       get_content = (output_mode != :name_only && output_mode != :name_status)
       found_match = false
-      diff(pattern, a_root, b_root, recurse_depth, get_content) do |type, old_entry, new_entry, old_value, new_value, error|
+      diff(pattern, a_root, b_root, recurse_depth, get_content).each do |type, old_entry, new_entry, old_value, new_value, error|
         found_match = true unless type == :both_nonexistent
         old_path = format_path.call(old_entry)
         new_path = format_path.call(new_entry)
@@ -124,112 +124,128 @@ module ChefFS
       error
     end
 
-    def self.diff(pattern, a_root, b_root, recurse_depth, get_content)
-      ChefFS::FileSystem.list_pairs(pattern, a_root, b_root) do |a, b|
-        diff_entries(a, b, recurse_depth, get_content) do |diff|
-          yield diff
-        end
+    def self.diff(pattern, old_root, new_root, recurse_depth, get_content)
+      ChefFS::Parallelizer.parallelize(ChefFS::FileSystem.list_pairs(pattern, old_root, new_root), :flatten => true) do |old_entry, new_entry|
+        diff_entries(old_entry, new_entry, recurse_depth, get_content)
       end
     end
 
     # Diff two known entries (could be files or dirs)
     def self.diff_entries(old_entry, new_entry, recurse_depth, get_content)
-      begin
-        # If both are directories
-        if old_entry.dir?
-          if new_entry.dir?
-            if recurse_depth == 0
-              yield [ :common_subdirectories, old_entry, new_entry ]
-            else
-              ChefFS::FileSystem.child_pairs(old_entry, new_entry).each do |old_child,new_child|
-                diff_entries(old_child, new_child,
-                             recurse_depth ? recurse_depth - 1 : nil, get_content) do |diff|
-                  yield diff
-                end
-              end
+      # If both are directories
+      if old_entry.dir?
+        if new_entry.dir?
+          if recurse_depth == 0
+            return [ [ :common_subdirectories, old_entry, new_entry ] ]
+          else
+            return ChefFS::Parallelizer.parallelize(ChefFS::FileSystem.child_pairs(old_entry, new_entry), :flatten => true) do |old_child, new_child|
+              ChefFS::CommandLine.diff_entries(old_child, new_child, recurse_depth ? recurse_depth - 1 : nil, get_content)
             end
+          end
 
         # If old is a directory and new is a file
-          elsif new_entry.exists?
-            yield [ :directory_to_file, old_entry, new_entry ]
+        elsif new_entry.exists?
+          return [ [ :directory_to_file, old_entry, new_entry ] ]
 
         # If old is a directory and new does not exist
-          elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
-            yield [ :deleted, old_entry, new_entry ]
-          end
+        elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+          return [ [ :deleted, old_entry, new_entry ] ]
 
-        # If new is a directory and old is a file
-        elsif new_entry.dir?
-          if old_entry.exists?
-            yield [ :file_to_directory, old_entry, new_entry ]
+        # If the new entry does not and *cannot* exist, report that.
+        else
+          return [ [ :new_cannot_upload, old_entry, new_entry ] ]
+        end
+
+      # If new is a directory and old is a file
+      elsif new_entry.dir?
+        if old_entry.exists?
+          return [ [ :file_to_directory, old_entry, new_entry ] ]
 
         # If new is a directory and old does not exist
-          elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
-            yield [ :added, old_entry, new_entry ]
-          end
+        elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+          return [ [ :added, old_entry, new_entry ] ]
 
-        # Neither is a directory, so they are diffable with file diff
+        # If the new entry does not and *cannot* exist, report that.
         else
-          are_same, old_value, new_value = ChefFS::FileSystem.compare(old_entry, new_entry)
-          if are_same
-            if old_value == :none
-              yield [ :both_nonexistent, old_entry, new_entry ]
-            else
-              yield [ :same, old_entry, new_entry ]
-            end
+          return [ [ :old_cannot_upload, old_entry, new_entry ] ]
+        end
+
+      # Neither is a directory, so they are diffable with file diff
+      else
+        are_same, old_value, new_value = ChefFS::FileSystem.compare(old_entry, new_entry)
+        if are_same
+          if old_value == :none
+            return [ [ :both_nonexistent, old_entry, new_entry ] ]
           else
-            if old_value == :none
-              old_exists = false
-            elsif old_value.nil?
-              old_exists = old_entry.exists?
-            else
-              old_exists = true
-            end
+            return [ [ :same, old_entry, new_entry ] ]
+          end
+        else
+          if old_value == :none
+            old_exists = false
+          elsif old_value.nil?
+            old_exists = old_entry.exists?
+          else
+            old_exists = true
+          end
 
-            if new_value == :none
-              new_exists = false
-            elsif new_value.nil?
-              new_exists = new_entry.exists?
-            else
-              new_exists = true
-            end
+          if new_value == :none
+            new_exists = false
+          elsif new_value.nil?
+            new_exists = new_entry.exists?
+          else
+            new_exists = true
+          end
 
-            # If one of the files doesn't exist, we only want to print the diff if the
-            # other file *could be uploaded/downloaded*.
-            if !old_exists && !old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
-              yield [ :old_cannot_upload, old_entry, new_entry ]
-              return
-            end
-            if !new_exists && !new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
-              yield [ :new_cannot_upload, old_entry, new_entry ]
-              return
-            end
+          # If one of the files doesn't exist, we only want to print the diff if the
+          # other file *could be uploaded/downloaded*.
+          if !old_exists && !old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+            return [ [ :old_cannot_upload, old_entry, new_entry ] ]
+          end
+          if !new_exists && !new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+            return [ [ :new_cannot_upload, old_entry, new_entry ] ]
+          end
 
-            if get_content
-              # If we haven't read the values yet, get them now so that they can be diffed
-              begin
-                old_value = old_entry.read if old_value.nil?
-              rescue ChefFS::FileSystem::NotFoundError
-                old_value = :none
-              end
-              begin
-                new_value = new_entry.read if new_value.nil?
-              rescue ChefFS::FileSystem::NotFoundError
-                new_value = :none
-              end
+          if get_content
+            # If we haven't read the values yet, get them now so that they can be diffed
+            begin
+              old_value = old_entry.read if old_value.nil?
+            rescue ChefFS::FileSystem::NotFoundError
+              old_value = :none
             end
-
-            if old_value == :none || (old_value == nil && !old_entry.exists?)
-              yield [ :added, old_entry, new_entry, old_value, new_value ]
-            elsif new_value == :none
-              yield [ :deleted, old_entry, new_entry, old_value, new_value ]
-            else
-              yield [ :modified, old_entry, new_entry, old_value, new_value ]
+            begin
+              new_value = new_entry.read if new_value.nil?
+            rescue ChefFS::FileSystem::NotFoundError
+              new_value = :none
             end
           end
+
+          if old_value == :none || (old_value == nil && !old_entry.exists?)
+            return [ [ :added, old_entry, new_entry, old_value, new_value ] ]
+          elsif new_value == :none
+            return [ [ :deleted, old_entry, new_entry, old_value, new_value ] ]
+          else
+            return [ [ :modified, old_entry, new_entry, old_value, new_value ] ]
+          end
         end
-      rescue ChefFS::FileSystem::FileSystemError => e
-        yield [ :error, old_entry, new_entry, nil, nil, e ]
+      end
+    rescue ChefFS::FileSystem::FileSystemError => e
+      return [ [ :error, old_entry, new_entry, nil, nil, e ] ]
+    end
+
+    class Differ
+      def initialize(old_entry, new_entry, recurse_depth, get_content)
+        @old_entry = old_entry
+        @new_entry = new_entry
+        @recurse_depth = recurse_depth
+        @get_content = get_content
+      end
+
+      attr_reader :old_entry
+      attr_reader :new_entry
+      attr_reader :recurse_depth
+      attr_reader :get_content
+
+      def each
       end
     end
 

data/lib/chef_fs/file_system.rb

@@ -20,39 +20,59 @@ require 'chef_fs/path_utils'
 require 'chef_fs/file_system/default_environment_cannot_be_modified_error'
 require 'chef_fs/file_system/operation_failed_error'
 require 'chef_fs/file_system/operation_not_allowed_error'
+require 'chef_fs/parallelizer'
 
 module ChefFS
   module FileSystem
-    # Yields a list of all things under (and including) this entry that match the
+    # Returns a list of all things under (and including) this entry that match the
     # given pattern.
     #
     # ==== Attributes
     #
-    # * +entry+ - Entry to start listing under
+    # * +root+ - Entry to start listing under
     # * +pattern+ - ChefFS::FilePattern to match children under
     #
-    def self.list(entry, pattern, &block)
-      # Include self in results if it matches
-      if pattern.match?(entry.path)
-        block.call(entry)
+    def self.list(root, pattern)
+      Lister.new(root, pattern)
+    end
+
+    class Lister
+      include Enumerable
+
+      def initialize(root, pattern)
+        @root = root
+        @pattern = pattern
       end
 
-      if pattern.could_match_children?(entry.path)
-        # If it's possible that our children could match, descend in and add matches.
-        exact_child_name = pattern.exact_child_name_under(entry.path)
+      attr_reader :root
+      attr_reader :pattern
 
-        # If we've got an exact name, don't bother listing children; just grab the
-        # child with the given name.
-        if exact_child_name
-          exact_child = entry.child(exact_child_name)
-          if exact_child
-            list(exact_child, pattern, &block)
-          end
+      def each(&block)
+        list_from(root, &block)
+      end
+
+      def list_from(entry, &block)
+        # Include self in results if it matches
+        if pattern.match?(entry.path)
+          block.call(entry)
+        end
+
+        if pattern.could_match_children?(entry.path)
+          # If it's possible that our children could match, descend in and add matches.
+          exact_child_name = pattern.exact_child_name_under(entry.path)
+
+          # If we've got an exact name, don't bother listing children; just grab the
+          # child with the given name.
+          if exact_child_name
+            exact_child = entry.child(exact_child_name)
+            if exact_child
+              list_from(exact_child, &block)
+            end
 
-        # Otherwise, go through all children and find any matches
-        elsif entry.dir?
-          entry.children.each do |child|
-            list(child, pattern, &block)
+          # Otherwise, go through all children and find any matches
+          elsif entry.dir?
+            results = Parallelizer::parallelize(entry.children, :flatten => true) { |child| ChefFS::FileSystem.list(child, pattern) }
+            results.each(&block)
           end
         end
       end
@@ -118,7 +138,7 @@ module ChefFS
     def self.copy_to(pattern, src_root, dest_root, recurse_depth, options, ui, format_path)
       found_result = false
       error = false
-      list_pairs(pattern, src_root, dest_root) do |src, dest|
+      parallel_do(list_pairs(pattern, src_root, dest_root)) do |src, dest|
         found_result = true
         new_dest_parent = get_or_create_parent(dest, options, ui, format_path)
         child_error = copy_entries(src, dest, new_dest_parent, recurse_depth, options, ui, format_path)
@@ -142,26 +162,44 @@ module ChefFS
     #
     # ==== Example
     #
-    #     ChefFS::FileSystem.list_pairs(FilePattern.new('**x.txt', a_root, b_root)) do |a, b|
+    #     ChefFS::FileSystem.list_pairs(FilePattern.new('**x.txt', a_root, b_root)).each do |a, b|
     #       ...
     #     end
     #
     def self.list_pairs(pattern, a_root, b_root)
-      # Make sure everything on the server is also on the filesystem, and diff
-      found_paths = Set.new
-      ChefFS::FileSystem.list(a_root, pattern) do |a|
-        found_paths << a.path
-        b = ChefFS::FileSystem.resolve_path(b_root, a.path)
-        yield [ a, b ]
+      PairLister.new(pattern, a_root, b_root)
+    end
+
+    class PairLister
+      include Enumerable
+
+      def initialize(pattern, a_root, b_root)
+        @pattern = pattern
+        @a_root = a_root
+        @b_root = b_root
       end
 
-      # Check the outer regex pattern to see if it matches anything on the
-      # filesystem that isn't on the server
-      ChefFS::FileSystem.list(b_root, pattern) do |b|
-        if !found_paths.include?(b.path)
-          a = ChefFS::FileSystem.resolve_path(a_root, b.path)
+      attr_reader :pattern
+      attr_reader :a_root
+      attr_reader :b_root
+
+      def each
+        # Make sure everything on the server is also on the filesystem, and diff
+        found_paths = Set.new
+        ChefFS::FileSystem.list(a_root, pattern).each do |a|
+          found_paths << a.path
+          b = ChefFS::FileSystem.resolve_path(b_root, a.path)
           yield [ a, b ]
         end
+
+        # Check the outer regex pattern to see if it matches anything on the
+        # filesystem that isn't on the server
+        ChefFS::FileSystem.list(b_root, pattern).each do |b|
+          if !found_paths.include?(b.path)
+            a = ChefFS::FileSystem.resolve_path(a_root, b.path)
+            yield [ a, b ]
+          end
+        end
       end
     end
 
@@ -204,6 +242,7 @@ module ChefFS
         are_same, b_value, a_value = b.compare_to(a)
       end
       if are_same.nil?
+        # TODO these reads can be parallelized
         begin
           a_value = a.read if a_value.nil?
         rescue ChefFS::FileSystem::NotFoundError
@@ -276,7 +315,7 @@ module ChefFS
               end
               # Directory creation is recursive.
               if recurse_depth != 0
-                src_entry.children.each do |src_child|
+                parallel_do(src_entry.children) do |src_child|
                   new_dest_child = new_dest_dir.child(src_child.name)
                   child_error = copy_entries(src_child, new_dest_child, new_dest_dir, recurse_depth ? recurse_depth - 1 : recurse_depth, options, ui, format_path)
                   error ||= child_error
@@ -313,7 +352,7 @@ module ChefFS
             if dest_entry.dir?
               # If both are directories, recurse into their children
               if recurse_depth != 0
-                child_pairs(src_entry, dest_entry).each do |src_child, dest_child|
+                parallel_do(child_pairs(src_entry, dest_entry)) do |src_child, dest_child|
                   child_error = copy_entries(src_child, dest_child, dest_entry, recurse_depth ? recurse_depth - 1 : recurse_depth, options, ui, format_path)
                   error ||= child_error
                 end
@@ -376,5 +415,8 @@ module ChefFS
       return parent
     end
 
+    def self.parallel_do(enum, options = {}, &block)
+      ChefFS::Parallelizer.parallelize(enum, options, &block).to_a
+    end
   end
 end