chef 11.4.4 → 11.6.0.hotfix.1

data/lib/chef/chef_fs/file_system/operation_not_allowed_error.rb

@@ -0,0 +1,48 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/file_system_error'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class OperationNotAllowedError < FileSystemError
+        def initialize(operation, entry, cause = nil)
+          super(entry, cause)
+          @operation = operation
+        end
+
+        attr_reader :operation
+        attr_reader :entry
+
+        def reason
+          case operation
+          when :delete
+            "cannot be deleted"
+          when :write
+            "cannot be updated"
+          when :create_child
+            "cannot have a child created under it"
+          when :read
+            "cannot be read"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef/chef_fs/file_system/rest_list_dir.rb

@@ -24,12 +24,14 @@ class Chef
   module ChefFS
     module FileSystem
       class RestListDir < BaseFSDir
-        def initialize(name, parent, api_path = nil)
+        def initialize(name, parent, api_path = nil, data_handler = nil)
           super(name, parent)
           @api_path = api_path || (parent.api_path == "" ? name : "#{parent.api_path}/#{name}")
+          @data_handler = data_handler
         end
 
         attr_reader :api_path
+        attr_reader :data_handler
 
         def child(name)
           result = @children.select { |child| child.name == name }.first if @children
@@ -43,28 +45,55 @@ class Chef
 
         def children
           begin
-            @children ||= rest.get_rest(api_path).keys.map do |key|
+            @children ||= Chef::ChefFS::RawRequest.raw_json(rest, api_path).keys.sort.map do |key|
               _make_child_entry("#{key}.json", true)
             end
-          rescue Net::HTTPServerException
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "Timeout retrieving children: #{e}"
+          rescue Net::HTTPServerException => e
             if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:children, self, e), "HTTP error retrieving children: #{e}"
             end
           end
         end
 
-        # NOTE if you change this significantly, you will likely need to change
-        # DataBagDir.create_child as well.
         def create_child(name, file_contents)
-          json = Chef::JSONCompat.from_json(file_contents).to_hash
-          base_name = name[0,name.length-5]
-          if json.include?('name') && json['name'] != base_name
-            raise "Name in #{path_for_printing}/#{name} must be '#{base_name}' (is '#{json['name']}')"
+          begin
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Parse error reading JSON creating child '#{name}': #{e}"
           end
-          rest.post_rest(api_path, json)
-          _make_child_entry(name, true)
+
+          result = _make_child_entry(name, true)
+
+          if data_handler
+            object = data_handler.normalize_for_post(object, result)
+            data_handler.verify_integrity(object, result) do |error|
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self), "Error creating '#{name}': #{error}"
+            end
+          end
+
+          begin
+            rest.post_rest(api_path, object)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Timeout creating '#{name}': #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+            elsif $!.response.code == "409"
+              raise Chef::ChefFS::FileSystem::AlreadyExistsError.new(:create_child, self, e), "Failure creating '#{name}': #{path}/#{name} already exists"
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:create_child, self, e), "Failure creating '#{name}': #{e.message}"
+            end
+          end
+
+          result
+        end
+
+        def org
+          parent.org
         end
 
         def environment

data/lib/chef/chef_fs/file_system/rest_list_entry.rb

@@ -18,6 +18,8 @@
 
 require 'chef/chef_fs/file_system/base_fs_object'
 require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/raw_request'
 require 'chef/role'
 require 'chef/node'
 
@@ -30,14 +32,25 @@ class Chef
           @exists = exists
         end
 
-        def api_path
+        def data_handler
+          parent.data_handler
+        end
+
+        def api_child_name
           if name.length < 5 || name[-5,5] != ".json"
             raise "Invalid name #{path}: must end in .json"
           end
-          api_child_name = name[0,name.length-5]
+          name[0,name.length-5]
+        end
+
+        def api_path
           "#{parent.api_path}/#{api_child_name}"
         end
 
+        def org
+          parent.org
+        end
+
         def environment
           parent.environment
         end
@@ -56,45 +69,76 @@ class Chef
         def delete(recurse)
           begin
             rest.delete_rest(api_path)
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
             end
           end
         end
 
         def read
-          Chef::JSONCompat.to_json_pretty(chef_object.to_hash)
+          Chef::JSONCompat.to_json_pretty(_read_hash)
         end
 
-        def chef_object
+        def _read_hash
           begin
-            # REST will inflate the Chef object using json_class
-            rest.get_rest(api_path)
-          rescue Net::HTTPServerException
+            json = Chef::ChefFS::RawRequest.raw_request(rest, api_path)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "Timeout reading: #{e}"
+          rescue Net::HTTPServerException => e
             if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "HTTP error reading: #{e}"
             end
           end
+          # Minimize the value (get rid of defaults) so the results don't look terrible
+          minimize_value(JSON.parse(json, :create_additions => false))
+        end
+
+        def chef_object
+          # REST will inflate the Chef object using json_class
+          data_handler.json_class.json_create(read)
+        end
+
+        def minimize_value(value)
+          data_handler.minimize(data_handler.normalize(value, self), self)
         end
 
         def compare_to(other)
+          # TODO this pair of reads can be parallelized
+
+          # Grab the other value
           begin
-            other_value = other.read
+            other_value_json = other.read
           rescue Chef::ChefFS::FileSystem::NotFoundError
             return [ nil, nil, :none ]
           end
+
+          # Grab this value
           begin
-            value = chef_object.to_hash
+            value = _read_hash
           rescue Chef::ChefFS::FileSystem::NotFoundError
-            return [ false, :none, other_value ]
+            return [ false, :none, other_value_json ]
           end
-          are_same = (value == Chef::JSONCompat.from_json(other_value, :create_additions => false))
-          [ are_same, Chef::JSONCompat.to_json_pretty(value), other_value ]
+
+          # Minimize (and normalize) both values for easy and beautiful diffs
+          value = minimize_value(value)
+          value_json = Chef::JSONCompat.to_json_pretty(value)
+          begin
+            other_value = JSON.parse(other_value_json, :create_additions => false)
+          rescue JSON::ParserError => e
+            Chef::Log.warn("Parse error reading #{other.path_for_printing} as JSON: #{e}")
+            return [ nil, value_json, other_value_json ]
+          end
+          other_value = minimize_value(other_value)
+          other_value_json = Chef::JSONCompat.to_json_pretty(other_value)
+
+          [ value == other_value, value_json, other_value_json ]
         end
 
         def rest
@@ -102,18 +146,28 @@ class Chef
         end
 
         def write(file_contents)
-          json = Chef::JSONCompat.from_json(file_contents).to_hash
-          base_name = name[0,name.length-5]
-          if json['name'] != base_name
-            raise "Name in #{path_for_printing}/#{name} must be '#{base_name}' (is '#{json['name']}')"
+          begin
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Parse error reading JSON: #{e}"
+          end
+
+          if data_handler
+            object = data_handler.normalize_for_put(object, self)
+            data_handler.verify_integrity(object, self) do |error|
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self), "#{error}"
+            end
           end
+
           begin
-            rest.put_rest(api_path, json)
-          rescue Net::HTTPServerException
-            if $!.response.code == "404"
-              raise Chef::ChefFS::FileSystem::NotFoundError.new($!), "#{path_for_printing} not found"
+            rest.put_rest(api_path, object)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Timeout writing: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
             else
-              raise
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "HTTP error writing: #{e}"
             end
           end
         end

data/lib/chef/chef_fs/knife.rb

@@ -16,52 +16,74 @@
 # limitations under the License.
 #
 
-require 'chef/chef_fs/file_system/chef_server_root_dir'
-require 'chef/chef_fs/file_system/chef_repository_file_system_root_dir'
-require 'chef/chef_fs/file_pattern'
-require 'chef/chef_fs/path_utils'
-require 'chef/config'
+require 'chef/knife'
 
 class Chef
   module ChefFS
     class Knife < Chef::Knife
-      def self.common_options
+      # Workaround for CHEF-3932
+      def self.deps
+        super do
+          require 'chef/config'
+          require 'chef/chef_fs/parallelizer'
+          require 'chef/chef_fs/config'
+          require 'chef/chef_fs/file_pattern'
+          require 'chef/chef_fs/path_utils'
+          yield
+        end
+      end
+
+      def self.inherited(c)
+        super
+        # Ensure we always get to do our includes, whether subclass calls deps or not
+        c.deps do
+        end
+
         option :repo_mode,
           :long => '--repo-mode MODE',
-          :default => "default",
-          :description => "Specifies the local repository layout.  Values: default or full"
+          :description => "Specifies the local repository layout.  Values: static, everything, hosted_everything.  Default: everything/hosted_everything"
+
+        option :chef_repo_path,
+          :long => '--chef-repo-path PATH',
+          :description => 'Overrides the location of chef repo. Default is specified by chef_repo_path in the config'
+
+        option :concurrency,
+          :long => '--concurrency THREADS',
+          :description => 'Maximum number of simultaneous requests to send (default: 10)'
       end
 
-      def base_path
-        @base_path ||= begin
-          relative_to_base = Chef::ChefFS::PathUtils::relative_to(File.expand_path(Dir.pwd), chef_repo)
-          relative_to_base == '.' ? '/' : "/#{relative_to_base}"
+      def configure_chef
+        super
+        Chef::Config[:repo_mode] = config[:repo_mode] if config[:repo_mode]
+        Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
+
+        # --chef-repo-path overrides all other paths
+        if config[:chef_repo_path]
+          Chef::Config[:chef_repo_path] = config[:chef_repo_path]
+          Chef::ChefFS::Config::PATH_VARIABLES.each do |variable_name|
+            Chef::Config[variable_name.to_sym] = chef_repo_paths.map { |path| File.join(path, "#{variable_name[0..-6]}s") }
+          end
         end
+
+        @chef_fs_config = Chef::ChefFS::Config.new(Chef::Config)
+
+        Chef::ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
       end
 
       def chef_fs
-        @chef_fs ||= Chef::ChefFS::FileSystem::ChefServerRootDir.new("remote", Chef::Config, config[:repo_mode])
+        @chef_fs_config.chef_fs
       end
 
-      def chef_repo
-        @chef_repo ||= File.expand_path(File.join(Chef::Config.cookbook_path, ".."))
+      def create_chef_fs
+        @chef_fs_config.create_chef_fs
       end
 
-      def format_path(path)
-        if path[0,base_path.length] == base_path
-          if path == base_path
-            return "."
-          elsif path[base_path.length] == "/"
-            return path[base_path.length + 1, path.length - base_path.length - 1]
-          elsif base_path == "/" && path[0] == "/"
-            return path[1, path.length - 1]
-          end
-        end
-        path
+      def local_fs
+        @chef_fs_config.local_fs
       end
 
-      def local_fs
-        @local_fs ||= Chef::ChefFS::FileSystem::ChefRepositoryFileSystemRootDir.new(chef_repo)
+      def create_local_fs
+        @chef_fs_config.create_local_fs
       end
 
       def pattern_args
@@ -69,9 +91,26 @@ class Chef
       end
 
       def pattern_args_from(args)
-        args.map { |arg| Chef::ChefFS::FilePattern::relative_to(base_path, arg) }.to_a
+        # TODO support absolute file paths and not just patterns?  Too much?
+        # Could be super useful in a world with multiple repo paths
+        args.map do |arg|
+          if !@chef_fs_config.base_path && !Chef::ChefFS::PathUtils.is_absolute?(arg)
+            # Check if chef repo path is specified to give a better error message
+            @chef_fs_config.require_chef_repo_path
+            ui.error("Attempt to use relative path '#{arg}' when current directory is outside the repository path")
+            exit(1)
+          end
+          Chef::ChefFS::FilePattern.relative_to(@chef_fs_config.base_path, arg)
+        end
       end
 
+      def format_path(entry)
+        @chef_fs_config.format_path(entry)
+      end
+
+      def parallelize(inputs, options = {}, &block)
+        Chef::ChefFS::Parallelizer.parallelize(inputs, options, &block)
+      end
     end
   end
 end

data/lib/chef/chef_fs/parallelizer.rb

@@ -0,0 +1,129 @@
+class Chef
+  module ChefFS
+    class Parallelizer
+      @@parallelizer = nil
+      @@threads = 0
+
+      def self.threads=(value)
+        if @@threads != value
+          @@threads = value
+          @@parallelizer = nil
+        end
+      end
+
+      def self.parallelize(enumerator, options = {}, &block)
+        @@parallelizer ||= Parallelizer.new(@@threads)
+        @@parallelizer.parallelize(enumerator, options, &block)
+      end
+
+      def initialize(threads)
+        @tasks_mutex = Mutex.new
+        @tasks = []
+        @threads = []
+        1.upto(threads) do
+          @threads << Thread.new { worker_loop }
+        end
+      end
+
+      def parallelize(enumerator, options = {}, &block)
+        task = ParallelizedResults.new(enumerator, options, &block)
+        @tasks_mutex.synchronize do
+          @tasks << task
+        end
+        task
+      end
+
+      class ParallelizedResults
+        include Enumerable
+
+        def initialize(enumerator, options, &block)
+          @inputs = enumerator.to_a
+          @options = options
+          @block = block
+
+          @mutex = Mutex.new
+          @outputs = []
+          @status = []
+        end
+
+        def each
+          next_index = 0
+          while true
+            # Report any results that already exist
+            while @status.length > next_index && ([:finished, :exception].include?(@status[next_index]))
+              if @status[next_index] == :finished
+                if @options[:flatten]
+                  @outputs[next_index].each do |entry|
+                    yield entry
+                  end
+                else
+                  yield @outputs[next_index]
+                end
+              else
+                raise @outputs[next_index]
+              end
+              next_index = next_index + 1
+            end
+
+            # Pick up a result and process it, if there is one.  This ensures we
+            # move forward even if there are *zero* worker threads available.
+            if !process_input
+              # Exit if we're done.
+              if next_index >= @status.length
+                break
+              else
+                # Ruby 1.8 threading sucks.  Wait till we process more things.
+                sleep(0.05)
+              end
+            end
+          end
+        end
+
+        def process_input
+          # Grab the next one to process
+          index, input = @mutex.synchronize do
+            index = @status.length
+            if index >= @inputs.length
+              return nil
+            end
+            input = @inputs[index]
+            @status[index] = :started
+            [ index, input ]
+          end
+
+          begin
+            @outputs[index] = @block.call(input)
+            @status[index] = :finished
+          rescue Exception
+            @outputs[index] = $!
+            @status[index] = :exception
+          end
+          index
+        end
+      end
+
+      private
+
+      def worker_loop
+        while true
+          begin
+            task = @tasks[0]
+            if task
+              if !task.process_input
+                @tasks_mutex.synchronize do
+                  @tasks.delete(task)
+                end
+              end
+            else
+              # Ruby 1.8 threading sucks.  Wait a bit to see if another task comes in.
+              sleep(0.05)
+            end
+          rescue
+            puts "ERROR #{$!}"
+            puts $!.backtrace
+          end
+        end
+      end
+    end
+  end
+end