chef 10.34.6-x86-mingw32 → 11.6.2-x86-mingw32

data/lib/chef/chef_fs/file_system/rest_list_entry.rb

@@ -0,0 +1,177 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs/file_system/base_fs_object'
+require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/raw_request'
+require 'chef/role'
+require 'chef/node'
+
+class Chef
+  module ChefFS
+    module FileSystem
+      class RestListEntry < BaseFSObject
+        def initialize(name, parent, exists = nil)
+          super(name, parent)
+          @exists = exists
+        end
+
+        def data_handler
+          parent.data_handler
+        end
+
+        def api_child_name
+          if name.length < 5 || name[-5,5] != ".json"
+            raise "Invalid name #{path}: must end in .json"
+          end
+          name[0,name.length-5]
+        end
+
+        def api_path
+          "#{parent.api_path}/#{api_child_name}"
+        end
+
+        def org
+          parent.org
+        end
+
+        def environment
+          parent.environment
+        end
+
+        def exists?
+          if @exists.nil?
+            begin
+              @exists = parent.children.any? { |child| child.name == name }
+            rescue Chef::ChefFS::FileSystem::NotFoundError
+              @exists = false
+            end
+          end
+          @exists
+        end
+
+        def delete(recurse)
+          begin
+            rest.delete_rest(api_path)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:delete, self, e), "Timeout deleting: #{e}"
+            end
+          end
+        end
+
+        def read
+          Chef::JSONCompat.to_json_pretty(_read_hash)
+        end
+
+        def _read_hash
+          begin
+            json = Chef::ChefFS::RawRequest.raw_request(rest, api_path)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "Timeout reading: #{e}"
+          rescue Net::HTTPServerException => e
+            if $!.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:read, self, e), "HTTP error reading: #{e}"
+            end
+          end
+          # Minimize the value (get rid of defaults) so the results don't look terrible
+          minimize_value(JSON.parse(json, :create_additions => false))
+        end
+
+        def chef_object
+          # REST will inflate the Chef object using json_class
+          data_handler.json_class.json_create(read)
+        end
+
+        def minimize_value(value)
+          data_handler.minimize(data_handler.normalize(value, self), self)
+        end
+
+        def compare_to(other)
+          # TODO this pair of reads can be parallelized
+
+          # Grab the other value
+          begin
+            other_value_json = other.read
+          rescue Chef::ChefFS::FileSystem::NotFoundError
+            return [ nil, nil, :none ]
+          end
+
+          # Grab this value
+          begin
+            value = _read_hash
+          rescue Chef::ChefFS::FileSystem::NotFoundError
+            return [ false, :none, other_value_json ]
+          end
+
+          # Minimize (and normalize) both values for easy and beautiful diffs
+          value = minimize_value(value)
+          value_json = Chef::JSONCompat.to_json_pretty(value)
+          begin
+            other_value = JSON.parse(other_value_json, :create_additions => false)
+          rescue JSON::ParserError => e
+            Chef::Log.warn("Parse error reading #{other.path_for_printing} as JSON: #{e}")
+            return [ nil, value_json, other_value_json ]
+          end
+          other_value = minimize_value(other_value)
+          other_value_json = Chef::JSONCompat.to_json_pretty(other_value)
+
+          [ value == other_value, value_json, other_value_json ]
+        end
+
+        def rest
+          parent.rest
+        end
+
+        def write(file_contents)
+          begin
+            object = JSON.parse(file_contents, :create_additions => false)
+          rescue JSON::ParserError => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Parse error reading JSON: #{e}"
+          end
+
+          if data_handler
+            object = data_handler.normalize_for_put(object, self)
+            data_handler.verify_integrity(object, self) do |error|
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self), "#{error}"
+            end
+          end
+
+          begin
+            rest.put_rest(api_path, object)
+          rescue Timeout::Error => e
+            raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "Timeout writing: #{e}"
+          rescue Net::HTTPServerException => e
+            if e.response.code == "404"
+              raise Chef::ChefFS::FileSystem::NotFoundError.new(self, e)
+            else
+              raise Chef::ChefFS::FileSystem::OperationFailedError.new(:write, self, e), "HTTP error writing: #{e}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef/chef_fs/knife.rb

@@ -0,0 +1,116 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+
+class Chef
+  module ChefFS
+    class Knife < Chef::Knife
+      # Workaround for CHEF-3932
+      def self.deps
+        super do
+          require 'chef/config'
+          require 'chef/chef_fs/parallelizer'
+          require 'chef/chef_fs/config'
+          require 'chef/chef_fs/file_pattern'
+          require 'chef/chef_fs/path_utils'
+          yield
+        end
+      end
+
+      def self.inherited(c)
+        super
+        # Ensure we always get to do our includes, whether subclass calls deps or not
+        c.deps do
+        end
+
+        option :repo_mode,
+          :long => '--repo-mode MODE',
+          :description => "Specifies the local repository layout.  Values: static, everything, hosted_everything.  Default: everything/hosted_everything"
+
+        option :chef_repo_path,
+          :long => '--chef-repo-path PATH',
+          :description => 'Overrides the location of chef repo. Default is specified by chef_repo_path in the config'
+
+        option :concurrency,
+          :long => '--concurrency THREADS',
+          :description => 'Maximum number of simultaneous requests to send (default: 10)'
+      end
+
+      def configure_chef
+        super
+        Chef::Config[:repo_mode] = config[:repo_mode] if config[:repo_mode]
+        Chef::Config[:concurrency] = config[:concurrency].to_i if config[:concurrency]
+
+        # --chef-repo-path overrides all other paths
+        if config[:chef_repo_path]
+          Chef::Config[:chef_repo_path] = config[:chef_repo_path]
+          Chef::ChefFS::Config::PATH_VARIABLES.each do |variable_name|
+            Chef::Config[variable_name.to_sym] = chef_repo_paths.map { |path| File.join(path, "#{variable_name[0..-6]}s") }
+          end
+        end
+
+        @chef_fs_config = Chef::ChefFS::Config.new(Chef::Config)
+
+        Chef::ChefFS::Parallelizer.threads = (Chef::Config[:concurrency] || 10) - 1
+      end
+
+      def chef_fs
+        @chef_fs_config.chef_fs
+      end
+
+      def create_chef_fs
+        @chef_fs_config.create_chef_fs
+      end
+
+      def local_fs
+        @chef_fs_config.local_fs
+      end
+
+      def create_local_fs
+        @chef_fs_config.create_local_fs
+      end
+
+      def pattern_args
+        @pattern_args ||= pattern_args_from(name_args)
+      end
+
+      def pattern_args_from(args)
+        # TODO support absolute file paths and not just patterns?  Too much?
+        # Could be super useful in a world with multiple repo paths
+        args.map do |arg|
+          if !@chef_fs_config.base_path && !Chef::ChefFS::PathUtils.is_absolute?(arg)
+            # Check if chef repo path is specified to give a better error message
+            @chef_fs_config.require_chef_repo_path
+            ui.error("Attempt to use relative path '#{arg}' when current directory is outside the repository path")
+            exit(1)
+          end
+          Chef::ChefFS::FilePattern.relative_to(@chef_fs_config.base_path, arg)
+        end
+      end
+
+      def format_path(entry)
+        @chef_fs_config.format_path(entry)
+      end
+
+      def parallelize(inputs, options = {}, &block)
+        Chef::ChefFS::Parallelizer.parallelize(inputs, options, &block)
+      end
+    end
+  end
+end

data/lib/chef/chef_fs/parallelizer.rb

@@ -0,0 +1,129 @@
+class Chef
+  module ChefFS
+    class Parallelizer
+      @@parallelizer = nil
+      @@threads = 0
+
+      def self.threads=(value)
+        if @@threads != value
+          @@threads = value
+          @@parallelizer = nil
+        end
+      end
+
+      def self.parallelize(enumerator, options = {}, &block)
+        @@parallelizer ||= Parallelizer.new(@@threads)
+        @@parallelizer.parallelize(enumerator, options, &block)
+      end
+
+      def initialize(threads)
+        @tasks_mutex = Mutex.new
+        @tasks = []
+        @threads = []
+        1.upto(threads) do
+          @threads << Thread.new { worker_loop }
+        end
+      end
+
+      def parallelize(enumerator, options = {}, &block)
+        task = ParallelizedResults.new(enumerator, options, &block)
+        @tasks_mutex.synchronize do
+          @tasks << task
+        end
+        task
+      end
+
+      class ParallelizedResults
+        include Enumerable
+
+        def initialize(enumerator, options, &block)
+          @inputs = enumerator.to_a
+          @options = options
+          @block = block
+
+          @mutex = Mutex.new
+          @outputs = []
+          @status = []
+        end
+
+        def each
+          next_index = 0
+          while true
+            # Report any results that already exist
+            while @status.length > next_index && ([:finished, :exception].include?(@status[next_index]))
+              if @status[next_index] == :finished
+                if @options[:flatten]
+                  @outputs[next_index].each do |entry|
+                    yield entry
+                  end
+                else
+                  yield @outputs[next_index]
+                end
+              else
+                raise @outputs[next_index]
+              end
+              next_index = next_index + 1
+            end
+
+            # Pick up a result and process it, if there is one.  This ensures we
+            # move forward even if there are *zero* worker threads available.
+            if !process_input
+              # Exit if we're done.
+              if next_index >= @status.length
+                break
+              else
+                # Ruby 1.8 threading sucks.  Wait till we process more things.
+                sleep(0.05)
+              end
+            end
+          end
+        end
+
+        def process_input
+          # Grab the next one to process
+          index, input = @mutex.synchronize do
+            index = @status.length
+            if index >= @inputs.length
+              return nil
+            end
+            input = @inputs[index]
+            @status[index] = :started
+            [ index, input ]
+          end
+
+          begin
+            @outputs[index] = @block.call(input)
+            @status[index] = :finished
+          rescue Exception
+            @outputs[index] = $!
+            @status[index] = :exception
+          end
+          index
+        end
+      end
+
+      private
+
+      def worker_loop
+        while true
+          begin
+            task = @tasks[0]
+            if task
+              if !task.process_input
+                @tasks_mutex.synchronize do
+                  @tasks.delete(task)
+                end
+              end
+            else
+              # Ruby 1.8 threading sucks.  Wait a bit to see if another task comes in.
+              sleep(0.05)
+            end
+          rescue
+            puts "ERROR #{$!}"
+            puts $!.backtrace
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef/chef_fs/path_utils.rb

@@ -0,0 +1,90 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/chef_fs'
+require 'pathname'
+
+class Chef
+  module ChefFS
+    class PathUtils
+
+      # If you are in 'source', this is what you would have to type to reach 'dest'
+      # relative_to('/a/b/c/d/e', '/a/b/x/y') == '../../c/d/e'
+      # relative_to('/a/b', '/a/b') == '.'
+      def self.relative_to(dest, source)
+        # Skip past the common parts
+        source_parts = Chef::ChefFS::PathUtils.split(source)
+        dest_parts = Chef::ChefFS::PathUtils.split(dest)
+        i = 0
+        until i >= source_parts.length || i >= dest_parts.length || source_parts[i] != dest_parts[i]
+          i+=1
+        end
+        # dot-dot up from 'source' to the common ancestor, then
+        # descend to 'dest' from the common ancestor
+        result = Chef::ChefFS::PathUtils.join(*(['..']*(source_parts.length-i) + dest_parts[i,dest.length-i]))
+        result == '' ? '.' : result
+      end
+
+      def self.join(*parts)
+        return "" if parts.length == 0
+        # Determine if it started with a slash
+        absolute = parts[0].length == 0 || parts[0].length > 0 && parts[0] =~ /^#{regexp_path_separator}/
+        # Remove leading and trailing slashes from each part so that the join will work (and the slash at the end will go away)
+        parts = parts.map { |part| part.gsub(/^\/|\/$/, "") }
+        # Don't join empty bits
+        result = parts.select { |part| part != "" }.join("/")
+        # Put the / back on
+        absolute ? "/#{result}" : result
+      end
+
+      def self.split(path)
+        path.split(Regexp.new(regexp_path_separator))
+      end
+
+      def self.regexp_path_separator
+        Chef::ChefFS::windows? ? '[\/\\\\]' : '/'
+      end
+
+      # Given a path which may only be partly real (i.e. /x/y/z when only /x exists,
+      # or /x/y/*/blah when /x/y/z/blah exists), call File.realpath on the biggest
+      # part that actually exists.
+      #
+      # If /x is a symlink to /blarghle, and has no subdirectories, then:
+      # PathUtils.realest_path('/x/y/z') == '/blarghle/y/z'
+      # PathUtils.realest_path('/x/*/z') == '/blarghle/*/z'
+      # PathUtils.realest_path('/*/y/z') == '/*/y/z'
+      def self.realest_path(path)
+        path = Pathname.new(path)
+        begin
+          path.realpath.to_s
+        rescue Errno::ENOENT
+          dirname = path.dirname
+          if dirname
+            PathUtils.join(realest_path(dirname), path.basename.to_s)
+          else
+            path.to_s
+          end
+        end
+      end
+
+      def self.is_absolute?(path)
+        path =~ /^#{regexp_path_separator}/
+      end
+    end
+  end
+end