chef 11.4.4 → 11.6.0.hotfix.1

data/lib/chef/chef_fs.rb

@@ -1,11 +1,9 @@
-require 'chef/chef_fs/file_system/chef_server_root_dir'
-require 'chef/config'
-require 'chef/rest'
+require 'chef/platform'
 
 class Chef
   module ChefFS
     def self.windows?
-      false
+      Chef::Platform.windows?
     end
   end
 end

data/lib/chef/chef_fs/chef_fs_data_store.rb

@@ -0,0 +1,371 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef_zero/data_store/memory_store'
+require 'chef_zero/data_store/data_already_exists_error'
+require 'chef_zero/data_store/data_not_found_error'
+require 'chef/chef_fs/file_pattern'
+require 'chef/chef_fs/file_system'
+require 'chef/chef_fs/file_system/not_found_error'
+require 'chef/chef_fs/file_system/memory_root'
+
+class Chef
+  module ChefFS
+    class ChefFSDataStore
+      def initialize(chef_fs)
+        @chef_fs = chef_fs
+        @memory_store = ChefZero::DataStore::MemoryStore.new
+      end
+
+      def publish_description
+        "Reading and writing data to #{chef_fs.fs_description}"
+      end
+
+      attr_reader :chef_fs
+
+      def create_dir(path, name, *options)
+        if use_memory_store?(path)
+          @memory_store.create_dir(path, name, *options)
+        else
+          with_dir(path) do |parent|
+            parent.create_child(chef_fs_filename(path + [name]), nil)
+          end
+        end
+      end
+
+      def create(path, name, data, *options)
+        if use_memory_store?(path)
+          @memory_store.create(path, name, data, *options)
+
+        elsif path[0] == 'cookbooks' && path.length == 2
+          # Do nothing.  The entry gets created when the cookbook is created.
+
+        else
+          if !data.is_a?(String)
+            raise "set only works with strings"
+          end
+
+          with_dir(path) do |parent|
+            parent.create_child(chef_fs_filename(path + [name]), data)
+          end
+        end
+      end
+
+      def get(path, request=nil)
+        if use_memory_store?(path)
+          @memory_store.get(path)
+
+        elsif path[0] == 'file_store' && path[1] == 'repo'
+          entry = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path[2..-1].join('/'))
+          begin
+            entry.read
+          rescue Chef::ChefFS::FileSystem::NotFoundError => e
+            raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+          end
+
+        else
+          with_entry(path) do |entry|
+            if path[0] == 'cookbooks' && path.length == 3
+              # get /cookbooks/NAME/version
+              result = entry.chef_object.to_hash
+              result.each_pair do |key, value|
+                if value.is_a?(Array)
+                  value.each do |file|
+                    if file.is_a?(Hash) && file.has_key?('checksum')
+                      relative = ['file_store', 'repo', 'cookbooks']
+                      if Chef::Config.versioned_cookbooks
+                        relative << "#{path[1]}-#{path[2]}"
+                      else
+                        relative << path[1]
+                      end
+                      relative = relative + file[:path].split('/')
+                      file['url'] = ChefZero::RestBase::build_uri(request.base_uri, relative)
+                    end
+                  end
+                end
+              end
+              JSON.pretty_generate(result)
+
+            else
+              entry.read
+            end
+          end
+        end
+      end
+
+      def set(path, data, *options)
+        if use_memory_store?(path)
+          @memory_store.set(path, data, *options)
+        else
+          if !data.is_a?(String)
+            raise "set only works with strings: #{path} = #{data.inspect}"
+          end
+
+          # Write out the files!
+          if path[0] == 'cookbooks' && path.length == 3
+            write_cookbook(path, data, *options)
+          else
+            with_dir(path[0..-2]) do |parent|
+              parent.create_child(chef_fs_filename(path), data)
+            end
+          end
+        end
+      end
+
+      def delete(path)
+        if use_memory_store?(path)
+          @memory_store.delete(path)
+        else
+          with_entry(path) do |entry|
+            if path[0] == 'cookbooks' && path.length >= 3
+              entry.delete(true)
+            else
+              entry.delete
+            end
+          end
+        end
+      end
+
+      def delete_dir(path, *options)
+        if use_memory_store?(path)
+          @memory_store.delete_dir(path, *options)
+        else
+          with_entry(path) do |entry|
+            entry.delete(options.include?(:recursive))
+          end
+        end
+      end
+
+      def list(path)
+        if use_memory_store?(path)
+          @memory_store.list(path)
+
+        elsif path[0] == 'cookbooks' && path.length == 1
+          with_entry(path) do |entry|
+            begin
+              if Chef::Config.versioned_cookbooks
+                # /cookbooks/name-version -> /cookbooks/name
+                entry.children.map { |child| split_name_version(child.name)[0] }.uniq
+              else
+                entry.children.map { |child| child.name }
+              end
+            rescue Chef::ChefFS::FileSystem::NotFoundError
+              # If the cookbooks dir doesn't exist, we have no cookbooks (not 404)
+              []
+            end
+          end
+
+        elsif path[0] == 'cookbooks' && path.length == 2
+          if Chef::Config.versioned_cookbooks
+            # list /cookbooks/name = filter /cookbooks/name-version down to name
+            entry.children.map { |child| split_name_version(child.name) }.
+                           select { |name, version| name == path[1] }.
+                           map { |name, version| version }.to_a
+          else
+            # list /cookbooks/name = <single version>
+            version = get_single_cookbook_version(path)
+            [version]
+          end
+
+        else
+          with_entry(path) do |entry|
+            begin
+              entry.children.map { |c| zero_filename(c) }.sort
+            rescue Chef::ChefFS::FileSystem::NotFoundError
+              # /cookbooks, /data, etc. never return 404
+              if path_always_exists?(path)
+                []
+              else
+                raise
+              end
+            end
+          end
+        end
+      end
+
+      def exists?(path)
+        if use_memory_store?(path)
+          @memory_store.exists?(path)
+        else
+          path_always_exists?(path) || Chef::ChefFS::FileSystem.resolve_path(chef_fs, to_chef_fs_path(path)).exists?
+        end
+      end
+
+      def exists_dir?(path)
+        if use_memory_store?(path)
+          @memory_store.exists_dir?(path)
+        elsif path[0] == 'cookbooks' && path.length == 2
+          list([ path[0] ]).include?(path[1])
+        else
+          Chef::ChefFS::FileSystem.resolve_path(chef_fs, to_chef_fs_path(path)).exists?
+        end
+      end
+
+      private
+
+      def use_memory_store?(path)
+        return path[0] == 'sandboxes' || path[0] == 'file_store' && path[1] == 'checksums' || path == [ 'environments', '_default' ]
+      end
+
+      def write_cookbook(path, data, *options)
+        # Create a little Chef::ChefFS memory filesystem with the data
+        if Chef::Config.versioned_cookbooks
+          cookbook_path = "cookbooks/#{path[1]}-#{path[2]}"
+        else
+          cookbook_path = "cookbooks/#{path[1]}"
+        end
+        cookbook_fs = Chef::ChefFS::FileSystem::MemoryRoot.new('uploading')
+        cookbook = JSON.parse(data, :create_additions => false)
+        cookbook.each_pair do |key, value|
+          if value.is_a?(Array)
+            value.each do |file|
+              if file.is_a?(Hash) && file.has_key?('checksum')
+                file_data = @memory_store.get(['file_store', 'checksums', file['checksum']])
+                cookbook_fs.add_file("#{cookbook_path}/#{file['path']}", file_data)
+              end
+            end
+          end
+        end
+
+        # Use the copy/diff algorithm to copy it down so we don't destroy
+        # chefignored data.  This is terribly un-thread-safe.
+        Chef::ChefFS::FileSystem.copy_to(Chef::ChefFS::FilePattern.new("/#{cookbook_path}"), cookbook_fs, chef_fs, nil, {:purge => true})
+      end
+
+      def split_name_version(entry_name)
+        name_version = entry_name.split('-')
+        name = name_version[0..-2].join('-')
+        version = name_version[-1]
+        [name,version]
+      end
+
+      def to_chef_fs_path(path)
+        _to_chef_fs_path(path).join('/')
+      end
+
+      def chef_fs_filename(path)
+        _to_chef_fs_path(path)[-1]
+      end
+
+      def _to_chef_fs_path(path)
+        if path[0] == 'data'
+          path = path.dup
+          path[0] = 'data_bags'
+          if path.length >= 3
+            path[2] = "#{path[2]}.json"
+          end
+        elsif path[0] == 'cookbooks'
+          if path.length == 2
+            raise ChefZero::DataStore::DataNotFoundError.new(path)
+          elsif Chef::Config.versioned_cookbooks
+            if path.length >= 3
+              # cookbooks/name/version -> cookbooks/name-version
+              path = [ path[0], "#{path[1]}-#{path[2]}" ] + path[3..-1]
+            end
+          else
+            if path.length >= 3
+              # cookbooks/name/version/... -> /cookbooks/name/... iff metadata says so
+              version = get_single_cookbook_version(path)
+              if path[2] == version
+                path = path[0..1] + path[3..-1]
+              else
+                raise ChefZero::DataStore::DataNotFoundError.new(path)
+              end
+            end
+          end
+        elsif path.length == 2
+          path = path.dup
+          path[1] = "#{path[1]}.json"
+        end
+        path
+      end
+
+      def to_zero_path(entry)
+        path = entry.path.split('/')[1..-1]
+        if path[0] == 'data_bags'
+          path = path.dup
+          path[0] = 'data'
+          if path.length >= 3
+            path[2] = path[2][0..-6]
+          end
+
+        elsif path[0] == 'cookbooks'
+          if Chef::Config.versioned_cookbooks
+            # cookbooks/name-version/... -> cookbooks/name/version/...
+            if path.length >= 2
+              name, version = split_name_version(path[1])
+              path = [ path[0], name, version ] + path[2..-1]
+            end
+          else
+            if path.length >= 2
+              # cookbooks/name/... -> cookbooks/name/version/...
+              version = get_single_cookbook_version(path)
+              path = path[0..1] + [version] + path[2..-1]
+            end
+          end
+
+        elsif path.length == 2 && path[0] != 'cookbooks'
+          path = path.dup
+          path[1] = path[1][0..-6]
+        end
+        path
+      end
+
+      def zero_filename(entry)
+        to_zero_path(entry)[-1]
+      end
+
+      def path_always_exists?(path)
+        return path.length == 1 && %w(clients cookbooks data environments nodes roles users).include?(path[0])
+      end
+
+      def with_entry(path)
+        begin
+          yield Chef::ChefFS::FileSystem.resolve_path(chef_fs, to_chef_fs_path(path))
+        rescue Chef::ChefFS::FileSystem::NotFoundError => e
+          raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+        end
+      end
+
+      def with_dir(path)
+        begin
+          yield get_dir(_to_chef_fs_path(path), true)
+        rescue Chef::ChefFS::FileSystem::NotFoundError => e
+          raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e)
+        end
+      end
+
+      def get_dir(path, create=false)
+        result = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path.join('/'))
+        if result.exists?
+          result
+        elsif create
+          get_dir(path[0..-2], create).create_child(result.name, nil)
+        else
+          raise ChefZero::DataStore::DataNotFoundError.new(path)
+        end
+      end
+
+      def get_single_cookbook_version(path)
+        dir = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path[0..1].join('/'))
+        metadata = ChefZero::CookbookData.metadata_from(dir, path[1], nil, [])
+        metadata[:version] || '0.0.0'
+      end
+    end
+  end
+end

data/lib/chef/chef_fs/command_line.rb

@@ -17,89 +17,171 @@
 #
 
 require 'chef/chef_fs/file_system'
+require 'chef/chef_fs/file_system/operation_failed_error'
+require 'chef/chef_fs/file_system/operation_not_allowed_error'
 
 class Chef
   module ChefFS
     module CommandLine
-      def self.diff(pattern, a_root, b_root, recurse_depth, output_mode)
-        found_result = false
-        Chef::ChefFS::FileSystem.list_pairs(pattern, a_root, b_root) do |a, b|
-          existed = diff_entries(a, b, recurse_depth, output_mode) do |diff|
-            yield diff
-          end
-          found_result = true if existed
-        end
-        if !found_result && pattern.exact_path
-          yield "#{pattern}: No such file or directory on remote or local"
+
+      def self.diff_print(pattern, a_root, b_root, recurse_depth, output_mode, format_path = nil, diff_filter = nil, ui = nil)
+        if format_path.nil?
+          format_path = proc { |entry| entry.path_for_printing }
         end
-      end
 
-      # Diff two known entries (could be files or dirs)
-      def self.diff_entries(old_entry, new_entry, recurse_depth, output_mode)
-        # If both are directories
-        if old_entry.dir?
-          if new_entry.dir?
-            if recurse_depth == 0
-              if output_mode != :name_only && output_mode != :name_status
-                yield "Common subdirectories: #{old_entry.path}\n"
-              end
+        get_content = (output_mode != :name_only && output_mode != :name_status)
+        found_match = false
+        diff(pattern, a_root, b_root, recurse_depth, get_content).each do |type, old_entry, new_entry, old_value, new_value, error|
+          found_match = true unless type == :both_nonexistent
+          old_path = format_path.call(old_entry)
+          new_path = format_path.call(new_entry)
+
+          case type
+          when :common_subdirectories
+            if output_mode != :name_only && output_mode != :name_status
+              yield "Common subdirectories: #{new_path}\n"
+            end
+
+          when :directory_to_file
+            next if diff_filter && diff_filter !~ /T/
+            if output_mode == :name_only
+              yield "#{new_path}\n"
+            elsif output_mode == :name_status
+              yield "T\t#{new_path}\n"
             else
-              Chef::ChefFS::FileSystem.child_pairs(old_entry, new_entry).each do |old_child,new_child|
-                diff_entries(old_child, new_child,
-                             recurse_depth ? recurse_depth - 1 : nil, output_mode) do |diff|
-                  yield diff
-                end
-              end
+              yield "File #{old_path} is a directory while file #{new_path} is a regular file\n"
             end
 
-        # If old is a directory and new is a file
-          elsif new_entry.exists?
+          when :file_to_directory
+            next if diff_filter && diff_filter !~ /T/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "T\t#{new_entry.path_for_printing}\n"
+              yield "T\t#{new_path}\n"
             else
-              yield "File #{new_entry.path_for_printing} is a directory while file #{new_entry.path_for_printing} is a regular file\n"
+              yield "File #{old_path} is a regular file while file #{new_path} is a directory\n"
             end
 
-        # If old is a directory and new does not exist
-          elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+          when :deleted
+            next if diff_filter && diff_filter !~ /D/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "D\t#{new_entry.path_for_printing}\n"
+              yield "D\t#{new_path}\n"
+            elsif old_value
+              result = "diff --knife #{old_path} #{new_path}\n"
+              result << "deleted file\n"
+              result << diff_text(old_path, '/dev/null', old_value, '')
+              yield result
             else
-              yield "Only in #{old_entry.parent.path_for_printing}: #{old_entry.name}\n"
+              yield "Only in #{format_path.call(old_entry.parent)}: #{old_entry.name}\n"
             end
-          end
 
-        # If new is a directory and old is a file
-        elsif new_entry.dir?
-          if old_entry.exists?
+          when :added
+            next if diff_filter && diff_filter !~ /A/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "T\t#{new_entry.path_for_printing}\n"
+              yield "A\t#{new_path}\n"
+            elsif new_value
+              result = "diff --knife #{old_path} #{new_path}\n"
+              result << "new file\n"
+              result << diff_text('/dev/null', new_path, '', new_value)
+              yield result
             else
-              yield "File #{old_entry.path_for_printing} is a regular file while file #{old_entry.path_for_printing} is a directory\n"
+              yield "Only in #{format_path.call(new_entry.parent)}: #{new_entry.name}\n"
             end
 
-        # If new is a directory and old does not exist
-          elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+          when :modified
+            next if diff_filter && diff_filter !~ /M/
             if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
+              yield "#{new_path}\n"
             elsif output_mode == :name_status
-              yield "A\t#{new_entry.path_for_printing}\n"
+              yield "M\t#{new_path}\n"
+            else
+              result = "diff --knife #{old_path} #{new_path}\n"
+              result << diff_text(old_path, new_path, old_value, new_value)
+              yield result
+            end
+
+          when :both_nonexistent
+          when :added_cannot_upload
+          when :deleted_cannot_download
+          when :same
+            # Skip these silently
+          when :error
+            if error.is_a?(Chef::ChefFS::FileSystem::OperationFailedError)
+              ui.error "#{format_path.call(error.entry)} failed to #{error.operation}: #{error.message}" if ui
+              error = true
+            elsif error.is_a?(Chef::ChefFS::FileSystem::OperationNotAllowedError)
+              ui.error "#{format_path.call(error.entry)} #{error.reason}." if ui
             else
-              yield "Only in #{new_entry.parent.path_for_printing}: #{new_entry.name}\n"
+              raise error
             end
           end
+        end
+        if !found_match
+          ui.error "#{pattern}: No such file or directory on remote or local" if ui
+          error = true
+        end
+        error
+      end
+
+      def self.diff(pattern, old_root, new_root, recurse_depth, get_content)
+        Chef::ChefFS::Parallelizer.parallelize(Chef::ChefFS::FileSystem.list_pairs(pattern, old_root, new_root), :flatten => true) do |old_entry, new_entry|
+          diff_entries(old_entry, new_entry, recurse_depth, get_content)
+        end
+      end
+
+      # Diff two known entries (could be files or dirs)
+      def self.diff_entries(old_entry, new_entry, recurse_depth, get_content)
+        # If both are directories
+        if old_entry.dir?
+          if new_entry.dir?
+            if recurse_depth == 0
+              return [ [ :common_subdirectories, old_entry, new_entry ] ]
+            else
+              return Chef::ChefFS::Parallelizer.parallelize(Chef::ChefFS::FileSystem.child_pairs(old_entry, new_entry), :flatten => true) do |old_child, new_child|
+                Chef::ChefFS::CommandLine.diff_entries(old_child, new_child, recurse_depth ? recurse_depth - 1 : nil, get_content)
+              end
+            end
+
+          # If old is a directory and new is a file
+          elsif new_entry.exists?
+            return [ [ :directory_to_file, old_entry, new_entry ] ]
+
+          # If old is a directory and new does not exist
+          elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+            return [ [ :deleted, old_entry, new_entry ] ]
+
+          # If the new entry does not and *cannot* exist, report that.
+          else
+            return [ [ :new_cannot_upload, old_entry, new_entry ] ]
+          end
+
+        # If new is a directory and old is a file
+        elsif new_entry.dir?
+          if old_entry.exists?
+            return [ [ :file_to_directory, old_entry, new_entry ] ]
+
+          # If new is a directory and old does not exist
+          elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+            return [ [ :added, old_entry, new_entry ] ]
+
+          # If the new entry does not and *cannot* exist, report that.
+          else
+            return [ [ :old_cannot_upload, old_entry, new_entry ] ]
+          end
 
         # Neither is a directory, so they are diffable with file diff
         else
           are_same, old_value, new_value = Chef::ChefFS::FileSystem.compare(old_entry, new_entry)
           if are_same
-            return old_value != :none
+            if old_value == :none
+              return [ [ :both_nonexistent, old_entry, new_entry ] ]
+            else
+              return [ [ :same, old_entry, new_entry ] ]
+            end
           else
             if old_value == :none
               old_exists = false
@@ -108,6 +190,7 @@ class Chef
             else
               old_exists = true
             end
+
             if new_value == :none
               new_exists = false
             elsif new_value.nil?
@@ -119,24 +202,14 @@ class Chef
             # If one of the files doesn't exist, we only want to print the diff if the
             # other file *could be uploaded/downloaded*.
             if !old_exists && !old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
-              return true
+              return [ [ :old_cannot_upload, old_entry, new_entry ] ]
             end
             if !new_exists && !new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
-              return true
+              return [ [ :new_cannot_upload, old_entry, new_entry ] ]
             end
 
-            if output_mode == :name_only
-              yield "#{new_entry.path_for_printing}\n"
-            elsif output_mode == :name_status
-              if old_value == :none || (old_value == nil && !old_entry.exists?)
-                yield "A\t#{new_entry.path_for_printing}\n"
-              elsif new_value == :none
-                yield "D\t#{new_entry.path_for_printing}\n"
-              else
-                yield "M\t#{new_entry.path_for_printing}\n"
-              end
-            else
-              # If we haven't read the values yet, get them now.
+            if get_content
+              # If we haven't read the values yet, get them now so that they can be diffed
               begin
                 old_value = old_entry.read if old_value.nil?
               rescue Chef::ChefFS::FileSystem::NotFoundError
@@ -147,27 +220,19 @@ class Chef
               rescue Chef::ChefFS::FileSystem::NotFoundError
                 new_value = :none
               end
+            end
 
-              old_path = old_entry.path_for_printing
-              new_path = new_entry.path_for_printing
-              result = ''
-              result << "diff --knife #{old_path} #{new_path}\n"
-              if old_value == :none
-                result << "new file\n"
-                old_path = "/dev/null"
-                old_value = ''
-              end
-              if new_value == :none
-                result << "deleted file\n"
-                new_path = "/dev/null"
-                new_value = ''
-              end
-              result << diff_text(old_path, new_path, old_value, new_value)
-              yield result
+            if old_value == :none || (old_value == nil && !old_entry.exists?)
+              return [ [ :added, old_entry, new_entry, old_value, new_value ] ]
+            elsif new_value == :none
+              return [ [ :deleted, old_entry, new_entry, old_value, new_value ] ]
+            else
+              return [ [ :modified, old_entry, new_entry, old_value, new_value ] ]
             end
           end
         end
-        return true
+      rescue Chef::ChefFS::FileSystem::FileSystemError => e
+        return [ [ :error, old_entry, new_entry, nil, nil, e ] ]
       end
 
       private
@@ -191,19 +256,6 @@ class Chef
       end
 
       def self.diff_text(old_path, new_path, old_value, new_value)
-        # Reformat JSON for a nicer diff.
-        if old_path =~ /\.json$/
-          begin
-            reformatted_old_value = canonicalize_json(old_value)
-            reformatted_new_value = canonicalize_json(new_value)
-            old_value = reformatted_old_value
-            new_value = reformatted_new_value
-          rescue
-            # If JSON parsing fails, we just won't change any values and fall back
-            # to normal diff.
-          end
-        end
-
         # Copy to tempfiles before diffing
         # TODO don't copy things that are already in files!  Or find an in-memory diff algorithm
         begin