knife-essentials 0.1

data/lib/chef/knife/show.rb

@@ -0,0 +1,30 @@
+require 'chef_fs/knife'
+require 'chef_fs/file_system'
+
+class Chef
+  class Knife
+    class Show < ChefFS::Knife
+      banner "show [PATTERN1 ... PATTERNn]"
+
+      def run
+        # Get the matches (recursively)
+        pattern_args.each do |pattern|
+          ChefFS::FileSystem.list(chef_fs, pattern) do |result|
+            if result.dir?
+              STDERR.puts "#{format_path(result.path)}: is a directory" if pattern.exact_path
+            else
+              begin
+                value = result.read
+                puts "#{format_path(result.path)}:"
+                output(format_for_display(result.read))
+              rescue ChefFS::FileSystem::NotFoundException
+                STDERR.puts "#{format_path(result.path)}: No such file or directory" if pattern.exact_path
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/chef_fs.rb

@@ -0,0 +1,9 @@
+require 'chef_fs/file_system/chef_server_root_dir'
+require 'chef/config'
+require 'chef/rest'
+
+module ChefFS
+  def self.windows?
+    false
+  end
+end

data/lib/chef_fs/command_line.rb

@@ -0,0 +1,91 @@
+require 'chef_fs/diff'
+
+module ChefFS
+  module CommandLine
+    def self.diff(pattern, a_root, b_root, recurse_depth)
+      found_result = false
+      ChefFS::Diff::diffable_leaves_from_pattern(pattern, a_root, b_root, recurse_depth) do |a_leaf, b_leaf|
+        found_result = true
+        diff = diff_leaves(a_leaf, b_leaf)
+        yield diff if diff != ''
+      end
+      if !found_result && pattern.exact_path
+        yield "#{pattern}: No such file or directory on remote or local"
+      end
+    end
+
+    private
+
+    # Diff two known leaves (could be files or dirs)
+    def self.diff_leaves(old_file, new_file)
+      result = ''
+      # If both are directories
+      # If old is a directory and new is a file
+      # If old is a directory and new does not exist
+      if old_file.dir?
+        if new_file.dir?
+          result << "Common subdirectories: #{old_file.path}\n"
+        elsif new_file.exists?
+          result << "File #{new_file.path_for_printing} is a directory while file #{new_file.path_for_printing} is a regular file\n"
+        else
+          result << "Only in #{old_file.parent.path_for_printing}: #{old_file.name}\n"
+        end
+
+      # If new is a directory and old does not exist
+      # If new is a directory and old is a file
+      elsif new_file.dir?
+        if old_file.exists?
+          result << "File #{old_file.path_for_printing} is a regular file while file #{old_file.path_for_printing} is a directory\n"
+        else
+          result << "Only in #{new_file.parent.path_for_printing}: #{new_file.name}\n"
+        end
+
+      else
+        # Neither is a directory, so they are diffable with file diff
+        different, old_value, new_value = ChefFS::Diff::diff_files(old_file, new_file)
+        if different
+          old_path = old_file.path_for_printing
+          new_path = new_file.path_for_printing
+          result << "diff --knife #{old_path} #{new_path}\n"
+          if !old_value
+            result << "new file\n"
+            old_path = "/dev/null"
+            old_value = ''
+          end
+          if !new_value
+            result << "deleted file\n"
+            new_path = "/dev/null"
+            new_value = ''
+          end
+          result << diff_text(old_path, new_path, old_value, new_value)
+        end
+      end
+      return result
+    end
+
+    def self.diff_text(old_path, new_path, old_value, new_value)
+      # Copy to tempfiles before diffing
+      # TODO don't copy things that are already in files!  Or find an in-memory diff algorithm
+      begin
+        new_tempfile = Tempfile.new("new")
+        new_tempfile.write(new_value)
+        new_tempfile.close
+
+        begin
+          old_tempfile = Tempfile.new("old")
+          old_tempfile.write(old_value)
+          old_tempfile.close
+
+          result = `diff -u #{old_tempfile.path} #{new_tempfile.path}`
+          result = result.gsub(/^--- #{old_tempfile.path}/, "--- #{old_path}")
+          result = result.gsub(/^\+\+\+ #{new_tempfile.path}/, "+++ #{new_path}")
+          result
+        ensure
+          old_tempfile.close!
+        end
+      ensure
+        new_tempfile.close!
+      end
+    end
+  end
+end

data/lib/chef_fs/diff.rb

@@ -0,0 +1,158 @@
+require 'chef_fs/file_system'
+require 'chef/json_compat'
+require 'tempfile'
+require 'fileutils'
+require 'digest/md5'
+require 'set'
+
+module ChefFS
+  class Diff
+    def self.calc_checksum(value)
+      return nil if value == nil
+      Digest::MD5.hexdigest(value)
+    end
+
+    def self.diff_files(old_file, new_file)
+      #
+      # Short-circuit expensive comparison (could be an extra network
+      # request) if a pre-calculated checksum is there
+      #
+      if new_file.respond_to?(:checksum)
+        new_checksum = new_file.checksum
+      end
+      if old_file.respond_to?(:checksum)
+        old_checksum = old_file.checksum
+      end
+
+      old_value = :not_retrieved
+      new_value = :not_retrieved
+
+      if old_checksum || new_checksum
+        if !old_checksum
+          old_value = read_file_value(old_file)
+          if old_value
+            old_checksum = calc_checksum(old_value)
+          end
+        end
+        if !new_checksum
+          new_value = read_file_value(new_file)
+          if new_value
+            new_checksum = calc_checksum(new_value)
+          end
+        end
+
+        # If the checksums are the same, they are the same.  Return.
+        return false if old_checksum == new_checksum
+      end
+
+      #
+      # Grab the values if we don't have them already from calculating checksum
+      #
+      old_value = read_file_value(old_file) if old_value == :not_retrieved
+      new_value = read_file_value(new_file) if new_value == :not_retrieved
+
+      return false if old_value == new_value
+      return false if old_value && new_value && !context_aware_diff(old_file, new_file, old_value, new_value)
+      return [ true, old_value, new_value ]
+    end
+
+    def self.context_aware_diff(old_file, new_file, old_value, new_value)
+      # TODO handle errors in reading JSON
+      if old_file.content_type == :json || new_file.content_type == :json
+        new_value = Chef::JSONCompat.from_json(new_value)
+        old_value = Chef::JSONCompat.from_json(old_value)
+
+        old_value != new_value
+      else
+        true
+      end
+    end
+
+    # Gets all common leaves, recursively, starting from the results of
+    # a pattern search on two roots.
+    #
+    # ==== Attributes
+    #
+    # * +pattern+ - a ChefFS::FilePattern representing the search you want to
+    #   do on both roots.
+    # * +a_root+ - the first root.
+    # * +b_root+ - 
+    # * +recurse_depth+ - the maximum number of directories to recurse from each
+    #   pattern result.  +0+ will cause pattern results to be immediately returned.
+    #   +nil+ means recurse infinitely to find all leaves.
+    #
+    def self.diffable_leaves_from_pattern(pattern, a_root, b_root, recurse_depth)
+      # Make sure everything on the server is also on the filesystem, and diff
+      found_paths = Set.new
+      ChefFS::FileSystem.list(a_root, pattern) do |a|
+        found_paths << a.path
+        b = ChefFS::FileSystem.resolve_path(b_root, a.path)
+        diffable_leaves(a, b, recurse_depth) do |a_leaf, b_leaf|
+          yield [ a_leaf, b_leaf ]
+        end
+      end
+
+      # Check the outer regex pattern to see if it matches anything on the
+      # filesystem that isn't on the server
+      ChefFS::FileSystem.list(b_root, pattern) do |b|
+        if !found_paths.include?(b.path)
+          a = ChefFS::FileSystem.resolve_path(a_root, b.path)
+          yield [ a, b ]
+        end
+      end
+    end
+
+    # Gets all common leaves, recursively, from a pair of directories or files.  It
+    # recursively descends into all children of +a+ and +b+, yielding equivalent
+    # pairs (common children with the same name) when it finds:
+    # * +a+ or +b+ is not a directory.
+    # * Both +a+ and +b+ are empty.
+    # * It reaches +recurse_depth+ depth in the tree.
+    #
+    # This method will *not* check whether files exist, nor will it actually diff
+    # the contents of files.
+    #
+    # ==== Attributes
+    #
+    # +a+ - the first directory to recursively scan
+    # +b+ - the second directory to recursively scan, in tandem with +a+
+    # +recurse_depth - the maximum number of directories to go down.  +0+ will
+    # cause +a+ and +b+ to be immediately returned.  +nil+ means recurse
+    # infinitely.
+    #
+    def self.diffable_leaves(a, b, recurse_depth)
+      # If we have children, recurse into them and diff the children instead of returning ourselves.
+      if recurse_depth != 0 && a.dir? && b.dir? && a.children.length > 0 && b.children.length > 0
+        a_children_names = Set.new
+        a.children.each do |a_child|
+          a_children_names << a_child.name
+          diffable_leaves(a_child, b.child(a_child.name), recurse_depth ? recurse_depth - 1 : nil) do |a_leaf, b_leaf|
+            yield [ a_leaf, b_leaf ]
+          end
+        end
+
+        # Check b for children that aren't in a
+        b.children.each do |b_child|
+          if !a_children_names.include?(b_child.name)
+            yield [ a.child(b_child.name), b_child ]
+          end
+        end
+        return
+      end
+
+      # Otherwise, this is a leaf we must diff.
+      yield [a, b]
+    end
+
+    private
+
+    def self.read_file_value(file)
+      begin
+        return file.read
+      rescue ChefFS::FileSystem::NotFoundException
+        return nil
+      end
+    end
+  end
+end
+

data/lib/chef_fs/file_pattern.rb

@@ -0,0 +1,292 @@
+require 'chef_fs'
+require 'chef_fs/path_utils'
+
+module ChefFS
+  #
+  # Represents a glob pattern.  This class is designed so that it can
+  # match arbitrary strings, and tell you about partial matches.
+  #
+  # Examples:
+  # * <tt>a*z</tt>
+  #   - Matches <tt>abcz</tt>
+  #   - Does not match <tt>ab/cd/ez</tt>
+  #   - Does not match <tt>xabcz</tt>
+  # * <tt>a**z</tt>
+  #   - Matches <tt>abcz</tt>
+  #   - Matches <tt>ab/cd/ez</tt>
+  #
+  # Special characters supported:
+  # * <tt>/</tt> (and <tt>\\</tt> on Windows) - directory separators
+  # * <tt>\*</tt> - match zero or more characters (but not directory separators)
+  # * <tt>\*\*</tt> - match zero or more characters, including directory separators
+  # * <tt>?</tt> - match exactly one character (not a directory separator)
+  # Only on Unix:
+  # * <tt>[abc0-9]</tt> - match one of the included characters
+  # * <tt>\\<character></tt> - escape character: match the given character
+  #
+  class FilePattern
+    # Initialize a new FilePattern with the pattern string.
+    #
+    # Raises +ArgumentError+ if empty file pattern is specified
+    def initialize(pattern)
+      @pattern = pattern
+    end
+
+    # The pattern string.
+    attr_reader :pattern
+
+    # Reports whether this pattern could match children of <tt>path</tt>.
+    # If the pattern doesn't match the path up to this point or
+    # if it matches and doesn't allow further children, this will
+    # return <tt>false</tt>.
+    #
+    # ==== Attributes
+    #
+    # * +path+ - a path to check
+    #
+    # ==== Examples
+    #
+    #   abc/def.could_match_children?('abc') == true
+    #   abc.could_match_children?('abc') == false
+    #   abc/def.could_match_children?('x') == false
+    #   a**z.could_match_children?('ab/cd') == true
+    def could_match_children?(path)
+      return false if path == '' # Empty string is not a path
+
+      argument_is_absolute = !!(path[0] =~ /^#{ChefFS::PathUtils::regexp_path_separator}/)
+      return false if is_absolute != argument_is_absolute
+      path = path[1,path.length-1] if argument_is_absolute
+
+      path_parts = ChefFS::PathUtils::split(path)
+      # If the pattern is shorter than the path (or same size), children will be larger than the pattern, and will not match.
+      return false if regexp_parts.length <= path_parts.length && !has_double_star
+      # If the path doesn't match up to this point, children won't match either.
+      return false if path_parts.zip(regexp_parts).any? { |part,regexp| !regexp.nil? && !regexp.match(part) }
+      # Otherwise, it's possible we could match: the path matches to this point, and the pattern is longer than the path.
+      # TODO There is one edge case where the double star comes after some characters like abc**def--we could check whether the next
+      # bit of path starts with abc in that case.
+      return true
+    end
+
+    # Returns the immediate child of a path that would be matched
+    # if this FilePattern was applied.  If more than one child
+    # could match, this method returns nil.
+    #
+    # ==== Attributes
+    #
+    # * +path+ - The path to look for an exact child name under.
+    #
+    # ==== Returns
+    #
+    # The next directory in the pattern under the given path.
+    # If the directory part could match more than one child, it
+    # returns +nil+.
+    #
+    # ==== Examples
+    #
+    #   abc/def.exact_child_name_under('abc') == 'def'
+    #   abc/def/ghi.exact_child_name_under('abc') == 'def'
+    #   abc/*/ghi.exact_child_name_under('abc') == nil
+    #   abc/*/ghi.exact_child_name_under('abc/def') == 'ghi'
+    #   abc/**/ghi.exact_child_name_under('abc/def') == nil
+    # 
+    # This method assumes +could_match_children?(path)+ is +true+.
+    def exact_child_name_under(path)
+      path = path[1,path.length-1] if !!(path[0] =~ /^#{ChefFS::PathUtils::regexp_path_separator}/)
+      dirs_in_path = ChefFS::PathUtils::split(path).length
+      return nil if exact_parts.length <= dirs_in_path
+      return exact_parts[dirs_in_path]
+    end
+
+    # If this pattern represents an exact path, returns the exact path.
+    #
+    #   abc/def.exact_path == 'abc/def'
+    #   abc/*def.exact_path == 'abc/def'
+    #   abc/x\\yz.exact_path == 'abc/xyz'
+    def exact_path
+      return nil if has_double_star || exact_parts.any? { |part| part.nil? }
+      result = ChefFS::PathUtils::join(*exact_parts)
+      is_absolute ? ChefFS::PathUtils::join('', result) : result
+    end
+
+    # Returns the normalized version of the pattern, with / as the directory
+    # separator, and "." and ".." removed.
+    #
+    # This does not presently change things like \b to b, but in the future
+    # it might.
+    def normalized_pattern
+      calculate
+      @normalized_pattern
+    end
+
+    # Tell whether this pattern matches absolute, or relative paths
+    def is_absolute
+      calculate
+      @is_absolute
+    end
+
+    # Returns <tt>true+ if this pattern matches the path, <tt>false+ otherwise.
+    #
+    #   abc/*/def.match?('abc/foo/def') == true
+    #   abc/*/def.match?('abc/foo') == false
+    def match?(path)
+      argument_is_absolute = !!(path[0] =~ /^#{ChefFS::PathUtils::regexp_path_separator}/)
+      return false if is_absolute != argument_is_absolute
+      path = path[1,path.length-1] if argument_is_absolute
+      !!regexp.match(path)
+    end
+
+    # Returns the string pattern
+    def to_s
+      pattern
+    end
+
+    # Given a relative file pattern and a directory, makes a new file pattern
+    # starting with the directory.
+    #
+    #   FilePattern.relative_to('/usr/local', 'bin/*grok') == FilePattern.new('/usr/local/bin/*grok')
+    #
+    # BUG: this does not support patterns starting with <tt>..</tt>
+    def self.relative_to(dir, pattern)
+      return FilePattern.new(pattern) if pattern =~ /^#{ChefFS::PathUtils::regexp_path_separator}/
+      FilePattern.new(ChefFS::PathUtils::join(dir, pattern))
+    end
+
+  private
+
+    def regexp
+      calculate
+      @regexp
+    end
+
+    def regexp_parts
+      calculate
+      @regexp_parts
+    end
+
+    def exact_parts
+      calculate
+      @exact_parts
+    end
+
+    def has_double_star
+      calculate
+      @has_double_star
+    end
+
+    def calculate
+      if !@regexp
+        @is_absolute = !!(@pattern =~ /^#{ChefFS::PathUtils::regexp_path_separator}/)
+
+        full_regexp_parts = []
+        normalized_parts = []
+        @regexp_parts = []
+        @exact_parts = []
+        @has_double_star = false
+
+        ChefFS::PathUtils::split(pattern).each do |part|
+          regexp, exact, has_double_star = FilePattern::pattern_to_regexp(part)
+          if has_double_star
+            @has_double_star = true
+          end
+
+          # Skip // and /./ (pretend it's not there)
+          if exact == '' || exact == '.'
+            next
+          end
+
+          # Back up when you see .. (unless the prior part has ** in it, in which case .. must be preserved)
+          if exact == '..'
+            if @is_absolute && normalized_parts.length == 0
+              # If we are at the root, just pretend the .. isn't there
+              next
+            elsif normalized_parts.length > 0
+              regexp_prev, exact_prev, has_double_star_prev = FilePattern.pattern_to_regexp(normalized_parts[-1])
+              if has_double_star_prev
+                raise ArgumentError, ".. overlapping a ** is unsupported"
+              end
+              full_regexp_parts.pop
+              normalized_parts.pop
+              if !@has_double_star
+                @regexp_parts.pop
+                @exact_parts.pop
+              end
+              next
+            end
+          end
+
+          # Build up the regexp
+          full_regexp_parts << regexp
+          normalized_parts << part
+          if !@has_double_star
+            @regexp_parts << Regexp.new("^#{regexp}$")
+            @exact_parts << exact
+          end
+        end
+
+        @regexp = Regexp.new("^#{full_regexp_parts.join(ChefFS::PathUtils::regexp_path_separator)}$")
+        @normalized_pattern = ChefFS::PathUtils.join(*normalized_parts)
+        @normalized_pattern = ChefFS::PathUtils.join('', @normalized_pattern) if @is_absolute
+      end
+    end
+
+    def self.pattern_special_characters
+      if ChefFS::windows?
+        @pattern_special_characters ||= /(\*\*|\*|\?|[\*\?\.\|\(\)\[\]\{\}\+\\\\\^\$])/
+      else
+        # Unix also supports character regexes and backslashes
+        @pattern_special_characters ||= /(\\.|\[[^\]]+\]|\*\*|\*|\?|[\*\?\.\|\(\)\[\]\{\}\+\\\\\^\$])/
+      end
+      @pattern_special_characters
+    end
+
+    def self.regexp_escape_characters
+      [ '[', '\\', '^', '$', '.', '|', '?', '*', '+', '(', ')', '{', '}' ]
+    end
+
+    def self.pattern_to_regexp(pattern)
+      regexp = ""
+      exact = ""
+      has_double_star = false
+      pattern.split(pattern_special_characters).each_with_index do |part, index|
+        # Odd indexes from the split are symbols.  Even are normal bits.
+        if index % 2 == 0
+          exact << part if !exact.nil?
+          regexp << part
+        else
+          case part
+          # **, * and ? happen on both platforms.
+          when '**'
+            exact = nil
+            has_double_star = true
+            regexp << '.*'
+          when '*'
+            exact = nil
+            regexp << '[^\/]*'
+          when '?'
+            exact = nil
+            regexp << '.'
+          else
+            if part[0] == '\\' && part.length == 2
+              # backslash escapes are only supported on Unix, and are handled here by leaving the escape on (it means the same thing in a regex)
+              exact << part[1] if !exact.nil?
+              if regexp_escape_characters.include?(part[1])
+                regexp << part
+              else
+                regexp << part[1]
+              end
+            elsif part[0] == '[' && part.length > 1
+              # [...] happens only on Unix, and is handled here by *not* backslashing (it means the same thing in and out of regex)
+              exact = nil
+              regexp << part
+            else
+              exact += part if !exact.nil?
+              regexp << "\\#{part}"
+            end
+          end
+        end
+      end
+      [regexp, exact, has_double_star]
+    end
+  end
+end