chef 0.9.6 → 0.9.8.beta.1

data/lib/chef/cookbook/metadata.rb

@@ -26,6 +26,11 @@ require 'chef/cookbook/metadata/version'
 class Chef
   class Cookbook
     class Metadata
+
+      COMPARISON_FIELDS = [ :name, :description, :long_description, :maintainer,
+                            :maintainer_email, :license, :platforms, :dependencies,
+                            :recommendations, :suggestions, :conflicting, :providing,
+                            :replacing, :attributes, :groupings, :recipes, :version]
     
       include Chef::Mixin::CheckHelper
       include Chef::Mixin::ParamsValidate
@@ -83,6 +88,12 @@ class Chef
         end
       end
 
+      def ==(other)
+        COMPARISON_FIELDS.inject(true) do |equal_so_far, field|
+          equal_so_far && other.respond_to?(field) && (other.send(field) == send(field))
+        end
+      end
+
       # Sets the cookbooks maintainer, or returns it.
       #
       # === Parameters
@@ -330,7 +341,7 @@ class Chef
             :description => { :kind_of => String },
             :choice => { :kind_of => [ Array ], :default => [] },
             :calculated => { :equal_to => [ true, false ], :default => false },
-            :type => { :equal_to => [ "string", "array", "hash" ], :default => "string" },
+            :type => { :equal_to => [ "string", "array", "hash", "symbol" ], :default => "string" },
             :required => { :equal_to => [ "required", "recommended", "optional", true, false ], :default => "optional" },
             :recipes => { :kind_of => [ Array ], :default => [] },
             :default => { :kind_of => [ String, Array, Hash ] }

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -0,0 +1,211 @@
+#
+# Author:: Stanislav Vitvitskiy
+# Author:: Nuo Yan (nuo@opscode.com)
+# Author:: Christopher Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2009, 2010 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'net/http'
+require 'mixlib/authentication/signedheaderauth'
+require 'openssl'
+
+# inspired by from http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
+class Chef
+  class CookbookSiteStreamingUploader
+
+    DefaultHeaders = { 'accept' => 'application/json', 'x-chef-version' => ::Chef::VERSION }
+
+    class << self
+
+      def post(to_url, user_id, secret_key_filename, params = {}, headers = {})
+        make_request(:post, to_url, user_id, secret_key_filename, params, headers)
+      end
+
+      def put(to_url, user_id, secret_key_filename, params = {}, headers = {})
+        make_request(:put, to_url, user_id, secret_key_filename, params, headers)
+      end
+
+      def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {})
+        boundary = '----RubyMultipartClient' + rand(1000000).to_s + 'ZZZZZ'
+        parts = []
+        content_file = nil
+
+        timestamp = Time.now.utc.iso8601
+        secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename))
+
+        unless params.nil? || params.empty?
+          params.each do |key, value|
+            if value.kind_of?(File)
+              content_file = value
+              filepath = value.path
+              filename = File.basename(filepath)
+              parts << StringPart.new( "--" + boundary + "\r\n" +
+                                       "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" +
+                                       "Content-Type: application/octet-stream\r\n\r\n")
+              parts << StreamPart.new(value, File.size(filepath))
+              parts << StringPart.new("\r\n")
+            else
+              parts << StringPart.new( "--" + boundary + "\r\n" +
+                                       "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n")
+              parts << StringPart.new(value.to_s + "\r\n")
+            end
+          end
+          parts << StringPart.new("--" + boundary + "--\r\n")
+        end
+
+        body_stream = MultipartStream.new(parts)
+
+        timestamp = Time.now.utc.iso8601
+
+        url = URI.parse(to_url)
+
+        Chef::Log.logger.debug("Signing: method: #{http_verb}, path: #{url.path}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}")
+
+        # We use the body for signing the request if the file parameter
+        # wasn't a valid file or wasn't included. Extract the body (with
+        # multi-part delimiters intact) to sign the request.
+        # TODO: tim: 2009-12-28: It'd be nice to remove this special case, and
+        # always hash the entire request body. In the file case it would just be
+        # expanded multipart text - the entire body of the POST.
+        content_body = parts.inject("") { |result,part| result + part.read(0, part.size) }
+        content_file.rewind if content_file # we consumed the file for the above operation, so rewind it.
+
+        signing_options = {
+          :http_method=>http_verb,
+          :path=>url.path,
+          :user_id=>user_id,
+          :timestamp=>timestamp}
+        (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || ""))
+
+        headers.merge!(Mixlib::Authentication::SignedHeaderAuth.signing_object(signing_options).sign(secret_key))
+
+        content_file.rewind if content_file
+
+        # net/http doesn't like symbols for header keys, so we'll to_s each one just in case
+        headers = DefaultHeaders.merge(Hash[*headers.map{ |k,v| [k.to_s, v] }.flatten])
+
+        req = case http_verb
+              when :put
+                Net::HTTP::Put.new(url.path, headers)
+              when :post
+                Net::HTTP::Post.new(url.path, headers)
+              end
+        req.content_length = body_stream.size
+        req.content_type = 'multipart/form-data; boundary=' + boundary unless parts.empty?
+        req.body_stream = body_stream
+
+        http = Net::HTTP.new(url.host, url.port)
+        if url.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        res = http.request(req)
+        #res = http.start {|http_proc| http_proc.request(req) }
+
+        # alias status to code and to_s to body for test purposes
+        # TODO: stop the following madness!
+        class << res
+          alias :to_s :body
+
+          # BUGBUG this makes the response compatible with what respsonse_steps expects to test headers (response.headers[] -> response[])
+          def headers
+            self
+          end
+
+          def status
+            code.to_i
+          end
+        end
+        res
+      end
+
+    end
+
+    class StreamPart
+      def initialize(stream, size)
+        @stream, @size = stream, size
+      end
+
+      def size
+        @size
+      end
+
+      # read the specified amount from the stream
+      def read(offset, how_much)
+        @stream.read(how_much)
+      end
+    end
+
+    class StringPart
+      def initialize(str)
+        @str = str
+      end
+
+      def size
+        @str.length
+      end
+
+      # read the specified amount from the string startiung at the offset
+      def read(offset, how_much)
+        @str[offset, how_much]
+      end
+    end
+
+    class MultipartStream
+      def initialize(parts)
+        @parts = parts
+        @part_no = 0
+        @part_offset = 0
+      end
+
+      def size
+        @parts.inject(0) {|size, part| size + part.size}
+      end
+
+      def read(how_much)
+        return nil if @part_no >= @parts.size
+
+        how_much_current_part = @parts[@part_no].size - @part_offset
+
+        how_much_current_part = if how_much_current_part > how_much
+                                  how_much
+                                else
+                                  how_much_current_part
+                                end
+
+        how_much_next_part = how_much - how_much_current_part
+
+        current_part = @parts[@part_no].read(@part_offset, how_much_current_part)
+
+        # recurse into the next part if the current one was not large enough
+        if how_much_next_part > 0
+          @part_no += 1
+          @part_offset = 0
+          next_part = read(how_much_next_part)
+          current_part + if next_part
+                           next_part
+                         else
+                           ''
+                         end
+        else
+          @part_offset += how_much_current_part
+          current_part
+        end
+      end
+    end
+
+  end
+end

data/lib/chef/cookbook_uploader.rb

@@ -0,0 +1,102 @@
+require 'rest_client'
+require 'chef/cookbook_loader'
+require 'chef/cache/checksum'
+require 'chef/sandbox'
+require 'chef/cookbook_version'
+require 'chef/cookbook/syntax_check'
+require 'chef/cookbook/file_system_file_vendor'
+
+class Chef
+  class CookbookUploader
+    class << self
+
+      def upload_cookbook(cookbook)
+        Chef::Log.info("Saving #{cookbook.name}")
+
+        rest = Chef::REST.new(Chef::Config[:chef_server_url])
+
+        # Syntax Check
+        validate_cookbook(cookbook)
+        # Generate metadata.json from metadata.rb
+        build_metadata(cookbook)
+
+        # generate checksums of cookbook files and create a sandbox
+        checksum_files = cookbook.checksums
+        checksums = checksum_files.inject({}){|memo,elt| memo[elt.first]=nil ; memo}
+        new_sandbox = rest.post_rest("/sandboxes", { :checksums => checksums })
+
+        Chef::Log.info("Uploading files")
+        # upload the new checksums and commit the sandbox
+        new_sandbox['checksums'].each do |checksum, info|
+          if info['needs_upload'] == true
+            Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
+
+            # Checksum is the hexadecimal representation of the md5,
+            # but we need the base64 encoding for the content-md5
+            # header
+            checksum64 = Base64.encode64([checksum].pack("H*")).strip
+            timestamp = Time.now.utc.iso8601
+            file_contents = File.read(checksum_files[checksum])
+            # TODO - 5/28/2010, cw: make signing and sending the request streaming
+            sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
+                                                                               :http_method => :put,
+                                                                               :path        => URI.parse(info['url']).path,
+                                                                               :body        => file_contents,
+                                                                               :timestamp   => timestamp,
+                                                                               :user_id     => rest.client_name
+                                                                               )
+            headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
+            headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
+            begin
+              RestClient::Request.execute(:method => :put, :url => info['url'], :headers => headers, :payload => file_contents)
+            rescue RestClient::RequestFailed => e
+              Chef::Log.error("Upload failed: #{e.message}\n#{e.response.body}")
+              raise
+            end
+          else
+            Chef::Log.debug("#{checksum_files[checksum]} has not changed")
+          end
+        end
+        sandbox_url = new_sandbox['uri']
+        Chef::Log.debug("Committing sandbox")
+        # Retry if S3 is claims a checksum doesn't exist (the eventual
+        # in eventual consistency)
+        retries = 0
+        begin
+          rest.put_rest(sandbox_url, {:is_completed => true})
+        rescue Net::HTTPServerException => e
+          if e.message =~ /^400/ && (retries += 1) <= 5
+            sleep 2
+            retry
+          else
+            raise
+          end
+        end
+
+        # files are uploaded, so save the manifest
+        cookbook.save
+
+        Chef::Log.info("Upload complete!")
+      end
+
+      def build_metadata(cookbook)
+        Chef::Log.debug("Generating metadata")
+        kcm = Chef::Knife::CookbookMetadata.new
+        kcm.config[:cookbook_path] = Chef::Config[:cookbook_path]
+        kcm.name_args = [ cookbook.name.to_s ]
+        kcm.run
+      end
+
+      def validate_cookbook(cookbook)
+        syntax_checker = Chef::Cookbook::SyntaxCheck.for_cookbook(cookbook.name, @user_cookbook_path)
+        Chef::Log.info("Validating ruby files")
+        exit(1) unless syntax_checker.validate_ruby_files
+        Chef::Log.info("Validating templates")
+        exit(1) unless syntax_checker.validate_templates
+        Chef::Log.info("Syntax OK")
+        true
+      end
+
+    end
+  end
+end

data/lib/chef/cookbook_version.rb

@@ -30,22 +30,6 @@ class Chef
   class CookbookVersion
     include Chef::IndexQueue::Indexable
 
-    attr_accessor :definition_filenames, :template_filenames, :file_filenames, 
-      :library_filenames, :resource_filenames, :provider_filenames, :root_filenames, :name,
-      :metadata, :metadata_filenames, :status, :couchdb_rev, :couchdb
-    attr_reader :couchdb_id
-
-    # attribute_filenames also has a setter that has non-default
-    # functionality.
-    attr_reader :attribute_filenames
-
-    # recipe_filenames also has a setter that has non-default
-    # functionality.
-    attr_reader :recipe_filenames
-
-    attr_reader :recipe_filenames_by_name
-    attr_reader :attribute_filenames_by_short_filename
-    
     COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ]
     
     DESIGN_DOCUMENT = {
@@ -167,6 +151,33 @@ class Chef
         },
       }
     }
+
+    attr_accessor :definition_filenames
+    attr_accessor :template_filenames
+    attr_accessor :file_filenames
+    attr_accessor :library_filenames
+    attr_accessor :resource_filenames
+    attr_accessor :provider_filenames
+    attr_accessor :root_filenames
+    attr_accessor :name
+    attr_accessor :metadata
+    attr_accessor :metadata_filenames
+    attr_accessor :status
+    attr_accessor :couchdb_rev
+    attr_accessor :couchdb
+
+    attr_reader :couchdb_id
+
+    # attribute_filenames also has a setter that has non-default
+    # functionality.
+    attr_reader :attribute_filenames
+
+    # recipe_filenames also has a setter that has non-default
+    # functionality.
+    attr_reader :recipe_filenames
+
+    attr_reader :recipe_filenames_by_name
+    attr_reader :attribute_filenames_by_short_filename
     
     # This is the one and only method that knows how cookbook files'
     # checksums are generated.
@@ -200,7 +211,7 @@ class Chef
       @status = :ready
       @manifest = nil
       @file_vendor = nil
-      @metadata = {}
+      @metadata = Chef::Cookbook::Metadata.new
     end
 
     def version
@@ -308,17 +319,23 @@ class Chef
       unless recipe_filenames_by_name.has_key?(recipe_name)
         raise ArgumentError, "Cannot find a recipe matching #{recipe_name} in cookbook #{name}"
       end
+
       Chef::Log.debug("Found recipe #{recipe_name} in cookbook #{name}")
       recipe = Chef::Recipe.new(name, recipe_name, run_context)
       recipe_filename = recipe_filenames_by_name[recipe_name]
-      raise Chef::Exceptions::RecipeNotFound, "could not find recipe #{recipe_name} for cookbook #{name}" unless recipe_filename
+
+      unless recipe_filename
+        raise Chef::Exceptions::RecipeNotFound, "could not find recipe #{recipe_name} for cookbook #{name}"
+      end
       
       recipe.from_file(recipe_filename)
       recipe
     end
 
     def segment_filenames(segment)
-      raise ArgumentError, "invalid segment #{segment}: must be one of #{COOKBOOK_SEGMENTS.join(', ')}" unless COOKBOOK_SEGMENTS.include?(segment)
+      unless COOKBOOK_SEGMENTS.include?(segment)
+        raise ArgumentError, "invalid segment #{segment}: must be one of #{COOKBOOK_SEGMENTS.join(', ')}"
+      end
 
       case segment.to_sym
       when :resources
@@ -398,9 +415,9 @@ class Chef
         # we're just going to make cookbook_files out of these and make the
         # cookbook find them by filespecificity again. but it's the shortest
         # path to "success" for now.
-        if manifest_record_path =~ /(#{segment}\/[^\/]+\/#{dirname})\/.+$/
+        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
           specificity_dirname = $1
-          non_specific_path = manifest_record_path[/#{segment}\/[^\/]+\/#{dirname}\/(.+)$/, 1]
+          non_specific_path = manifest_record_path[/#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)}\/(.+)$/, 1]
           # Record the specificity_dirname only if it's in the list of
           # valid preferences
           if filenames_by_pref[specificity_dirname]
@@ -429,7 +446,7 @@ class Chef
         manifest_record_path = manifest_record[:path]
 
         # extract the preference part from the path.
-        if manifest_record_path =~ /(#{segment}\/[^\/]+\/#{dirname})\/.+$/
+        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
           # Note the specificy_dirname includes the segment and
           # dirname argument as above, which is what
           # preferences_for_path returns. It could be
@@ -461,7 +478,7 @@ class Chef
           platform, version = Chef::Platform.find_platform_and_version(node)
         rescue ArgumentError => e
           # Skip platform/version if they were not found by find_platform_and_version
-          if e.message =~ /Cannot find a (platform|version)/
+          if e.message =~ /Cannot find a (?:platform|version)/
             platform = "/unknown_platform/"
             version = "/unknown_platform_version/"
           else
@@ -504,9 +521,9 @@ class Chef
         # we're just going to make cookbook_files out of these and make the
         # cookbook find them by filespecificity again. but it's the shortest
         # path to "success" for now.
-        if manifest_record_path =~ /(#{segment}\/[^\/]+\/#{dirname})\/.+$/
+        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
           specificity_dirname = $1
-          non_specific_path = manifest_record_path[/#{segment}\/[^\/]+\/#{dirname}\/(.+)$/, 1]
+          non_specific_path = manifest_record_path[/#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)}\/(.+)$/, 1]
           # Record the specificity_dirname only if it's in the list of
           # valid preferences
           if filenames_by_pref[specificity_dirname]
@@ -535,7 +552,7 @@ class Chef
         manifest_record_path = manifest_record[:path]
 
         # extract the preference part from the path.
-        if manifest_record_path =~ /(#{segment}\/[^\/]+\/#{dirname})\/.+$/
+        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
           # Note the specificy_dirname includes the segment and
           # dirname argument as above, which is what
           # preferences_for_path returns. It could be
@@ -567,7 +584,7 @@ class Chef
           platform, version = Chef::Platform.find_platform_and_version(node)
         rescue ArgumentError => e
           # Skip platform/version if they were not found by find_platform_and_version
-          if e.message =~ /Cannot find a (platform|version)/
+          if e.message =~ /Cannot find a (?:platform|version)/
             platform = "/unknown_platform/"
             version = "/unknown_platform_version/"
           else
@@ -780,11 +797,11 @@ class Chef
           specificity = "default"
           
           if segment == :root_files
-            matcher = segment_file.match(".+/#{name}/(.+)")
+            matcher = segment_file.match(".+/#{Regexp.escape(name.to_s)}/(.+)")
             file_name = matcher[1]
             path = file_name
           elsif segment == :templates || segment == :files
-            matcher = segment_file.match("/#{name}/(#{segment}/(.+?)/(.+))")
+            matcher = segment_file.match("/#{Regexp.escape(name.to_s)}/(#{Regexp.escape(segment.to_s)}/(.+?)/(.+))")
             unless matcher
               Chef::Log.debug("Skipping file #{segment_file}, as it doesn't have a proper segment.")
               next
@@ -793,7 +810,7 @@ class Chef
             specificity = matcher[2]
             file_name = matcher[3]
           else
-            matcher = segment_file.match("/#{name}/(#{segment}/(.+))")
+            matcher = segment_file.match("/#{Regexp.escape(name.to_s)}/(#{Regexp.escape(segment.to_s)}/(.+))")
             path = matcher[1]
             file_name = matcher[2]
           end