TrueCar-chef 0.10.0.beta.3

data/lib/chef/cookbook/remote_file_vendor.rb

@@ -0,0 +1,87 @@
+#
+# Author:: Tim Hinderliter (<tim@opscode.com>)
+# Copyright:: Copyright (c) 2010 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/cookbook/file_vendor'
+
+class Chef
+  class Cookbook
+    # == Chef::Cookbook::RemoteFileVendor
+    # This FileVendor loads files by either fetching them from the local cache, or
+    # if not available, loading them from the remote server.
+    class RemoteFileVendor < FileVendor
+      
+      def initialize(manifest, rest)
+        @manifest = manifest
+        @cookbook_name = @manifest[:cookbook_name]
+        @rest = rest
+      end
+      
+      # Implements abstract base's requirement. It looks in the
+      # Chef::Config.cookbook_path file hierarchy for the requested
+      # file.
+      def get_filename(filename)
+        if filename =~ /([^\/]+)\/(.+)$/
+          segment = $1
+        else
+          raise "get_filename: Cannot determine segment/filename for incoming filename #{filename}"
+        end
+        
+        raise "No such segment #{segment} in cookbook #{@cookbook_name}" unless @manifest[segment]
+        found_manifest_record = @manifest[segment].find {|manifest_record| manifest_record[:path] == filename }
+        raise "No such file #{filename} in #{@cookbook_name}" unless found_manifest_record
+        
+        cache_filename = File.join("cookbooks", @cookbook_name, found_manifest_record['path'])
+        
+        # update valid_cache_entries so the upstream cache cleaner knows what
+        # we've used.
+        validate_cached_copy(cache_filename)
+
+        current_checksum = nil
+        if Chef::FileCache.has_key?(cache_filename)
+          current_checksum = Chef::CookbookVersion.checksum_cookbook_file(Chef::FileCache.load(cache_filename, false))
+        end
+
+        # If the checksums are different between on-disk (current) and on-server
+        # (remote, per manifest), do the update. This will also execute if there
+        # is no current checksum.
+        if current_checksum != found_manifest_record['checksum']
+          raw_file = @rest.get_rest(found_manifest_record[:url], true)
+
+          Chef::Log.info("Storing updated #{cache_filename} in the cache.")
+          Chef::FileCache.move_to(raw_file.path, cache_filename)
+        else
+          Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.")
+        end
+
+        full_path_cache_filename = Chef::FileCache.load(cache_filename, false)
+
+        # return the filename, not the contents (second argument= false)
+        full_path_cache_filename
+      end
+
+      def validate_cached_copy(cache_filename)
+        valid_cache_entries[cache_filename] = true
+      end
+
+      def valid_cache_entries
+        Chef::CookbookVersion.valid_cache_entries
+      end
+      
+    end
+  end
+end

data/lib/chef/cookbook/syntax_check.rb

@@ -0,0 +1,136 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2010 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/checksum_cache'
+require 'chef/mixin/shell_out'
+
+class Chef
+  class Cookbook
+    # == Chef::Cookbook::SyntaxCheck
+    # Encapsulates the process of validating the ruby syntax of files in Chef
+    # cookbooks.
+    class SyntaxCheck
+      include Chef::Mixin::ShellOut
+
+      attr_reader :cookbook_path
+
+      # Creates a new SyntaxCheck given the +cookbook_name+ and a +cookbook_path+.
+      # If no +cookbook_path+ is given, +Chef::Config.cookbook_path+ is used.
+      def self.for_cookbook(cookbook_name, cookbook_path=nil)
+        cookbook_path ||= Chef::Config.cookbook_path
+        unless cookbook_path
+          raise ArgumentError, "Cannot find cookbook #{cookbook_name} unless Chef::Config.cookbook_path is set or an explicit cookbook path is given"
+        end
+        new(File.join(cookbook_path, cookbook_name.to_s))
+      end
+
+      # Create a new SyntaxCheck object
+      # === Arguments
+      # cookbook_path::: the (on disk) path to the cookbook
+      def initialize(cookbook_path)
+        @cookbook_path = cookbook_path
+      end
+
+      def cache
+        Chef::ChecksumCache.instance
+      end
+
+      def ruby_files
+        Dir[File.join(cookbook_path, '**', '*.rb')]
+      end
+
+      def untested_ruby_files
+        ruby_files.reject do |file|
+          if validated?(file)
+            Chef::Log.debug("ruby file #{file} is unchanged, skipping syntax check")
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def template_files
+        Dir[File.join(cookbook_path, '**', '*.erb')]
+      end
+
+      def untested_template_files
+        template_files.reject do |file| 
+          if validated?(file)
+            Chef::Log.debug("template #{file} is unchanged, skipping syntax check")
+            true
+          else
+            false
+          end
+        end
+      end
+
+      def validated?(file)
+        !!cache.lookup_checksum(cache_key(file), File.stat(file))
+      end
+
+      def validated(file)
+        cache.generate_checksum(cache_key(file), file, File.stat(file))
+      end
+
+      def cache_key(file)
+        @cache_keys ||= {}
+        @cache_keys[file] ||= cache.generate_key(file, "chef-test")
+      end
+
+      def validate_ruby_files
+        untested_ruby_files.each do |ruby_file|
+          return false unless validate_ruby_file(ruby_file)
+          validated(ruby_file)
+        end
+      end
+
+      def validate_templates
+        untested_template_files.each do |template|
+          return false unless validate_template(template)
+          validated(template)
+        end
+      end
+
+      def validate_template(erb_file)
+        Chef::Log.debug("Testing template #{erb_file} for syntax errors...")
+        result = shell_out("sh -c 'erubis -x #{erb_file} | ruby -c'")
+        result.error!
+        true
+      rescue Chef::Exceptions::ShellCommandFailed
+        file_relative_path = erb_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1]
+        Chef::Log.fatal("Erb template #{file_relative_path} has a syntax error:")
+        result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
+        false
+      end
+      
+      def validate_ruby_file(ruby_file)
+        Chef::Log.debug("Testing #{ruby_file} for syntax errors...")
+        result = shell_out("ruby -c #{ruby_file}")
+        result.error!
+        true
+      rescue Chef::Exceptions::ShellCommandFailed
+        file_relative_path = ruby_file[/^#{Regexp.escape(cookbook_path+File::Separator)}(.*)/, 1]
+        Chef::Log.fatal("Cookbook file #{file_relative_path} has a ruby syntax error:")
+        result.stderr.each_line { |l| Chef::Log.fatal(l.chomp) }
+        false
+      end
+      
+    end
+  end
+end

data/lib/chef/cookbook_loader.rb

@@ -0,0 +1,103 @@
+#
+# Author:: Adam Jacob (<adam@opscode.com>)
+# Author:: Christopher Walters (<cw@opscode.com>)
+# Author:: Daniel DeLeo (<dan@kallistec.com>)
+# Copyright:: Copyright (c) 2008 Opscode, Inc.
+# Copyright:: Copyright (c) 2009 Daniel DeLeo
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'chef/config'
+require 'chef/exceptions'
+require 'chef/cookbook/cookbook_version_loader'
+require 'chef/cookbook_version'
+require 'chef/cookbook/chefignore'
+require 'chef/cookbook/metadata'
+
+class Chef
+  class CookbookLoader
+
+    attr_accessor :metadata
+    attr_reader :cookbooks_by_name
+
+    include Enumerable
+
+    def initialize(*repo_paths)
+      @repo_paths = repo_paths.flatten
+      raise ArgumentError, "You must specify at least one cookbook repo path" if @repo_paths.empty?
+      @cookbooks_by_name = Mash.new
+      @loaded_cookbooks = {}
+      @metadata = Mash.new
+      load_cookbooks
+    end
+
+    def load_cookbooks
+      cookbook_settings = Hash.new
+      @repo_paths.each do |repo_path|
+        repo_path = File.expand_path(repo_path)
+        chefignore = Cookbook::Chefignore.new(repo_path)
+        Dir[File.join(repo_path, "*")].each do |cookbook_path|
+          next unless File.directory?(cookbook_path)
+          loader = Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore)
+          loader.load_cookbooks
+          next if loader.empty?
+          if @loaded_cookbooks.key?(loader.cookbook_name)
+            @loaded_cookbooks[loader.cookbook_name].merge!(loader)
+          else
+            @loaded_cookbooks[loader.cookbook_name] = loader
+          end
+        end
+      end
+
+      @loaded_cookbooks.each do |cookbook, loader|
+        cookbook_version = loader.cookbook_version
+        @cookbooks_by_name[cookbook] = cookbook_version
+        @metadata[cookbook] = cookbook_version.metadata
+      end
+      @cookbooks_by_name
+    end
+
+    def [](cookbook)
+      if @cookbooks_by_name.has_key?(cookbook.to_sym)
+        @cookbooks_by_name[cookbook.to_sym]
+      else
+        raise Exceptions::CookbookNotFoundInRepo, "Cannot find a cookbook named #{cookbook.to_s}; did you forget to add metadata to a cookbook? (http://wiki.opscode.com/display/chef/Metadata)"
+      end
+    end
+
+    alias :fetch :[]
+
+    def has_key?(cookbook_name)
+      @cookbooks_by_name.has_key?(cookbook_name)
+    end
+    alias :cookbook_exists? :has_key?
+    alias :key? :has_key?
+
+    def each
+      @cookbooks_by_name.keys.sort { |a,b| a.to_s <=> b.to_s }.each do |cname|
+        yield(cname, @cookbooks_by_name[cname])
+      end
+    end
+
+    def cookbook_names
+      @cookbooks_by_name.keys.sort
+    end
+
+    def values
+      @cookbooks_by_name.values
+    end
+    alias :cookbooks :values
+
+  end
+end

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -0,0 +1,244 @@
+#
+# Author:: Stanislav Vitvitskiy
+# Author:: Nuo Yan (nuo@opscode.com)
+# Author:: Christopher Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2009, 2010 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'net/http'
+require 'mixlib/authentication/signedheaderauth'
+require 'openssl'
+
+class Chef
+  # == Chef::CookbookSiteStreamingUploader
+  # A streaming multipart HTTP upload implementation. Used to upload cookbooks
+  # (in tarball form) to http://cookbooks.opscode.com
+  #
+  # inspired by http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
+  class CookbookSiteStreamingUploader
+
+    DefaultHeaders = { 'accept' => 'application/json', 'x-chef-version' => ::Chef::VERSION }
+
+    class << self
+
+       def create_build_dir(cookbook)
+         tmp_cookbook_path = Tempfile.new("chef-#{cookbook.name}-build")
+         tmp_cookbook_path.close
+         tmp_cookbook_dir = tmp_cookbook_path.path
+         File.unlink(tmp_cookbook_dir)
+         FileUtils.mkdir_p(tmp_cookbook_dir)
+         Chef::Log.debug("Staging at #{tmp_cookbook_dir}")
+         checksums_to_on_disk_paths = cookbook.checksums
+         Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment|
+           cookbook.manifest[segment].each do |manifest_record|
+             path_in_cookbook = manifest_record[:path]
+             on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+             dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
+             FileUtils.mkdir_p(File.dirname(dest))
+             Chef::Log.debug("Staging #{on_disk_path} to #{dest}")
+             FileUtils.cp(on_disk_path, dest)
+          end
+        end
+
+        # First, generate metadata
+        Chef::Log.debug("Generating metadata")
+        kcm = Chef::Knife::CookbookMetadata.new
+        kcm.config[:cookbook_path] = [ tmp_cookbook_dir ]
+        kcm.name_args = [ cookbook.name.to_s ]
+        kcm.run
+
+        tmp_cookbook_dir
+      end
+
+      def post(to_url, user_id, secret_key_filename, params = {}, headers = {})
+        make_request(:post, to_url, user_id, secret_key_filename, params, headers)
+      end
+
+      def put(to_url, user_id, secret_key_filename, params = {}, headers = {})
+        make_request(:put, to_url, user_id, secret_key_filename, params, headers)
+      end
+
+      def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {})
+        boundary = '----RubyMultipartClient' + rand(1000000).to_s + 'ZZZZZ'
+        parts = []
+        content_file = nil
+
+        timestamp = Time.now.utc.iso8601
+        secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename))
+
+        unless params.nil? || params.empty?
+          params.each do |key, value|
+            if value.kind_of?(File)
+              content_file = value
+              filepath = value.path
+              filename = File.basename(filepath)
+              parts << StringPart.new( "--" + boundary + "\r\n" +
+                                       "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" +
+                                       "Content-Type: application/octet-stream\r\n\r\n")
+              parts << StreamPart.new(value, File.size(filepath))
+              parts << StringPart.new("\r\n")
+            else
+              parts << StringPart.new( "--" + boundary + "\r\n" +
+                                       "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n")
+              parts << StringPart.new(value.to_s + "\r\n")
+            end
+          end
+          parts << StringPart.new("--" + boundary + "--\r\n")
+        end
+
+        body_stream = MultipartStream.new(parts)
+
+        timestamp = Time.now.utc.iso8601
+
+        url = URI.parse(to_url)
+
+        Chef::Log.logger.debug("Signing: method: #{http_verb}, path: #{url.path}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}")
+
+        # We use the body for signing the request if the file parameter
+        # wasn't a valid file or wasn't included. Extract the body (with
+        # multi-part delimiters intact) to sign the request.
+        # TODO: tim: 2009-12-28: It'd be nice to remove this special case, and
+        # always hash the entire request body. In the file case it would just be
+        # expanded multipart text - the entire body of the POST.
+        content_body = parts.inject("") { |result,part| result + part.read(0, part.size) }
+        content_file.rewind if content_file # we consumed the file for the above operation, so rewind it.
+
+        signing_options = {
+          :http_method=>http_verb,
+          :path=>url.path,
+          :user_id=>user_id,
+          :timestamp=>timestamp}
+        (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || ""))
+
+        headers.merge!(Mixlib::Authentication::SignedHeaderAuth.signing_object(signing_options).sign(secret_key))
+
+        content_file.rewind if content_file
+
+        # net/http doesn't like symbols for header keys, so we'll to_s each one just in case
+        headers = DefaultHeaders.merge(Hash[*headers.map{ |k,v| [k.to_s, v] }.flatten])
+
+        req = case http_verb
+              when :put
+                Net::HTTP::Put.new(url.path, headers)
+              when :post
+                Net::HTTP::Post.new(url.path, headers)
+              end
+        req.content_length = body_stream.size
+        req.content_type = 'multipart/form-data; boundary=' + boundary unless parts.empty?
+        req.body_stream = body_stream
+
+        http = Net::HTTP.new(url.host, url.port)
+        if url.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        res = http.request(req)
+        #res = http.start {|http_proc| http_proc.request(req) }
+
+        # alias status to code and to_s to body for test purposes
+        # TODO: stop the following madness!
+        class << res
+          alias :to_s :body
+
+          # BUGBUG this makes the response compatible with what respsonse_steps expects to test headers (response.headers[] -> response[])
+          def headers
+            self
+          end
+
+          def status
+            code.to_i
+          end
+        end
+        res
+      end
+
+    end
+
+    class StreamPart
+      def initialize(stream, size)
+        @stream, @size = stream, size
+      end
+
+      def size
+        @size
+      end
+
+      # read the specified amount from the stream
+      def read(offset, how_much)
+        @stream.read(how_much)
+      end
+    end
+
+    class StringPart
+      def initialize(str)
+        @str = str
+      end
+
+      def size
+        @str.length
+      end
+
+      # read the specified amount from the string startiung at the offset
+      def read(offset, how_much)
+        @str[offset, how_much]
+      end
+    end
+
+    class MultipartStream
+      def initialize(parts)
+        @parts = parts
+        @part_no = 0
+        @part_offset = 0
+      end
+
+      def size
+        @parts.inject(0) {|size, part| size + part.size}
+      end
+
+      def read(how_much)
+        return nil if @part_no >= @parts.size
+
+        how_much_current_part = @parts[@part_no].size - @part_offset
+
+        how_much_current_part = if how_much_current_part > how_much
+                                  how_much
+                                else
+                                  how_much_current_part
+                                end
+
+        how_much_next_part = how_much - how_much_current_part
+
+        current_part = @parts[@part_no].read(@part_offset, how_much_current_part)
+
+        # recurse into the next part if the current one was not large enough
+        if how_much_next_part > 0
+          @part_no += 1
+          @part_offset = 0
+          next_part = read(how_much_next_part)
+          current_part + if next_part
+                           next_part
+                         else
+                           ''
+                         end
+        else
+          @part_offset += how_much_current_part
+          current_part
+        end
+      end
+    end
+
+  end
+end