chef 0.7.16 → 0.8.2

data/lib/chef/streaming_cookbook_uploader.rb

@@ -0,0 +1,187 @@
+require 'net/http'
+require 'mixlib/authentication/signedheaderauth'
+require 'openssl'
+
+# inspired by/cargo-culted from http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
+# TODO: confirm that code is public domain
+class Chef
+  class StreamingCookbookUploader
+
+    DefaultHeaders = { 'accept' => 'application/json' }
+    
+    class << self
+
+      def post(to_url, user_id, secret_key_filename, params = {}, headers = {})
+        make_request(:post, to_url, user_id, secret_key_filename, params, headers)
+      end
+      
+      def put(to_url, user_id, secret_key_filename, params = {}, headers = {})
+        make_request(:put, to_url, user_id, secret_key_filename, params, headers)
+      end
+      
+      def make_request(http_verb, to_url, user_id, secret_key_filename, params = {}, headers = {})
+        boundary = '----RubyMultipartClient' + rand(1000000).to_s + 'ZZZZZ'
+        parts = []
+        content_file = nil
+        content_body = nil
+        
+        timestamp = Time.now.utc.iso8601
+        secret_key = OpenSSL::PKey::RSA.new(File.read(secret_key_filename))
+
+        unless params.nil? || params.empty?
+          params.each do |key, value|
+            if value.kind_of?(File)
+              content_file = value
+              filepath = value.path
+              filename = File.basename(filepath)
+              parts << StringPart.new( "--" + boundary + "\r\n" +
+                                       "Content-Disposition: form-data; name=\"" + key.to_s + "\"; filename=\"" + filename + "\"\r\n" +
+                                       "Content-Type: application/octet-stream\r\n\r\n")
+              parts << StreamPart.new(value, File.size(filepath))
+              parts << StringPart.new("\r\n")
+            else
+              content_body = value.to_s
+              parts << StringPart.new( "--" + boundary + "\r\n" +
+                                       "Content-Disposition: form-data; name=\"" + key.to_s + "\"\r\n\r\n")
+              parts << StringPart.new(content_body + "\r\n")
+            end
+          end
+          parts << StringPart.new("--" + boundary + "--\r\n")
+        end
+        
+        body_stream = MultipartStream.new(parts)
+        
+        timestamp = Time.now.utc.iso8601
+        
+        url = URI.parse(to_url)
+        
+        Chef::Log.logger.debug("Signing: method: #{http_verb}, path: #{url.path}, file: #{content_file}, User-id: #{user_id}, Timestamp: #{timestamp}")
+        
+        signing_options = {
+          :http_method=>http_verb,
+          :path=>url.path,
+          :user_id=>user_id,
+          :timestamp=>timestamp}
+        (content_file && signing_options[:file] = content_file) || (signing_options[:body] = (content_body || ""))
+        
+        headers.merge!(Mixlib::Authentication::SignedHeaderAuth.signing_object(signing_options).sign(secret_key))
+
+        content_file.rewind if content_file
+
+        # net/http doesn't like symbols for header keys, so we'll to_s each one just in case
+        headers = DefaultHeaders.merge(Hash[*headers.map{ |k,v| [k.to_s, v] }.flatten])
+
+        req = case http_verb
+              when :put
+                Net::HTTP::Put.new(url.path, headers)
+              when :post
+                Net::HTTP::Post.new(url.path, headers)
+              end              
+        req.content_length = body_stream.size
+        req.content_type = 'multipart/form-data; boundary=' + boundary unless parts.empty?
+        req.body_stream = body_stream
+        
+        http = Net::HTTP.new(url.host, url.port)
+        if url.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        res = http.request(req)
+        #res = http.start {|http_proc| http_proc.request(req) }
+
+        # alias status to code and to_s to body for test purposes
+        # TODO: stop the following madness!
+        class << res
+          alias :to_s :body
+          
+          # BUGBUG this makes the response compatible with what respsonse_steps expects to test headers (response.headers[] -> response[])
+          def headers
+            self
+          end
+          
+          def status
+            code.to_i
+          end
+        end
+        res
+      end
+      
+    end
+
+    class StreamPart
+      def initialize(stream, size)
+        @stream, @size = stream, size
+      end
+      
+      def size
+        @size
+      end
+      
+      # read the specified amount from the stream
+      def read(offset, how_much)
+        @stream.read(how_much)
+      end
+    end
+
+    class StringPart
+      def initialize(str)
+        @str = str
+      end
+      
+      def size
+        @str.length
+      end
+
+      # read the specified amount from the string startiung at the offset
+      def read(offset, how_much)
+        @str[offset, how_much]
+      end
+    end
+
+    class MultipartStream
+      def initialize(parts)
+        @parts = parts
+        @part_no = 0
+        @part_offset = 0
+      end
+      
+      def size
+        @parts.inject(0) {|size, part| size + part.size}
+      end
+      
+      def read(how_much)
+        return nil if @part_no >= @parts.size
+
+        how_much_current_part = @parts[@part_no].size - @part_offset
+        
+        how_much_current_part = if how_much_current_part > how_much
+                                  how_much
+                                else
+                                  how_much_current_part
+                                end
+        
+        how_much_next_part = how_much - how_much_current_part
+
+        current_part = @parts[@part_no].read(@part_offset, how_much_current_part)
+        
+        # recurse into the next part if the current one was not large enough
+        if how_much_next_part > 0
+          @part_no += 1
+          @part_offset = 0
+          next_part = read(how_much_next_part)
+          current_part + if next_part
+                           next_part
+                         else
+                           ''
+                         end
+        else
+          @part_offset += how_much_current_part
+          current_part
+        end
+      end
+    end
+    
+  end
+
+
+end

data/lib/chef/tasks/chef_repo.rake

@@ -24,6 +24,20 @@ require 'chef/cookbook/metadata'
 require 'tempfile'
 require 'rake'
 
+# Allow REMOTE options to be overridden on the command line
+REMOTE_HOST = ENV["REMOTE_HOST"] if ENV["REMOTE_HOST"] != nil
+REMOTE_SUDO = ENV["REMOTE_SUDO"] if ENV["REMOTE_SUDO"] != nil
+if defined? REMOTE_HOST
+  REMOTE_PATH_PREFIX = "#{REMOTE_HOST}:"
+  REMOTE_EXEC_PREFIX = "ssh #{REMOTE_HOST}"
+  REMOTE_EXEC_PREFIX += " sudo" if defined? REMOTE_SUDO
+  LOCAL_EXEC_PREFIX = ""
+else
+  REMOTE_PATH_PREFIX = ""
+  REMOTE_EXEC_PREFIX = ""
+  LOCAL_EXEC_PREFIX = "sudo"
+end
+
 desc "Update your repository from source control"
 task :update do
   puts "** Updating your repository"
@@ -33,7 +47,6 @@ task :update do
     sh %{svn up}
   when :git
     pull = false
-    pull = true if File.join(TOPDIR, ".git", "remotes", "origin")
     IO.foreach(File.join(TOPDIR, ".git", "config")) do |line|
       pull = true if line =~ /\[remote "origin"\]/
     end
@@ -47,124 +60,15 @@ task :update do
   end
 end
 
-desc "Test your cookbooks for syntax errors"
-task :test_recipes do
-  puts "** Testing your cookbooks for syntax errors"
-
-  if File.exists?(TEST_CACHE)
-    cache = JSON.load(open(TEST_CACHE).read)
-    trap("INT") { puts "INT received, flushing test cache"; write_cache(cache) }
-  else
-    cache = {}
-  end
-
-  recipes = ["*cookbooks"].map { |folder|
-    Dir[File.join(TOPDIR, folder, "**", "*.rb")]
-  }.flatten
-
-  recipes.each do |recipe|
-    print "Testing recipe #{recipe}: "
-
-    recipe_mtime = File.stat(recipe).mtime.to_s
-    if cache.has_key?(recipe)
-      if cache[recipe]["mtime"] == recipe_mtime 
-         puts "No modification since last test."
-         next
-      end
-    else
-      cache[recipe] = {}
-    end
-
-
-    sh %{ruby -c #{recipe}} do |ok, res|
-      if ok
-        cache[recipe]["mtime"] = recipe_mtime
-      else
-        write_cache(cache)
-        raise "Syntax error in #{recipe}"
-      end
-    end
-  end
-
-  write_cache(cache)
-end
-
-desc "Test your templates for syntax errors"
-task :test_templates do
-  puts "** Testing your cookbooks for syntax errors"
-
-  if File.exists?(TEST_CACHE)
-    cache = JSON.load(open(TEST_CACHE).read)
-    trap("INT") { puts "INT received, flushing test cache"; write_cache(cache) }
-  else
-    cache = {}
-  end
-
-  templates = ["*cookbooks"].map { |folder|
-    Dir[File.join(TOPDIR, folder, "**", "*.erb")]
-  }.flatten
-
-  templates.each do |template|
-    print "Testing template #{template}: "
-
-    template_mtime = File.stat(template).mtime.to_s
-    if cache.has_key?(template)
-      if cache[template]["mtime"] == template_mtime 
-         puts "No change since last test."
-         next
-      end
-    else
-      cache[template] = {}
-    end
-
-    sh %{erubis -x #{template} | ruby -c} do |ok, res|
-      if ok
-        cache[template]["mtime"] = template_mtime
-      else
-        write_cache(cache)
-        raise "Syntax error in #{template}"
-      end
-    end
-
-  end
-
-  write_cache(cache)
-end
-
-desc "Test your cookbooks for syntax errors"
-task :test => [ :test_recipes , :test_templates ]
-
-def write_cache(cache)
-  File.open(TEST_CACHE, "w") { |f| JSON.dump(cache, f) }
-end
-
 desc "Install the latest copy of the repository on this Chef Server"
-task :install => [ :update, :test, :metadata, :roles ] do
-  puts "** Installing your cookbooks"  
-  directories = [ 
-    COOKBOOK_PATH,
-    SITE_COOKBOOK_PATH,
-    CHEF_CONFIG_PATH
-  ]
-  puts "* Creating Directories"
-  directories.each do |dir|
-    sh "sudo mkdir -p #{dir}"
-    sh "sudo chown root #{dir}"
-  end
-  puts "* Installing new Cookbooks"
-  sh "sudo rsync -rlP --delete --exclude '.svn' --exclude '.git*' cookbooks/ #{COOKBOOK_PATH}"
-  puts "* Installing new Site Cookbooks"
-  sh "sudo rsync -rlP --delete --exclude '.svn' --exclude '.git*' site-cookbooks/ #{SITE_COOKBOOK_PATH}"
-  puts "* Installing new Node Roles"
-  sh "sudo rsync -rlP --delete --exclude '.svn' --exclude '.git*' roles/ #{ROLE_PATH}"
-  
-  if File.exists?(File.join(File.dirname(__FILE__), "config", "server.rb"))
+task :install => [ :update, :roles, :upload_cookbooks ] do
+  if File.exists?(File.join(TOPDIR, "config", "server.rb"))
     puts "* Installing new Chef Server Config"
-    sh "sudo cp config/server.rb #{CHEF_SERVER_CONFIG}"
+    sh "#{LOCAL_EXEC_PREFIX} rsync -rlt --delete --exclude '.svn' --exclude '.git*' config/server.rb #{REMOTE_PATH_PREFIX}#{CHEF_SERVER_CONFIG}"
   end
-  if File.exists?(File.join(File.dirname(__FILE__), "config", "client.rb"))
+  if File.exists?(File.join(TOPDIR, "config", "client.rb"))
     puts "* Installing new Chef Client Config"
-    sh "sudo cp config/client.rb #{CHEF_CLIENT_CONFIG}"
+    sh "#{LOCAL_EXEC_PREFIX} rsync -rlt --delete --exclude '.svn' --exclude '.git*' config/client.rb #{REMOTE_PATH_PREFIX}#{CHEF_CLIENT_CONFIG}"
   end
 end
 
@@ -279,12 +183,13 @@ task :ssl_cert do
   fqdn =~ /^(.+?)\.(.+)$/
   hostname = $1
   domain = $2
+  keyfile = fqdn.gsub("*", "wildcard")
   raise "Must provide FQDN!" unless fqdn && hostname && domain
   puts "** Creating self signed SSL Certificate for #{fqdn}"
-  sh("(cd #{CADIR} && openssl genrsa 2048 > #{fqdn}.key)")
-  sh("(cd #{CADIR} && chmod 644 #{fqdn}.key)")
+  sh("(cd #{CADIR} && openssl genrsa 2048 > #{keyfile}.key)")
+  sh("(cd #{CADIR} && chmod 644 #{keyfile}.key)")
   puts "* Generating Self Signed Certificate Request"
-  tf = Tempfile.new("#{fqdn}.ssl-conf")
+  tf = Tempfile.new("#{keyfile}.ssl-conf")
   ssl_config = <<EOH
 [ req ]
 distinguished_name = req_distinguished_name
@@ -301,47 +206,40 @@ emailAddress           = #{SSL_EMAIL_ADDRESS}
 EOH
   tf.puts(ssl_config)
   tf.close
-  sh("(cd #{CADIR} && openssl req -config '#{tf.path}' -new -x509 -nodes -sha1 -days 3650 -key #{fqdn}.key > #{fqdn}.crt)")
-  sh("(cd #{CADIR} && openssl x509 -noout -fingerprint -text < #{fqdn}.crt > #{fqdn}.info)")
-  sh("(cd #{CADIR} && cat #{fqdn}.crt #{fqdn}.key > #{fqdn}.pem)")
-  sh("(cd #{CADIR} && chmod 644 #{fqdn}.pem)")
+  sh("(cd #{CADIR} && openssl req -config '#{tf.path}' -new -x509 -nodes -sha1 -days 3650 -key #{keyfile}.key > #{keyfile}.crt)")
+  sh("(cd #{CADIR} && openssl x509 -noout -fingerprint -text < #{keyfile}.crt > #{keyfile}.info)")
+  sh("(cd #{CADIR} && cat #{keyfile}.crt #{keyfile}.key > #{keyfile}.pem)")
+  sh("(cd #{CADIR} && chmod 644 #{keyfile}.pem)")
+end
+
+rule(%r{\b(?:site-)?cookbooks/[^/]+/metadata\.json\Z} => [ proc { |task_name| task_name.sub(/\.[^.]+$/, '.rb') } ]) do |t|
+  system("knife cookbook metadata #{t.source}")
 end
 
 desc "Build cookbook metadata.json from metadata.rb"
-task :metadata do
-  Chef::Config[:cookbook_path] = [ File.join(TOPDIR, 'cookbooks'), File.join(TOPDIR, 'site-cookbooks') ]
-  cl = Chef::CookbookLoader.new
-  cl.each do |cookbook|
-    if ENV['COOKBOOK']
-      next unless cookbook.name.to_s == ENV['COOKBOOK']
-    end
-    cook_meta = Chef::Cookbook::Metadata.new(cookbook)
-    Chef::Config.cookbook_path.each do |cdir|
-      metadata_rb_file = File.join(cdir, cookbook.name.to_s, 'metadata.rb')
-      metadata_json_file = File.join(cdir, cookbook.name.to_s, 'metadata.json')
-      if File.exists?(metadata_rb_file)
-        puts "Generating metadata for #{cookbook.name}"
-        cook_meta.from_file(metadata_rb_file)
-        File.open(metadata_json_file, "w") do |f| 
-          f.write(JSON.pretty_generate(cook_meta))
-        end
-      end
-    end
-  end
+task :metadata => FileList[File.join(TOPDIR, '*cookbooks', ENV['COOKBOOK'] || '*', 'metadata.rb')].pathmap('%X.json')
+
+rule(%r{\broles/\S+\.json\Z} => [ proc { |task_name| task_name.sub(/\.[^.]+$/, '.rb') } ]) do |t|
+  system("knife role from file #{t.source}")
 end
 
-desc "Build roles from roles/role_name.json from role_name.rb"
-task :roles do
-  Chef::Config[:role_path] = File.join(TOPDIR, 'roles')
-  Dir[File.join(TOPDIR, 'roles', '**', '*.rb')].each do |role_file|
-    short_name = File.basename(role_file, '.rb')
-    puts "Generating role JSON for #{short_name}"
-    role = Chef::Role.new
-    role.name(short_name)
-    role.from_file(role_file)
-    File.open(File.join(TOPDIR, 'roles', "#{short_name}.json"), "w") do |f|
-      f.write(JSON.pretty_generate(role))
-    end
-  end
+desc "Update roles"
+task :roles  => FileList[File.join(TOPDIR, 'roles', '**', '*.rb')].pathmap('%X.json')
+
+desc "Update a specific role"
+task :role, :role_name do |t, args|
+  system("knife role from file #{args.cookbook}")
+end
+
+desc "Upload all cookbooks"
+task :upload_cookbooks => [ :metadata ]
+task :upload_cookbooks do
+  system("knife cookbook upload --all")
+end
+
+desc "Upload a single cookbook"
+task :upload_cookbook => [ :metadata ]
+task :upload_cookbook, :cookbook do |t, args|
+  system("knife cookbook upload #{args.cookbook}")
 end
 

data/lib/chef/util/file_edit.rb

@@ -20,106 +20,104 @@ require 'fileutils'
 require 'tempfile'
 
 class Chef
-	class Util
-		class FileEdit
+  class Util
+  	class FileEdit
 
-			private
-			
-			attr_accessor :original_pathname, :contents, :file_edited
+  		private
+  		
+  		attr_accessor :original_pathname, :contents, :file_edited
 
-			public
-			
-			def initialize(filepath)
-				@original_pathname = filepath
-				@file_edited = false
-				
-				raise ArgumentError, "File doesn't exist" unless File.exist? @original_pathname
-				raise ArgumentError, "File is blank" unless (@contents = File.new(@original_pathname).readlines).length > 0
-			end
-			
-			#search the file line by line and match each line with the given regex
-			#if matched, replace the whole line with newline.
-			def search_file_replace_line(regex, newline)
-				search_match(regex, newline, 'r', 1)
-			end
+  		public
+  		
+  		def initialize(filepath)
+  			@original_pathname = filepath
+  			@file_edited = false
+  			
+  			raise ArgumentError, "File doesn't exist" unless File.exist? @original_pathname
+  			raise ArgumentError, "File is blank" unless (@contents = File.new(@original_pathname).readlines).length > 0
+  		end
+  		
+  		#search the file line by line and match each line with the given regex
+  		#if matched, replace the whole line with newline.
+  		def search_file_replace_line(regex, newline)
+  			search_match(regex, newline, 'r', 1)
+  		end
 
-			#search the file line by line and match each line with the given regex
-			#if matched, replace the match (all occurances)  with the replace parameter
-			def search_file_replace(regex, replace)
-				search_match(regex, replace, 'r', 2)
-			end
-			
-			#search the file line by line and match each line with the given regex
-			#if matched, delete the line
-			def search_file_delete_line(regex)
-				search_match(regex, " ", 'd', 1)
-			end
-			
-			#search the file line by line and match each line with the given regex
-			#if matched, delete the match (all occurances) from the line
-			def search_file_delete(regex)
-				search_match(regex, " ", 'd', 2)
-			end
+  		#search the file line by line and match each line with the given regex
+  		#if matched, replace the match (all occurances)  with the replace parameter
+  		def search_file_replace(regex, replace)
+  			search_match(regex, replace, 'r', 2)
+  		end
+  		
+  		#search the file line by line and match each line with the given regex
+  		#if matched, delete the line
+  		def search_file_delete_line(regex)
+  			search_match(regex, " ", 'd', 1)
+  		end
+  		
+  		#search the file line by line and match each line with the given regex
+  		#if matched, delete the match (all occurances) from the line
+  		def search_file_delete(regex)
+  			search_match(regex, " ", 'd', 2)
+  		end
 
-			#search the file line by line and match each line with the given regex
-			#if matched, insert newline after each matching line
-			def insert_line_after_match(regex, newline)
-				search_match(regex, newline, 'i', 0)
-			end
-			 
-			#Make a copy of old_file and write new file out (only if file changed)
-			def write_file
-				
-				# file_edited is false when there was no match in the whole file and thus no contents have changed.
-				if file_edited
-					backup_pathname = original_pathname + ".old"
-					File.copy(original_pathname, backup_pathname)
-					Tempfile.open("w") do |newfile|
-						contents.each do |line|
-							newfile.puts(line)
-						end
-						newfile.flush
-						FileUtils.mv(newfile.path, original_pathname)
-					end
-				end
-				self.file_edited = false
+  		#search the file line by line and match each line with the given regex
+  		#if matched, insert newline after each matching line
+  		def insert_line_after_match(regex, newline)
+  			search_match(regex, newline, 'i', 0)
+  		end
+  		 
+  		#Make a copy of old_file and write new file out (only if file changed)
+  		def write_file
+  			
+  			# file_edited is false when there was no match in the whole file and thus no contents have changed.
+        if file_edited
+          backup_pathname = original_pathname + ".old"
+          FileUtils.cp(original_pathname, backup_pathname, :preserve => true)
+          File.open(original_pathname, "w") do |newfile|
+            contents.each do |line|
+              newfile.puts(line)
+            end
+            newfile.flush
+          end
+        end
+        self.file_edited = false
+  		end
+  		
+  		private
+  		
+  		#helper method to do the match, replace, delete, and insert operations
+  		#command is the switch of delete, replace, and insert ('d', 'r', 'i')
+  		#method is to control operation on whole line or only the match (1 for line, 2 for match)
+  		def search_match(regex, replace, command, method)
+  			
+  			#convert regex to a Regexp object (if not already is one) and store it in exp.
+  			exp = Regexp.new(regex)
 
-			end
-			
-			private
-			
-			#helper method to do the match, replace, delete, and insert operations
-			#command is the switch of delete, replace, and insert ('d', 'r', 'i')
-			#method is to control operation on whole line or only the match (1 for line, 2 for match)
-			def search_match(regex, replace, command, method)
-				
-				#convert regex to a Regexp object (if not already is one) and store it in exp.
-				exp = Regexp.new(regex)
+  			#loop through contents and do the appropriate operation depending on 'command' and 'method'
+  			new_contents = []
+  			
+  			contents.each do |line|
+  				if line.match(exp) 
+  					self.file_edited = true
+  					case
+  					when command == 'r'
+  						new_contents << ((method == 1) ? replace : line.gsub!(exp, replace))
+  					when command == 'd'
+  						if method == 2
+  							new_contents << line.gsub!(exp, "")
+  						end
+  					when command == 'i'
+  						new_contents << line
+  						new_contents << replace
+  					end
+  				else
+  					new_contents << line
+  				end
+  			end
 
-				#loop through contents and do the appropriate operation depending on 'command' and 'method'
-				new_contents = []
-				
-				contents.each do |line|
-					if line.match(exp) 
-						self.file_edited = true
-						case
-						when command == 'r'
-							new_contents << ((method == 1) ? replace : line.gsub!(exp, replace))
-						when command == 'd'
-							if method == 2
-								new_contents << line.gsub!(exp, "")
-							end
-						when command == 'i'
-							new_contents << line
-							new_contents << replace
-						end
-					else
-						new_contents << line
-					end
-				end
-
-				self.contents = new_contents
-			end
-		end
-	end
+  			self.contents = new_contents
+  		end
+  	end
+  end
 end