chef 0.9.18 → 0.10.0.beta.0

data/lib/chef/shell_out/unix.rb

@@ -0,0 +1,224 @@
+class Chef
+  class ShellOut
+    module Unix
+
+      # Run the command, writing the command's standard out and standard error
+      # to +stdout+ and +stderr+, and saving its exit status object to +status+
+      # === Returns
+      # returns   +self+; +stdout+, +stderr+, +status+, and +exitstatus+ will be
+      # populated with results of the command
+      # === Raises
+      # * Errno::EACCES  when you are not privileged to execute the command
+      # * Errno::ENOENT  when the command is not available on the system (or not
+      #   in the current $PATH)
+      # * Chef::Exceptions::CommandTimeout  when the command does not complete
+      #   within +timeout+ seconds (default: 60s)
+      def run_command
+        Chef::Log.debug("sh(#{@command})")
+        @child_pid = fork_subprocess
+
+        configure_parent_process_file_descriptors
+        propagate_pre_exec_failure
+
+        @result = nil
+        @execution_time = 0
+
+        # Ruby 1.8.7 and 1.8.6 from mid 2009 try to allocate objects during GC
+        # when calling IO.select and IO#read. Some OS Vendors are not interested
+        # in updating their ruby packages (Apple, *cough*) and we *have to*
+        # make it work. So I give you this epic hack:
+        GC.disable
+        until @status
+          ready = IO.select(open_pipes, nil, nil, READ_WAIT_TIME)
+          unless ready
+            @execution_time += READ_WAIT_TIME
+            if @execution_time >= timeout && !@result
+              raise Chef::Exceptions::CommandTimeout, "command timed out:\n#{format_for_exception}"
+            end
+          end
+
+          if ready && ready.first.include?(child_stdout)
+            read_stdout_to_buffer
+          end
+          if ready && ready.first.include?(child_stderr)
+            read_stderr_to_buffer
+          end
+
+          unless @status
+            # make one more pass to get the last of the output after the
+            # child process dies
+            if results = Process.waitpid2(@child_pid, Process::WNOHANG)
+              @status = results.last
+              redo
+            end
+          end
+        end
+        self
+      rescue Exception
+        # do our best to kill zombies
+        Process.waitpid2(@child_pid, Process::WNOHANG) rescue nil
+        raise
+      ensure
+        # no matter what happens, turn the GC back on, and hope whatever busted
+        # version of ruby we're on doesn't allocate some objects during the next
+        # GC run.
+        GC.enable
+        close_all_pipes
+      end
+
+      private
+
+      def set_user
+        if user
+          Process.euid = uid
+          Process.uid = uid
+        end
+      end
+
+      def set_group
+        if group
+          Process.egid = gid
+          Process.gid = gid
+        end
+      end
+
+      def set_environment
+        environment.each do |env_var,value|
+          ENV[env_var] = value
+        end
+      end
+
+      def set_umask
+        File.umask(umask) if umask
+      end
+
+      def set_cwd
+        Dir.chdir(cwd) if cwd
+      end
+
+      def initialize_ipc
+        @stdout_pipe, @stderr_pipe, @process_status_pipe = IO.pipe, IO.pipe, IO.pipe
+        @process_status_pipe.last.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+      end
+
+      def child_stdout
+        @stdout_pipe[0]
+      end
+
+      def child_stderr
+        @stderr_pipe[0]
+      end
+
+      def child_process_status
+        @process_status_pipe[0]
+      end
+
+      def close_all_pipes
+        child_stdout.close  unless child_stdout.closed?
+        child_stderr.close  unless child_stderr.closed?
+        child_process_status.close unless child_process_status.closed?
+      end
+
+      # replace stdout, and stderr with pipes to the parent, and close the
+      # reader side of the error marshaling side channel. Close STDIN so when we
+      # exec, the new program will know it's never getting input ever.
+      def configure_subprocess_file_descriptors
+        process_status_pipe.first.close
+
+        # HACK: for some reason, just STDIN.close isn't good enough when running
+        # under ruby 1.9.2, so make it good enough:
+        stdin_reader, stdin_writer = IO.pipe
+        stdin_writer.close
+        STDIN.reopen stdin_reader
+        stdin_reader.close
+
+        stdout_pipe.first.close
+        STDOUT.reopen stdout_pipe.last
+        stdout_pipe.last.close
+
+        stderr_pipe.first.close
+        STDERR.reopen stderr_pipe.last
+        stderr_pipe.last.close
+
+        STDOUT.sync = STDERR.sync = true
+      end
+
+      def configure_parent_process_file_descriptors
+        # Close the sides of the pipes we don't care about
+        stdout_pipe.last.close
+        stderr_pipe.last.close
+        process_status_pipe.last.close
+        # Get output as it happens rather than buffered
+        child_stdout.sync = true
+        child_stderr.sync = true
+
+        true
+      end
+
+      # Some patch levels of ruby in wide use (in particular the ruby 1.8.6 on OSX)
+      # segfault when you IO.select a pipe that's reached eof. Weak sauce.
+      def open_pipes
+        @open_pipes ||= [child_stdout, child_stderr]
+      end
+
+      def read_stdout_to_buffer
+        while chunk = child_stdout.read_nonblock(READ_SIZE)
+          @stdout << chunk
+          @live_stream << chunk if @live_stream
+        end
+      rescue Errno::EAGAIN
+      rescue EOFError
+        open_pipes.delete_at(0)
+      end
+
+      def read_stderr_to_buffer
+        while chunk = child_stderr.read_nonblock(READ_SIZE)
+          @stderr << chunk
+        end
+      rescue Errno::EAGAIN
+      rescue EOFError
+        open_pipes.delete_at(1)
+      end
+
+      def fork_subprocess
+        initialize_ipc
+
+        fork do
+          configure_subprocess_file_descriptors
+
+          set_user
+          set_group
+          set_environment
+          set_umask
+          set_cwd
+
+          begin
+            command.kind_of?(Array) ? exec(*command) : exec(command)
+
+            raise 'forty-two' # Should never get here
+          rescue Exception => e
+            Marshal.dump(e, process_status_pipe.last)
+            process_status_pipe.last.flush
+          end
+          process_status_pipe.last.close unless (process_status_pipe.last.closed?)
+          exit!
+        end
+      end
+
+      # Attempt to get a Marshaled error from the side-channel.
+      # If it's there, un-marshal it and raise. If it's not there,
+      # assume everything went well.
+      def propagate_pre_exec_failure
+        begin
+          e = Marshal.load child_process_status
+          raise(Exception === e ? e : "unknown failure: #{e.inspect}")
+        rescue EOFError # If we get an EOF error, then the exec was successful
+          true
+        ensure
+          child_process_status.close
+        end
+      end
+
+    end
+  end
+end

data/lib/chef/shell_out/windows.rb

@@ -0,0 +1,95 @@
+#--
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'timeout'
+require 'win32/open3'
+
+class Chef
+  class ShellOut
+    module Windows
+
+      #--
+      # Missing lots of features from the UNIX version, such as
+      # environment, cwd, etc.
+      def run_command
+        Chef::Log.debug("sh(#{@command})")
+        # win32 open4 is really just open3.
+        Open3.popen3(@command) do |stdin,stdout,stderr|
+          @finished_stdout = false
+          @finished_stderr = false
+          stdin.close
+          stdout.sync = true
+          stderr.sync = true
+
+          # TBH, I really don't know what this will do when it times out.
+          # However, I'm powerless to make windows have non-blocking IO, so
+          # thread party it is.
+          Timeout.timeout(timeout) do
+            out_reader = Thread.new do
+              loop do
+                read_stdout(stdout)
+                break if @finished_stdout
+              end
+            end
+            err_reader = Thread.new do
+              loop do
+                read_stderr(stderr)
+                break if @finished_stderr
+              end
+            end
+
+            out_reader.join
+            err_reader.join
+
+            @status = $?
+          end
+        end
+
+        self
+
+      rescue Timeout::Error
+        raise Chef::Exceptions::CommandTimeout, "command timed out:\n#{format_for_exception}"
+      end
+
+      def read_stdout(stdout)
+        return nil if @finished_stdout
+        if chunk = stdout.sysread(8096)
+          @stdout << chunk
+        else
+          @finished_stdout = true
+        end
+      rescue EOFError
+        @finished_stdout = true
+      rescue Errno::EAGAIN
+      end
+
+      def read_stderr(stderr)
+        return nil if @finished_stderr
+        if chunk = stderr.sysread(8096)
+          @stderr << chunk
+        else
+          @finished_stderr = true
+        end
+      rescue EOFError
+        @finished_stderr = true
+      rescue Errno::EAGAIN
+      end
+
+    end
+  end
+end

data/lib/chef/solr_query.rb

@@ -0,0 +1,187 @@
+#
+# Author:: Adam Jacob (<adam@opscode.com>)
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Copyright:: Copyright (c) 2009-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/mixin/xml_escape'
+require 'chef/log'
+require 'chef/config'
+require 'chef/couchdb'
+require 'chef/solr_query/solr_http_request'
+require 'chef/solr_query/query_transform'
+
+class Chef
+  class SolrQuery
+
+    ID_KEY = "X_CHEF_id_CHEF_X"
+    DEFAULT_PARAMS = Mash.new(:start => 0, :rows => 1000, :sort => "#{ID_KEY} asc", :wt => 'json', :indent => 'off').freeze
+    FILTER_PARAM_MAP = {:database => 'X_CHEF_database_CHEF_X', :type => "X_CHEF_type_CHEF_X", :data_bag  => 'data_bag'}
+    VALID_PARAMS = [:start,:rows,:sort,:q,:type]
+    BUILTIN_SEARCH_TYPES = ["role","node","client","environment"]
+    DATA_BAG_ITEM = 'data_bag_item'
+
+    include Chef::Mixin::XMLEscape
+
+    attr_accessor :query
+    attr_accessor :params
+
+    # Create a new Query object - takes the solr_url and optional
+    # Chef::CouchDB object to inflate objects into.
+    def initialize(couchdb = nil)
+      @filter_query = {}
+      @params = {}
+
+      if couchdb.nil?
+        @database = Chef::Config[:couchdb_database]
+        @couchdb = Chef::CouchDB.new(nil, Chef::Config[:couchdb_database])
+      else
+        unless couchdb.kind_of?(Chef::CouchDB)
+          Chef::Log.warn("Passing the database name to Chef::Solr::Query initialization is deprecated. Please pass in the Chef::CouchDB object instead.")
+          @database = couchdb
+          @couchdb = Chef::CouchDB.new(nil, couchdb)
+        else
+          @database = couchdb.couchdb_database
+          @couchdb = couchdb
+        end
+      end
+    end
+
+    def self.from_params(params, couchdb=nil)
+      query = new(couchdb)
+      query.params = VALID_PARAMS.inject({}) do |p, param_name|
+        p[param_name] = params[param_name] if params.key?(param_name)
+        p
+      end
+      query.update_filter_query_from_params
+      query.update_query_from_params
+      query
+    end
+
+    def filter_by(filter_query_params)
+      filter_query_params.each do |key, value|
+        @filter_query[FILTER_PARAM_MAP[key]] = value
+      end
+    end
+
+    def filter_query
+      @filter_query.map { |param, value| "+#{param}:#{value}" }.join(' ')
+    end
+
+    def filter_by_type(type)
+      case type
+      when *BUILTIN_SEARCH_TYPES
+        filter_by(:type => type)
+      else
+        filter_by(:type => DATA_BAG_ITEM, :data_bag => type)
+      end
+    end
+
+    def update_filter_query_from_params
+      filter_by(:database => @database)
+      filter_by_type(params.delete(:type))
+    end
+
+    def update_query_from_params
+      original_query = URI.decode(params.delete(:q) || "*:*")
+      @query = Chef::SolrQuery::QueryTransform.transform(original_query)
+    end
+
+    def objects
+      if !object_ids.empty?
+        @bulk_objects ||= @couchdb.bulk_get(object_ids)
+        Chef::Log.debug { "bulk get of objects: #{@bulk_objects.inspect}" }
+        @bulk_objects
+      else
+        []
+      end
+    end
+
+    def object_ids
+      @object_ids ||= results["response"]["docs"].map { |d| d[ID_KEY] }
+    end
+
+    def results
+      @results ||= SolrHTTPRequest.select(self.to_hash)
+    end
+
+    # Search Solr for objects of a given type, for a given query.
+    def search
+      { "rows" => objects, "start" => results["response"]["start"],
+        "total" => results["response"]["numFound"] }
+    end
+
+    def to_hash
+      options = DEFAULT_PARAMS.merge(params)
+      options[:fq] = filter_query
+      options[:q] = @query
+      options
+    end
+
+    START_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n".freeze
+    START_DELETE_BY_QUERY = "<delete><query>".freeze
+    END_DELETE_BY_QUERY = "</query></delete>\n".freeze
+    COMMIT = "<commit/>\n".freeze
+
+    def commit(opts={})
+      SolrHTTPRequest.update("#{START_XML}#{COMMIT}")
+    end
+
+    def delete_database(db)
+      query_data = xml_escape("X_CHEF_database_CHEF_X:#{db}")
+      xml = "#{START_XML}#{START_DELETE_BY_QUERY}#{query_data}#{END_DELETE_BY_QUERY}"
+      SolrHTTPRequest.update(xml)
+      commit
+    end
+
+    def rebuild_index(db=Chef::Config[:couchdb_database])
+      delete_database(db)
+
+      results = {}
+      [Chef::ApiClient, Chef::Node, Chef::Role].each do |klass|
+        results[klass.name] = reindex_all(klass) ? "success" : "failed"
+      end
+      databags = Chef::DataBag.cdb_list(true)
+      Chef::Log.info("Reloading #{databags.size.to_s} #{Chef::DataBag} objects into the indexer")
+      databags.each { |i| i.add_to_index; i.list(true).each { |x| x.add_to_index } }
+      results[Chef::DataBag.name] = "success"
+      results
+    end
+
+    def reindex_all(klass, metadata={})
+      begin
+        items = klass.cdb_list(true)
+        Chef::Log.info("Reloading #{items.size.to_s} #{klass.name} objects into the indexer")
+        items.each { |i| i.add_to_index }
+      rescue Net::HTTPServerException => e
+        # 404s are okay, there might not be any of that kind of object...
+        if e.message =~ /Not Found/
+          Chef::Log.warn("Could not load #{klass.name} objects from couch for re-indexing (this is ok if you don't have any of these)")
+          return false
+        else
+          raise e
+        end
+      rescue Exception => e
+        Chef::Log.fatal("Chef encountered an error while attempting to load #{klass.name} objects back into the index")
+        raise e
+      end
+      true
+    end
+
+
+  end
+end