opscode-pushy-client 2.3.0

data/lib/pushy_client/cli.rb

@@ -0,0 +1,168 @@
+# @copyright Copyright 2014 Chef Software, Inc. All Rights Reserved.
+#
+# This file is provided to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+require 'chef/application'
+require 'chef/config'
+# This is needed for compat with chef-client >= 11.8.0.
+# To keep compat with older chef-client, rescue if not found
+require 'chef/config_fetcher' rescue 'assuming chef-client < 11.8.0'
+require 'chef/log'
+require_relative '../pushy_client'
+require_relative '../pushy_client/version'
+
+class PushyClient
+  class CLI < Chef::Application
+
+    def self.find_default_config
+      configs = ['chef-push-client.rb', 'push-jobs-client.rb', 'client.rb']
+      base = "/etc/chef"
+      paths = configs.map {|c| Chef::Config.platform_specific_path(File.join(base, c)) }
+      path = paths.detect {|p| File.exist?(p) }
+      # todo make debug before commit
+      Chef::Log.info("Push Client using default config file path: '#{path}'")
+      path
+    end
+
+    option :config_file,
+      :short => "-c CONFIG",
+      :long  => "--config CONFIG",
+      :default => find_default_config,
+      :description => "The configuration file to use"
+
+    option :log_level,
+      :short        => "-l LEVEL",
+      :long         => "--log_level LEVEL",
+      :description  => "Set the log level (debug, info, warn, error, fatal)",
+      :proc         => lambda { |l| l.to_sym }
+
+    option :log_location,
+      :short        => "-L LOGLOCATION",
+      :long         => "--logfile LOGLOCATION",
+      :description  => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
+      :proc         => nil
+
+    option :help,
+      :short        => "-h",
+      :long         => "--help",
+      :description  => "Show this message",
+      :on           => :tail,
+      :boolean      => true,
+      :show_options => true,
+      :exit         => 0
+
+    option :node_name,
+      :short => "-N NODE_NAME",
+      :long => "--node-name NODE_NAME",
+      :description => "The node name for this client",
+      :proc => nil
+
+    option :chef_server_url,
+      :short => "-S CHEFSERVERURL",
+      :long => "--server CHEFSERVERURL",
+      :description => "The chef server URL",
+      :proc => nil
+
+    option :client_key,
+      :short        => "-k KEY_FILE",
+      :long         => "--client_key KEY_FILE",
+      :description  => "Set the client key file location",
+      :proc         => nil
+
+    option :version,
+      :short        => "-v",
+      :long         => "--version",
+      :description  => "Show push client version",
+      :boolean      => true,
+      :proc         => lambda {|v| puts "Push Client: #{::PushyClient::VERSION}"},
+      :exit         => 0
+
+    option :file_dir,
+      :short        => "-d DIR",
+      :long         => "--file_dir DIR",
+      :description  => "Set the directory for temporary files",
+      :default      => "/tmp/chef-push",
+      :proc         => nil
+
+    option :allow_unencrypted,
+      :long        => "--allow_unencrypted",
+      :boolean     => true,
+      :description => "Allow unencrypted connections to 1.x servers"
+
+    def reconfigure
+      # We do not use Chef's formatters.
+      Chef::Config[:force_logger] = true
+      super
+    end
+
+    def setup_application
+    end
+
+    def shutdown(ret_code = 0)
+        @client.stop if @client
+        exit(ret_code)
+    end
+
+    def run_application
+      if Chef::Config[:version]
+        puts "Push Client version: #{::PushyClient::VERSION}"
+      end
+
+      ohai = Ohai::System.new
+      ohai.load_plugins
+      ohai.run_plugins(true, ['hostname'])
+
+      @client = PushyClient.new(
+        :chef_server_url => Chef::Config[:chef_server_url],
+        :client_key      => Chef::Config[:client_key],
+        :node_name       => Chef::Config[:node_name] || ohai[:fqdn] || ohai[:hostname],
+        :whitelist       => Chef::Config[:whitelist] || { 'chef-client' => 'chef-client' },
+        :hostname        => ohai[:hostname],
+        :filedir         => Chef::Config[:file_dir],
+        :allow_unencrypted => Chef::Config[:allow_unencrypted]
+      )
+
+      @client.start
+
+      # install signal handlers
+      # Windows does not support QUIT and USR1 signals
+      exit_signals = if Chef::Platform.windows?
+                       ["TERM", "INT"]
+                     else
+                       ["TERM", "QUIT", "INT"]
+                     end
+
+      exit_signals.each do |sig|
+        Signal.trap(sig) do
+          puts "received #{sig}, shutting down"
+          shutdown(0)
+        end
+      end
+
+      unless Chef::Platform.windows?
+        Signal.trap("USR1") do
+          puts "received USR1, reconfiguring"
+          @client.trigger_reconfigure
+        end
+      end
+
+      # Block forever so that client threads can run
+      while true
+        sleep 3600
+      end
+    end
+  end
+end

data/lib/pushy_client/heartbeater.rb

@@ -0,0 +1,153 @@
+# @copyright Copyright 2014 Chef Software, Inc. All Rights Reserved.
+#
+# This file is provided to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+class PushyClient
+  class Heartbeater
+    NUM_HEARTBEATS_TO_LOG = 3
+
+    def initialize(client)
+      @client = client
+      @online_mutex = Mutex.new
+      @heartbeat_sequence = 1
+      @on_server_availability_change = []
+    end
+
+    attr_reader :client
+    attr_reader :incarnation_id
+    attr_reader :online_threshold
+    attr_reader :offline_threshold
+    attr_reader :interval
+
+    def node_name
+      client.node_name
+    end
+
+    def online?
+      @online
+    end
+
+    def on_server_availability_change(&block)
+      @on_server_availability_change << block
+    end
+
+    def start
+      @incarnation_id = client.config['incarnation_id']
+      @online_threshold = client.config['push_jobs']['heartbeat']['online_threshold']
+      @offline_threshold = client.config['push_jobs']['heartbeat']['offline_threshold']
+      @interval = client.config['push_jobs']['heartbeat']['interval']
+
+      @online_counter = 0
+      @offline_counter = 0
+      # We optimistically declare the server online since we just got a config blob via http from it
+      # however, if the server is reachable via http but not zmq we'll go down after a few heartbeats.
+      set_online(true)
+
+      @heartbeat_thread = Thread.new do
+        Chef::Log.info "[#{node_name}] Starting heartbeat / offline detection thread on interval #{interval} ..."
+
+        while true
+          begin
+            # When the server goes more than <offline_threshold> intervals
+            # without sending us a heartbeat, treat it as offline
+            @online_mutex.synchronize do
+              if @online
+                if @offline_counter > offline_threshold
+                  Chef::Log.info "[#{node_name}] Server has missed #{@offline_counter} heartbeats in a row.  Considering it offline, and stopping heartbeat."
+                  set_online(false)
+                  @online_counter = 0
+                else
+                  @offline_counter += 1
+                end
+              end
+            end
+
+            # We only send heartbeats to online servers
+            if @online
+              client.send_heartbeat(@heartbeat_sequence)
+              if @heartbeat_sequence <= NUM_HEARTBEATS_TO_LOG
+                Chef::Log.info "[#{node_name}] Sending heartbeat #{@heartbeat_sequence} (logging first #{NUM_HEARTBEATS_TO_LOG})"
+              else
+                Chef::Log.debug "[#{node_name}] Sending heartbeat #{@heartbeat_sequence}"
+              end
+              @heartbeat_sequence += 1
+            end
+            sleep(interval)
+          rescue
+            client.log_exception("Error in heartbeat / offline detection thread", $!)
+          end
+        end
+      end
+    end
+
+    def stop
+      Chef::Log.info "[#{node_name}] Stopping heartbeat / offline detection thread ..."
+      @heartbeat_thread.kill
+      @heartbeat_thread.join
+    end
+
+    def reconfigure
+      stop
+      start # Start picks up new configuration
+    end
+
+    # TODO use the sequence for something?
+    def heartbeat_received(incarnation_id, sequence)
+      message = "[#{node_name}] Received server heartbeat (sequence ##{sequence})"
+      if @online_counter <= NUM_HEARTBEATS_TO_LOG
+        Chef::Log.info message + " logging #{@online_counter}/#{NUM_HEARTBEATS_TO_LOG}"
+      else
+        Chef::Log.debug message
+      end
+      # If the incarnation id has changed, we need to reconfigure.
+      if @incarnation_id != incarnation_id
+        if @incarnation_id.nil?
+          @incarnation_id = incarnation_id
+          Chef::Log.info "[#{node_name}] First heartbeat received.  Server is at incarnation ID #{incarnation_id}."
+        else
+          # We need to set incarnation id before we reconfigure; this thread will
+          # be killed by the reconfigure :)
+          splay = Random.new.rand(interval.to_f)
+          Chef::Log.info "[#{node_name}] Server restart detected (incarnation ID changed from #{@incarnation_id} to #{incarnation_id}).  Reconfiguring after a randomly chosen #{splay} second delay to avoid storming the server ..."
+          @incarnation_id = incarnation_id
+          sleep(splay)
+          client.trigger_reconfigure
+        end
+      end
+
+      @online_mutex.synchronize do
+        @offline_counter = 0
+
+        if !@online && @online_counter > online_threshold
+          Chef::Log.info "[#{node_name}] Server has heartbeated #{@online_counter} times without missing more than #{offline_threshold} heartbeats in a row."
+          set_online(true)
+        else
+          @online_counter += 1
+        end
+      end
+    end
+
+    private
+
+    def set_online(online)
+      @online = online
+      Chef::Log.info "[#{node_name}] Considering server online, and starting to heartbeat"
+      @on_server_availability_change.each do |block|
+        block.call(online)
+      end
+    end
+  end
+end

data/lib/pushy_client/job_runner.rb

@@ -0,0 +1,316 @@
+# @copyright Copyright 2014 Chef Software, Inc. All Rights Reserved.
+#
+# This file is provided to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This is needed to fix an issue in win32-process v. 0.6.5
+# where Process.wait blocks the entire Ruby interpreter
+# for the duration of the process.
+require 'chef/platform'
+require 'mixlib/shellout'
+if Chef::Platform.windows?
+  require 'pushy_client/win32'
+end
+
+class PushyClient
+  class JobRunner
+    def initialize(client)
+      @client = client
+      @on_job_state_change = []
+
+      set_job_state(:idle)
+      @pid = nil
+      @process_thread = nil
+
+      # Keep job state and process state in sync
+      @state_lock = Mutex.new
+    end
+
+    attr_reader :client
+    attr_reader :state
+    attr_reader :job_id
+    attr_reader :command
+    attr_reader :lockfile
+
+    def safe_to_reconfigure?
+      @state_lock.synchronize do
+        @state == :idle
+      end
+    end
+
+    def node_name
+      client.node_name
+    end
+
+    def start
+    end
+
+    def stop
+      if @state == :running
+        kill_process
+      end
+      set_job_state(:idle)
+    end
+
+    def reconfigure
+      # We have no configuration, and keep state between reconfigures
+    end
+
+    def commit(job_id, command, opts)
+      @opts = opts
+      @state_lock.synchronize do
+        if @state == :idle
+          # If we're being asked to lock
+          if client.whitelist[command] &&
+             client.whitelist[command].is_a?(Hash) &&
+             client.whitelist[command][:lock]
+            # If the command is chef-client
+            # We don't want to run if there is already another instance of chef-client going,
+            # so we check to see if there is a runlock on chef-client before committing. This
+            # currently only works in versions of chef where runlock has been implemented.
+
+            # The location of our lockfile
+            if client.whitelist[command][:lock] == true
+              lockfile_location = Chef::Config[:lockfile] || "#{Chef::Config[:file_cache_path]}/chef-client-running.pid"
+            else
+              lockfile_location = client.whitelist[command][:lock]
+            end
+            # Open the Lockfile
+            begin
+              @lockfile = File.open(lockfile_location, 'w')
+              locked = lockfile.flock(File::LOCK_EX|File::LOCK_NB)
+              unless locked
+                Chef::Log.info("[#{node_name}] Received commit #{job_id} but is already running '#{command}'")
+                client.send_command(:nack_commit, job_id)
+                return false
+              end
+            rescue Errno::ENOENT
+            end
+          elsif client.whitelist[command]
+            user_ok = check_user(job_id)
+            dir_ok = check_dir(job_id)
+            file_ok = check_file(job_id)
+            if user_ok && dir_ok && file_ok
+              Chef::Log.info("[#{node_name}] Received commit #{job_id}")
+              set_job_state(:committed, job_id, command)
+              client.send_command(:ack_commit, job_id)
+              true
+            else
+              client.send_command(:nack_commit, job_id)
+            end
+          else
+            Chef::Log.error("[#{node_name}] Received commit #{job_id}, but command '#{command}' is not in the whitelist!")
+            client.send_command(:nack_commit, job_id)
+            false
+          end
+        else
+          Chef::Log.warn("[#{node_name}] Received commit #{job_id} but current state is #{@state} #{@job_id}")
+          client.send_command(:nack_commit, job_id)
+          false
+        end
+      end
+    end
+
+    def run(job_id)
+      @state_lock.synchronize do
+        if @state == :committed && @job_id == job_id
+          Chef::Log.info("[#{node_name}] Received run #{job_id}")
+          pid, process_thread = start_process
+          set_job_state(:running, job_id, @command, pid, process_thread)
+          client.send_command(:ack_run, job_id)
+          true
+        else
+          Chef::Log.warn("[#{node_name}] Received run #{job_id} but current state is #{@state} #{@job_id}")
+          client.send_command(:nack_run, job_id)
+          false
+        end
+      end
+    end
+
+    def abort
+      Chef::Log.info("[#{node_name}] Received abort")
+      @state_lock.synchronize do
+        _job_id = job_id
+        stop
+        client.send_command(:aborted, _job_id)
+      end
+    end
+
+    def job_state
+      @state_lock.synchronize do
+        get_job_state
+      end
+    end
+
+    def on_job_state_change(&block)
+      @on_job_state_change << block
+    end
+
+    private
+
+    def get_job_state
+      {
+        :state => @state,
+        :job_id => @job_id,
+        :command => @command
+      }
+    end
+
+    def set_job_state(state, job_id = nil, command = nil, pid = nil, process_thread = nil)
+      if state == :idle || state == :running
+        if @lockfile
+          # If there is a lockfile Release the lock to allow chef-client to run
+          lockfile.flock(File::LOCK_UN)
+          lockfile.close
+        end
+      end
+      @state = state
+      @job_id = job_id
+      @command = command
+      @pid = pid
+      @process_thread = process_thread
+
+      Chef::Log.debug("[] Job #{job_id}: command '#{command}' state '#{state}'")
+
+      # Notify people of the change
+      @on_job_state_change.each { |block| block.call(get_job_state) }
+    end
+
+    def completed(job_id, exit_code, stdout, stderr)
+      Chef::Log.info("[#{node_name}] Job #{job_id} completed with exit code #{exit_code}")
+      @state_lock.synchronize do
+        if @state == :running && @job_id == job_id
+          set_job_state(:idle)
+          status = exit_code == 0 ? :succeeded : :failed
+          params = {}
+          params[:stdout] = stdout if stdout
+          params[:stderr] = stderr if stderr
+          client.send_command(status, job_id, params)
+        end
+      end
+    end
+
+    def start_process
+      # _pid and _job_id are local variables so that if @pid or @job_id change
+      # for any reason (for example, they become nil), the thread we create
+      # still tracks the correct pid.
+      if client.whitelist[command].is_a?(Hash)
+        command_line = client.whitelist[command][:command_line]
+      else
+        command_line = client.whitelist[command]
+      end
+      user = @opts['user']
+      dir = @opts['dir']
+      env = @opts['env'] || {}
+      capture = @opts['capture'] || false
+      path = extract_file
+      env.merge!({'CHEF_PUSH_JOB_FILE' => path}) if path
+      std_env = {'CHEF_PUSH_NODE_NAME' => node_name, 'CHEF_PUSH_JOB_ID' => @job_id}
+      env.merge!(std_env)
+      # XXX We set the timeout to 86400, because the time in ShellOut is
+      # 60 seconds, and that might be too slow.  But we currently don't
+      # have the timeout from the pushy-server.  Instead of changing it from
+      # a hard-coded value to a config option, we should expand the protocol
+      # to support sending the timeout.
+      command = Mixlib::ShellOut.new(command_line,
+                                      :user => user,
+                                      :cwd => dir,
+                                      :env => env,
+                                      :timeout => 86400)
+      _job_id = @job_id
+      # Can't get the _pid from the ShellOut command.  So
+      # we can't kill it, either.
+      _pid = nil
+      Chef::Log.info("[#{node_name}] Job #{job_id}: started command '#{command_line}' with PID '#{_pid}'")
+
+      # Wait for the job to complete and close it out.
+      process_thread = Thread.new do
+        begin
+          command.run_command
+          stdout = command.stdout if capture
+          stderr = command.stderr if capture
+          completed(_job_id, command.status.exitstatus, stdout, stderr)
+        rescue
+          client.log_exception("Exception raised while waiting for job #{_job_id} to complete", $!)
+          abort
+        end
+      end
+
+      [ _pid, process_thread ]
+    end
+
+    def kill_process
+      Chef::Log.info("[#{node_name}] Killing process #{@pid}")
+      @process_thread.kill
+      @process_thread.join
+      begin
+        Process.kill(1, @pid) if @pid
+      rescue
+        client.log_exception("Exception in Process.kill(1, #{@pid})", $!)
+      end
+    end
+
+    def check_user(job_id)
+      user = @opts['user']
+      if user
+        begin
+          Etc.getpwnam(user)
+          true
+        rescue
+          Chef::Log.error("[#{node_name}] Received commit #{job_id}, but user '#{user}' does not exist!")
+          false
+        end
+      else
+        true
+      end
+    end
+
+    def check_dir(job_id)
+      # XX Perhaps should be stricted, e.g. forking a process to actually try to chdir
+      dir = @opts['dir']
+      dir_ok = !dir || Dir.exists?(dir)
+      Chef::Log.error("[#{node_name}] Received commit #{job_id}, but dir '#{dir}' does not exist!") unless dir_ok
+      dir_ok
+    end
+
+    def check_file(job_id)
+      file = @opts['file']
+      file_ok = !file || file.start_with?('base64:', 'raw:')
+      Chef::Log.error("[#{node_name}] Received commit #{job_id}, but file '#{file}' is a bad format!") unless file_ok
+      file_ok
+    end
+
+    def extract_file
+      file = @opts['file']
+      return nil unless file
+      require 'tmpdir'
+      dir = client.file_dir
+      Dir.mkdir(dir) unless Dir.exists?(dir)
+      path = Dir::Tmpname.create('pushy_file', dir){|p| p}
+      File.open(path, 'w') do |f|
+        type, filedata = file.split(/:/, 2)
+        case type
+        when "raw"
+          f.write(filedata)
+        when "base64"
+          f.write(Base64.decode64(filedata))
+        else
+          Chef::Log.error("[#{node_name}] Received commit #{job_id}, but file starting with '#{file.slice(0,80)}' has a bad format!")
+        end
+      end
+      path
+    end
+  end
+end