opscode-pushy-client 2.100.0 → 3.0.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44b1f6febf06aecd9bffcece1c65126a9e58d9a4fdd1925506782f0400cdf189
-  data.tar.gz: 8e288e08d58790b64be013bfea14ca6b1838c4e136aa9a19cc1c116a71ccb1ec
+  metadata.gz: c77b2f636067d317b9810cc3d5835084e6eb9ce4973af66c40a340caa44e534a
+  data.tar.gz: f01c7d95292f4d1d4b6beef2bc695ace5eca8e65b999f0708875b5557950b286
 SHA512:
-  metadata.gz: 52ab8e2aa49d1b0af72a1474d798ca711b878fdc1d8f8686c8b3ec9d182770996afd18559af44987a497a3257c4ff38c767881b776d5a830ae664efc8f520801
-  data.tar.gz: d01ca100485a00b6cdfb8eff100993de90791ef133ac92117d18dca5ac121bbd65a309f1ac9aecf9230340dd023370723b0b3fb1bbb8b618c4178cc0febf32d1
+  metadata.gz: bad3691458ecc47aa5167e2f0fa80b039ad318287e0d412f1daf26039a858f78134fe855efcb80bfd0e935eb977ce328a7c31bbcb19142f7013fa36992096fb9
+  data.tar.gz: 8f830c8e0af5bfd83e6879cacb7759f95e9835f368421997f1cac034b3587564eb6e38e5cbb63861cf40da6068fcf403711c26467502ef12ed563de83934c907

data/Gemfile

@@ -9,6 +9,7 @@ end
 platforms :mswin, :mingw do
   gem "ffi"
   gem "rdp-ruby-wmi"
+  gem "windows-api"
   gem "windows-pr"
   gem "win32-api"
   gem "win32-dir"

data/lib/pushy_client.rb

@@ -41,6 +41,7 @@ class PushyClient
     @file_dir        = options[:file_dir] || '/tmp/pushy'
     @file_dir_expiry = options[:file_dir_expiry] || 86400
     @allowed_overwritable_env_vars = options[:allowed_overwritable_env_vars]
+    @max_body_size   = options[:max_body_size] || 63000
 
     @allow_unencrypted = options[:allow_unencrypted] || false
     @client_curve_pub_key, @client_curve_sec_key = ZMQ::Util.curve_keypair
@@ -79,6 +80,7 @@ class PushyClient
   attr_accessor :node_name
   attr_accessor :hostname
   attr_accessor :whitelist
+  attr_accessor :max_body_size
   attr_reader :incarnation_id
   attr_reader :legacy_mode # indicate we've fallen back to 1.x
   attr_reader :allowed_overwritable_env_vars

data/lib/pushy_client/cli.rb

@@ -70,6 +70,12 @@ class PushyClient
       :description => "The node name for this client",
       :proc => nil
 
+    option :max_body_size,
+      :short => "-M MAX_BODY_SIZE",
+      :long => "--max-body-size MAX_BODY_SIZE",
+      :description => "MAX_BODY_SIZE of STDER/SDTOUT for this client",
+      :proc => nil
+
     option :chef_server_url,
       :short => "-S CHEFSERVERURL",
       :long => "--server CHEFSERVERURL",
@@ -133,7 +139,8 @@ class PushyClient
         :hostname        => ohai[:hostname],
         :filedir         => Chef::Config[:file_dir],
         :allow_unencrypted => Chef::Config[:allow_unencrypted],
-        :allowed_overwritable_env_vars => Chef::Config[:allowed_overwritable_env_vars]
+        :allowed_overwritable_env_vars => Chef::Config[:allowed_overwritable_env_vars],
+        :max_body_size   => Chef::Config[:max_body_size]
       )
 
       @client.start

data/lib/pushy_client/protocol_handler.rb

@@ -60,14 +60,6 @@ class PushyClient
       end
     end
 
-    # The maximum size, in bytes, allowed for a message body. This is not
-    # configurable because it is configurable (though not documented) on the
-    # server, and we don't want to make users have to sync the two values.
-    # The max on the server is actually 65536, but we leave a little room since
-    # the server is measuring the signed message and we're just counting
-    # the size of the stderr and stdout.
-    MAX_BODY_SIZE = 63000
-
     def initialize(client)
       @client = client
       # We synchronize on this when we change the socket (so if you want a
@@ -114,6 +106,7 @@ class PushyClient
       @server_public_key = OpenSSL::PKey::RSA.new(client.config['public_key'])
       @client_private_key = ProtocolHandler::load_key(client.client_key)
       @max_message_skew = client.config['max_message_skew']
+      @max_body_size = client.config['max_body_size']
 
       if client.using_curve
         server_curve_pub_key = client.config['curve_public_key']
@@ -417,9 +410,8 @@ class PushyClient
     def validate_params(params = {})
       stdout_bytes = params[:stdout].to_s.bytesize
       stderr_bytes = params[:stderr].to_s.bytesize
-
-      if (stdout_bytes + stderr_bytes) > MAX_BODY_SIZE
-        Chef::Log.warn("Command output too long. Will not be sent to server.")
+      if (stdout_bytes + stderr_bytes) > client.max_body_size.to_i
+        Chef::Log.warn("Command output #{stdout_bytes + stderr_bytes} is larger than the Client MAX_BODY_SIZE of #{client.max_body_size.to_i}")
         params.delete_if { |k| [:stdout, :stderr].include? k }
       else
         params
@@ -471,7 +463,7 @@ class PushyClient
       @socket_lock.synchronize do
         @command_socket_outgoing_seq += 1
         json[:sequence] = @command_socket_outgoing_seq
-        message = JSON.generate(json)
+        message = validate_and_generate_json(json)
         if @command_socket
           ProtocolHandler::send_signed_message(@command_socket, method, @client_private_key, @session_key, message, @client)
         else
@@ -480,6 +472,25 @@ class PushyClient
       end
     end
 
+    # This method validates and generates the JSON message
+    # params user packet
+    # Returns Waring or JSON message
+    def validate_and_generate_json(json)
+      message = JSON.generate(json)
+      if message.to_s.bytesize > client.max_body_size.to_i
+        deleted_packet = json.select { |k| [:stdout, :stderr].include? k }
+        json.delete_if { |k| [:stdout, :stderr].include? k }
+        Chef::Log.warn("Deleted packet #{deleted_packet} from Client packet") if deleted_packet.to_s.size > 1
+        # Re-generate json message
+        message = JSON.generate(json)
+      end
+      if message.to_s.bytesize > client.max_body_size.to_i
+        Chef::Log.warn("Client message size #{message.to_s.bytesize} is larger than the Client MAX_BODY_SIZE of #{client.max_body_size.to_i}")
+      else
+        message
+      end
+    end
+
     def self.send_signed_message(socket, method, client_private_key, session_key, message, client)
       auth = case method
              when :rsa2048_sha1

data/lib/pushy_client/status_monitor.rb~

@@ -0,0 +1,308 @@
+# @copyright Copyright 2014 Chef Software, Inc. All Rights Reserved.
+#
+# This file is provided to you under the Apache License,
+# Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This is needed to fix an issue in win32-process v. 0.6.5
+# where Process.wait blocks the entire Ruby interpreter
+# for the duration of the process.
+require 'chef/platform'
+require 'mixlib/shellout'
+if Chef::Platform.windows?
+  require 'pushy_client/win32'
+end
+
+class PushyClient
+  class JobRunner
+    def initialize(client)
+      @client = client
+      @on_job_state_change = []
+
+      set_job_state(:idle)
+      @pid = nil
+      @process_thread = nil
+
+      # Keep job state and process state in sync
+      @state_lock = Mutex.new
+    end
+
+    attr_reader :client
+    attr_reader :state
+    attr_reader :job_id
+    attr_reader :command
+    attr_reader :lockfile
+
+    def node_name
+      client.node_name
+    end
+
+    def start
+    end
+
+    def stop
+      if @state == :running
+        kill_process
+      end
+      set_job_state(:idle)
+    end
+
+    def reconfigure
+      # We have no configuration, and keep state between reconfigures
+    end
+
+    def commit(job_id, command, opts)
+      @opts = opts
+      @state_lock.synchronize do
+        if @state == :idle
+          # If we're being asked to lock
+          if client.whitelist[command] &&
+             client.whitelist[command].is_a?(Hash) &&
+             client.whitelist[command][:lock]
+            # If the command is chef-client
+            # We don't want to run if there is already another instance of chef-client going,
+            # so we check to see if there is a runlock on chef-client before committing. This
+            # currently only works in versions of chef where runlock has been implemented.
+
+            # The location of our lockfile
+            if client.whitelist[command][:lock] == true
+              lockfile_location = Chef::Config[:lockfile] || "#{Chef::Config[:file_cache_path]}/chef-client-running.pid"
+            else
+              lockfile_location = client.whitelist[command][:lock]
+            end
+            # Open the Lockfile
+            begin
+              @lockfile = File.open(lockfile_location, 'w')
+              locked = lockfile.flock(File::LOCK_EX|File::LOCK_NB)
+              unless locked
+                Chef::Log.info("[#{node_name}] Received commit #{job_id} but is already running '#{command}'")
+                client.send_command(:nack_commit, job_id)
+                return false
+              end
+            rescue Errno::ENOENT
+            end
+          elsif client.whitelist[command]
+            user_ok = check_user(job_id)
+            dir_ok = check_dir(job_id)
+            file_ok = check_file(job_id)
+            if user_ok && dir_ok && file_ok
+              Chef::Log.info("[#{node_name}] Received commit #{job_id}")
+              set_job_state(:committed, job_id, command)
+              client.send_command(:ack_commit, job_id)
+              true
+            else
+              client.send_command(:nack_commit, job_id)
+            end
+          else
+            Chef::Log.error("[#{node_name}] Received commit #{job_id}, but command '#{command}' is not in the whitelist!")
+            client.send_command(:nack_commit, job_id)
+            false
+          end
+        else
+          Chef::Log.warn("[#{node_name}] Received commit #{job_id} but current state is #{@state} #{@job_id}")
+          client.send_command(:nack_commit, job_id)
+          false
+        end
+      end
+    end
+
+    def run(job_id)
+      @state_lock.synchronize do
+        if @state == :committed && @job_id == job_id
+          Chef::Log.info("[#{node_name}] Received run #{job_id}")
+          pid, process_thread = start_process
+          set_job_state(:running, job_id, @command, pid, process_thread)
+          client.send_command(:ack_run, job_id)
+          true
+        else
+          Chef::Log.warn("[#{node_name}] Received run #{job_id} but current state is #{@state} #{@job_id}")
+          client.send_command(:nack_run, job_id)
+          false
+        end
+      end
+    end
+
+    def abort
+      Chef::Log.info("[#{node_name}] Received abort")
+      @state_lock.synchronize do
+        _job_id = job_id
+        stop
+        client.send_command(:aborted, _job_id)
+      end
+    end
+
+    def job_state
+      @state_lock.synchronize do
+        get_job_state
+      end
+    end
+
+    def on_job_state_change(&block)
+      @on_job_state_change << block
+    end
+
+    private
+
+    def get_job_state
+      {
+        :state => @state,
+        :job_id => @job_id,
+        :command => @command
+      }
+    end
+
+    def set_job_state(state, job_id = nil, command = nil, pid = nil, process_thread = nil)
+      if state == :idle || state == :running
+        if @lockfile
+          # If there is a lockfile Release the lock to allow chef-client to run
+          lockfile.flock(File::LOCK_UN)
+          lockfile.close
+        end
+      end
+      @state = state
+      @job_id = job_id
+      @command = command
+      @pid = pid
+      @process_thread = process_thread
+
+      # Notify people of the change
+      @on_job_state_change.each { |block| block.call(get_job_state) }
+    end
+
+    def completed(job_id, exit_code, stdout, stderr)
+      Chef::Log.info("[#{node_name}] Job #{job_id} completed with exit code #{exit_code}")
+      @state_lock.synchronize do
+        if @state == :running && @job_id == job_id
+          set_job_state(:idle)
+          status = exit_code == 0 ? :succeeded : :failed
+          params = {}
+          params[:stdout] = stdout if stdout
+          params[:stderr] = stderr if stderr
+          client.send_command(status, job_id, params)
+        end
+      end
+    end
+
+    def start_process
+      # _pid and _job_id are local variables so that if @pid or @job_id change
+      # for any reason (for example, they become nil), the thread we create
+      # still tracks the correct pid.
+      if client.whitelist[command].is_a?(Hash)
+        command_line = client.whitelist[command][:command_line]
+      else
+        command_line = client.whitelist[command]
+      end
+      user = @opts['user']
+      dir = @opts['dir']
+      env = @opts['env'] || {}
+      capture = @opts['capture'] || false
+      path = extract_file
+      env.merge!({'CHEF_PUSH_JOB_FILE' => path}) if path
+      std_env = {'CHEF_PUSH_NODE_NAME' => node_name, 'CHEF_PUSH_JOB_ID' => @job_id}
+      env.merge!(std_env)
+      # XXX We set the timeout to 86400, because the time in ShellOut is
+      # 60 seconds, and that might be too slow.  But we currently don't
+      # have the timeout from the pushy-server.  Instead of changing it from
+      # a hard-coded value to a config option, we should expand the protocol
+      # to support sending the timeout.
+      command = Mixlib::ShellOut.new(command_line,
+                                      :user => user,
+                                      :cwd => dir,
+                                      :env => env,
+                                      :timeout => 86400)
+      _job_id = @job_id
+      # Can't get the _pid from the ShellOut command.  So
+      # we can't kill it, either.
+      _pid = nil
+      Chef::Log.info("[#{node_name}] Job #{job_id}: started command '#{command_line}' with PID '#{_pid}'")
+
+      # Wait for the job to complete and close it out.
+      process_thread = Thread.new do
+        begin
+          command.run_command
+          stdout = command.stdout if capture
+          stderr = command.stderr if capture
+          completed(_job_id, command.status.exitstatus, stdout, stderr)
+        rescue
+          client.log_exception("Exception raised while waiting for job #{_job_id} to complete", $!)
+          abort
+        end
+      end
+
+      [ _pid, process_thread ]
+    end
+
+    def kill_process
+      Chef::Log.info("[#{node_name}] Killing process #{@pid}")
+      @process_thread.kill
+      @process_thread.join
+      begin
+        Process.kill(1, @pid) if @pid
+      rescue
+        client.log_exception("Exception in Process.kill(1, #{@pid})", $!)
+      end
+    end
+
+    def check_user(job_id)
+      user = @opts['user']
+      if user
+        begin
+          Etc.getpwnam(user)
+          true
+        rescue
+          Chef::Log.error("[#{node_name}] Received commit #{job_id}, but user '#{user}' does not exist!")
+          false
+        end
+      else
+        true
+      end
+    end
+
+    def check_dir(job_id)
+      # XX Perhaps should be stricted, e.g. forking a process to actually try to chdir
+      dir = @opts['dir']
+      dir_ok = !dir || Dir.exists?(dir)
+      Chef::Log.error("[#{node_name}] Received commit #{job_id}, but dir '#{dir}' does not exist!") unless dir_ok
+      dir_ok
+    end
+
+    def check_file(job_id)
+      file = @opts['file']
+      file_ok = !file || file.start_with?('base64:', 'raw:')
+      Chef::Log.error("[#{node_name}] Received commit #{job_id}, but file '#{file}' is a bad format!") unless file_ok
+      file_ok
+    end
+
+    def extract_file
+      file = @opts['file']
+      return nil unless file
+      require 'tmpdir'
+      dir = client.file_dir
+      Dir.mkdir(dir) unless Dir.exists?(dir)
+      path = Dir::Tmpname.create('pushy_file', dir){|p| p}
+      File.open(path, 'w') do |f|
+        type, filedata = file.split(/:/, 2)
+        case type
+        when "raw"
+          f.write(filedata)
+        when "base64"
+          f.write(Base64.decode64(filedata))
+        else
+          Chef::Log.error("[#{node_name}] Received commit #{job_id}, but file starting with '#{file.slice(0,80)}' has a bad format!") 
+        end
+      end
+      path
+    end
+  end
+end

data/lib/pushy_client/version.rb

@@ -18,6 +18,6 @@
 # Note: the version must also be updated in
 # omnibus/config/projects/push-jobs-client.rb
 class PushyClient
-  VERSION = "2.100.0"
+  VERSION = "3.0.0.pre"
   PROTOCOL_VERSION = "2.0"
 end

data/opscode-pushy-client.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.5'
 
-  gem.add_dependency "chef", ">= 15.0", "< 17.0"
-  gem.add_dependency "ohai", ">= 15.0", "< 17.0"
+  gem.add_dependency "chef", ">= 14.0", "< 16.0"
+  gem.add_dependency "ohai", ">= 14.0", "< 16.0"
   gem.add_dependency "ffi-rzmq"
   gem.add_dependency "uuidtools"
 

data/spec/pushy_client/protocol_handler_spec.rb

@@ -18,6 +18,7 @@ describe PushyClient::ProtocolHandler do
       allow(client).to receive(:hostname)
       allow(client).to receive(:org_name)
       allow(client).to receive(:incarnation_id)
+      allow(client).to receive(:max_body_size).and_return(63000)
       allow_any_instance_of(described_class).to receive(:send_signed_json_command)
       allow(Chef::Log).to receive(:warn)
     end
@@ -32,7 +33,7 @@ describe PushyClient::ProtocolHandler do
 
       it "logs a warning" do
         expect(Chef::Log).to receive(:warn).with(
-          "Command output too long. Will not be sent to server."
+          "Command output #{params[:stdout].to_s.bytesize + params[:stderr].to_s.bytesize} is larger than the Client MAX_BODY_SIZE of #{client.max_body_size.to_i}"
         )
         send_command
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opscode-pushy-client
 version: !ruby/object:Gem::Version
-  version: 2.100.0
+  version: 3.0.0.pre
 platform: ruby
 authors:
 - Mark Anderson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-09 00:00:00.000000000 Z
+date: 2019-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -16,40 +16,40 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '15.0'
+        version: '14.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '17.0'
+        version: '16.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '15.0'
+        version: '14.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '17.0'
+        version: '16.0'
 - !ruby/object:Gem::Dependency
   name: ohai
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '15.0'
+        version: '14.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '17.0'
+        version: '16.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '15.0'
+        version: '14.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '17.0'
+        version: '16.0'
 - !ruby/object:Gem::Dependency
   name: ffi-rzmq
   requirement: !ruby/object:Gem::Requirement
@@ -124,10 +124,11 @@ description: Client for Chef push jobs server
 email:
 - mark@chef.io
 executables:
-- pushy-service-manager
+- pushy-client
 - print_execution_environment
 - push-apply
-- pushy-client
+- pushy-service-manager
+- print_execution_environment~
 extensions: []
 extra_rdoc_files: []
 files:
@@ -135,6 +136,7 @@ files:
 - LICENSE
 - Rakefile
 - bin/print_execution_environment
+- bin/print_execution_environment~
 - bin/push-apply
 - bin/pushy-client
 - bin/pushy-service-manager
@@ -144,6 +146,7 @@ files:
 - lib/pushy_client/job_runner.rb
 - lib/pushy_client/periodic_reconfigurer.rb
 - lib/pushy_client/protocol_handler.rb
+- lib/pushy_client/status_monitor.rb~
 - lib/pushy_client/version.rb
 - lib/pushy_client/whitelist.rb
 - lib/pushy_client/win32.rb
@@ -167,16 +170,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.8
+rubyforge_project: 
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: Client for Chef push jobs server
 test_files:
-- spec/pushy_client/job_runner_spec.rb
+- spec/spec_helper.rb
 - spec/pushy_client/whitelist_spec.rb
 - spec/pushy_client/protocol_handler_spec.rb
-- spec/spec_helper.rb
+- spec/pushy_client/job_runner_spec.rb