palo_alto 0.3.1 → 0.4.0

data/lib/palo_alto/log.rb

@@ -12,7 +12,7 @@ module PaloAlto
     class Log < Enumerator
       def initialize(client:, query:, log_type:, nlogs: 20, dir: :backward, show_detail: false, days: 7) # rubocop:disable Metrics/MethodLength,Metrics/ParameterLists
         @client = client
-        payload = {
+        @log_query_payload = {
           type: 'log',
           'log-type': log_type,
           nlogs: nlogs,
@@ -22,15 +22,22 @@ module PaloAlto
         }
 
         if days
-          payload[:query] += " AND (receive_time geq '#{(Time.now - days * 3600 * 24).strftime('%Y/%m/%d %H:%M:%S')}')"
+          @log_query_payload[:query] += " AND (receive_time geq '#{(Time.now - days * 3600 * 24).strftime('%Y/%m/%d %H:%M:%S')}')"
         end
 
-        result = @client.execute(payload)
+        run_query
+
+        @first_result = fetch_result
+        super
+      end
+
+      def run_query
+        result = @client.execute(@log_query_payload)
         @job_id = result.at_xpath('response/result/job').text
+        warn "#{@client.host} #{Time.now}: Got job id #{@job_id} for log query"
+
         @count = nil
         @skip = 0
-        @first_result = fetch_result
-        super
       end
 
       def restore_first
@@ -46,17 +53,24 @@ module PaloAlto
       def fetch_result # rubocop:disable Metrics/MethodLength
         return nil if @count && @skip == @count
 
-        payload = {
-          type: 'log',
-          action: 'get',
-          'job-id': @job_id,
-          skip: @skip
-        }
-
         i = 0
         loop do
           sleep 0.5 if i.positive?
-          @current_result = @client.execute(payload)
+          begin
+            payload = {
+              type: 'log',
+              action: 'get',
+              'job-id': @job_id,
+              skip: @skip
+            }
+            @current_result = @client.execute(payload)
+          rescue PaloAlto::UnknownErrorException => e
+            if e.message == 'Query timed out'
+              warn 'Retrying log query'
+              run_query
+              retry
+            end
+          end
           i += 1
           break if @current_result.at_xpath('response/result/job/status').text == 'FIN'
         end

data/lib/palo_alto/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PaloAlto
-  VERSION = '0.3.1'
+  VERSION = '0.4.0'
 end

data/lib/palo_alto.rb

@@ -30,7 +30,7 @@ module PaloAlto
   class UnknownCommandException < PermanentException
   end
 
-  class InternalErrorsException < TemporaryException
+  class UnknownErrorException < TemporaryException
   end
 
   class BadXpathException < PermanentException
@@ -39,6 +39,9 @@ module PaloAlto
   class ObjectNotPresentException < PermanentException
   end
 
+  class ObjectNotFoundException < PermanentException
+  end
+
   class ObjectNotUniqueException < PermanentException
   end
 
@@ -122,6 +125,8 @@ module PaloAlto
         case response.code
         when '200'
           return response.body
+        when '401'
+          raise SessionTimedOutException
         when '400', '403'
           begin
             data = Nokogiri::XML.parse(response.body)
@@ -136,7 +141,7 @@ module PaloAlto
         end
 
         nil
-      rescue Net::OpenTimeout, Errno::ECONNREFUSED => e
+      rescue Net::OpenTimeout, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
         raise ConnectionErrorException, e.message
       end
 
@@ -145,13 +150,12 @@ module PaloAlto
                 when 400 then BadRequestException
                 when 403 then ForbiddenException
                 when 1 then UnknownCommandException
-                when 2..5 then InternalErrorsException
                 when 6 then BadXpathException
                 when 7 then ObjectNotPresentException
                 when 8 then ObjectNotUniqueException
                 when 10 then ReferenceCountNotZeroException
-                when 0, 11, 21 then InternalErrorException # also if there is no code..
                 when 12 then InvalidObjectException
+                when 13 then ObjectNotFoundException
                 when 14 then OperationNotPossibleException
                 when 15 then OperationDeniedException
                 when 16 then UnauthorizedException
@@ -159,7 +163,8 @@ module PaloAlto
                 when 18 then MalformedCommandException
                 when 19..20 then SuccessException
                 when 22 then SessionTimedOutException
-                else InternalErrorException
+                when 2..5, 11, 21 then InternalErrorException
+                else UnknownErrorException
                 end
         raise error, message
       end
@@ -170,15 +175,39 @@ module PaloAlto
     attr_accessor :host, :username, :auth_key, :verify_ssl, :debug, :timeout
 
     def pretty_print_instance_variables
-      super - [:@password, :@subclasses, :@subclasses, :@expression, :@arguments]
+      super - [:@password, :@subclasses, :@subclasses, :@expression, :@arguments, :@cache, :@op, :@auth_key]
     end
 
-    def execute(payload, skip_authentication: false)
+    def execute(payload, skip_authentication: false, skip_cache: false)
       if !auth_key && !skip_authentication
         get_auth_key
       end
 
-      retried = false
+      if payload[:type] == 'config' && !skip_cache
+        if payload[:action] == 'get'
+          start_time = Time.now
+          @cache.each do |cached_xpath, cache|
+            search_xpath = payload[:xpath].sub('/descendant::device-group[1]/', '/device-group/')
+            next unless search_xpath.start_with?(cached_xpath)
+
+            remove = cached_xpath.split('/')[1...-1].join('/').length
+            new_xpath = 'response/result/' + search_xpath[(remove+2)..]
+
+            results = cache.xpath(new_xpath)
+						xml = Nokogiri.parse("<?xml version=\"1.0\"?><response><result>#{results.to_s}</result></response>")
+
+            if debug.include?(:statistics)
+              warn "Elapsed for parsing cache: #{Time.now - start_time} seconds"
+            end
+
+            return xml
+          end
+        elsif !@keep_cache_on_edit
+          @cache = {}
+        end
+      end
+
+      retried = 0
       begin
         # configure options for the request
         options = {}
@@ -198,47 +227,102 @@ module PaloAlto
         start_time = Time.now
         text = Helpers::Rest.make_request(options)
         if debug.include?(:statistics)
-          warn "Elapsed for API call #{payload[:type]}/#{payload[:action] || '(unknown action)'} on #{host}: #{Time.now - start_time} seconds"
+          warn "Elapsed for API call #{payload[:type]}/#{payload[:action] || '(unknown action)'} on #{host}: #{Time.now - start_time} seconds, #{text.length} bytes"
         end
 
         warn "received: #{Time.now}\n#{text}\n" if debug.include?(:received)
 
         data = Nokogiri::XML.parse(text)
         unless data.xpath('//response/@status').to_s == 'success'
-          warn "command failed on host #{host} at #{Time.now}"
-          warn "sent:\n#{options.inspect}\n" if debug.include?(:sent_on_error)
-          warn "received:\n#{text.inspect}\n" if debug.include?(:received_on_error)
+          unless %w[op commit].include?(payload[:type]) # here we fail silent
+            warn "command failed on host #{host} at #{Time.now}"
+            warn "sent:\n#{options.inspect}\n" if debug.include?(:sent_on_error)
+            warn "received:\n#{text.inspect}\n" if debug.include?(:received_on_error)
+          end
           code = data.at_xpath('//response/@code')&.value.to_i # sometimes there is no code :( e.g. for 'op' errors
           message = data.xpath('/response/msg/line').map(&:text).map(&:strip).join("\n")
           Helpers::Rest.raise_error(code, message)
         end
 
         data
+      rescue EOFError, Net::ReadTimeout => e
+        max_retries = if %w[keygen config].include?(payload[:type])
+                        # TODO: only retry on config, when it's get or edit, otherwise you may get strange errors
+                        3
+                      else
+                        0
+                      end
+
+        raise e if retried >= max_retries
+
+        retried += 1
+        warn "Got error #{e.inspect}; retrying (try #{retried})" if debug.include?(:warnings)
+        sleep 10
+        retry
       rescue TemporaryException => e
         dont_retry_at = [
           'Partial revert is not allowed. Full system commit must be completed.',
           'Config for scope ',
           'Config is not currently locked for scope ',
           'Commit lock is not currently held by',
-          'You already own a config lock for scope '
+          'You already own a config lock for scope ',
+          'This operation is blocked because of ',
+          'Other administrators are holding config locks ',
+          'Configuration is locked by ',
+          ' device-group' #  device-group -> ... is already in use
         ]
-        raise e if retried || dont_retry_at.any? { |x| e.message.start_with?(x) }
 
-        warn "Got error #{e.inspect}; retrying" if debug.include?(:warnings)
-        retried = true
+        max_retries = if dont_retry_at.any? { |str| e.message.start_with?(str) }
+                        0
+                      elsif e.message.start_with?('Timed out while getting config lock. Please try again.')
+                        10
+                      else
+                        1
+                      end
+
+        raise e if retried >= max_retries
+
+        retried += 1
+        warn "Got error #{e.inspect}; retrying (try #{retried})" if debug.include?(:warnings)
+
         get_auth_key if e.is_a?(SessionTimedOutException)
         retry
       end
     end
 
-    def commit!(all: false, device_groups: nil, templates: nil, wait_for_completion: true, wait: 5, timeout: 480)
+    def clear_cache!
+      @cache = {}
+      @keep_cache_on_edit = nil
+    end
+
+    def keep_cache_on_edit!
+      @keep_cache_on_edit = true
+    end
+
+    def cache!(xpath)
+      cached_xpath = xpath.is_a?(String) ? xpath : xpath.to_xpath
+
+      payload = {
+        type: 'config',
+        action: 'get',
+        xpath: cached_xpath
+      }
+
+      @cache[cached_xpath] = execute(payload, skip_cache: true)
+      true
+    end
+
+    def commit!(all: false, device_groups: nil, templates: nil,
+                admins: [username],
+                raw_result: false,
+                wait_for_completion: true, wait: 5, timeout: 60 * 20)
       return nil if device_groups.is_a?(Array) && device_groups.empty? && templates.is_a?(Array) && templates.empty?
 
       cmd = if all
               'commit'
             else
               { commit: { partial: [
-                { 'admin': [username] },
+                { 'admin': admins },
                 if device_groups
                   device_groups.empty? ? 'no-device-group' : { 'device-group': device_groups }
                 end,
@@ -256,8 +340,9 @@ module PaloAlto
             end
       result = op.execute(cmd)
 
-      job_id = result.at_xpath('response/result/job')&.text
+      return result if raw_result
 
+      job_id = result.at_xpath('response/result/job')&.text
       return result unless job_id && wait_for_completion
 
       wait_for_job_completion(job_id, wait: wait, timeout: timeout) if job_id
@@ -294,6 +379,7 @@ module PaloAlto
 
     # will execute block if given and unlock afterwards. returns false if lock could not be aquired
     def lock(area:, comment: nil, type: nil, location: nil)
+      raise MalformedCommandException, 'No type specified' if location && !type
       if block_given?
         return false unless lock(area: area, comment: comment, type: type, location: location)
 
@@ -308,8 +394,10 @@ module PaloAlto
         cmd = { request: { "#{area}-lock": { add: { comment: comment || '(null)' } } } }
         op.execute(cmd, type: type, location: location)
         true
-      rescue PaloAlto::InternalErrorException => e
-        return true if e.message.start_with?('You already own a config lock for scope ')
+      rescue PaloAlto::UnknownErrorException => e
+        return true if e.message.start_with?('You already own a config lock for scope ') ||
+                       e.message == "Config for scope shared is currently locked by #{username}" ||
+                       e.message == "Config for scope #{location} is currently locked by #{username}"
 
         false
       end
@@ -323,7 +411,9 @@ module PaloAlto
                 { request: { "#{area}-lock": 'remove' } }
               end
         op.execute(cmd, type: type, location: location)
-      rescue PaloAlto::InternalErrorException
+      rescue PaloAlto::UnknownErrorException => e
+        return true if e.message.start_with?('Config is not currently locked')
+
         return false
       end
       true
@@ -337,9 +427,9 @@ module PaloAlto
       end
     end
 
-    def check_for_changes(usernames: [username])
-      cmd = if usernames
-              { show: { config: { list: { 'change-summary': { partial: { admin: usernames } } } } } }
+    def check_for_changes(admins: [username])
+      cmd = if admins
+              { show: { config: { list: { 'change-summary': { partial: { admin: admins } } } } } }
             else
               { show: { config: { list: 'change-summary' } } }
             end
@@ -350,7 +440,43 @@ module PaloAlto
       }
     end
 
-    def wait_for_job_completion(job_id, wait: 5, timeout: 600)
+    # returns nil if job isn't finished yet, otherwise the job result
+    def query_and_parse_job(job_id)
+      cmd = { show: { jobs: { id: job_id } } }
+      result = op.execute(cmd)
+      status = result.at_xpath('response/result/job/status')&.text
+      return result unless %w[ACT PEND].include?(status)
+
+      nil
+    rescue => e
+      warn [:job_query_error, @host, e].inspect
+      false
+    end
+
+    # returns true if successful
+    # returns nil if not completed yet
+    # otherwise returns the error
+    def commit_successful?(commit_result)
+      if commit_result.at_xpath('response/msg')&.text&.start_with?('The result of this commit would be the same as the previous commit queued/processed') ||
+         commit_result.at_xpath('response/msg')&.text == 'There are no changes to commit.'
+        return true
+      end
+
+      job_id = commit_result.at_xpath('response/result/job')&.text
+      unless job_id
+        warn [:no_job_id, result].inspect
+        return false
+      end
+
+      job_result = query_and_parse_job(job_id)
+      return job_result if !job_result # can be either nil or false (errored)
+
+      return true if job_result.xpath('response/result/job/details/line').text&.include?('Configuration committed successfully')
+
+      job_result
+    end
+
+    def wait_for_job_completion(job_id, wait: 5, timeout: 20 * 60)
       cmd = { show: { jobs: { id: job_id } } }
       start = Time.now
       loop do
@@ -359,11 +485,13 @@ module PaloAlto
           status = result.at_xpath('response/result/job/status')&.text
           return result unless %w[ACT PEND].include?(status)
         rescue => e
-          warn [:job_query_error, e].inspect
+          warn [:job_query_error, Time.now, @host, e].inspect
+          return false if e.message =~ /\Ajob \d+ not found\z/
         end
         sleep wait
         break unless start + timeout > Time.now
       end
+      warn [:job_query_error, Time.now, @host, :timeout].inspect
       false
     end
 
@@ -406,6 +534,8 @@ module PaloAlto
 
       @subclasses = {}
 
+      @cache = {}
+
       # xpath
       @expression = :root
       @arguments = [Expression.new(:this_node), []]

data/palo_alto.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = 'Palo Alto API for Ruby'
   spec.homepage      = 'https://github.com/Sebbb/'
-  spec.license       = 'artistic-2.0'
+  spec.license       = 'Artistic-2.0'
   spec.required_ruby_version = '>= 2.7.0'
 
   spec.metadata['homepage_uri'] = spec.homepage

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: palo_alto
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Sebastian Roesner
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-09 00:00:00.000000000 Z
+date: 2023-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -52,7 +52,7 @@ files:
 - palo_alto.gemspec
 homepage: https://github.com/Sebbb/
 licenses:
-- artistic-2.0
+- Artistic-2.0
 metadata:
   homepage_uri: https://github.com/Sebbb/
   source_code_uri: https://github.com/Sebbb/palo_alto/