smart_proxy_openscap 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: feab59ae3e8ce1e915f22d558c1160c55373fa16
-  data.tar.gz: e7de38b8eae9570f604850f129beb66a69e27060
+  metadata.gz: 17c8ac2a2ead069f4efa7282704a31bfc92b248a
+  data.tar.gz: bfe816a4639d132847e7ba97848aeed06829c80b
 SHA512:
-  metadata.gz: 37138364fff28b9677e4c2f401aad31c0c2fce71c18b5ca26cb25c0e6f8081ad0cf98caf8b092061b4241e9a9c2ce98a2ee800e025b4645c798e5ecb0c9ff7f7
-  data.tar.gz: aac42840356d8a85c7f74e08fe050584c45af8c5e857ab30d0b14004e71eb0e26afa88dfdb3fe207edad60669701ab2c9f55ffb997f32ac8869f93ef3f667f41
+  metadata.gz: db6215253f5ddd5f6e862b5fa01a23a06f2eaeacb16d0a82bd037b493bc5c4cdcdabd21034f425aede1013ca7189c7b98b83ef81e06c6fa5684d75402d8008e8
+  data.tar.gz: cf76b07956f6d50a101ea9ec26432c2dd7a0206763db8732149e9d97bb97ae6114eb8b481354ecff08941e0253e90006673014913b2d9f0014cda393a27d7546

data/lib/smart_proxy_openscap/fetch_tailoring_file.rb

@@ -10,7 +10,7 @@ module Proxy::OpenSCAP
       create_store_dir store_dir
 
       scap_file = policy_content_file(policy_tailoring_file)
-      clean_store_folder(policy_store_dir) unless scap_file
+      clean_store_folder(store_dir) unless scap_file
       scap_file ||= save_or_serve_scap_file(policy_tailoring_file, file_download_path)
     end
   end

data/lib/smart_proxy_openscap/foreman_forwarder.rb

@@ -11,6 +11,9 @@ module Proxy::OpenSCAP
         response.value
         res = JSON.parse(response.body)
         raise StandardError, "Received response: #{response.code} #{response.msg}" unless res['result'] == 'OK'
+      rescue OpenSCAP::OpenSCAPError => e
+        logger.debug e.backtrace.join("\n\t")
+        raise e
       rescue StandardError => e
         logger.debug response.body if response
         logger.debug e.backtrace.join("\n\t")

data/lib/smart_proxy_openscap/openscap_api.rb

@@ -42,6 +42,9 @@ module Proxy::OpenSCAP
         Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.failed_dir, cn, post_to_foreman['id'], date).store_failed(request.body.string)
         logger.error "Failed to save Report in reports directory (#{Proxy::OpenSCAP::Plugin.settings.reportsdir}). Failed with: #{e.message}.
                       Saving file in #{Proxy::OpenSCAP::Plugin.settings.failed_dir}. Please copy manually to #{Proxy::OpenSCAP::Plugin.settings.reportsdir}"
+      rescue OpenSCAP::OpenSCAPError => e
+        logger.error "Failed to parse Arf Report, moving to #{Proxy::OpenSCAP::Plugin.settings.corrupted_dir}"
+        Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.corrupted_dir, cn, policy, date).store_corrupted(request.body.string)
       rescue *HTTP_ERRORS => e
         ### If the upload to foreman fails then store it in the spooldir
         logger.error "Failed to upload to Foreman, saving in spool. Failed with: #{e.message}"

data/lib/smart_proxy_openscap/openscap_exception.rb

@@ -4,4 +4,5 @@ module Proxy::OpenSCAP
   class StoreSpoolError < StandardError; end
   class StoreFailedError < StandardError; end
   class FileNotFound < StandardError; end
+  class StoreCorruptedError < StandardError; end
 end

data/lib/smart_proxy_openscap/spool_forwarder.rb

@@ -3,19 +3,11 @@ module Proxy::OpenSCAP
     include ::Proxy::Log
 
     def post_arf_from_spool(arf_dir)
-      failed = nil
       Dir.foreach(arf_dir) do |cname|
-        begin
-          next if cname == '.' || cname == '..'
-          cname_dir = File.join(arf_dir, cname)
-          forward_cname_dir(cname, cname_dir) if File.directory?(cname_dir)
-        rescue StandardError => e
-          logger.debug e.backtrace.join("\n\t") 
-          logger.error "Failed to send SCAP results for #{cname} to the Foreman server: #{e}" 
-          failed = true
-        end
+        next if cname == '.' || cname == '..'
+        cname_dir = File.join(arf_dir, cname)
+        forward_cname_dir(cname, cname_dir) if File.directory?(cname_dir)
       end
-      raise "Failed to send SCAP results for one or more hosts." if failed
     end
 
     private
@@ -28,7 +20,7 @@ module Proxy::OpenSCAP
           forward_policy_dir(cname, policy_id, policy_dir)
         end
       end
-      remove(cname_dir)
+      remove_if_empty(cname_dir)
     end
 
     def forward_policy_dir(cname, policy_id, policy_dir)
@@ -39,7 +31,7 @@ module Proxy::OpenSCAP
           forward_date_dir(cname, policy_id, date, date_dir)
         end
       end
-      remove(policy_dir)
+      remove_if_empty(policy_dir)
     end
 
     def forward_date_dir(cname, policy_id, date, date_dir)
@@ -51,7 +43,7 @@ module Proxy::OpenSCAP
           forward_arf_file(cname, policy_id, date, arf_path)
         end
       end
-      remove(date_dir)
+      remove_if_empty(date_dir)
     end
 
     def forward_arf_file(cname, policy_id, date, arf_file_path)
@@ -59,11 +51,19 @@ module Proxy::OpenSCAP
       post_to_foreman = ForemanForwarder.new.post_arf_report(cname, policy_id, date, data)
       Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.reportsdir, cname, post_to_foreman['id'], date).store_archive(data)
       File.delete arf_file_path
+    rescue OpenSCAP::OpenSCAPError => e
+      logger.error "Failed to parse Arf Report at #{arf_file_path}, moving to #{Proxy::OpenSCAP::Plugin.settings.corrupted_dir}"
+
+      Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.corrupted_dir, cname, policy_id, date)
+                                .move_corrupted(arf_file_path.split('/').last)
+    rescue StandardError => e
+      logger.error "smart-proxy-openscap-send failed to upload Compliance report for #{cname}, generated on #{Time.at date.to_i}. Cause: #{e}"
     end
 
-    def remove(dir)
+    def remove_if_empty(dir)
       begin
-        Dir.delete dir
+        Dir.delete dir if Dir["#{dir}/*"].empty?
+        logger.debug "Removing directory: #{dir}"
       rescue StandardError => e
         logger.error "Could not remove directory: #{e.message}"
       end

data/lib/smart_proxy_openscap/storage_fs.rb

@@ -14,6 +14,15 @@ module Proxy::OpenSCAP
       store(data, StoreFailedError)
     end
 
+    def store_corrupted(data)
+      store(data, StoreCorruptedError)
+    end
+
+    def move_corrupted(digest)
+      source = "#{Proxy::OpenSCAP::Plugin.settings.spooldir}/#{@namespace}/#{@cname}/#{@id}/#{@date}"
+      move "#{source}/#{digest}", StoreCorruptedError
+    end
+
     def get_arf_xml(digest)
       get_arf_file(digest)[:xml]
     end
@@ -54,6 +63,15 @@ module Proxy::OpenSCAP
       @path
     end
 
+    def move(source, error_type)
+      begin
+        create_directory
+        FileUtils.mv source, @path
+      rescue StandardError => e
+        raise error_type, "Could not move file: #{e.message}"
+      end
+    end
+
     def store(data, error_type)
       begin
         create_directory

data/lib/smart_proxy_openscap/version.rb

@@ -10,6 +10,6 @@
 
 module Proxy
   module OpenSCAP
-    VERSION = '0.6.1'
+    VERSION = '0.6.2'
   end
 end

data/settings.d/openscap.yml.example

@@ -22,3 +22,7 @@
 # Directory where OpenSCAP report XML are stored
 # In case sending to Foreman succeeded, yet failed to save to reportsdir
 #:failed_dir: /usr/share/foreman-proxy/openscap/failed
+
+# Directory where corrupted OpenSCAP report XML are stored
+# When proxy cannot parse the report sent by client
+#:corrupted_dir: /var/lib/foreman-proxy/openscap/corrupted

data/test/post_report_api_test.rb

@@ -16,10 +16,13 @@ class OpenSCAPApiTest < Test::Unit::TestCase
     Proxy::OpenSCAP::Plugin.settings.stubs(:spooldir).returns(@results_path + "/spool")
     Proxy::OpenSCAP::Plugin.settings.stubs(:reportsdir).returns(@results_path + "/reports")
     Proxy::OpenSCAP::Plugin.settings.stubs(:failed_dir).returns(@results_path + "/failed")
+    Proxy::OpenSCAP::Plugin.settings.stubs(:corrupted_dir).returns(@results_path + "/corrupted")
     @arf_report = File.open("#{Dir.getwd}/test/data/arf_report").read
+    @corrupted_arf_report = File.open("#{Dir.getwd}/test/data/corrupted_arf_report").read
     @policy_id = 1
     @arf_id = 145
     @filename = Digest::SHA256.hexdigest(@arf_report)
+    @corrupted_filename = Digest::SHA256.hexdigest(@corrupted_arf_report)
     @cname = 'node.example.org'
     @date = Time.now.to_i
     # Bypass common_name as it requires ssl certificate
@@ -72,4 +75,12 @@ class OpenSCAPApiTest < Test::Unit::TestCase
     log_file = File.read('logs/test.log')
     assert(log_file.include?('Failed to save Report in reports directory'), 'Logger should notify that failed to save in reports dir')
   end
+
+  def test_post_corrupted_should_move_to_corrupted
+    stub_request(:post, "#{@foreman_url}/api/v2/compliance/arf_reports/#{@cname}/#{@policy_id}/#{@date}")
+      .to_return(:status => 200, :body => "{\"result\":\"OK\",\"id\":\"#{@arf_id}\"}")
+    post "/arf/#{@policy_id}", @corrupted_arf_report, 'CONTENT_TYPE' => 'text/xml', 'CONTENT_ENCODING' => 'x-bzip2'
+    assert(File.file?("#{@results_path}/corrupted/arf/#{@cname}/#{@policy_id}/#{@date}/#{@corrupted_filename}"), "File should be in Corrupted directory")
+    refute(File.file?("#{@results_path}/spool/arf/#{@cname}/#{@policy_id}/#{@date}/#{@corrupted_filename}"), "File should not be in Spool directory")
+  end
 end

data/test/spool_forwarder_test.rb

@@ -0,0 +1,82 @@
+require 'test_helper'
+require 'smart_proxy_openscap'
+require 'smart_proxy_openscap/openscap_lib'
+
+class SpoolForwarderTest < Test::Unit::TestCase
+  def setup
+    @foreman_url = 'https://foreman.example.com'
+    Proxy::SETTINGS.stubs(:foreman_url).returns(@foreman_url)
+    @base_spool_for_test = ("#{Dir.getwd}/test/data/spool")
+    @results_path = ("#{Dir.getwd}/test/test_run_files")
+    FileUtils.mkdir_p(@results_path)
+    Proxy::OpenSCAP::Plugin.settings.stubs(:contentdir).returns(@results_path)
+    @spooldir = @results_path + "/spool"
+    Proxy::OpenSCAP::Plugin.settings.stubs(:spooldir).returns(@spooldir)
+    Proxy::OpenSCAP::Plugin.settings.stubs(:reportsdir).returns(@results_path + "/reports")
+    Proxy::OpenSCAP::Plugin.settings.stubs(:corrupted_dir).returns(@results_path + "/corrupted")
+    @policy_id = 1
+    @date_1 = 1484309984
+    @date_2 = 1484313035
+    @id_1 = 42
+    @id_2 = 84
+    @valid_digest = "fa2f68ffb944c917332a284dc63ec7f8fa76990cb815ddcad3318b5d9457f8a1"
+    @corrupted_digest = "a4dfba5db27b21795e6fa401b8dce7a70faeb25b7963891f07f6f4baaf052afb"
+    @cname = "e20b9695-f655-401a-9dda-8cca7a47a8c0"
+    @cname_2 = "2c101b95-033f-4b15-b490-f50bf9090dae"
+
+    @arf_dir = File.join(Proxy::OpenSCAP::Plugin.settings.spooldir, "/arf")
+
+    stub_request(:post, "#{@foreman_url}/api/v2/compliance/arf_reports/#{@cname}/#{@policy_id}/#{@date_1}")
+      .to_return(:status => 200, :body => "{\"result\":\"OK\",\"id\":\"#{@id_1}\"}")
+
+    stub_request(:post, "#{@foreman_url}/api/v2/compliance/arf_reports/#{@cname}/#{@policy_id}/#{@date_2}")
+      .to_return(:status => 200, :body => "{\"result\":\"OK\",\"id\":\"#{@id_2}\"}")
+
+  end
+
+  def teardown
+    FileUtils.rm_rf @results_path
+  end
+
+  def test_send_spool_to_foreman
+    test_spool = @base_spool_for_test + "/valid_spool/"
+    FileUtils.cp_r test_spool, @spooldir
+
+    Proxy::OpenSCAP::SpoolForwarder.new.post_arf_from_spool(@arf_dir)
+    assert(File.file?("#{@results_path}/reports/arf/#{@cname}/#{@id_1}/#{@date_1}/#{@valid_digest}"), "File should be in reports directory")
+    assert(File.file?("#{@results_path}/reports/arf/#{@cname}/#{@id_2}/#{@date_2}/#{@valid_digest}"), "File should be in reports directory")
+    refute(File.file?("#{@spooldir}/arf/#{@cname}/#{@policy_id}/#{@date_1}/#{@valid_digest}"), "File should not be in spool directory")
+    refute(File.file?("#{@spooldir}/arf/#{@cname}/#{@policy_id}/#{@date_2}/#{@valid_digest}"), "File should not be in spool directory")
+  end
+
+  def test_send_corrupted_spool_to_foreman
+    test_spool = @base_spool_for_test + "/corrupted_spool/"
+    FileUtils.cp_r test_spool, @spooldir
+
+    Proxy::OpenSCAP::SpoolForwarder.new.post_arf_from_spool(@arf_dir)
+
+    assert(File.file?("#{@results_path}/corrupted/arf/#{@cname}/#{@policy_id}/#{@date_1}/#{@corrupted_digest}"), "File should be in corrupted directory")
+    assert(File.file?("#{@results_path}/reports/arf/#{@cname}/#{@id_2}/#{@date_2}/#{@valid_digest}"), "File should be in reports directory")
+
+    refute(File.file?("#{@spooldir}/arf/#{@cname}/#{@policy_id}/#{@date_1}/#{@corrupted_digest}"), "File should not be in spool directory")
+    refute(File.file?("#{@spooldir}/arf/#{@cname}/#{@policy_id}/#{@date_2}/#{@valid_digest}"), "File should not be in spool directory")
+  end
+
+  def test_send_spool_cleans_up
+    test_spool = @base_spool_for_test + "/cleanup_spool/"
+    FileUtils.cp_r test_spool, @spooldir
+
+    stub_request(:post, "#{@foreman_url}/api/v2/compliance/arf_reports/#{@cname}/#{@policy_id}/#{@date_1}")
+      .to_return(:status => 200, :body => "{\"result\":\"OK\",\"id\":\"#{@id_1}\"}")
+
+    stub_request(:post, "#{@foreman_url}/api/v2/compliance/arf_reports/#{@cname_2}/#{@policy_id}/#{@date_2}")
+      .to_return(:status => 500)
+
+    Proxy::OpenSCAP::SpoolForwarder.new.post_arf_from_spool(@arf_dir)
+
+    assert(File.file?("#{@results_path}/reports/arf/#{@cname}/#{@id_1}/#{@date_1}/#{@valid_digest}"), "File should be in reports directory")
+    assert(File.file?("#{@spooldir}/arf/#{@cname_2}/#{@policy_id}/#{@date_2}/#{@valid_digest}"), "File should be in spool directory")
+
+    refute(File.exist?("#{@spooldir}/arf/#{@cname}"), "Folder should not exist")
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_openscap
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - "Šimon Lukašík"
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-14 00:00:00.000000000 Z
+date: 2017-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -121,6 +121,13 @@ files:
 - settings.d/openscap.yml.example
 - smart_proxy_openscap.gemspec
 - test/data/arf_report
+- test/data/corrupted_arf_report
+- test/data/spool/cleanup_spool/arf/2c101b95-033f-4b15-b490-f50bf9090dae/1/1484313035/fa2f68ffb944c917332a284dc63ec7f8fa76990cb815ddcad3318b5d9457f8a1
+- test/data/spool/cleanup_spool/arf/e20b9695-f655-401a-9dda-8cca7a47a8c0/1/1484309984/fa2f68ffb944c917332a284dc63ec7f8fa76990cb815ddcad3318b5d9457f8a1
+- test/data/spool/corrupted_spool/arf/e20b9695-f655-401a-9dda-8cca7a47a8c0/1/1484309984/a4dfba5db27b21795e6fa401b8dce7a70faeb25b7963891f07f6f4baaf052afb
+- test/data/spool/corrupted_spool/arf/e20b9695-f655-401a-9dda-8cca7a47a8c0/1/1484313035/fa2f68ffb944c917332a284dc63ec7f8fa76990cb815ddcad3318b5d9457f8a1
+- test/data/spool/valid_spool/arf/e20b9695-f655-401a-9dda-8cca7a47a8c0/1/1484309984/fa2f68ffb944c917332a284dc63ec7f8fa76990cb815ddcad3318b5d9457f8a1
+- test/data/spool/valid_spool/arf/e20b9695-f655-401a-9dda-8cca7a47a8c0/1/1484313035/fa2f68ffb944c917332a284dc63ec7f8fa76990cb815ddcad3318b5d9457f8a1
 - test/data/ssg-rhel7-ds.xml
 - test/data/tailoring.xml
 - test/fetch_scap_api_test.rb
@@ -128,6 +135,7 @@ files:
 - test/get_report_xml_html_test.rb
 - test/post_report_api_test.rb
 - test/scap_content_parser_api_test.rb
+- test/spool_forwarder_test.rb
 - test/test_helper.rb
 homepage: http://github.com/OpenSCAP/smart_proxy_openscap
 licenses: