smart_proxy_openscap 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc0f9e535cc7359af74ea374ced6f071f6af4745
-  data.tar.gz: 7d0741cb41875f74908d3383dcfaea081db68366
+  metadata.gz: 52077a3d7e9a009b0ab9d8abba663b3ab7191b78
+  data.tar.gz: db0401879d67124ea9361d48800dad7a66675d37
 SHA512:
-  metadata.gz: 66e1c835c86977103e63102e6764bcd906e659d5ab02df882cef8dca1b58a6ec8de60fd9559e1625ec78aa7124f0abc103cc4e44228034efce0b26b3149d1e87
-  data.tar.gz: 30610e832fe4cb4a5c8a9db0203b956b6b090eb67c4f6f86350de8a1cb1b302c25c82a957510aa8ac607a1f13ce02ba21f77346a6acc173b7bb9c5a0ebcd2458
+  metadata.gz: 8133075f66c34e22e0da4c431dd49bcd4a3530f8fb8a2f8d7b564b3e781d06eb251ce4db830ad1d87a739d646d55427ec18a60ee06f196b3b318eb55a7a6ac36
+  data.tar.gz: 12d22393be694c5f68e3b3ce5fa48afc592c5baee07fab7c3c760e358755bb96d07f32e84eaaf4a4a238b432c3d41ac98994d16f906c7b08c63032b0d0a60e6d

data/.rubocop.yml

@@ -0,0 +1,41 @@
+inherit_from: .rubocop_todo.yml
+
+# Offense count: 11
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+
+# Offense count: 14
+# Configuration parameters: AllowSafeAssignment.
+Lint/AssignmentInCondition:
+  Enabled: false
+
+# Offense count: 3
+Lint/Eval:
+  Enabled: false
+
+# Offense count: 13
+# Cop supports --auto-correct.
+Lint/RescueException:
+  Enabled: false
+
+# Offense count: 9
+# Cop supports --auto-correct.
+Lint/StringConversionInInterpolation:
+  Enabled: true
+
+# Offense count: 9
+# Cop supports --auto-correct.
+Lint/UnusedBlockArgument:
+  Enabled: false
+
+# Offense count: 18
+# Cop supports --auto-correct.
+Lint/UnusedMethodArgument:
+  Enabled: false
+
+# Offense count: 5
+Metrics/BlockNesting:
+  Max: 4
+
+Style/SymbolProc:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,111 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2015-09-11 14:29:44 +0300 using RuboCop version 0.32.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 9
+Metrics/AbcSize:
+  Max: 29
+
+# Offense count: 54
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 184
+
+# Offense count: 7
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 24
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: IndentWhenRelativeTo, SupportedStyles, IndentOneStep.
+Style/CaseIndentation:
+  Enabled: false
+
+# Offense count: 9
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+# Offense count: 1
+Style/ClassVars:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Style/ColonMethodCall:
+  Enabled: false
+
+# Offense count: 13
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: Exclude.
+Style/FileName:
+  Enabled: false
+
+# Offense count: 25
+# Cop supports --auto-correct.
+# Configuration parameters: SupportedStyles, UseHashRocketsWithSymbolValues.
+Style/HashSyntax:
+  EnforcedStyle: hash_rockets
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: Width.
+Style/IndentationWidth:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/NegatedIf:
+  Enabled: false
+
+# Offense count: 4
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/RaiseArgs:
+  Enabled: false
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SignalException:
+  Enabled: false
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/SpaceAfterComma:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: MultiSpaceAllowedForOperators.
+Style/SpaceAroundOperators:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles.
+Style/SpaceInsideHashLiteralBraces:
+  Enabled: false
+
+# Offense count: 43
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/StringLiterals:
+  Enabled: false
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/TrailingBlankLines:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Style/TrailingWhitespace:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'rubocop'
+  gem 'smart_proxy', :github => "theforeman/smart-proxy", :branch => 'develop'
+end

data/README.md

@@ -1,19 +1,20 @@
 # OpenSCAP plug-in for Foreman Proxy
 
 A plug-in to the Foreman Proxy which receives bzip2ed ARF files
-and forwards them to the Foreman.
+and forwards them as JSON to the Foreman.
+smart_proxy_openscap plugin is required for the normal operation of OpenSCAP in the Foreman.
 
-Incoming ARF files are authenticated using puppet certificate of
-the client machine. Proxy caches collected ARF files until they
-are forwarded to Foreman.
+## How it works
 
-Learn more about [Foreman-OpenSCAP](https://github.com/OpenSCAP/foreman_openscap) workflow.
+Incoming ARF files are authenticated using either puppet certificate or Katello certificate of
+the client machine. The ARF files are parsed on the proxy and posted to the Foreman as JSON reports 
+and the ARF files are saved in a reports directory, where they can be accessed for full HTML report or 
+downloaded as bzip2ed xml report.
+If posting is failed, the ARF files are saved in queue for later retry.
 
-## Installation from RPMs
-
-- Install foreman-proxy from Foreman-proxy upstream
+Learn more about [Foreman-OpenSCAP](https://github.com/theforeman/foreman_openscap) workflow.
 
-- Enable [isimluk/OpenSCAP](https://copr.fedoraproject.org/coprs/isimluk/OpenSCAP/) COPR repository
+## Installation from RPMs
 
 - Install smart_proxy_openscap
 
@@ -22,40 +23,40 @@ Learn more about [Foreman-OpenSCAP](https://github.com/OpenSCAP/foreman_openscap
   ```
 
 ## Installation from upstream git
-
-- Install foreman-proxy from Foreman-proxy upstream
-- Download smart_proxy_openscap
-
-  ```
-  ~$ git clone https://github.com/OpenSCAP/smart_proxy_openscap.git
-  ```
-
-- Build smart_proxy_openscap RPM
-
-  ```
-  ~$ cd smart_proxy_openscap
-  ~$ gem build smart_proxy_openscap.gemspec
-  ~# yum install yum-utils rpm-build
-  ~# yum-builddep extra/rubygem-smartproxy_openscap.spec
-  ~# rpmbuild  --define "_sourcedir `pwd`" -ba extra/rubygem-smart_proxy_openscap.spec
-  ```
-
-- Install rubygem-smart_proxy_openscap
-
+- Add smart_proxy_openscap to your smart proxy `bundler.d/openscap.rb` gemfile:
+ 
   ```
-  ~$ yum local install ~/rpmbuild/RPMS/noarch/rubygem-smart_proxy_openscap*
+  ~$ gem 'smart_proxy_openscap', :git => https://github.com/theforeman/smart_proxy_openscap.git
   ```
 
-If you don't install through RPM but you are using bundler, you may need to create 
-/var/spool/foreman-proxy directory manually and set it's owner to the user under which 
-foreman-proxy runs.
+If you don't install through RPM and you are using bundler, you may need to create 
+/var/spool/foreman-proxy & /usr/share/foreman-proxy/openscap directories manually and 
+set it's owner to the user under which foreman-proxy runs.
 
 ## Configuration
 
   ```
   cp /etc/foreman-proxy/settings.d/openscap.yml{.example,}
-  vim /etc/foreman-proxy/settings.d/openscap.yml
-  echo ":foreman_url: https://my-foreman.local.lan" >> /etc/foreman-proxy/settings.yml
+  ```
+Configure the following parameters so it would look like:
+  
+  ```
+  ---
+  :enabled: true
+  
+  # Log file for the forwarding script.
+  :openscap_send_log_file: /var/log/foreman-proxy/openscap-send.log
+  
+  # Directory where OpenSCAP audits are stored
+  # before they are forwarded to Foreman
+  :spooldir: /var/spool/foreman-proxy/openscap
+  
+  # Directory where OpenSCAP content XML are stored
+  # So we will not request the XML from Foreman each time
+  :contentdir: /var/lib/openscap/content
+  # Directory where OpenSCAP report XML are stored
+  # So Foreman can request arf xml reports
+  :reportsdir: /usr/share/foreman-proxy/openscap/content
   ```
 
 - Deploy
@@ -64,9 +65,29 @@ foreman-proxy runs.
   ~# service foreman-proxy restart
   ```
 
-- Usage:
+## Usage
+
+![Openscap design](http://shlomizadok.github.io/foreman_openscap/static/images/reports_design.png)
+
+### Exposed APIs
+
+* POST "/compliance/arf/:policy" - API to recieve ARF files, parse them and post to the Foreman. `:policy` is the policy ID from Foreman. expects ARF bzip2 file as POST body
+
+* GET "/compliance/arf/:id/:cname/:date/:digest/xml" - API to download bizped2 ARF file. `:id` - ArfReport id, `:cname` - Host name, `:date` - Report date, `:digest` - Digest of the Arf file
+
+* GET "/compliance/arf/:id/:cname/:date/:digest/html" - API to fetch full HTML report. `:id` - ArfReport id, `:cname` - Host name, `:date` - Report date, `:digest` - Digest of the Arf file
+
+* GET "/compliance/policies/:policy_id/content" - API to download and serve SCAP content file for policy. `:policy_id` - Policy id from Foreman
+
+* POST "/compliance/scap_content/policies" - API to extract policies from SCAP content. expects SCAP content posted as the POST body
+
+* POST "/compliance/scap_content/validator" - API to validate SCAP content. expects SCAP content posted as the POST body
+
+* POST "/compliance/scap_content/guide/:policy" - API to return Policy's HTML guide. `:policy` - policy name. expects SCAP content posted as the POST body
+
+### Binaries
 
-  Learn more about [Foreman-OpenSCAP](https://github.com/OpenSCAP/foreman_openscap) workflow.
+* `smart_proxy_openscap_send` - Sends failed ARF files to Foreman (in case first try failed). When installed with RPM, a cron jobs is configured to run every 30 minutes.
 
 ## Copyright
 

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rake'
+require 'rake/testtask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the Smart Proxy OpenSCAP.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << '.'
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+end

data/bin/smart-proxy-openscap-send

@@ -23,7 +23,7 @@ exit unless Proxy::OpenSCAP::Plugin.settings.enabled == true
 
 module Proxy
   module Log
-    @@logger = ::Logger.new(Proxy::OpenSCAP::Plugin.settings.openscap_send_log_file, 6, 1024*1024*10)
+    @@logger = ::Logger.new(Proxy::OpenSCAP.fullpath(Proxy::OpenSCAP::Plugin.settings.openscap_send_log_file), 6, 1024*1024*10)
     @@logger.level = ::Logger.const_get(Proxy::SETTINGS.log_level.upcase)
   end
 end

data/lib/smart_proxy_openscap/fetch_scap_content.rb

@@ -0,0 +1,58 @@
+module Proxy::OpenSCAP
+  class FetchScapContent
+    include ::Proxy::Log
+    def get_policy_content(policy_id)
+      policy_store_dir = File.join(Proxy::OpenSCAP.fullpath(Proxy::OpenSCAP::Plugin.settings.contentdir), policy_id.to_s)
+      policy_scap_file = File.join(policy_store_dir, "#{policy_id}_scap_content.xml")
+      begin
+        logger.info "Creating directory to store SCAP file: #{policy_store_dir}"
+        FileUtils.mkdir_p(policy_store_dir) # will fail silently if exists
+      rescue Errno::EACCES => e
+        logger.error "No permission to create directory #{policy_store_dir}"
+        raise e
+      rescue StandardError => e
+        logger.error "Could not create '#{policy_store_dir}' directory: #{e.message}"
+        raise e
+      end
+
+      scap_file = policy_content_file(policy_scap_file)
+      scap_file ||= save_or_serve_scap_file(policy_id, policy_scap_file)
+      scap_file
+    end
+
+    private
+
+    def policy_content_file(policy_scap_file)
+      return nil if !File.file?(policy_scap_file) || File.zero?(policy_scap_file)
+      File.open(policy_scap_file, 'rb').read
+    end
+
+    def save_or_serve_scap_file(policy_id, policy_scap_file)
+      lock = Proxy::FileLock::try_locking(policy_scap_file)
+      response = fetch_scap_content_xml(policy_id, policy_scap_file)
+      if lock.nil?
+        return response
+      else
+        begin
+          File.open(policy_scap_file, 'wb') do |file|
+            file << response
+          end
+        ensure
+          Proxy::FileLock::unlock(lock)
+        end
+        scap_file = policy_content_file(policy_scap_file)
+        raise FileNotFound if scap_file.nil?
+        return scap_file
+      end
+    end
+
+    def fetch_scap_content_xml(policy_id, policy_scap_file)
+      foreman_request = Proxy::HttpRequest::ForemanRequest.new
+      policy_content_path = "api/v2/compliance/policies/#{policy_id}/content"
+      req = foreman_request.request_factory.create_get(policy_content_path)
+      response = foreman_request.send_request(req)
+      response.value
+      response.body
+    end
+  end
+end

data/lib/smart_proxy_openscap/foreman_forwarder.rb

@@ -0,0 +1,38 @@
+module Proxy::OpenSCAP
+  class ForemanForwarder < Proxy::HttpRequest::ForemanRequest
+    include ::Proxy::Log
+
+    def post_arf_report(cname, policy_id, date, data)
+      begin
+        json_data = Proxy::OpenSCAP::Parse.new(data).as_json
+        foreman_api_path = upload_path(cname, policy_id, date)
+        response = send_request(foreman_api_path, json_data)
+        # Raise an HTTP error if the response is not 2xx (success).
+        response.value
+        res = JSON.parse(response.body)
+        raise StandardError, "Received response: #{response.code} #{response.msg}" unless res['result'] == 'OK'
+      rescue StandardError => e
+        logger.debug response.body if response
+        logger.debug e.backtrace.join("\n\t")
+        raise e
+      end
+      res
+    end
+
+    private
+
+    def upload_path(cname, policy_id, date)
+      "/api/v2/compliance/arf_reports/#{cname}/#{policy_id}/#{date}"
+    end
+
+    def send_request(path, body)
+      # Override the parent method to set the right headers
+      path = [uri.path, path].join('/') unless uri.path.empty?
+      req = Net::HTTP::Post.new(URI.join(uri.to_s, path).path)
+      req.add_field('Accept', 'application/json,version=2')
+      req.content_type = 'application/json'
+      req.body = body
+      http.request(req)
+    end
+  end
+end

data/lib/smart_proxy_openscap/openscap_api.rb

@@ -7,49 +7,108 @@
 # FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv3
 # along with this software; if not, see http://www.gnu.org/licenses/gpl.txt
 #
-
 require 'smart_proxy_openscap/openscap_lib'
 
 module Proxy::OpenSCAP
+  HTTP_ERRORS = [
+    EOFError,
+    Errno::ECONNRESET,
+    Errno::EINVAL,
+    Net::HTTPBadResponse,
+    Net::HTTPHeaderSyntaxError,
+    Net::ProtocolError,
+    Timeout::Error
+  ]
+
   class Api < ::Sinatra::Base
     include ::Proxy::Log
     helpers ::Proxy::Helpers
     authorize_with_ssl_client
 
-    put "/arf/:policy" do
+    post "/arf/:policy" do
       # first let's verify client's certificate
       begin
         cn = Proxy::OpenSCAP::common_name request
       rescue Proxy::Error::Unauthorized => e
         log_halt 403, "Client authentication failed: #{e.message}"
       end
+      date = Time.now.to_i
+      policy = params[:policy]
 
-      # validate the url (i.e. avoid malformed :policy)
       begin
-        target_dir = Proxy::OpenSCAP::spool_arf_dir(cn, params[:policy])
-      rescue Proxy::Error::BadRequest => e
-        log_halt 400, "Requested URI is malformed: #{e.message}"
-      rescue StandardError => e
-        log_halt 500, "Could not fulfill request: #{e.message}"
+        post_to_foreman = ForemanForwarder.new.post_arf_report(cn, policy, date, request.body.string)
+        Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.reportsdir, cn, post_to_foreman['id'], date)
+          .store_archive(request.body.string)
+      rescue Proxy::OpenSCAP::StoreReportError => e
+        Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.failed_dir, cn, post_to_foreman['id'], date)
+          .store_failed(request.body.string)
+        logger.error "Failed to save Report in reports directory (#{Proxy::OpenSCAP::Plugin.settings.reportsdir}). Failed with: #{e.message}.
+                      Saving file in #{Proxy::OpenSCAP::Plugin.settings.failed_dir}. Please copy manually to #{Proxy::OpenSCAP::Plugin.settings.reportsdir}"
+      rescue *HTTP_ERRORS => e
+        ### If the upload to foreman fails then store it in the spooldir
+        logger.error "Failed to upload to Foreman, saving in spool. Failed with: #{e.message}"
+        Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.spooldir, cn, policy, date)
+          .store_spool(request.body.string)
+      rescue Proxy::OpenSCAP::StoreSpoolError => e
+        log_halt 500, e.message
       end
+    end
 
+    get "/arf/:id/:cname/:date/:digest/xml" do
+      content_type 'application/x-bzip2'
       begin
-        target_path = Proxy::OpenSCAP::store_arf(target_dir, request.body.string)
-      rescue StandardError => e
-        log_halt 500, "Could not store file: #{e.message}"
+        Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.reportsdir, params[:cname], params[:id], params[:date])
+          .get_arf_xml(params[:digest])
+      rescue FileNotFound => e
+        log_halt 500, "Could not find requested file, #{e.message}"
       end
+    end
 
-      logger.debug "File #{target_path} stored successfully."
-
-      {"created" => true}.to_json
+    get "/arf/:id/:cname/:date/:digest/html" do
+      begin
+        Proxy::OpenSCAP::StorageFS.new(Proxy::OpenSCAP::Plugin.settings.reportsdir, params[:cname], params[:id], params[:date])
+          .get_arf_html(params[:digest])
+      rescue FileNotFound => e
+        log_halt 500, "Could not find requested file, #{e.message}"
+      end
     end
 
     get "/policies/:policy_id/content" do
       content_type 'application/xml'
       begin
-        Proxy::OpenSCAP::get_policy_content(params[:policy_id])
-      rescue OpenSCAPException => e
-        log_halt e.http_code, "Error fetching xml file: #{e.message}"
+        Proxy::OpenSCAP::FetchScapContent.new.get_policy_content(params[:policy_id])
+      rescue *HTTP_ERRORS => e
+        log_halt e.response.code.to_i, "File not found on Foreman. Wrong policy id?"
+      rescue StandardError => e
+        log_halt 500, "Error occurred: #{e.message}"
+      end
+    end
+
+    post "/scap_content/policies" do
+      begin
+        Proxy::OpenSCAP::ContentParser.new(request.body.string).extract_policies
+      rescue *HTTP_ERRORS => e
+        log_halt 500, e.message
+      rescue StandardError => e
+        log_halt 500, "Error occurred: #{e.message}"
+      end
+    end
+
+    post "/scap_content/validator" do
+      begin
+        Proxy::OpenSCAP::ContentParser.new(request.body.string).validate
+      rescue *HTTP_ERRORS => e
+        log_halt 500, e.message
+      rescue StandardError => e
+        log_halt 500, "Error occurred: #{e.message}"
+      end
+    end
+
+    post "/scap_content/guide/:policy" do
+      begin
+        Proxy::OpenSCAP::ContentParser.new(request.body.string).guide(params[:policy])
+      rescue *HTTP_ERRORS => e
+        log_halt 500, e.message
       rescue StandardError => e
         log_halt 500, "Error occurred: #{e.message}"
       end