smart_proxy_abrt 0.0.1

data/README

@@ -0,0 +1,4 @@
+This is an ABRT plugin for Foreman's smart proxy. Please see the README of the
+Foreman ABRT plugin for more information:
+
+https://github.com/abrt/foreman_abrt/blob/master/README.md

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rake'
+require 'rake/testtask'
+
+# Test for 1.9
+if (RUBY_VERSION.split('.').map{|s|s.to_i} <=> [1,9,0]) > 0 then
+  PLATFORM = RUBY_PLATFORM
+end
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the Foreman Proxy plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << '.'
+  t.libs << 'lib'
+  t.libs << 'test'
+  files = FileList['test/**/*_test.rb']
+  if PLATFORM =~ /mingw/
+    files = FileList['test/**/server_ms_test*']
+  else
+    files = FileList['test/**/*_test.rb'].delete_if{|f| f =~ /_ms_/}
+  end
+  t.test_files = files
+  t.verbose = true
+end

data/bin/smart-proxy-abrt-send

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift '/usr/share/foreman-proxy/lib' #XXX!
+$LOAD_PATH.unshift '/usr/share/foreman-proxy/modules' #XXX!
+
+# We rely on the main smart_proxy module to initialize the plugins so that we
+# can access our module's settings. Also used from smart-proxy code: global
+# settings, logging, foreman requests.
+require 'smart_proxy'
+
+require 'smart_proxy_abrt'
+require 'smart_proxy_abrt/abrt_lib'
+
+# Substitute our own logger so that we don't log to the main smart-proxy logfile.
+module Proxy
+  module Log
+    @@logger = ::Logger.new(Proxy::Abrt::Plugin.settings.abrt_send_log_file, 6, 1024*1024*10)
+    @@logger.level = ::Logger.const_get(Proxy::SETTINGS.log_level.upcase)
+  end
+end
+include Proxy::Log
+
+# Don't run if ABRT plugin is disabled.
+exit unless Proxy::Abrt::Plugin.settings.enabled == true
+
+if !Proxy::SETTINGS.foreman_url
+  logger.error "Foreman URL not configured"
+  exit false
+end
+
+begin
+  require 'satyr' if Proxy::Abrt::Plugin.settings.aggregate_reports
+rescue LoadError
+  logger.error "The satyr gem required for report aggregation was not found. "\
+               "You need to either install it or disable the aggregation."
+end
+
+def send_reports_from_spool
+  reports_by_host = {}
+
+  # load reports from disk
+  reports = Proxy::Abrt::HostReport.load_from_spool
+
+  # aggregate reports by host and by duplication hash if possible
+  reports.each do |report|
+    if reports_by_host.has_key? report.host
+      begin
+        reports_by_host[report.host].merge report
+      rescue StandardError => e
+        logger.error "Failed to merge #{report.files[0]} " \
+                     "into #{reports_by_host[report.host].files}: #{e}"
+      end
+    else
+      reports_by_host[report.host] = report
+    end
+  end
+
+  # send reports
+  reports_by_host.each_value do |hr|
+    begin
+      result = hr.send_to_foreman
+    rescue StandardError => e
+      logger.error "Unable to forward to Foreman server: #{e}"
+      next
+    end
+    unless result.is_a? Net::HTTPSuccess
+      logger.error "Foreman server rejected report (status #{result.code}): #{result.body}"
+    end
+    begin
+      hr.unlink
+    rescue StandardError => e
+      logger.error "Cannot delete #{hr.files}: #{e}"
+    end
+  end
+end
+
+if ARGV[0] == "--daemon"
+  if PLATFORM =~ /mingw/
+    puts "Daemon mode is not supported on Windows."
+    exit false
+  end
+  require 'daemon'
+  Process.daemon true
+
+  sleep_interval = Proxy::Abrt::Plugin.settings.abrt_send_period || 600
+  loop do
+    send_reports_from_spool
+    sleep sleep_interval
+  end
+else
+  send_reports_from_spool
+end

data/bundler.d/abrt.rb

@@ -0,0 +1,6 @@
+# Drop this file into bundler.d/ in smart-proxy root
+gem 'smart_proxy_abrt'
+
+group :abrt do
+  # plugin dependencies go here
+end

data/extra/foreman-proxy-abrt-send.cron

@@ -0,0 +1,2 @@
+# Send collected ABRT reports once every 10 minutes
+*/30 * * * * foreman-proxy smart-proxy-abrt-send

data/extra/rubygem-smart_proxy_abrt.spec

@@ -0,0 +1,110 @@
+%global gem_name smart_proxy_abrt
+
+%global foreman_proxy_bundlerd_dir /usr/share/foreman-proxy/bundler.d
+%global foreman_proxy_pluginconf_dir /etc/foreman-proxy/settings.d
+%global spool_dir /var/spool/foreman-proxy-abrt
+
+Name: rubygem-%{gem_name}
+Version: 0.0.1
+Release: 1%{?dist}
+Summary: Automatic Bug Reporting Tool plugin for Foreman's smart proxy
+Group: Applications/Internet
+License: GPLv3
+URL: http://github.com/abrt/smart-proxy-abrt
+Source0: https://fedorahosted.org/released/abrt/%{gem_name}-%{version}.gem
+Requires: ruby(release)
+Requires: ruby(rubygems)
+Requires: rubygem(ffi)
+Requires: foreman-proxy >= 1.6.0
+Requires: crontabs
+## does not exist in repository yet
+#Requires: rubygem-satyr
+BuildRequires: ruby(release)
+BuildRequires: rubygems-devel
+BuildRequires: ruby
+BuildRequires: rubygem(ffi)
+BuildRequires: rubygem(minitest)
+BuildArch: noarch
+Provides: rubygem(%{gem_name}) = %{version}
+
+%description
+This smart proxy plugin, together with a Foreman plugin, add the capability to
+send ABRT micro-reports from your managed hosts to Foreman.
+
+%package doc
+Summary: Documentation for %{name}
+Group: Documentation
+Requires:%{name} = %{version}-%{release}
+
+%description doc
+Documentation for %{name}
+
+%prep
+gem unpack %{SOURCE0}
+%setup -q -D -T -n  %{gem_name}-%{version}
+gem spec %{SOURCE0} -l --ruby > %{gem_name}.gemspec
+
+%build
+# Create the gem as gem install only works on a gem file
+gem build %{gem_name}.gemspec
+
+# %%gem_install compiles any C extensions and installs the gem into ./%gem_dir
+# by default, so that we can move it into the buildroot in %%install
+%gem_install
+
+%install
+# Packaging guidelines say: Do not ship tests
+rm -r .%{gem_instdir}/test .%{gem_instdir}/Rakefile
+rm .%{gem_instdir}/extra/*.spec
+
+mkdir -p %{buildroot}%{gem_dir}
+cp -a .%{gem_dir}/* \
+       %{buildroot}%{gem_dir}/
+
+# executables
+mkdir -p %{buildroot}%{_bindir}
+cp -a .%{_bindir}/* \
+       %{buildroot}%{_bindir}
+
+# bundler file
+mkdir -p %{buildroot}%{foreman_proxy_bundlerd_dir}
+mv %{buildroot}%{gem_instdir}/bundler.d/abrt.rb \
+   %{buildroot}%{foreman_proxy_bundlerd_dir}
+
+# sample config
+mkdir -p %{buildroot}%{foreman_proxy_pluginconf_dir}
+mv %{buildroot}%{gem_instdir}/settings.d/abrt.yml.example \
+   %{buildroot}%{foreman_proxy_pluginconf_dir}/
+
+# crontab
+mkdir -p %{buildroot}%{_sysconfdir}/cron.d/
+mv %{buildroot}%{gem_instdir}/extra/foreman-proxy-abrt-send.cron \
+   %{buildroot}%{_sysconfdir}/cron.d/%{name}
+
+# create spool directory
+mkdir -p %{buildroot}%{spool_dir}
+
+#%check
+#testrb -Ilib test
+
+%files
+%dir %{gem_instdir}
+%{gem_libdir}
+%exclude %{gem_cache}
+%{gem_spec}
+%{gem_instdir}/bin
+
+%dir %attr(0755, foreman-proxy, foreman-proxy) %{spool_dir}
+%{foreman_proxy_bundlerd_dir}/abrt.rb
+%{_bindir}/smart-proxy-abrt-send
+%doc %{foreman_proxy_pluginconf_dir}/abrt.yml.example
+%config(noreplace) %{_sysconfdir}/cron.d/%{name}
+
+%files doc
+%{gem_docdir}
+%{gem_instdir}/README
+%{gem_instdir}/LICENSE
+
+%changelog
+* Tue Jul 15 2014 Martin Milata <mmilata@redhat.com> - 0.0.1-1
+- Initial package

data/lib/smart_proxy_abrt.rb

@@ -0,0 +1,3 @@
+require 'smart_proxy_abrt/abrt_plugin'
+
+module Proxy::Abrt; end

data/lib/smart_proxy_abrt/abrt_api.rb

@@ -0,0 +1,76 @@
+require 'openssl'
+require 'json'
+
+require 'smart_proxy_abrt/abrt_lib'
+
+STATUS_ACCEPTED = 202
+
+module Proxy::Abrt
+  class Api < ::Sinatra::Base
+    include ::Proxy::Log
+    helpers ::Proxy::Helpers
+
+    post '/reports/new/' do
+      begin
+        cn = Proxy::Abrt::common_name request
+      rescue Proxy::Error::Unauthorized => e
+        log_halt 403, "Client authentication failed: #{e.message}"
+      end
+
+      ureport_json = request['file'][:tempfile].read
+      ureport = JSON.parse(ureport_json)
+
+      #forward to FAF
+      response = nil
+      if Proxy::Abrt::Plugin.settings.server_url
+        begin
+          result = Proxy::Abrt::faf_request "/reports/new/", ureport_json
+          response = result.body if result.code.to_s == STATUS_ACCEPTED.to_s
+        rescue StandardError => e
+          logger.error "Unable to forward to ABRT server: #{e}"
+        end
+      end
+      unless response
+        # forwarding is not configured or failed
+        # FAF source that generates replies is in src/webfaf/reports/views.py
+        response = { "result" => false,
+                     "message" => "Report queued" }
+        if Proxy::SETTINGS.foreman_url
+          foreman_url = Proxy::SETTINGS.foreman_url
+          foreman_url += "/" if url[-1] != "/"
+          foreman_url += "hosts/#{cn}/abrt_reports"
+          response["reported_to"] = [{ "reporter" => "Foreman",
+                                       "type" => "url",
+                                       "value" => foreman_url }]
+        end
+        response = response.to_json
+      end
+
+      #save report to disk
+      begin
+        Proxy::Abrt::HostReport.save cn, ureport
+      rescue StandardError => e
+        log_halt 500, "Failed to save the report: #{e}"
+      end
+
+      status STATUS_ACCEPTED
+      response
+    end
+
+    post '/reports/:action/' do
+      # pass through to real FAF if configured
+      if Proxy::Abrt::Plugin.settings.server_url
+        body = request['file'][:tempfile].read
+        begin
+          result = Proxy::Abrt::faf_request "/reports/#{params[:action]}/", body
+        rescue StandardError => e
+          log_halt 503, "ABRT server unavailable: #{e}"
+        end
+        status result.code
+        result.body
+      else
+        log_halt 501, "foreman-proxy does not implement /reports/#{params[:action]}/"
+      end
+    end
+  end
+end

data/lib/smart_proxy_abrt/abrt_lib.rb

@@ -0,0 +1,244 @@
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'time'
+require 'fileutils'
+
+require 'proxy/log'
+require 'proxy/request'
+
+module Proxy::Abrt
+  def self.random_alpha_string(length)
+    base = ('a'..'z').to_a
+    result = ""
+    length.times { result << base[rand(base.size)] }
+    result
+  end
+
+  # Generate multipart boundary separator
+  def self.suggest_separator
+      separator = "-"*28
+      separator + self.random_alpha_string(16)
+  end
+
+  # It seems that Net::HTTP does not support multipart/form-data - this function
+  # is adapted from http://stackoverflow.com/a/213276 and lib/proxy/request.rb
+  def self.form_data_file(content, file_content_type)
+    # Assemble the request body using the special multipart format
+    thepart =  "Content-Disposition: form-data; name=\"file\"; filename=\"*buffer*\"\r\n" +
+               "Content-Type: #{ file_content_type }\r\n\r\n#{ content }\r\n"
+
+    boundary = self.suggest_separator
+    while thepart.include? boundary
+      boundary = self.suggest_separator
+    end
+
+    body = "--" + boundary + "\r\n" + thepart + "--" + boundary + "--\r\n"
+    headers = {
+      "User-Agent"     => "foreman-proxy/#{Proxy::VERSION}",
+      "Content-Type"   => "multipart/form-data; boundary=#{ boundary }",
+      "Content-Length" => body.length.to_s
+    }
+
+    return headers, body
+  end
+
+  def self.faf_request(path, content, content_type="application/json")
+    uri              = URI.parse(Proxy::Abrt::Plugin.settings.server_url.to_s)
+    http             = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl     = uri.scheme == 'https'
+    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+    if Proxy::Abrt::Plugin.settings.server_ssl_noverify
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+
+    if Proxy::Abrt::Plugin.settings.server_ssl_cert && !Proxy::Abrt::Plugin.settings.server_ssl_cert.to_s.empty? \
+        && Proxy::Abrt::Plugin.settings.server_ssl_key && !Proxy::Abrt::Plugin.settings.server_ssl_key.to_s.empty?
+      http.cert = OpenSSL::X509::Certificate.new(File.read(Proxy::Abrt::Plugin.settings.server_ssl_cert))
+      http.key  = OpenSSL::PKey::RSA.new(File.read(Proxy::Abrt::Plugin.settings.server_ssl_key), nil)
+    end
+
+    headers, body = self.form_data_file content, content_type
+
+    path = [uri.path, path].join unless uri.path.empty?
+    response = http.start { |con| con.post(path, body, headers) }
+
+    response
+  end
+
+  def self.common_name(request)
+    client_cert = request.env['SSL_CLIENT_CERT']
+    raise Proxy::Error::Unauthorized, "Client certificate required" if client_cert.to_s.empty?
+
+    begin
+      client_cert = OpenSSL::X509::Certificate.new(client_cert)
+    rescue OpenSSL::OpenSSLError => e
+      raise Proxy::Error::Unauthorized, e.message
+    end
+
+    cn = client_cert.subject.to_a.detect { |name, value| name == 'CN' }
+    cn = cn[1] unless cn.nil?
+    raise Proxy::Error::Unauthorized, "Common Name not found in the certificate" unless cn
+
+    return cn
+  end
+
+  class HostReport
+    include Proxy::Log
+
+    class AggregatedReport
+      attr_accessor :report, :count, :hash, :reported_at
+      def initialize(report, count, hash, reported_at)
+        @report = report
+        @count = count
+        @hash = hash
+        @reported_at = Time.parse reported_at
+      end
+    end
+
+    class Error < RuntimeError; end
+
+    attr_reader :host, :reports, :files, :by_hash
+
+    def initialize(fname)
+      contents = IO.read(fname)
+      json = JSON.parse(contents)
+
+      report = json["report"]
+      hash = HostReport.duphash report
+      ar = AggregatedReport.new(json["report"], 1, hash, json["reported_at"])
+      @reports = [ar]
+      # index the array elements by duphash, if they have one
+      @by_hash = {}
+      @by_hash[hash] = ar unless hash.nil?
+      @files = [fname]
+      @host = json["host"]
+    end
+
+    def merge(other)
+      raise HostReport::Error, "Host names do not match" unless @host == other.host
+
+      other.reports.each do |ar|
+        if !ar.hash.nil? && @by_hash.has_key?(ar.hash)
+          # we already have this report, just increment the counter
+          found_report = @by_hash[ar.hash]
+          found_report.count += ar.count
+          found_report.reported_at = [found_report.reported_at, ar.reported_at].min
+        else
+          # we either don't have this report or it has no hash
+          @reports << ar
+          @by_hash[ar.hash] = ar unless ar.hash.nil?
+        end
+      end
+      @files += other.files
+    end
+
+    def send_to_foreman
+      foreman_report = create_foreman_report
+      logger.debug "Sending #{foreman_report}"
+      Proxy::HttpRequest::ForemanRequest.new.send_request("/api/abrt_reports", foreman_report.to_json)
+    end
+
+    def unlink
+      @files.each do |fname|
+        logger.debug "Deleting #{fname}"
+        File.unlink(fname)
+      end
+    end
+
+    def self.save(host, report, reported_at=nil)
+      # create the spool dir if it does not exist
+      FileUtils.mkdir_p HostReport.spooldir
+
+      reported_at ||= Time.now.utc
+      on_disk_report = { "host" => host, "report" => report , "reported_at" => reported_at.to_s }
+
+      # write report to temporary file
+      temp_fname = with_unique_filename "new-" do |temp_fname|
+        File.open temp_fname, File::WRONLY|File::CREAT|File::EXCL do |tmpfile|
+          tmpfile.write(on_disk_report.to_json)
+        end
+      end
+
+      # rename it
+      with_unique_filename ("ureport-" + DateTime.now.iso8601 + "-") do |final_fname|
+        File.link temp_fname, final_fname
+        File.unlink temp_fname
+      end
+    end
+
+    def self.load_from_spool
+      reports = []
+      report_files = Dir[File.join(HostReport.spooldir, "ureport-*")]
+      report_files.each do |fname|
+        begin
+          reports << new(fname)
+        rescue StandardError => e
+          logger.error "Failed to parse report #{fname}: #{e}"
+        end
+      end
+      reports
+    end
+
+    private
+
+    def format_reports
+      @reports.collect do |ar|
+        r = {
+          "count"       => ar.count,
+          "reported_at" => ar.reported_at.utc.to_s,
+          "full"        => ar.report
+        }
+        r["duphash"] = ar.hash unless ar.hash.nil?
+        r
+      end
+    end
+
+    # http://projects.theforeman.org/projects/foreman/wiki/Json-report-format
+    # To be replaced once Foreman understands other report types than from Puppet.
+    def create_foreman_report
+      { "abrt_report" => {
+            "host"        => @host,
+            "reports"     => format_reports
+        }
+      }
+    end
+
+    def self.duphash(report)
+      return nil if !Proxy::Abrt::Plugin.settings.aggregate_reports
+
+      begin
+        satyr_report = Satyr::Report.new report.to_json
+        stacktrace = satyr_report.stacktrace
+        thread = stacktrace.find_crash_thread
+        thread.duphash
+      rescue StandardError => e
+        logger.error "Error computing duphash: #{e}"
+        nil
+      end
+    end
+
+    def self.unique_filename(prefix)
+      File.join(HostReport.spooldir, prefix + Proxy::Abrt::random_alpha_string(8))
+    end
+
+    def self.with_unique_filename(prefix)
+      filename = unique_filename prefix
+      tries_left = 5
+      begin
+        yield filename
+      rescue Errno::EEXIST => e
+        filename = unique_filename prefix
+        tries_left -= 1
+        retry if tries_left > 0
+        raise HostReport::Error, "Unable to create unique file"
+      end
+      filename
+    end
+
+    def self.spooldir
+      Proxy::Abrt::Plugin.settings.spooldir || File.join(APP_ROOT, "spool/foreman-proxy-abrt")
+    end
+  end
+end