sonar_connector 0.8.5

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Trampoline Systems Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,18 @@
+Trampoline SONAR Connector framework
+
+== Dependencies
+
+=== Jruby
+
+Here's how to install jruby 1.4.0 on OS X:
+  
+  wget http://jruby.kenai.com/downloads/1.4.0/jruby-bin-1.4.0.tar.gz
+  sudo tar xfvz jruby-bin-1.4.0.tar.gz -C /usr/local/
+  ln -s /usr/local/jruby/bin/jruby /usr/bin/jruby
+  ln -s /usr/local/jruby/bin/rake /usr/bin/jrake
+  ln -s /usr/local/jruby/bin/gem /usr/bin/jgem
+  ln -s /usr/local/jruby/bin/jirb /usr/bin/jirb
+
+== Copyright
+
+Copyright (c) 2010 Trampoline Systems Ltd. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,41 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "sonar_connector"
+    gem.summary = %Q{A behind-the-firewall connector for Trampoline SONAR}
+    gem.description = %Q{Framework that allows arbitrary push and pull connectors to send data to an instance of the Trampoline SONAR server}
+    gem.email = "hello@empire42.com"
+    gem.homepage = "http://github.com/trampoline/sonar-connector"
+    gem.authors = ["Peter MacRobert", "Mark Meyer"]
+    
+    gem.add_dependency "actionmailer", "= 2.3.10"
+    gem.add_dependency "actionmailer_extensions", ">= 0.4.2"
+    gem.add_dependency "json_pure", ">= 1.2.2"
+    gem.add_dependency "uuidtools", ">= 2.1.1"
+    gem.add_dependency "sonar_connector_filestore", ">= 0.1.0"
+    
+    gem.add_development_dependency "rspec", ">= 1.2.8"
+    gem.add_development_dependency "rr", ">= 0.10.5"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+task :spec => :check_dependencies

data/VERSION

@@ -0,0 +1 @@
+0.8.5

data/bin/sonar-connector

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require File.join(File.dirname(__FILE__), '..', 'lib', 'sonar_connector')
+require 'optparse'
+
+config_filename = File.expand_path File.join(Dir.pwd, "config", "config.json")
+mode = nil
+install_path = nil
+
+ARGV.options do |opts|
+  script_name = File.basename($0)
+
+  opts.banner = "Usage: #{script_name} OPTION"
+  
+  opts.separator "Run modes:"
+  opts.on("--start", "Start the connector") { mode = :start }
+  opts.on("--check", "Validate the connector config") { mode = :check } 
+  opts.on("--install=PATH", String, "Install the connector working dir and default config to the file system") {|p|
+    install_path = p
+    mode = :install
+  }
+  opts.on("--console", "Run IRB console in connector framework environment") { mode = :console }
+  
+  opts.separator "Options:"
+  
+  opts.on("-c", "--config=FILE", String, "Override the path to the config file.") {|s| config_filename = File.expand_path s}
+  
+  opts.separator "Misc:"
+  opts.on_tail("-v", "--version", "Show version") { mode = :version }
+  opts.on_tail("-h", "--help", "Show this message")
+  opts.parse!
+end
+
+case mode
+when :start
+  puts "Starting SONAR Connector from config file: #{config_filename}"
+  connector = Sonar::Connector::Controller.new(config_filename)
+  puts "Connector bootstrapped successfully, check log files for details."
+  connector.start
+  exit
+  
+when :check
+  puts "Checking config file: #{config_filename}"
+  Sonar::Connector::Controller.new(config_filename)
+  puts "...clean."
+  exit
+when :install
+  path = File.expand_path install_path
+  
+  if File.directory?(path)
+    puts "Error: Directory '#{path}' already exists, aborting."
+    exit(1)
+  end
+  
+  %W{config log var}.each {|dir| FileUtils.mkdir_p File.join(path, dir)}
+  FileUtils.cp File.join(Sonar::Connector::ROOT, '..', "config", "config.example.json"), File.join(path, 'config', 'config.json')
+  puts "Success: Set up working directory '#{path}' and associated subdirs."
+  exit
+when :console
+  lib_path = File.expand_path File.join(File.dirname(__FILE__), '..', 'lib')
+  Kernel.system "irb -rubygems -I #{lib_path} -r sonar_connector.rb"
+  exit
+when :version
+  version_file = File.join File.expand_path(File.dirname(__FILE__)), "..", "VERSION"
+  puts "SONAR Connector Framework #{File.read(version_file)}"
+  exit
+else
+  puts ARGV.options
+end

data/config/config.example.json

@@ -0,0 +1,82 @@
+{
+  /* log level must be one of: "debug", "info", "warn", "error", "fatal" */
+  "log_level" : "debug",
+  
+  /* max log file size in megabytes */
+  "log_file_max_size" : "10",
+  
+  /* number of log files to keep */
+  "log_files_to_keep" : "7",
+  
+  "email_settings": {
+    "admin_recipients": ["admin@server.local"],
+    "admin_sender": "Sonar Connector <noreply@server.local>",
+    "perform_deliveries": false,
+    
+    /* options are ["smtp", "sendmail", "test"] */
+    "delivery_method": "smtp",
+    "save_emails_to_disk": true,
+    
+    "smtp_settings": {
+      "address": "127.0.0.1",
+      "port": 25,
+      "domain": "server.local",
+      "user_name": null,
+      "password": null,
+      
+      /* options are ["plain", "login", "cram_md5"] */
+      "authentication": null
+    },
+    
+    "sendmail_settings": {
+      "location": "/usr/sbin/sendmail",
+      "arguments": "-i -t -f nobody@localhost"
+    }
+  },
+  
+  /* 
+    Specific configuration for each connector. Each connector must have 
+    a class and a unique name. The require load path can also be specified if necessary.
+    Note that each connector type may have further configuration options
+    that are specific to the connector class.
+  */
+  "connectors": [
+    {
+      "class": "Sonar::Connector::ImapPullConnector",
+      "require": "sonar_imap_pull_connector",
+      "name": "gmail_1",
+      "repeat_delay": 10,
+      "host": "imap.gmail.com",
+      "user": "foo@bar.com",
+      "password": "---",
+      "folders": "[Gmail]/All Mail"
+    }
+    ,
+   {
+      "class": "Sonar::Connector::ImapPullConnector",
+      "require": "sonar_imap_pull_connector",
+      "name": "gmail_2",
+      "repeat_delay": 10,
+      "host": "imap.gmail.com",
+      "user": "baz@bar.com",
+      "password": "---",
+      "folders": "[Google Mail]/All Mail"
+    }
+     ,
+    {
+     "class": "Sonar::Connector::SonarPushConnector",
+     "require": "sonar_push_connector",
+     "name": "sonar_push",
+     "repeat_delay": 10,
+     "source_connectors": ["gmail_1", "gmail_2"],
+     "uri": "http://localhost:3000/api/1_0/rfc822_messages",
+     "connector_credentials": "---"
+    },
+    {
+     "class": "Sonar::Connector::SeppukuConnector",
+     "name": "seppuku",
+     "enabled": true,
+     "repeat_delay": 43200
+    }
+  ] 
+}

data/lib/sonar_connector.rb

@@ -0,0 +1,40 @@
+module Sonar
+  module Connector
+    ROOT = File.dirname(__FILE__) unless Sonar::Connector.const_defined?("ROOT")
+  end
+end
+
+require 'rubygems'
+$:.unshift(File.expand_path("..", __FILE__))
+
+# Load external deps
+require 'active_support'
+require 'json'
+require 'yaml'
+require 'thread'
+require 'logger'
+require 'action_mailer'
+require 'actionmailer_extensions'
+require 'fileutils'
+require 'sonar_connector_filestore'
+
+# Load internal classes
+%W(
+  controller
+  config
+  status
+  consumer
+  emailer
+  utils
+  connectors/base
+  connectors/dummy_connector
+  connectors/seppuku_connector
+  commands/command
+  commands/update_status_command
+  commands/send_admin_email_command
+  commands/update_disk_usage_command
+  commands/increment_status_value_command
+  commands/commit_seppuku_command
+).each do |file|
+  require File.join('sonar_connector', file)
+end

data/lib/sonar_connector/commands/command.rb

@@ -0,0 +1,21 @@
+module Sonar
+  module Connector
+    
+    ##
+    # Base command class that all commands should subclass.
+    
+    class Command
+      
+      attr_accessor :proc
+      
+      def initialize(proc)
+        @proc = proc
+      end
+      
+      def execute(context = nil)
+        context.instance_eval(&@proc)
+      end
+      
+    end
+  end
+end

data/lib/sonar_connector/commands/commit_seppuku_command.rb

@@ -0,0 +1,15 @@
+module Sonar
+  module Connector
+    
+    class CommitSeppukuCommand < Sonar::Connector::Command
+      def initialize
+        l = lambda do
+          # controller is in scope here because we've jumped thru some serious hoops
+          # and shaved the hell out of a yak or three.
+          Thread.new {controller.shutdown_lambda.call}
+        end
+        super(l)
+      end
+    end
+  end
+end

data/lib/sonar_connector/commands/increment_status_value_command.rb

@@ -0,0 +1,14 @@
+module Sonar
+  module Connector
+    
+    class IncrementStatusValueCommand < Sonar::Connector::Command
+      def initialize(connector, field, value = 1)
+        l = lambda do
+          current = status[connector.name] ? status[connector.name][field].to_i : 0
+          status.set connector.name, field, current+value
+        end
+        super(l)
+      end
+    end
+  end
+end

data/lib/sonar_connector/commands/send_admin_email_command.rb

@@ -0,0 +1,12 @@
+module Sonar
+  module Connector
+    class SendAdminEmailCommand < Sonar::Connector::Command
+      def initialize(connector, message)
+        l = lambda do
+          Sonar::Connector::Emailer.deliver_admin_message(connector, message)
+        end
+        super(l)
+      end
+    end
+  end
+end

data/lib/sonar_connector/commands/update_disk_usage_command.rb

@@ -0,0 +1,13 @@
+module Sonar
+  module Connector
+    class UpdateDiskUsageCommand < Sonar::Connector::Command
+      def initialize(connector)
+        l = lambda do
+          du = (Sonar::Connector::Utils.du(connector.connector_dir).to_f / 1024).round
+          status.set connector.name, 'disk_usage', "#{du} Kb"
+        end
+        super(l)
+      end
+    end
+  end
+end

data/lib/sonar_connector/commands/update_status_command.rb

@@ -0,0 +1,16 @@
+module Sonar
+  module Connector
+    
+    ACTION_OK = 'ok'
+    ACTION_FAILED = 'failed'
+    
+    class UpdateStatusCommand < Sonar::Connector::Command
+      def initialize(connector, field, value)
+        l = lambda do
+          status.set connector.name, field, value
+        end
+        super(l)
+      end
+    end
+  end
+end

data/lib/sonar_connector/config.rb

@@ -0,0 +1,166 @@
+module Sonar
+  module Connector
+    class InvalidConfig < RuntimeError; end
+    
+    class Config
+
+      # base params
+      attr_reader :base_dir
+      attr_reader :log_dir
+      attr_reader :connectors_dir
+      attr_reader :controller_log_file
+      attr_reader :status_file
+      attr_reader :connectors
+      attr_reader :email_settings
+      
+      # configurable: logger params
+      attr_reader :log_level
+      attr_reader :log_file_max_size
+      attr_reader :log_files_to_keep
+      
+      # Entry-point for creating and setting the CONFIG instance.
+      # Give it a path to the JSON settings file and it'll do the rest.
+      def self.load(config_file)
+        config = Config.new(config_file).parse
+        Sonar::Connector.const_set("CONFIG", config)
+      end
+      
+      # Helper method to read and parse JSON file from disk. Abstracted for testing purposes.
+      def self.read_json_file(config_file)
+        JSON.parse IO.read(config_file)
+      end
+      
+      def initialize(config_file)
+        @config_file = config_file
+      end
+      
+      def parse
+        @raw_config = Config.read_json_file(config_file)
+        
+        # extract the core config params
+        @base_dir = parse_base_dir @raw_config["base_dir"]
+        @log_dir = File.join @base_dir, 'log'
+        @connectors_dir = File.join @base_dir, 'var'
+        @controller_log_file = File.join @log_dir, 'controller.log'
+        @status_file = File.join @base_dir, 'status.yml'
+        @log_level = parse_log_level @raw_config["log_level"]
+        @log_file_max_size = parse_log_file_max_size @raw_config["log_file_max_size"]
+        @log_files_to_keep = parse_log_files_to_keep @raw_config["log_files_to_keep"]
+        @email_settings = parse_email_settings @raw_config["email_settings"]
+        
+        # extract each connector, locate its class and attempt to parse its config
+        @connectors = parse_connectors @raw_config["connectors"]
+        
+        associate_connector_dependencies! @connectors
+        
+        self
+      rescue JSON::ParserError => e
+        raise InvalidConfig.new("Config file #{config_file} is not in a valid JSON format. Please check the contents carefully. This is the exact error: \n#{e.message}")
+      end
+
+      private
+      
+      attr_reader :config_file
+      attr_reader :raw_config
+      
+      def parse_base_dir(base_dir)
+        d = base_dir.blank? ? File.dirname(File.dirname(@config_file)) : base_dir
+        raise InvalidConfig.new("#{d} not a valid directory") unless File.directory?(d)
+        d
+      end
+      
+      def parse_log_level(log_level)
+        raise InvalidConfig.new("Config option 'log_level' is a required parameter.") if log_level.blank?
+        valid_log_levels = ["debug", "info", "warn", "error", "fatal"]
+        raise InvalidConfig.new("unknown log_level #{log_level}") unless valid_log_levels.include?(log_level)
+        Logger.const_get log_level.upcase
+      end
+      
+      def parse_log_file_max_size(log_file_max_size)
+        raise InvalidConfig.new("invalid log_file_max_size #{log_file_max_size}") if !log_file_max_size.blank? && log_file_max_size.to_i == 0
+        log_file_max_size.blank? ? 10*1024*1024 : log_file_max_size.to_i*1024*1024
+      end
+
+      def parse_log_files_to_keep(log_files_to_keep)
+        raise InvalidConfig.new("invalid log_files_to_keep #{log_files_to_keep}") if !log_files_to_keep.blank? && log_files_to_keep.to_i == 0
+        log_files_to_keep.blank? ? 10 : log_files_to_keep.to_i
+      end
+      
+      def parse_email_settings(settings)
+        ActionMailer::Base.perform_deliveries = settings["perform_deliveries"]
+        ActionMailer::Base.delivery_method = settings["delivery_method"].to_sym
+        ActionMailer::Base.raise_delivery_errors = true
+        
+        # ActionMailer needs the smtp and sendmail settings hashes to have symbols for keys
+        ActionMailer::Base.smtp_settings  = symbolise_hash_keys settings["smtp_settings"]
+        ActionMailer::Base.sendmail_settings = symbolise_hash_keys settings["sendmail_settings"]
+        
+        ActionMailer::Base.save_emails_to_disk = settings["save_emails_to_disk"]
+        ActionMailer::Base.email_output_dir = File.join @base_dir, 'sent_administrator_emails'
+        ActionMailer::Base.safe_recipients = settings["admin_recipients"].to_a
+        settings
+      end
+      
+      def parse_connectors(connectors_config)
+        raise InvalidConfig.new("Connector parameter must be an array and cannot be empty") unless connectors_config.instance_of?(Array) && !connectors_config.empty?
+        
+        c = []
+        connectors_config.each do |config|
+          c << parse_connector(config)
+        end
+        
+        raise InvalidConfig.new("Connector names must be unique. You supplied: #{c.map(&:name).inspect}") if c.map(&:name).uniq.size != c.size
+        c
+      end
+      
+      def parse_connector(config)
+        
+        # Load the require first, if specified
+        begin
+          require config["require"] unless config["require"].blank?
+        rescue MissingSourceFile
+          raise InvalidConfig.new("Error with parameter 'require' in connector settings '#{config.inspect}': require failed - check that the path is correct.")
+        end
+        
+        # Insist that class is specified
+        raise InvalidConfig.new("Error with parameter 'class' in connector settings '#{config.inspect}': class must be specified.") if config["class"].blank?
+        
+        # Attempt to load the class definition
+        begin
+          klass = config["class"].constantize
+        rescue
+          raise InvalidConfig.new("Error with parameter 'class' in connector settings '#{config.inspect}': could not load class.")
+        end
+        
+        # sanity-check that the connector class subclasses the base
+        raise InvalidConfig.new("Connector class #{klass.name} must subclass Sonar::Connector::Base") unless klass.ancestors.include?(Sonar::Connector::Base)
+        klass.new(config, self)
+      end
+      
+      def symbolise_hash_keys(hash)
+        return nil unless hash
+        hash.keys.inject({}){|acc, k| acc[k.to_sym] = hash[k]; acc}
+      end
+      
+      # Find all connectors with "source_connectors" specified in config, and map these 
+      # associations to the connector instances.
+      def associate_connector_dependencies!(connectors)
+        connectors.each do |connector|
+          source_names = [*connector.raw_config["source_connectors"]].compact
+          next if source_names.blank?
+          
+          source_connectors = source_names.map do |source_name|
+            c = connectors.select{|connector2| connector2.name == source_name}.first
+            raise InvalidConfig.new("Connector '#{connector.name}' references a source connector '#{source_name}' but no such connector name is defined.") unless c
+            raise InvalidConfig.new("Connector '#{connector.name}' cannot have itself as a source connector.") if c == connector
+            c
+          end
+          
+          connector.send :source_connectors=, source_connectors
+        end
+      end
+      
+    end
+  end
+end
+