sonar_connector 0.8.5

data/lib/sonar_connector/connectors/base.rb

@@ -0,0 +1,243 @@
+module Sonar
+  module Connector
+    class Base
+      
+      # every connector has a unique name
+      attr_reader :name
+
+      # this connector
+      attr_reader :connector
+      
+      # Connector-specific config hash
+      attr_reader :raw_config
+      
+      # each connector instance has a working dir for its state and files
+      attr_reader :connector_dir
+      
+      # logger instance
+      attr_reader :log
+      
+      # state hash that is serialized and persisted to disk every cycle of the run loop
+      attr_reader :state
+      
+      # repeat delay which is waited out on each cycle of the run loop
+      attr_reader :repeat_delay
+      
+      # central command queue for sending messages back to the controller
+      attr_reader :queue
+      
+      # run loop flag
+      attr_reader :run
+
+      # filestore for whole connector
+      attr_reader :connector_filestore
+
+      # filestore for an action run
+      attr_reader :filestore
+      
+      # Array of associated connectors that provide source data via the file system
+      attr_reader :source_connectors
+      
+      def initialize(connector_config, base_config)
+        @base_config = base_config
+        @raw_config = connector_config
+        
+        @name = connector_config["name"]
+        @connector = self
+        
+        # Create STDOUT logger and inherit the logger settings from the base controller config
+        @log_file = File.join base_config.log_dir, "connector_#{@name}.log"
+        @log = Sonar::Connector::Utils.stdout_logger base_config
+        
+        # every connector instance must set the repeat delay
+        raise InvalidConfig.new("Connector '#{@name}': repeat_delay is missing or blank") if connector_config["repeat_delay"].blank?
+        @repeat_delay = connector_config["repeat_delay"].to_i
+        raise InvalidConfig.new("Connector '#{@name}': repeat_delay must be >= 1 second") if @repeat_delay < 1
+        
+        @connector_dir = File.join(base_config.connectors_dir, @name)
+        FileUtils.mkdir_p(@connector_dir)
+        @state_file = File.join(@connector_dir, "state.yml")
+        
+        # empty state hash which will get written to by parse, and then potentially over-written by load_state
+        @state = {}
+
+        @connector_filestore = Sonar::Connector::FileStore.new(@connector_dir, 
+                                                               "#{@name}_filestore", 
+                                                               [:working, :error, :complete, :actions],
+                                                               :logger=>@log)
+
+        parse connector_config
+        load_state
+        
+        @run = true
+      end
+
+      def prepare(queue)
+        @queue = queue
+        switch_to_log_file
+        
+        cleanup_old_action_filestores # in case we were interrupted mid-action
+        cleanup # before we begin
+      end
+
+      # Logging defaults to use STDOUT. After initialization we need to switch the 
+      # logger to use an output file.
+      def switch_to_log_file
+        @log = Sonar::Connector::Utils.disk_logger(log_file, base_config)
+        @connector_filestore.logger = @log if @connector_filestore
+      end
+      
+      # Load the state hash from YAML file
+      def load_state
+        @state.merge! read_state
+      end
+      
+      # Read state file
+      def read_state
+        s = {}
+        if File.exist?(state_file)
+          ds = YAML.load_file state_file
+          raise "State file did not contain a serialised hash." unless ds.is_a?(Hash)
+          s = ds  # only return the parsed value if it is actually a hash
+        end
+      rescue Exception => e
+        log.error "Error loading #{state_file} so it was ignored. Original error: #{e.message}\n" + e.backtrace.join("\n")
+      ensure 
+        return s
+      end
+      
+      # Save the state hash to a YAML file
+      def save_state
+        make_dir
+        File.open(state_file, "w"){|f| f << state.to_yaml }
+      end
+      
+      # Cleanup routine after connector shutdown
+      def cleanup
+      end
+      
+      # the main run loop that every connector executes indefinitely 
+      # until Thread.raise is called on this instance.
+      def start
+        begin
+          run_loop
+
+          @run = false
+          cleanup
+          true
+        rescue Exception=>e
+          log.error([e.class.to_s, e.message, *e.backtrace].join("\n"))
+        end
+      end
+
+      def run_loop
+        while run
+          begin
+            run_once
+            sleep_for repeat_delay
+          rescue ThreadTerminator
+            break
+          rescue Exception => e
+            log.error ["Connector '#{name} raised an unhandled exception:",
+                       e.class.to_s,
+                       e.message,
+                       *e.backtrace].join("\n")
+            log.info "Connector blew up with an exception - waiting 5 seconds before retrying."
+            queue << Sonar::Connector::UpdateStatusCommand.new(connector, 'last_action', Sonar::Connector::ACTION_FAILED)
+            sleep_for 5
+          end
+        end
+      end
+
+      def run_once
+        log.info "beginning action"
+
+        with_action_filestore do
+          action
+
+          save_state
+          log.info "finished action,saved state"
+          log.info "working: #{filestore.count(:working)}, error: #{filestore.count(:error)}, complete: #{filestore.count(:complete)}"
+          
+          queue << Sonar::Connector::UpdateStatusCommand.new(connector, 'last_action', Sonar::Connector::ACTION_OK)
+          queue << Sonar::Connector::UpdateDiskUsageCommand.new(connector)
+          queue << Sonar::Connector::UpdateStatusCommand.new(connector, 'working_count', filestore.count(:working))
+          queue << Sonar::Connector::UpdateStatusCommand.new(connector, 'error_count', filestore.count(:error))
+          queue << Sonar::Connector::UpdateStatusCommand.new(connector, 'complete_count', filestore.count(:complete))
+        end
+      end
+      
+      # Connector subclasses can overload the parse method.
+      def parse(config)
+        log.warn "Method #parse called on connector base class. Connector #{name} should define #parse method."
+      end
+      
+      def to_s
+        "#{self.class} '#{name}'"
+      end
+      alias :inspect :to_s
+      
+      private
+      
+      attr_reader :state_file, :base_config, :log_file
+      attr_writer :source_connectors
+      
+      def sleep_for(seconds=0)
+        sleep seconds
+      end
+      
+      def make_dir
+        FileUtils.mkdir_p(@connector_dir) unless File.directory?(@connector_dir)
+      end
+
+      def with_action_filestore
+        fs = create_action_filestore
+        begin
+          initialize_action_filestore(fs)
+          @filestore = fs
+          yield
+        ensure
+          @filestore = nil
+          finalize_action_filestore(fs)
+        end
+      end
+      
+      def create_action_filestore
+        now = Time.new
+        fs_name = now.strftime("action_%Y%m%d_%H%M%S_") + UUIDTools::UUID.timestamp_create.to_s.gsub('-','_')
+        action_fs_root = connector_filestore.area_path(:actions)
+        fs=FileStore.new(action_fs_root, fs_name, [:working, :error, :complete], :logger=>@log)
+        log.info("action_filestore path: #{fs.filestore_path}")
+        fs
+      end
+
+      def initialize_action_filestore(fs)
+        # grab any unfinished work for this action
+        connector_filestore.flip(:working, fs, :working)
+        fs.scrub!(:working)
+      end
+
+      def finalize_action_filestore(fs)
+        [:complete, :error].each do |area|
+          fs.scrub!(area)
+          fs.flip(area, connector_filestore, area, false) # flip with uuid folders
+        end
+        fs.scrub!(:working)
+        fs.flip(:working, connector_filestore, :working) # flip without folders
+        fs.destroy!
+      end
+
+      def cleanup_old_action_filestores
+        actionfs_root = connector_filestore.area_path(:actions)
+        
+        Dir.foreach(actionfs_root) do |fs_name|
+          fs_path = File.join(actionfs_root, fs_name)
+          if File.directory?(fs_path) && FileStore.valid_filestore_name?(fs_name)
+            fs = FileStore.new(actionfs_root, fs_name, [:working, :error, :complete], :logger=>@log)
+            finalize_action_filestore(fs)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sonar_connector/connectors/dummy_connector.rb

@@ -0,0 +1,17 @@
+module Sonar
+  module Connector
+    
+    ## 
+    # Dummy connector type. Does nothing except report back to the controller.
+    class DummyConnector < Sonar::Connector::Base
+      
+      def parse(config)
+      end
+      
+      def action
+        log.debug "#{name} mumbled incoherently to itself."
+      end
+      
+    end
+  end
+end

data/lib/sonar_connector/connectors/seppuku_connector.rb

@@ -0,0 +1,26 @@
+module Sonar
+  module Connector
+    
+    # Suicide connector.Submits a command to cause the death of the entire connector framework. 
+    # The connector will then get restarted by god or by the Windows service wrapper.
+    class SeppukuConnector < Sonar::Connector::Base
+      
+      attr_accessor :run_count
+      
+      def parse(config)
+        @run_count = 0
+      end
+      
+      def action
+        
+        if @run_count > 0 
+          log.info "切腹! #{name} committing honourable suicide and terminating the connector service."
+          queue << Sonar::Connector::CommitSeppukuCommand.new
+        end
+        
+        @run_count += 1
+      end
+      
+    end
+  end
+end

data/lib/sonar_connector/consumer.rb

@@ -0,0 +1,94 @@
+module Sonar
+  module Connector
+    
+    # Command execution context, whereby commands can be injected with consumer context
+    # variables for the logger and status objects.
+    class ExecutionContext
+        attr_reader :log
+        attr_reader :status
+        attr_reader :controller
+        
+        def initialize(params)
+          @log = params[:log]
+          @status = params[:status]
+          
+          # pass the controller all the way from the consumer initialisation into the command execution context,
+          # so that we can call privileged methods such as shutdown.
+          @controller = params[:controller]
+        end
+    end
+    
+    
+    ## 
+    # Listens to thread message queue and processes messages.
+    class Consumer
+      
+      attr_reader :base_config
+      attr_reader :queue
+      attr_reader :controller
+      attr_reader :status
+      attr_reader :log
+      attr_reader :run
+      
+      def initialize(controller, base_config)
+        @controller = controller
+        @base_config = base_config
+        
+        # Consumer holds the status object because changes 
+        # to status should be centrally moderated.
+        @status = Sonar::Connector::Status.new(@base_config)
+        
+        # Creat logger and inherit the logger settings from the base controller config
+        @log_file = File.join(base_config.log_dir, "consumer.log")
+        @log = Sonar::Connector::Utils.stdout_logger base_config
+        
+        @run = true
+      end
+
+      # It's kinda evil to be passing in the controller here. The better option is to 
+      # refactor the consumer to be part of the controller.
+      def prepare(queue)
+        @queue = queue
+        switch_to_log_file
+      end
+      
+      def switch_to_log_file
+        FileUtils.mkdir_p(base_config.log_dir) unless File.directory?(base_config.log_dir)
+        @log = Sonar::Connector::Utils.disk_logger(@log_file, base_config)
+      end
+      
+      def cleanup
+        log.info "Shut down consumer"
+        log.close
+      end
+      
+      ##
+      # Main loop to watch the command queue and process commands.
+      def watch
+        while run
+          begin
+            run_once
+          rescue ThreadTerminator
+            break
+          end
+        end
+        
+        @run = false
+        cleanup
+        true
+      end
+
+      def run_once
+        begin
+          command = queue.pop
+          command.execute ExecutionContext.new(:log=>log, :status=>status, :controller=>controller)
+        rescue ThreadTerminator => e
+          raise
+        rescue Exception => e
+          log.error ["Command #{command.class} raised an unhandled exception: ",
+                     e.class.to_s, e.message, *e.backtrace].join('\n')
+        end
+      end
+    end
+  end
+end

data/lib/sonar_connector/controller.rb

@@ -0,0 +1,164 @@
+module Sonar
+  module Connector
+    
+    class ThreadTerminator < Exception; end
+    
+    class Controller
+      
+      DEFAULT_CONFIG_FILENAME = File.join("config", "config.json")
+      
+      ##
+      # single command queue for threads to communicate with the controller
+      attr_reader :queue
+      
+      ##
+      # array of instantiated connector instances
+      attr_reader :connectors
+
+      ##
+      # instance of Sonar::Connector::Consumer
+      attr_reader :consumer
+      
+      ##
+      # instance of Sonar::Connector::Config
+      attr_reader :config
+      
+      ## 
+      # instance of Sonar::Connector::Status
+      attr_reader :status
+      
+      ##
+      # controller logger
+      attr_reader :log
+      
+      ##
+      # array of threads
+      attr_reader :threads
+      
+      ##
+      # Parse the config file and create instances of each connector, 
+      # parsing their config in turn.
+      def initialize(config_filename = DEFAULT_CONFIG_FILENAME)
+        @config = Sonar::Connector::Config.load config_filename
+        @log = Sonar::Connector::Utils.stdout_logger @config
+
+        @connectors = @config.connectors
+        @consumer = Sonar::Connector::Consumer.new(self, @config)
+        
+        @threads = []
+        
+        @queue = Queue.new
+        
+        create_startup_dirs_and_files
+      rescue Sonar::Connector::InvalidConfig => e
+        $stderr << ([e.class.to_s, e.message, *e.backtrace].join("\n")) << "\n"
+        raise RuntimeError, "Invalid configuration in #{config_filename}: \n #{e.message}"
+      end
+      
+      def switch_to_log_file
+        @log = Sonar::Connector::Utils.disk_logger(config.controller_log_file, config)
+      end
+      
+      def start
+        prepare_connector
+        start_threads
+
+        # let the controlling thread go into an endless sleep
+        puts "Ctrl-C to stop."
+        
+        # Standardize shutdown via ctrl-c and SIGTERM (from god)
+        trap "SIGINT", shutdown_lambda
+        trap "SIGTERM", shutdown_lambda
+        
+        endless_sleep
+      end
+
+      # prepare the connector, start an IRB console, but don't start any threads
+      def start_console
+        prepare_connector
+        # make the Controller globally visible
+        Connector.const_set("CONTROLLER", self)
+
+        require 'irb'
+        IRB.start
+      end
+
+      def prepare_connector
+        switch_to_log_file
+        log_startup_params
+        
+        connectors.each do |connector|
+          log.info "preparing connector '#{connector.name}'"
+          connector.prepare(queue)
+        end
+
+        log.info "preparing message queue consumer"
+        consumer.prepare(queue)
+      end
+
+      ##
+      # Main framework loop. Fire up one thread per connector, 
+      # plus the message queue consumer. Then wait for quit signal.
+      def start_threads
+        # fire up the connector threads
+        connectors.each do |connector|
+          log.info "starting connector '#{connector.name}'"
+          threads << Thread.new { connector.start }
+        end
+        
+        log.info "starting the message queue consumer"
+        threads << Thread.new{ consumer.watch }
+      end
+      
+      
+      def shutdown_lambda
+        lambda do
+          puts "\nGiving threads 10 seconds to shut down..."
+          threads.each{|t| t.raise(ThreadTerminator)}
+          begin
+            Timeout::timeout(10) { 
+              threads.map(&:join)
+            }
+          rescue Timeout::Error
+            puts "...couldn't stop all threads cleanly."
+            log.info "Could not cleanly terminate all threads."
+            log.close
+            exit(1)
+          rescue ThreadTerminator
+            # ignore it, since it's come from one of the recently-nuked threads.
+          rescue Exception => e
+            log.debug ["Caught unhandled exception: ",
+                       e.class.to_s,
+                       e.message,
+                       *e.backtrace].join("\n")
+          end
+          
+          puts "...exited cleanly."
+          log.info "Terminated all threads cleanly."
+          log.close
+          exit(0)
+        end
+      end
+      
+      private
+      
+      def log_startup_params
+        log.info "Startup: base directory is #{config.base_dir}"
+        log.info "Startup: logging directory is #{config.log_dir}"
+        log.info "Startup: log level is " + config.send(:raw_config)['log_level']
+        log.info "Startup: controller logging to #{config.controller_log_file}"
+      end
+      
+      def create_startup_dirs_and_files
+        FileUtils.mkdir_p(config.base_dir) unless File.directory?(config.base_dir)
+        FileUtils.mkdir_p(config.log_dir) unless File.directory?(config.log_dir)
+        FileUtils.touch config.controller_log_file
+      end
+      
+      def endless_sleep
+        sleep
+      end
+      
+    end
+  end
+end