lyber-core 1.3.0 → 3.2.2

data/lib/lyber_core/robots/service_controller.rb

@@ -1,174 +0,0 @@
-require 'daemons'
-require 'logger'
-require 'fileutils'
-
-module LyberCore
-  module Robots
-    class ServiceController < Daemons::ApplicationGroup
-      attr_reader :logger
-      
-      def initialize(opts = {})
-        if opts[:logger]
-          @logger = opts[:logger]
-        else
-          @logger = Logger.new($stdout)
-          @logger.level = opts[:log_level] || Logger::WARN
-        end
-        @sleep_time = opts[:sleep_time] || (15*60)
-        @working_dir = opts[:working_dir] || ENV['ROBOT_ROOT'] || Dir.pwd
-        @pid_dir = opts[:pid_dir] || File.join(@working_dir, 'pid')
-        @pid_dir = File.expand_path(@pid_dir)
-        FileUtils.mkdir(@pid_dir) unless(File.directory? @pid_dir)
-        @argv = (opts[:argv] || []).dup
-        @logger.debug "Initializing application group."
-        @logger.debug "Writing pids to #{@pid_dir}"
-        super('robot_service_controller', :dir_mode => :normal, :dir => @pid_dir, :multiple => true, :backtrace => true)
-      end
-
-      def qname(workflow, robot_name)
-        [workflow,robot_name].join(':')
-      end
-      
-      def start(workflow, robot_name)
-        result = false
-        app = find_app(workflow, robot_name).first
-        process_name = qname(workflow,robot_name)
-        if app.nil? or (app.running? == false)
-          @logger.info "Starting #{process_name}..."
-          with_app_name("#{process_name}") do
-            app, message = capture_stdout do
-              raw_module_name = workflow.split('WF').first
-              module_name = raw_module_name[0].chr.upcase << raw_module_name.slice(1, raw_module_name.size - 1)
-              robot_klass = Module.const_get(module_name).const_get(robot_name.split(/-/).collect { |w| w.capitalize }.join(''))
-              log_state = marshal_logger(@logger)
-              robot_proc = lambda {
-                Dir.chdir(@working_dir) do
-                  begin
-                    logger = restore_logger(log_state)
-                    robot = robot_klass.new(:argv => @argv.dup)
-                    loop { 
-                      case robot.start 
-                      when LyberCore::Robots::SLEEP
-                        logger.info "SLEEP condition reached in #{process_name}. Sleeping for #{@sleep_time} seconds."
-                        sleep(@sleep_time)
-                      when LyberCore::Robots::HALT
-                        logger.error "HALT condition reached in #{process_name}. Shutting down."
-                        break
-                      end
-                    }
-                  ensure
-                    logger.info "Shutting down."
-                  end
-                end
-              }
-              new_app = self.new_application({:mode => :proc, :proc => robot_proc, :dir_mode => :normal, :log_output => true, :log_dir => @pid_dir})
-              new_app.start
-              new_app
-            end
-          end
-          
-          if app.running?
-            @logger.info "#{process_name} [#{app.pid.pid}] started."
-            result = true
-          else
-            @logger.error "Unable to start #{process_name}"
-          end
-        else app.running?
-          @logger.warn "Robot #{process_name} [#{app.pid.pid}] is already running"
-        end
-        return result
-      end
-
-      def stop(workflow, robot_name)
-        apps = find_app(workflow, robot_name)
-        process_name = qname(workflow,robot_name)
-        result = false
-        if apps.empty?
-          @logger.info "Robot #{process_name} not found"
-        else
-          apps.each do |app|
-            if app.running?
-              @logger.info "Shutting down #{process_name} [#{app.pid.pid}]..."
-              result, message = capture_stdout { app.stop }
-              if app.running?
-                @logger.error "Unable to stop #{process_name} [#{app.pid.pid}]."
-              else
-                @logger.info "#{process_name} [#{app.pid.pid}] shut down."
-                result = true
-              end
-            else
-              @logger.warn "Robot #{process_name} [#{app.pid.pid}] is not running but pidfile exists"
-              app.zap!
-            end
-          end
-        end
-        result
-      end
-  
-      def status(workflow, robot_name)
-        apps = find_app(workflow, robot_name)
-        apps.collect do |app|
-          { :pid => app.pid.pid, :status => app.running? ? :running : :stopped }
-        end
-      end
-  
-      def status_message(workflow, robot_name)
-        app_status = status(workflow, robot_name)
-        process_name = qname(workflow,robot_name)
-        if app_status.empty?
-          ["Robot #{process_name} not found"]
-        else
-          app_status.collect do |s|
-            case s[:status]
-            when :running
-              "Robot #{process_name} [#{s[:pid]}] is running"
-            when :stopped
-              "Robot #{process_name} [#{s[:pid]}] is not running but pidfile exists"
-            end
-          end
-        end
-      end
-  
-#      private
-      def with_app_name(name)
-        old_name, @app_name = @app_name, name
-        begin
-          return yield
-        ensure
-          @app_name = old_name
-        end
-      end
-      
-      def capture_stdout
-        old_io = $stdout
-        begin
-          new_io = StringIO.new('')
-          $stdout = new_io
-          result = yield
-          @logger.debug new_io.string
-          return result, new_io.string
-        ensure
-          $stdout = old_io
-        end
-      end
-  
-      def find_app(workflow, robot_name)
-        with_app_name(qname(workflow,robot_name)) {
-          self.find_applications_by_pidfiles(@pid_dir)
-        }
-      end
-      
-      def marshal_logger(l)
-        log_device = l.instance_variable_get('@logdev')
-        { :dev => log_device, :file => log_device.filename, :level => l.level }
-      end
-      
-      def restore_logger(params)
-        result = Logger.new(params[:file] || params[:log_device])
-        result.level = params[:level]
-        return result
-      end
-  
-    end
-  end
-end

data/lib/lyber_core/robots/work_item.rb

@@ -1,112 +0,0 @@
-require 'dor_service'
-require "xml_models/identity_metadata/identity_metadata"
-require "xml_models/identity_metadata/dublin_core"
-
-
-# Represents a single object being processed as part of a workflow queue
-module LyberCore
-  module Robots
-    class WorkItem
-
-      # The queue that this workitem is a member of
-      attr_reader :work_queue
-      # The primary id for the object being processed
-      attr_accessor :druid
-      # An object used to hold unmarshalled XML from the identityMetadata datastream
-      attr_accessor :identity_metadata
-      # Timings for this workitem's processing
-      attr_reader :start_time
-      attr_reader :end_time
-      attr_reader :elapsed_time
-
-      # Create a new WorkItem object, save a pointer to the parent WorkQueue, and start the timer
-      def initialize(work_queue)
-        @work_queue = work_queue
-        @start_time = Time.new
-      end
-
-      # Inject an IdentityMetadata object (currently used for unit testing only)
-      def identity_metadata=(identity_metadata)
-        @identity_metadata = identity_metadata
-      end
-    
-      #save the IdentityMetadata object to identityMetadata datastream
-      def identity_metadata_save
-        unless DorService.get_datastream(@druid, 'identityMetadata')
-          DorService.add_datastream(@druid, 'identityMetadata', 'identityMetadata', self.identity_metadata.to_xml)
-        else
-          DorService.update_datastream(@druid, 'identityMetadata', self.identity_metadata.to_xml, content_type='application/xml', versionable = false)
-        end #unless
-      end #identity_metadata_save
-
-      # Return the IdentityMetadata object bound to identityMetadata datastream XML
-      def identity_metadata
-        if (@identity_metadata == nil)
-          if (@druid == nil)
-            @identity_metadata = IdentityMetadata.new
-          else
-            idmd_str = DorService.get_datastream(@druid, 'identityMetadata')
-            @identity_metadata = IdentityMetadata.from_xml(idmd_str)
-          end
-        end
-        return @identity_metadata
-      end
-
-      # Return the identifier value for the specified identier name
-      def identifier(key)
-        return self.identity_metadata.get_identifier_value(key)
-      end
-
-      # Add a new name,value pair to the set of identifiers
-      def identifier_add(key, value)
-        self.identity_metadata.add_identifier(key, value)
-      end
-
-      # Return an array of strings where each entry consists of name:value
-      def id_pairs
-        self.identity_metadata.get_id_pairs
-      end
-
-      # Return the druid for the work item if it exists, else the first identifier value
-      def item_id
-        return @druid if @druid
-        pairs = self.identity_metadata.get_id_pairs
-        return pairs[0] if (pairs.size > 0)
-      end
-
-      # Record a non-error status for the workstep operation
-      def set_status(status)
-        @elapsed_time = Time.new - @start_time
-        @end_time = Time.new
-        @elapsed_time = @end_time - @start_time
-        LyberCore::Log.info("#{item_id} #{status} in #{@elapsed_time} seconds")
-        if (@druid)
-          Dor::WorkflowService.update_workflow_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, status, @elapsed_time)
-        end
-      end
-
-      # Record the successful outcome of the workstep operation for this workitem
-      def set_success
-        @work_queue.success_count += 1
-        self.set_status('completed')
-      end
-
-      # Record the unsuccessful outcome of the workstep operation for this workitem
-      def set_error(e)
-        @work_queue.error_count += 1
-        @end_time = Time.new
-        @elapsed_time = @end_time - @start_time
-        if (e.is_a?(LyberCore::Exceptions::ItemError) )
-          item_error = e
-        else
-          item_error = LyberCore::Exceptions::ItemError.new(@druid, "Item error", e)
-        end
-        LyberCore::Log.exception(item_error)
-        if (@druid)
-          DorService.update_workflow_error_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, item_error.message)
-        end
-      end
-      
-    end
-  end
-end

data/lib/lyber_core/robots/work_queue.rb

@@ -1,177 +0,0 @@
-require 'dor_service'
-require 'dlss_service'
-require 'yaml'
-
-module LyberCore
-  module Robots
-    # Represents a set of workitem objects to be processed by a given step of a workflow
-    class WorkQueue
-
-      # The workflow that this queue is a part of
-      attr_reader :workflow
-      # The step in the workflow that is being processed against this queue
-      attr_reader :workflow_step
-      # The workflow step that should have already been completed for the workitem object
-      attr_reader :prerequisite
-      # The maximum number of workitem objects to process in one run of a robot
-      attr_reader :batch_limit
-      # The maximum number of errors to allow before terminating the batch run
-      attr_reader :error_limit
-      # The array of primary identifiers for the workitem objects to be processed
-      attr_reader :druids
-      # An alternative identitier to be used when druids are not yet available (e.g at registration)
-      attr_reader :identifier_name
-      attr_reader :identifier_values
-      # The tally of how many items have been processed
-      attr_reader :item_count
-      attr_accessor :success_count
-      attr_accessor :error_count
-      # The timings for the batch run
-      attr_reader :start_time
-      attr_reader :end_time
-      attr :elapsed_time
-      
-      attr_reader :config_file
-
-
-      # Create a new WorkQueue object for the specified step,
-      # save a pointer to the parent WorkFlow,
-      # start the timer,
-      # read in the configuration information for the work step
-      def initialize(workflow=nil, workflow_step=nil)
-        LyberCore::Log.debug("Initializing work queue with workflow #{workflow} and workflow_step #{workflow_step}")
-        @start_time = Time.new
-        LyberCore::Log.info("Starting #{workflow_step} at #{@start_time}")
-        @workflow = workflow
-        @workflow_step = workflow_step
-        @item_count = 0
-        @success_count = 0
-        @error_count = 0
-        # nil arguments should only be used if in test mode
-        if (workflow.nil? || workflow_step.nil?)
-          @batch_limit = 2
-          @error_limit = 1
-          return
-        end
-        
-        self.process_config_file
-
-      end
-      
-      def process_config_file
-          LyberCore::Log.debug("Processing config file ... ")
-          LyberCore::Log.debug("@workflow.workflow_config_dir = #{@workflow.workflow_config_dir}")
-          
-          @config_file = File.join(@workflow.workflow_config_dir, 'process-config.yaml')
-          LyberCore::Log.debug("I'm opening the config file at #{@config_file}")
-          
-          # Does the file exist?
-          raise "Can't open process-config file #{@config_file}" unless File.file? @config_file
-          
-          process_config = YAML.load_file(config_file)
-          LyberCore::Log.debug("process_config: #{process_config.inspect}")
-
-          @prerequisite = process_config[@workflow_step]["prerequisite"]
-          LyberCore::Log.debug("@prerequisite: #{@prerequisite}")
-          
-          @batch_limit = process_config[@workflow_step]['batch_limit']  
-          LyberCore::Log.debug("@batch_limit: #{@batch_limit}")
-          
-          @error_limit = process_config[@workflow_step]['error_limit']
-          LyberCore::Log.debug("@error_limit: #{@error_limit}")
-          
-      end
-
-      # Explicitly specify a set of druids to be processed by the workflow step
-      def enqueue_druids(druid_array)
-        LyberCore::Log.debug("\nEnqueing an array of druids...")
-        @druids = druid_array
-        LyberCore::Log.debug("\n@druids = #{@druids}")
-      end
-      
-      def fully_qualified_prerequisite?
-        if(@prerequisite.class == Array)
-          fully_qualified = @prerequisite.all? {|p| p =~ /.+:.+:.+/ }
-        else
-          fully_qualified = (@prerequisite =~ /.+:.+:.+/)
-        end
-        fully_qualified
-      end
-
-      # Obtain the set of druids to be processed using a database query
-      # to obtain the repository objects that are awaiting this step
-      def enqueue_workstep_waiting()
-        begin
-          LyberCore::Log.debug("\nEnqueing workstep waiting...")
-          
-          prerequisites = Array(@prerequisite)
-          qualified = fully_qualified_prerequisite?
-          druid_lists = prerequisites.collect do |prerequisite|
-            object_list_xml = qualified ? 
-              DorService.get_objects_for_qualified_workstep(prerequisite, "#{workflow.repository}:#{workflow.workflow_id}:#{@workflow_step}") :
-              DorService.get_objects_for_workstep(workflow.repository, workflow.workflow_id, prerequisite, @workflow_step)
-            LyberCore::Log.debug("\nobject_list_xml = #{object_list_xml}")
-            druid_list = DlssService.get_all_druids_from_object_list(object_list_xml)
-            LyberCore::Log.debug("\n@druids = #{@druids}")
-            druid_list
-          end
-          @druids = druid_lists.inject(druid_lists[0]) { |collector, list| collector & list }
-          @druids = @druids[0..(self.batch_limit-1)]
-        rescue Exception => e
-          raise e
-        end
-      end
-
-      # Use an alternative set of identifiers as the basis of this queue
-      # e.g. use array of barcodes as basis for google register-object robot
-      def enqueue_identifiers(identifier_name, identifier_values)
-        @identifier_name = identifier_name
-        @identifier_values = identifier_values
-      end
-
-      # Get the next WorkItem to be processed by the robot for the workflow step
-      def next_item()
-        if (@item_count >= @batch_limit )
-          LyberCore::Log.info "Batch limit of #{@batch_limit} items reached"
-          return nil
-        end
-        if (@error_count >= @error_limit )
-          LyberCore::Log.info "Error limit of #{@error_limit} items reached"
-          return nil
-        end
-        work_item =  LyberCore::Robots::WorkItem.new(self)
-        if (@druids)
-          return nil if (@item_count >= @druids.length)
-          work_item.druid= @druids[@item_count]
-        elsif (@identifier_values)
-          return nil if (@item_count >= @identifier_values.length)
-          work_item.identifier_add(@identifier_name,@identifier_values[@item_count])
-        else
-          return nil
-        end
-        @item_count += 1
-        return  work_item
-      end
-      
-      def max_errors_reached?
-        @error_count >= @error_limit
-      end
-
-      # Output the batch's timings and other statistics to the main log file
-      def print_stats
-        @end_time = Time.new
-        @elapsed_time = @end_time - @start_time
-        LyberCore::Log.info "Total time: " + @elapsed_time.to_s + "\n"
-        LyberCore::Log.info "Completed objects: " + self.success_count.to_s + "\n"
-        LyberCore::Log.info "Errors: " + self.error_count.to_s + "\n"    
-      end
-      
-      def print_empty_stats
-        @end_time = Time.new
-        @elapsed_time = @end_time - @start_time
-        LyberCore::Log.info "Total time: " + @elapsed_time.to_s + "\n"
-        LyberCore::Log.info "Empty queue"   
-      end
-    end
-  end
-end