lyber-core 0.9.6.2.3 → 1.3.0

data/lib/lyber_core/robots/service_controller.rb

@@ -0,0 +1,174 @@
+require 'daemons'
+require 'logger'
+require 'fileutils'
+
+module LyberCore
+  module Robots
+    class ServiceController < Daemons::ApplicationGroup
+      attr_reader :logger
+      
+      def initialize(opts = {})
+        if opts[:logger]
+          @logger = opts[:logger]
+        else
+          @logger = Logger.new($stdout)
+          @logger.level = opts[:log_level] || Logger::WARN
+        end
+        @sleep_time = opts[:sleep_time] || (15*60)
+        @working_dir = opts[:working_dir] || ENV['ROBOT_ROOT'] || Dir.pwd
+        @pid_dir = opts[:pid_dir] || File.join(@working_dir, 'pid')
+        @pid_dir = File.expand_path(@pid_dir)
+        FileUtils.mkdir(@pid_dir) unless(File.directory? @pid_dir)
+        @argv = (opts[:argv] || []).dup
+        @logger.debug "Initializing application group."
+        @logger.debug "Writing pids to #{@pid_dir}"
+        super('robot_service_controller', :dir_mode => :normal, :dir => @pid_dir, :multiple => true, :backtrace => true)
+      end
+
+      def qname(workflow, robot_name)
+        [workflow,robot_name].join(':')
+      end
+      
+      def start(workflow, robot_name)
+        result = false
+        app = find_app(workflow, robot_name).first
+        process_name = qname(workflow,robot_name)
+        if app.nil? or (app.running? == false)
+          @logger.info "Starting #{process_name}..."
+          with_app_name("#{process_name}") do
+            app, message = capture_stdout do
+              raw_module_name = workflow.split('WF').first
+              module_name = raw_module_name[0].chr.upcase << raw_module_name.slice(1, raw_module_name.size - 1)
+              robot_klass = Module.const_get(module_name).const_get(robot_name.split(/-/).collect { |w| w.capitalize }.join(''))
+              log_state = marshal_logger(@logger)
+              robot_proc = lambda {
+                Dir.chdir(@working_dir) do
+                  begin
+                    logger = restore_logger(log_state)
+                    robot = robot_klass.new(:argv => @argv.dup)
+                    loop { 
+                      case robot.start 
+                      when LyberCore::Robots::SLEEP
+                        logger.info "SLEEP condition reached in #{process_name}. Sleeping for #{@sleep_time} seconds."
+                        sleep(@sleep_time)
+                      when LyberCore::Robots::HALT
+                        logger.error "HALT condition reached in #{process_name}. Shutting down."
+                        break
+                      end
+                    }
+                  ensure
+                    logger.info "Shutting down."
+                  end
+                end
+              }
+              new_app = self.new_application({:mode => :proc, :proc => robot_proc, :dir_mode => :normal, :log_output => true, :log_dir => @pid_dir})
+              new_app.start
+              new_app
+            end
+          end
+          
+          if app.running?
+            @logger.info "#{process_name} [#{app.pid.pid}] started."
+            result = true
+          else
+            @logger.error "Unable to start #{process_name}"
+          end
+        else app.running?
+          @logger.warn "Robot #{process_name} [#{app.pid.pid}] is already running"
+        end
+        return result
+      end
+
+      def stop(workflow, robot_name)
+        apps = find_app(workflow, robot_name)
+        process_name = qname(workflow,robot_name)
+        result = false
+        if apps.empty?
+          @logger.info "Robot #{process_name} not found"
+        else
+          apps.each do |app|
+            if app.running?
+              @logger.info "Shutting down #{process_name} [#{app.pid.pid}]..."
+              result, message = capture_stdout { app.stop }
+              if app.running?
+                @logger.error "Unable to stop #{process_name} [#{app.pid.pid}]."
+              else
+                @logger.info "#{process_name} [#{app.pid.pid}] shut down."
+                result = true
+              end
+            else
+              @logger.warn "Robot #{process_name} [#{app.pid.pid}] is not running but pidfile exists"
+              app.zap!
+            end
+          end
+        end
+        result
+      end
+  
+      def status(workflow, robot_name)
+        apps = find_app(workflow, robot_name)
+        apps.collect do |app|
+          { :pid => app.pid.pid, :status => app.running? ? :running : :stopped }
+        end
+      end
+  
+      def status_message(workflow, robot_name)
+        app_status = status(workflow, robot_name)
+        process_name = qname(workflow,robot_name)
+        if app_status.empty?
+          ["Robot #{process_name} not found"]
+        else
+          app_status.collect do |s|
+            case s[:status]
+            when :running
+              "Robot #{process_name} [#{s[:pid]}] is running"
+            when :stopped
+              "Robot #{process_name} [#{s[:pid]}] is not running but pidfile exists"
+            end
+          end
+        end
+      end
+  
+#      private
+      def with_app_name(name)
+        old_name, @app_name = @app_name, name
+        begin
+          return yield
+        ensure
+          @app_name = old_name
+        end
+      end
+      
+      def capture_stdout
+        old_io = $stdout
+        begin
+          new_io = StringIO.new('')
+          $stdout = new_io
+          result = yield
+          @logger.debug new_io.string
+          return result, new_io.string
+        ensure
+          $stdout = old_io
+        end
+      end
+  
+      def find_app(workflow, robot_name)
+        with_app_name(qname(workflow,robot_name)) {
+          self.find_applications_by_pidfiles(@pid_dir)
+        }
+      end
+      
+      def marshal_logger(l)
+        log_device = l.instance_variable_get('@logdev')
+        { :dev => log_device, :file => log_device.filename, :level => l.level }
+      end
+      
+      def restore_logger(params)
+        result = Logger.new(params[:file] || params[:log_device])
+        result.level = params[:level]
+        return result
+      end
+  
+    end
+  end
+end

data/lib/lyber_core/robots/work_item.rb

@@ -1,6 +1,7 @@
 require 'dor_service'
-require "roxml_models/identity_metadata/identity_metadata"
-require "roxml_models/identity_metadata/dublin_core"
+require "xml_models/identity_metadata/identity_metadata"
+require "xml_models/identity_metadata/dublin_core"
+
 
 # Represents a single object being processed as part of a workflow queue
 module LyberCore
@@ -32,9 +33,9 @@ module LyberCore
       #save the IdentityMetadata object to identityMetadata datastream
       def identity_metadata_save
         unless DorService.get_datastream(@druid, 'identityMetadata')
-          DorService.add_datastream(@druid, 'identityMetadata', 'identityMetadata', self.identity_metadata.to_xml.to_xml)
+          DorService.add_datastream(@druid, 'identityMetadata', 'identityMetadata', self.identity_metadata.to_xml)
         else
-          DorService.update_datastream(@druid, 'identityMetadata', self.identity_metadata.to_xml.to_xml, content_type='application/xml', versionable = false)
+          DorService.update_datastream(@druid, 'identityMetadata', self.identity_metadata.to_xml, content_type='application/xml', versionable = false)
         end #unless
       end #identity_metadata_save
 
@@ -73,31 +74,39 @@ module LyberCore
         return pairs[0] if (pairs.size > 0)
       end
 
-      # Record the successful outcome of the workstep operation for this workitem
-      def set_success
-        @work_queue.success_count += 1
+      # Record a non-error status for the workstep operation
+      def set_status(status)
+        @elapsed_time = Time.new - @start_time
         @end_time = Time.new
         @elapsed_time = @end_time - @start_time
-        LyberCore::Log.info("#{item_id} completed in #{@elapsed_time} seconds")
+        LyberCore::Log.info("#{item_id} #{status} in #{@elapsed_time} seconds")
         if (@druid)
-          Dor::WorkflowService.update_workflow_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, 'completed', @elapsed_time)
+          Dor::WorkflowService.update_workflow_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, status, @elapsed_time)
         end
       end
 
+      # Record the successful outcome of the workstep operation for this workitem
+      def set_success
+        @work_queue.success_count += 1
+        self.set_status('completed')
+      end
+
       # Record the unsuccessful outcome of the workstep operation for this workitem
       def set_error(e)
         @work_queue.error_count += 1
         @end_time = Time.new
         @elapsed_time = @end_time - @start_time
-        LyberCore::Log.error("#{item_id} error - #{e.backtrace}")
-        # By default puts will output an array with a newline between each item.
+        if (e.is_a?(LyberCore::Exceptions::ItemError) )
+          item_error = e
+        else
+          item_error = LyberCore::Exceptions::ItemError.new(@druid, "Item error", e)
+        end
+        LyberCore::Log.exception(item_error)
         if (@druid)
-          DorService.update_workflow_error_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, e.message)
+          DorService.update_workflow_error_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, item_error.message)
         end
-        # We've caught and processed the error at this point, I don't think we want to raise it again. --bess
-        # raise e
       end
-
+      
     end
   end
 end

data/lib/lyber_core/robots/work_queue.rb

@@ -88,16 +88,35 @@ module LyberCore
         @druids = druid_array
         LyberCore::Log.debug("\n@druids = #{@druids}")
       end
+      
+      def fully_qualified_prerequisite?
+        if(@prerequisite.class == Array)
+          fully_qualified = @prerequisite.all? {|p| p =~ /.+:.+:.+/ }
+        else
+          fully_qualified = (@prerequisite =~ /.+:.+:.+/)
+        end
+        fully_qualified
+      end
 
       # Obtain the set of druids to be processed using a database query
       # to obtain the repository objects that are awaiting this step
       def enqueue_workstep_waiting()
         begin
           LyberCore::Log.debug("\nEnqueing workstep waiting...")
-          object_list_xml = DorService.get_objects_for_workstep(workflow.repository, workflow.workflow_id, @prerequisite, @workflow_step)
-          LyberCore::Log.debug("\nobject_list_xml = #{object_list_xml}")
-          @druids = DlssService.get_some_druids_from_object_list(object_list_xml,self.batch_limit)
-          LyberCore::Log.debug("\n@druids = #{@druids}")
+          
+          prerequisites = Array(@prerequisite)
+          qualified = fully_qualified_prerequisite?
+          druid_lists = prerequisites.collect do |prerequisite|
+            object_list_xml = qualified ? 
+              DorService.get_objects_for_qualified_workstep(prerequisite, "#{workflow.repository}:#{workflow.workflow_id}:#{@workflow_step}") :
+              DorService.get_objects_for_workstep(workflow.repository, workflow.workflow_id, prerequisite, @workflow_step)
+            LyberCore::Log.debug("\nobject_list_xml = #{object_list_xml}")
+            druid_list = DlssService.get_all_druids_from_object_list(object_list_xml)
+            LyberCore::Log.debug("\n@druids = #{@druids}")
+            druid_list
+          end
+          @druids = druid_lists.inject(druid_lists[0]) { |collector, list| collector & list }
+          @druids = @druids[0..(self.batch_limit-1)]
         rescue Exception => e
           raise e
         end
@@ -133,6 +152,10 @@ module LyberCore
         @item_count += 1
         return  work_item
       end
+      
+      def max_errors_reached?
+        @error_count >= @error_limit
+      end
 
       # Output the batch's timings and other statistics to the main log file
       def print_stats

data/lib/lyber_core/robots/workspace.rb

@@ -2,7 +2,7 @@ require 'fileutils'
 module LyberCore
   module Robots
     class Workspace
-
+      
       attr_reader :workflow_name
       attr_reader :collection_name
       attr_reader :workspace_base
@@ -20,7 +20,7 @@ module LyberCore
       def set_workspace_home
         begin
           if not (workspace_home = ENV['WORKSPACE_HOME'] )
-            workspace_home = WORKSPACE_HOME
+            workspace_home = Dor::Config.robots.workspace
           end
         rescue NameError => e
           LyberCore::Log.fatal("WORKSPACE_HOME is undefined. Do you need to set it in your config file?")

data/lib/lyber_core/utils/checksum_validate.rb

@@ -45,7 +45,7 @@ module LyberCore
       def self.md5_hash_from_content_metadata(content_md)
         content_md_checksum_hash = {}
         doc = Nokogiri::XML(content_md)
-        doc.xpath('/contentMetadata/resource/file').each do |filenode|
+        doc.xpath('/contentMetadata/resource[@type="page"]/file').each do |filenode|
            filename = filenode.attribute('id')
            if (filename)
              md5_element = filenode.xpath('checksum[@type="MD5"]').first

data/lib/lyber_core/utils/file_utilities.rb

@@ -3,21 +3,18 @@ require 'systemu'
 
 # File Utilities for use in transferring filesystem objects,
 # decrypting a file, unpacking a targz archive, and validating checksums
-# Author:: rnanders@stanford.edu
+# @author rnanders@stanford.edu
 module LyberCore
   module Utils
     class FileUtilities
 
 
-      # Executes a system command in a subprocess
-      #
-      # = Inputs:
-      # * command = the command to be executed
-      #
-      # = Return value:
-      # * The method will return stdout from the command if execution was successful.
-      # * The method will raise an exception if if execution fails
+      # Executes a system command in a subprocess. 
+      # The method will return stdout from the command if execution was successful.
+      # The method will raise an exception if if execution fails. 
       # The exception's message will contain the explaination of the failure.
+      # @param [String] command the command to be executed
+      # @return [String] stdout from the command if execution was successful
       def FileUtilities.execute(command)
         status, stdout, stderr = systemu(command)
         if (status.exitstatus != 0)
@@ -25,7 +22,10 @@ module LyberCore
         end
         return stdout
       rescue
-        raise "Command failed to execute: #{command}"
+        msg = "Command failed to execute: [#{command}] caused by <STDERR = #{stderr.split($/).join('; ')}>"
+        msg << " STDOUT = #{stdout.split($/).join('; ')}" if (stdout && (stdout.length > 0))
+        raise msg
+
       end
 
       # Generates a dirname for storing or retrieving a file in

data/lib/xml_models/identity_metadata/dublin_core.rb

@@ -0,0 +1,116 @@
+require 'nokogiri'
+
+#<oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+#xmlns:srw_dc="info:srw/schema/1/dc-schema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+#xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://cosimo.stanford.edu/standards/oai_dc/v2/oai_dc.xsd">
+#  <dc:title>Life of Abraham Lincoln, sixteenth president of the United States: Containing his early
+#history and political career; together with the speeches, messages, proclamations and other official
+#documents illus. of his eventful administration</dc:title>
+#  <dc:creator>Crosby, Frank.</dc:creator>
+#  <dc:format>text</dc:format>
+#  <dc:language>eng</dc:language>
+#  <dc:subject>E457 .C94</dc:subject>
+#  <dc:identifier>lccn:11030686</dc:identifier>
+#  <dc:identifier>callseq:1</dc:identifier>
+#  <dc:identifier>shelfseq:973.7111 .L731CR</dc:identifier>
+#  <dc:identifier>catkey:1206382</dc:identifier>
+#  <dc:identifier>barcode:36105005459602</dc:identifier>
+#  <dc:identifier>uuid:ddcf5f1a-0331-4345-beca-e66f7db276eb</dc:identifier>
+#  <dc:identifier>google:STANFORD_36105005459602</dc:identifier>
+#  <dc:identifier>druid:ng786kn0371</dc:identifier>
+#</oai_dc:dc>
+
+class DublinCore 
+  
+  
+  attr_accessor :xml
+  
+  attr_accessor :title
+  attr_accessor :creator
+  attr_accessor :subject
+  attr_accessor :description
+  attr_accessor :publisher
+  attr_accessor :contributor
+  attr_accessor :date
+  attr_accessor :type
+  attr_accessor :format
+  attr_accessor :identifier
+  attr_accessor :source
+  attr_accessor :language
+  attr_accessor :relation
+  attr_accessor :coverage
+  attr_accessor :rights
+  
+
+
+
+  def initialize(xml = nil)  
+   
+    @title ||= []  
+    @creator ||= []  
+    @subject ||= []  
+    @description ||= []  
+    @publisher ||= []  
+    @contributor ||= []  
+    @date ||= []  
+    @type ||= []  
+    @format ||= []  
+    @identifier ||= []  
+    @source ||= []  
+    @language ||= []  
+    @relation ||= []  
+    @coverage ||= []   
+    @rights ||= []
+    
+    # if the new is given an xml string, store that in the xml attr_accessor and don't rebuild.
+    # this will allow users to access the raw unprocessed  XML string via @xml. 
+    if xml.nil?
+      build_xml()
+    else
+      @xml = xml
+    end
+      
+  end #initalize
+  
+  
+  def build_xml()
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.dc('xmlns:dc' => 'http://purl.org/dc/elements/1.1/', 'xmlns:srw_dc' => 'info:srw/schema/1/dc-schema', "xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance"   ) {
+          xml.parent.namespace = xml.parent.add_namespace_definition('oai_dc','http://www.openarchives.org/OAI/2.0/oai_dc/')
+          xml.parent.add_namespace_definition("xsi:schemaLocation", "http://www.openarchives.org/OAI/2.0/oai_dc/ http://cosimo.stanford.edu/standards/oai_dc/v2/oai_dc.xsd")
+          self.instance_variables.each do |var|
+            unless var == "@xml"         
+              self.instance_variable_get(var).each { |v| xml['dc'].send("#{var.gsub('@','')}_", v) }
+            end #unless
+          end #instance_Variables.each
+        }
+      end
+      @xml = builder.to_xml
+  end
+  
+  
+  
+  # This method rebuilds the xml attr_accesor and returns it as a string.   
+  def to_xml
+    build_xml
+    return self.xml
+  end #to_xml
+
+
+  # This method takes DC XML as a string, and maps the root child node to their proper attr_accesor. 
+  
+  def self.from_xml(xml="")
+    dc = DublinCore.new(xml)
+    doc = Nokogiri::XML(xml)
+    children = doc.root.element_children
+    children.each do |c|
+      if dc.instance_variables.include?("@#{c.name}")
+        dc.send("#{c.name}").send("<<", c.text.strip)
+      end #if
+    end  #each
+    return dc
+  end #from_xml
+
+
+  
+end #dublin_core