lyber-core 0.9.6.2.3 → 1.3.0

data/README.rdoc

@@ -38,10 +38,28 @@ The BagitBag class requires the bagit gem
   http://github.com/flazz/bagit
 
 == Build and release procedure
+Modify the version number in lyber-core.gemspec, then push your commits to AFS.  DO NOT TAG!
 Run: 'rake dlss_release' to tag, build, and publish the lyber-core gem
 See the Rakefile and the LyberCore::DlssRelease task in lib/lyber_core/rake/dlss_release.rb for more details
 
 == Releases
+- <b>1.3</b> Started to use Dor::Config for workspace configuration
+- <b>1.2.1</b> Clean up logging of exceptions in LyberCore::Log
+- <b>1.2</b> Robots can now run as daemons via the LyberCore::Robots::ServiceController
+- <b>1.1.2</b> Can pass an array of "command line" arguments to the Robot constructor
+- <b>1.1.1</b> Robot#start now returns LyberCore::Robots::CONTINUE if it did work without error, LyberCore::Robots::SLEEP if it did no work,
+  and LyberCore::Robots::HALT if it reached its error limit while working on its queue
+- <b>1.1.0</b> Allow Robots::WorkQueue to resolve an arbitrary number of prerequisites
+- <b>1.0.0</b> Factored all Dor::* classes and object models out of lyber-core and into a separate dor-services gem. WARNING: MAY BREAK COMPATIBILITY WITH PREVIOUS DOR-ENABLED CODE.
+- <b>0.9.8</b> Created branch for legacy work "0.9-legacy".  Robots can now be configured with fully qualified workflows for prerequisites
+  eg <i>dor:googleScannedBookWF:register-object</i> 
+- <b>0.9.7.4</b> Untangled a couple development dependencies; fixed issue where "include REXML" was polluting the Object namespace
+- <b>0.9.7.3</b> Logging enhancements
+- <b>0.9.7.2</b> IdentityMetadata bugfixes
+- <b>0.9.7.1</b> Enhanced exception handling 
+- <b>0.9.7</b> ActiveMQ message-based robot parallelization as described here: https://consul.stanford.edu/x/tQjdBw . Removal of ROXML models. 
+- <b>0.9.6.3</b> Better error reporting for LyberCore::Utils::FileUtilities.execute, which means when a system command fails we have a better idea of why. 
+- <b>0.9.6.2</b> Handles new response from workflow service when there are no objects in the queue: <objects count="0"\>
 - <b>0.9.6</b> DorService.get_objects_for_workstep can handle one or two completed steps.  Trimmed-down gem dependencies now defined in lyber-core.gemspec. 'rake dlss_release' will tag, build and publish gem
 - <b>0.9.5.5</b> Robots now log to ROBOT_ROOT/log/robot_name.log unless specified in constructor
 - <b>0.9.5.4</b> Custom exception classes, more checking of error conditions

data/lib/dlss_service.rb

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'net/http'
 require 'net/https'
 require 'uri'

data/lib/dor_service.rb

@@ -4,10 +4,10 @@ require 'uri'
 require 'cgi'
 require 'rexml/document'
 
-include REXML
-
 class DorService
        
+  include REXML
+
     def DorService.get_https_connection(url)
       https = Net::HTTP.new(url.host, url.port)
       if(url.scheme == 'https')
@@ -31,7 +31,7 @@ class DorService
       druid = $1
       return druid
     rescue Exception => e
-      LyberCore::Log.error("Unable to create object #{e.backtrace}")
+      LyberCore::Log.debug("Unable to create object #{e.backtrace}")
       raise e
     end  
   end
@@ -55,7 +55,7 @@ class DorService
       LyberCore::Log.debug("new druid = #{druid}")
       return druid
     rescue Exception => e
-      LyberCore::Log.error("Unable to create object")
+      LyberCore::Log.debug("Unable to create object")
       raise e, "Unable to create object "
     end
   end
@@ -111,7 +111,8 @@ class DorService
     LyberCore::Log.debug("Fetching druid for dor_id #{dor_id} at url #{url_string}")
     url = URI.parse(url_string)
     req = Net::HTTP::Get.new(url.request_uri)
-    res = DorService.get_https_connection(url).start {|http| http.request(req) }
+    # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+    res = LyberCore::Connection.send_request(url,req)
       
     case res
       when Net::HTTPSuccess
@@ -121,10 +122,10 @@ class DorService
         LyberCore::Log.debug("Barcode does not yet exist in DOR: #{dor_id}") 
         return nil
       when Net::HTTPServerError
-        LyberCore::Log.error("Encountered HTTPServerError error when requesting #{url}: #{res.inspect}") 
+        LyberCore::Log.debug("Encountered HTTPServerError error when requesting #{url}: #{res.inspect}")
         raise "Encountered 500 error when requesting #{url}: #{res.inspect}"
       else
-        LyberCore::Log.error("Encountered unknown error when requesting #{url}: #{res.inspect}") 
+        LyberCore::Log.debug("Encountered unknown error when requesting #{url}: #{res.inspect}")
         raise "Encountered unknown error when requesting #{url}: #{res.inspect}"
       end
   end
@@ -132,7 +133,7 @@ class DorService
   #############################################  Start of Datastream methods
   # Until ActiveFedora supports client-side certificate configuration, we are stuck with our own methods to access datastreams
   
-  #/objects/{pid}/datastreams/{dsID} ? [controlGroup] [dsLocation] [altIDs] [dsLabel] [versionable] [dsState] [formatURI] [checksumType] [checksum] [logMessage]
+  #/objects/pid/datastreams/dsID ? [controlGroup] [dsLocation] [altIDs] [dsLabel] [versionable] [dsState] [formatURI] [checksumType] [checksum] [logMessage]
   def DorService.add_datastream(druid, ds_id, ds_label, xml, content_type='application/xml', versionable = false )
     DorService.add_datastream_managed(druid, ds_id, ds_label, xml, content_type, versionable)
   end
@@ -168,8 +169,9 @@ class DorService
       LyberCore::Log.debug("Connecting to #{url_string}...")
       req = Net::HTTP::Get.new(url.request_uri)
       LyberCore::Log.debug("request object: #{req.inspect}")
-      res = DorService.get_https_connection(url).start {|http| http.request(req) }  
-      
+      # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+      res = LyberCore::Connection.send_request(url,req)
+
       case res
         when Net::HTTPSuccess
           return res.body
@@ -177,10 +179,10 @@ class DorService
           LyberCore::Log.debug("Datastream not found at url #{url_string}") 
           return nil
         when Net::HTTPServerError
-          LyberCore::Log.error("Attempted to reach #{url_string} but failed")
+          LyberCore::Log.debug("Attempted to reach #{url_string} but failed")
           raise "Encountered 500 error when requesting #{url_string}: #{res.inspect}"
         else
-          LyberCore::Log.error("Encountered unknown error when requesting #{url}: #{res.inspect}")
+          LyberCore::Log.debug("Encountered unknown error when requesting #{url}: #{res.inspect}")
           raise "Encountered unknown error when requesting #{url}: #{res.inspect}"
         end
      rescue Exception => e
@@ -203,7 +205,8 @@ class DorService
       req = Net::HTTP::Get.new(url.request_uri)
       req.basic_auth FEDORA_USER, FEDORA_PASS
       LyberCore::Log.debug("request object: #{req.inspect}")
-      res = DorService.get_https_connection(url).start {|http| http.request(req) }  
+      # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+      res = LyberCore::Connection.send_request(url,req)
       case res
         when Net::HTTPSuccess
           return res.body
@@ -276,56 +279,69 @@ class DorService
   #     </objects>
   def DorService.get_objects_for_workstep(repository, workflow, completed, waiting)
     LyberCore::Log.debug("DorService.get_objects_for_workstep(#{repository}, #{workflow}, #{completed}, #{waiting})")
-    begin  
-      if repository.nil? or workflow.nil? or completed.nil? or waiting.nil?
-        LyberCore::Log.fatal("Can't execute DorService.get_objects_for_workstep: missing info")
-      end
-      
-      unless defined?(WORKFLOW_URI) and WORKFLOW_URI != nil
-        LyberCore::Log.fatal("WORKFLOW_URI is not set. ROBOT_ROOT = #{ROBOT_ROOT}")
-        raise "WORKFLOW_URI is not set"   
-      end
-      
-      uri_string = "#{WORKFLOW_URI}/workflow_queue?repository=#{repository}&workflow=#{workflow}&waiting=#{waiting}"
-      if(completed.class == Array)
-        raise "The workflow service can only handle queries with no more than 2 completed steps" if completed.size > 2
-        completed.each {|step| uri_string << "&completed=#{step}"}
-      else
-        uri_string << "&completed=#{completed}"
-      end
-      LyberCore::Log.info("Attempting to connect to #{uri_string}")
-      url = URI.parse(uri_string)
-      req = Net::HTTP::Get.new(url.request_uri)
-      res = DorService.get_https_connection(url).start {|http| http.request(req) }  
-      case res
-        when Net::HTTPSuccess
-          begin
-            doc = Nokogiri::XML(res.body)
-            count = doc.root.at_xpath("//objects/@count").content.to_i
-          rescue Exception => e
-            msg = "Could not parse response from Workflow Service"
-            LyberCore::Log.error(msg + "\n#{res.body}")
-            raise e, msg
-          end
-          
-          if(count == 0)
-            raise LyberCore::Exceptions::EmptyQueue.new, "empty queue" 
-          else
-            return res.body
-          end
-        else
-          LyberCore::Log.fatal("Workflow queue not found for #{workflow} : #{waiting}")
-          LyberCore::Log.debug("I am attempting to connect to WORKFLOW_URI #{WORKFLOW_URI}")
-          LyberCore::Log.debug("repository: #{repository}")
-          LyberCore::Log.debug("workflow: #{workflow}")
-          LyberCore::Log.debug("completed: #{completed}")
-          LyberCore::Log.debug("waiting: #{waiting}")
-          LyberCore::Log.debug(res.inspect)
-          raise "Could not connect to url #{uri_string}"
-       end
-    end 
+ 
+    if repository.nil? or workflow.nil? or completed.nil? or waiting.nil?
+      LyberCore::Log.fatal("Can't execute DorService.get_objects_for_workstep: missing info")
+    end
+    
+    unless defined?(WORKFLOW_URI) and WORKFLOW_URI != nil
+      LyberCore::Log.fatal("WORKFLOW_URI is not set. ROBOT_ROOT = #{ROBOT_ROOT}")
+      raise "WORKFLOW_URI is not set"   
+    end
+    
+    uri_string = "#{WORKFLOW_URI}/workflow_queue?repository=#{repository}&workflow=#{workflow}&waiting=#{waiting}"
+    if(completed.class == Array)
+      raise "The workflow service can only handle queries with no more than 2 completed steps" if completed.size > 2
+      completed.each {|step| uri_string << "&completed=#{step}"}
+    else
+      uri_string << "&completed=#{completed}"
+    end
+    
+    return DorService.execute_workflow_xml_query(uri_string)
   end
   
+  # Returns string containing object list XML from a workflow DOR query using fully qualified workflow step names
+  # eg <tt>dor:googleScannedBookWF:register-object</tt>
+  # 
+  # @param [String, Array] completed if only querying for one completed step, pass in a String of a fully qualified workflow step.
+  #   If querying for two completed steps, pass in an Array of the two completed steps
+  # @param [String] waiting the fully qualified name of the waiting step
+  # @raise [LyberCore::Exceptions::EmptyQueue] When the query is successful, but no objects are found in that queue 
+  # @raise [Exception] For other problems like connection failures or passing in non-qualified workflow names
+  # @return [String] XML containing all the objects that match the specific query. It looks like:
+  #     <objects>
+  #       <object druid="dr:123" url="http://localhost:9999/jersey-spring/objects/dr:123%5c" />
+  #       <object druid="dr:abc" url="http://localhost:9999/jersey-spring/objects/dr:abc%5c" />
+  #     </objects>
+  def DorService.get_objects_for_qualified_workstep(completed, waiting)
+    LyberCore::Log.debug("DorService.get_objects_for_qualified_workstep(#{completed}, #{waiting})")
+
+    if completed.nil? or waiting.nil?
+      LyberCore::Log.fatal("Can't execute DorService.get_objects_for_qualified_workstep: missing info")
+    end
+    
+    unless defined?(WORKFLOW_URI) and WORKFLOW_URI != nil
+      LyberCore::Log.fatal("WORKFLOW_URI is not set. ROBOT_ROOT = #{ROBOT_ROOT}")
+      raise "WORKFLOW_URI is not set"   
+    end
+    
+    unless(waiting =~ /.+:.+:.+/)
+      raise "The waiting step was not fully qualified or of the form: <repository>:<workflow>:<stepname>. Received #{waiting}"
+    end
+    uri_string = "#{WORKFLOW_URI}/workflow_queue?waiting=#{waiting}"
+    
+    completed_steps = Array(completed)
+    raise "The workflow service can only handle queries with no more than 2 completed steps" if completed_steps.size > 2
+    completed_steps.each do |step|
+      raise "A completed step was not fully qualified or of the form: <repository>:<workflow>:<stepname>. Received #{step}" unless(step =~ /.+:.+:.+/)
+      uri_string << "&completed=#{step}"
+    end
+
+    return DorService.execute_workflow_xml_query(uri_string)
+  end
+  
+  
+  
   def DorService.log_and_raise_workflow_connection_problem(repository, workflow, completed, waiting, response)
     
   end
@@ -400,18 +416,19 @@ class DorService
       req.body = DorService.construct_error_update_request(process, error_msg, error_txt)
       req.content_type = 'application/xml'
       LyberCore::Log::debug("Putting request: #{req.inspect}")
-      res = DorService.get_https_connection(url).start {|http| http.request(req) }
+      # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+      res = LyberCore::Connection.send_request(url,req)
       LyberCore::Log::debug("Got response: #{res.inspect}")
       case res
        when Net::HTTPSuccess
-         LyberCore::Log.error("#{workflow} - #{process} set to error for " + druid)
+         LyberCore::Log.info("#{workflow} - #{process} set to error for " + druid)
        else
-         LyberCore::Log.error(res.body)
+         LyberCore::Log.debug(res.body)
          raise res.error!, "Received error from the workflow service"
       end
     rescue Exception => e
       msg = "Unable to update workflow service at url #{url_string}"
-      LyberCore::Log.error(msg)
+      LyberCore::Log.debug(msg)
       raise e, msg
     end
   end
@@ -434,7 +451,7 @@ class DorService
             LyberCore::Log.debug("Successfully queried symphony for #{flexkey}")
             return res.body          
           else
-            LyberCore::Log.error("Encountered an error from symphony: #{res.body}")
+            LyberCore::Log.debug("Encountered an error from symphony: #{res.body}")
             raise res.error!
         end
       rescue Exception => e
@@ -445,6 +462,40 @@ class DorService
 
 
   private
+  
+  def DorService.execute_workflow_xml_query(uri_string)
+    LyberCore::Log.info("Attempting to connect to #{uri_string}")
+    url = URI.parse(uri_string)
+    req = Net::HTTP::Get.new(url.request_uri)
+    # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+    res = LyberCore::Connection.send_request(url,req)
+    case res
+      when Net::HTTPSuccess
+        begin
+          doc = Nokogiri::XML(res.body)
+          count = doc.root.at_xpath("//objects/@count").content.to_i
+        rescue Exception => e
+          msg = "Could not parse response from Workflow Service"
+          LyberCore::Log.debug(msg + "\n#{res.body}")
+          raise e, msg
+        end
+        
+        if(count == 0)
+          raise LyberCore::Exceptions::EmptyQueue.new, "empty queue" 
+        else
+          return res.body
+        end
+      else
+        LyberCore::Log.fatal("Workflow queue not found for #{workflow} : #{waiting}")
+        LyberCore::Log.debug("I am attempting to connect to WORKFLOW_URI #{WORKFLOW_URI}")
+        LyberCore::Log.debug("repository: #{repository}")
+        LyberCore::Log.debug("workflow: #{workflow}")
+        LyberCore::Log.debug("completed: #{completed}")
+        LyberCore::Log.debug("waiting: #{waiting}")
+        LyberCore::Log.debug(res.inspect)
+        raise "Could not connect to url #{uri_string}"
+     end
+  end
   # druid, ds, url, content_type, method, parms
   def DorService.set_datastream(druid, ds_id, parms, method, content = {})
     begin  
@@ -458,15 +509,16 @@ class DorService
       req.basic_auth FEDORA_USER, FEDORA_PASS
       req.body = content[:xml] if(content[:xml])
       req.content_type = content[:type]
-      res = DorService.get_https_connection(url).start {|http| http.request(req) }
+      # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+      res = LyberCore::Connection.send_request(url,req)
       case res
         when Net::HTTPSuccess
           return true
         when Net::HTTPServerError
-          LyberCore::Log.error("Attempted to set datastream #{url} but failed")
+          LyberCore::Log.debug("Attempted to set datastream #{url} but failed")
           raise "Encountered 500 error setting datastream #{url}: #{res.inspect}"
         else
-          LyberCore::Log.error("Encountered unknown error when setting datastream #{url}: #{res.inspect}")
+          LyberCore::Log.debug("Encountered unknown error when setting datastream #{url}: #{res.inspect}")
           raise "Encountered unknown error when setting datastream #{url}: #{res.inspect}"
       end
     rescue Exception => e
@@ -506,15 +558,16 @@ end
      req = Net::HTTP::Put.new(url.path)
      req.body = DorService.construct_xml_for_tag_array(tags)
      req.content_type = 'application/xml'
-     res = DorService.get_https_connection(url).start {|http| http.request(req) }
+     # res = DorService.get_https_connection(url).start {|http| http.request(req) }
+     res = LyberCore::Connection.send_request(url,req)
      case res
        when Net::HTTPSuccess
          return true
        when Net::HTTPServerError
-         LyberCore::Log.error("Attempted to add identity tags #{url} but failed")
+         LyberCore::Log.debug("Attempted to add identity tags #{url} but failed")
          raise "Encountered 500 error when adding identity tags #{url}: #{res.inspect}"
        else
-         LyberCore::Log.error("Encountered unknown error when adding identity tags #{url}: #{res.inspect}")
+         LyberCore::Log.debug("Encountered unknown error when adding identity tags #{url}: #{res.inspect}")
          raise "Encountered unknown error when adding identity tags #{url}: #{res.inspect}"
      end
    rescue Exception => e

data/lib/lyber_core.rb

@@ -1,14 +1,17 @@
+require 'lyber_core/config'
 require 'dlss_service'
-require 'dor/suri_service'
-require 'dor/workflow_service'
-require 'dor/base'
+require 'dor-services'
 require 'lyber_core/connection'
 require 'lyber_core/destroyer'
 require 'lyber_core/log'
 require 'lyber_core/robots/robot'
+require 'lyber_core/robots/service_controller'
 require 'lyber_core/robots/workflow'
 require 'lyber_core/robots/workspace'
 require 'lyber_core/robots/work_queue'
 require 'lyber_core/robots/work_item'
 require 'lyber_core/exceptions/empty_queue'
+require 'lyber_core/exceptions/fatal_error'
+require 'lyber_core/exceptions/service_error'
+require 'lyber_core/exceptions/item_error'
 

data/lib/lyber_core/config.rb

@@ -0,0 +1,13 @@
+require 'dor-services'
+
+module Dor
+  
+  Config.declare do
+    
+    robots do
+      workspace nil
+    end
+    
+  end
+  
+end

data/lib/lyber_core/connection.rb

@@ -2,6 +2,27 @@ require 'net/https'
 require 'uri'
 require 'cgi'
 
+# Extend the Integer class to facilitate retries of code blocks if specified exception(s) occur
+# see: http://blog.josh-nesbitt.net/2010/02/08/writing-contingent-ruby-code-with-retryable/
+RETRYABLE_SLEEP_VALUE = 300
+class Integer
+  def tries(options={}, &block)
+    attempts          = self
+    exception_classes = [*options[:on] || StandardError]
+    begin
+      # First attempt
+      return yield
+    rescue *exception_classes
+      sleep RETRYABLE_SLEEP_VALUE
+      # 2nd to n-1 attempts
+      retry if (attempts -= 1) > 1
+    end
+    # final (nth) attempt
+    yield
+  end
+end
+
+
 module LyberCore
   class Connection
     def Connection.get_https_connection(url)
@@ -62,7 +83,7 @@ module LyberCore
         req.basic_auth options[:auth_user], options[:auth_password]
       end
 
-      res = Connection.get_https_connection(url).start {|http| http.request(req) }
+      res = Connection.send_request(url, req)
       case res
       when Net::HTTPSuccess
         if(block_given?)
@@ -72,9 +93,21 @@ module LyberCore
         end
       else
         raise res.error!
+        # ??? raise LyberCore::Exceptions::ServiceError.new('HTTP Request failed',res.error!)
       end
 
     end
+
+
+    # Send the request to the server, with multiple retries if specified exceptions occur
+    def Connection.send_request(url, req)
+      3.tries :on => [Timeout::Error, EOFError, Errno::ECONNRESET] do
+        Connection.get_https_connection(url).start {|http| http.request(req) }
+      end
+    rescue Exception => e
+      raise LyberCore::Exceptions::ServiceError.new('HTTP Request failed',e)
+    end
+
   end