lyber-core 0.9.6.2

data/lib/lyber_core/robots/work_item.rb

@@ -0,0 +1,103 @@
+require 'dor_service'
+require "roxml_models/identity_metadata/identity_metadata"
+require "roxml_models/identity_metadata/dublin_core"
+
+# Represents a single object being processed as part of a workflow queue
+module LyberCore
+  module Robots
+    class WorkItem
+
+      # The queue that this workitem is a member of
+      attr_reader :work_queue
+      # The primary id for the object being processed
+      attr_accessor :druid
+      # An object used to hold unmarshalled XML from the identityMetadata datastream
+      attr_accessor :identity_metadata
+      # Timings for this workitem's processing
+      attr_reader :start_time
+      attr_reader :end_time
+      attr_reader :elapsed_time
+
+      # Create a new WorkItem object, save a pointer to the parent WorkQueue, and start the timer
+      def initialize(work_queue)
+        @work_queue = work_queue
+        @start_time = Time.new
+      end
+
+      # Inject an IdentityMetadata object (currently used for unit testing only)
+      def identity_metadata=(identity_metadata)
+        @identity_metadata = identity_metadata
+      end
+    
+      #save the IdentityMetadata object to identityMetadata datastream
+      def identity_metadata_save
+        unless DorService.get_datastream(@druid, 'identityMetadata')
+          DorService.add_datastream(@druid, 'identityMetadata', 'identityMetadata', self.identity_metadata.to_xml.to_xml)
+        else
+          DorService.update_datastream(@druid, 'identityMetadata', self.identity_metadata.to_xml.to_xml, content_type='application/xml', versionable = false)
+        end #unless
+      end #identity_metadata_save
+
+      # Return the IdentityMetadata object bound to identityMetadata datastream XML
+      def identity_metadata
+        if (@identity_metadata == nil)
+          if (@druid == nil)
+            @identity_metadata = IdentityMetadata.new
+          else
+            idmd_str = DorService.get_datastream(@druid, 'identityMetadata')
+            @identity_metadata = IdentityMetadata.from_xml(idmd_str)
+          end
+        end
+        return @identity_metadata
+      end
+
+      # Return the identifier value for the specified identier name
+      def identifier(key)
+        return self.identity_metadata.get_identifier_value(key)
+      end
+
+      # Add a new name,value pair to the set of identifiers
+      def identifier_add(key, value)
+        self.identity_metadata.add_identifier(key, value)
+      end
+
+      # Return an array of strings where each entry consists of name:value
+      def id_pairs
+        self.identity_metadata.get_id_pairs
+      end
+
+      # Return the druid for the work item if it exists, else the first identifier value
+      def item_id
+        return @druid if @druid
+        pairs = self.identity_metadata.get_id_pairs
+        return pairs[0] if (pairs.size > 0)
+      end
+
+      # Record the successful outcome of the workstep operation for this workitem
+      def set_success
+        @work_queue.success_count += 1
+        @end_time = Time.new
+        @elapsed_time = @end_time - @start_time
+        LyberCore::Log.info("#{item_id} completed in #{@elapsed_time} seconds")
+        if (@druid)
+          Dor::WorkflowService.update_workflow_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, 'completed', @elapsed_time)
+        end
+      end
+
+      # Record the unsuccessful outcome of the workstep operation for this workitem
+      def set_error(e)
+        @work_queue.error_count += 1
+        @end_time = Time.new
+        @elapsed_time = @end_time - @start_time
+        LyberCore::Log.error("#{item_id} error - #{e.backtrace}")
+        # By default puts will output an array with a newline between each item.
+        if (@druid)
+          DorService.update_workflow_error_status(@work_queue.workflow.repository, @druid, @work_queue.workflow.workflow_id, @work_queue.workflow_step, e.message)
+        end
+        # We've caught and processed the error at this point, I don't think we want to raise it again. --bess
+        # raise e
+      end
+
+    end
+  end
+end

data/lib/lyber_core/robots/work_queue.rb

@@ -0,0 +1,154 @@
+require 'dor_service'
+require 'dlss_service'
+require 'yaml'
+
+module LyberCore
+  module Robots
+    # Represents a set of workitem objects to be processed by a given step of a workflow
+    class WorkQueue
+
+      # The workflow that this queue is a part of
+      attr_reader :workflow
+      # The step in the workflow that is being processed against this queue
+      attr_reader :workflow_step
+      # The workflow step that should have already been completed for the workitem object
+      attr_reader :prerequisite
+      # The maximum number of workitem objects to process in one run of a robot
+      attr_reader :batch_limit
+      # The maximum number of errors to allow before terminating the batch run
+      attr_reader :error_limit
+      # The array of primary identifiers for the workitem objects to be processed
+      attr_reader :druids
+      # An alternative identitier to be used when druids are not yet available (e.g at registration)
+      attr_reader :identifier_name
+      attr_reader :identifier_values
+      # The tally of how many items have been processed
+      attr_reader :item_count
+      attr_accessor :success_count
+      attr_accessor :error_count
+      # The timings for the batch run
+      attr_reader :start_time
+      attr_reader :end_time
+      attr :elapsed_time
+      
+      attr_reader :config_file
+
+
+      # Create a new WorkQueue object for the specified step,
+      # save a pointer to the parent WorkFlow,
+      # start the timer,
+      # read in the configuration information for the work step
+      def initialize(workflow=nil, workflow_step=nil)
+        LyberCore::Log.debug("Initializing work queue with workflow #{workflow} and workflow_step #{workflow_step}")
+        @start_time = Time.new
+        LyberCore::Log.info("Starting #{workflow_step} at #{@start_time}")
+        @workflow = workflow
+        @workflow_step = workflow_step
+        @item_count = 0
+        @success_count = 0
+        @error_count = 0
+        # nil arguments should only be used if in test mode
+        if (workflow.nil? || workflow_step.nil?)
+          @batch_limit = 2
+          @error_limit = 1
+          return
+        end
+        
+        self.process_config_file
+
+      end
+      
+      def process_config_file
+          LyberCore::Log.debug("Processing config file ... ")
+          LyberCore::Log.debug("@workflow.workflow_config_dir = #{@workflow.workflow_config_dir}")
+          
+          @config_file = File.join(@workflow.workflow_config_dir, 'process-config.yaml')
+          LyberCore::Log.debug("I'm opening the config file at #{@config_file}")
+          
+          # Does the file exist?
+          raise "Can't open process-config file #{@config_file}" unless File.file? @config_file
+          
+          process_config = YAML.load_file(config_file)
+          LyberCore::Log.debug("process_config: #{process_config.inspect}")
+
+          @prerequisite = process_config[@workflow_step]["prerequisite"]
+          LyberCore::Log.debug("@prerequisite: #{@prerequisite}")
+          
+          @batch_limit = process_config[@workflow_step]['batch_limit']  
+          LyberCore::Log.debug("@batch_limit: #{@batch_limit}")
+          
+          @error_limit = process_config[@workflow_step]['error_limit']
+          LyberCore::Log.debug("@error_limit: #{@error_limit}")
+          
+      end
+
+      # Explicitly specify a set of druids to be processed by the workflow step
+      def enqueue_druids(druid_array)
+        LyberCore::Log.debug("\nEnqueing an array of druids...")
+        @druids = druid_array
+        LyberCore::Log.debug("\n@druids = #{@druids}")
+      end
+
+      # Obtain the set of druids to be processed using a database query
+      # to obtain the repository objects that are awaiting this step
+      def enqueue_workstep_waiting()
+        begin
+          LyberCore::Log.debug("\nEnqueing workstep waiting...")
+          object_list_xml = DorService.get_objects_for_workstep(workflow.repository, workflow.workflow_id, @prerequisite, @workflow_step)
+          LyberCore::Log.debug("\nobject_list_xml = #{object_list_xml}")
+          @druids = DlssService.get_some_druids_from_object_list(object_list_xml,self.batch_limit)
+          LyberCore::Log.debug("\n@druids = #{@druids}")
+        rescue Exception => e
+          raise e
+        end
+      end
+
+      # Use an alternative set of identifiers as the basis of this queue
+      # e.g. use array of barcodes as basis for google register-object robot
+      def enqueue_identifiers(identifier_name, identifier_values)
+        @identifier_name = identifier_name
+        @identifier_values = identifier_values
+      end
+
+      # Get the next WorkItem to be processed by the robot for the workflow step
+      def next_item()
+        if (@item_count >= @batch_limit )
+          LyberCore::Log.info "Batch limit of #{@batch_limit} items reached"
+          return nil
+        end
+        if (@error_count >= @error_limit )
+          LyberCore::Log.info "Error limit of #{@error_limit} items reached"
+          return nil
+        end
+        work_item =  LyberCore::Robots::WorkItem.new(self)
+        if (@druids)
+          return nil if (@item_count >= @druids.length)
+          work_item.druid= @druids[@item_count]
+        elsif (@identifier_values)
+          return nil if (@item_count >= @identifier_values.length)
+          work_item.identifier_add(@identifier_name,@identifier_values[@item_count])
+        else
+          return nil
+        end
+        @item_count += 1
+        return  work_item
+      end
+
+      # Output the batch's timings and other statistics to the main log file
+      def print_stats
+        @end_time = Time.new
+        @elapsed_time = @end_time - @start_time
+        LyberCore::Log.info "Total time: " + @elapsed_time.to_s + "\n"
+        LyberCore::Log.info "Completed objects: " + self.success_count.to_s + "\n"
+        LyberCore::Log.info "Errors: " + self.error_count.to_s + "\n"    
+      end
+      
+      def print_empty_stats
+        @end_time = Time.new
+        @elapsed_time = @end_time - @start_time
+        LyberCore::Log.info "Total time: " + @elapsed_time.to_s + "\n"
+        LyberCore::Log.info "Empty queue"   
+      end
+    end
+  end
+end

data/lib/lyber_core/robots/workflow.rb

@@ -0,0 +1,104 @@
+module LyberCore
+  module Robots
+    class Workflow
+
+      attr_reader :workflow_name
+      attr_reader :workflow_config_dir
+      attr_reader :workflow_config_file
+      attr_reader :workflow_config
+      
+      attr_reader :collection_name
+      attr_reader :collection_config_dir
+      
+
+      # @param [String] workflow_name name of the workflow
+      # @param [Hash] options a hash of optional arguments
+      # @return [LyberCore::Robots::Workflow] a workflow object
+      # @example Create a new workflow object with a collection_name
+      #   @wf = LyberCore::Robots::Workflow.new(workflow_name, {:collection_name => collection})
+      def initialize(workflow_name, options = {})
+        # ROBOT_ROOT must be set before invoking a robot
+        raise "ROBOT_ROOT isn't set. Please set it to point to where your config files live." unless defined? ROBOT_ROOT
+        
+        @workflow_name = workflow_name
+        @collection_name = options[:collection_name]
+        self.load_workflow_config
+      end
+      
+      def load_workflow_config
+        # # can override the default location of workflow config files
+        # # by setting WORKFLOW_CONFIG_HOME environmental variable
+        unless ROBOT_ROOT        
+          if not (config_home = ENV['WORKFLOW_CONFIG_HOME'] )
+            config_home = File.join(File.dirname(__FILE__), "..", "..", "config")
+          end
+        else
+          config_home = File.join(ROBOT_ROOT, "config", "workflows")
+        end
+        
+        @workflow_config_dir = File.join(config_home, @workflow_name )
+        LyberCore::Log.debug("@workflow_config_dir = #{@workflow_config_dir}")
+        @collection_config_dir = File.join(@workflow_config_dir, @collection_name ) if(@collection_name)
+        @workflow_config_file = File.join(@workflow_config_dir, 'workflow-config.yaml')
+        if (File.exist?(@workflow_config_file))
+          @workflow_config = YAML.load_file(workflow_config_file)
+        else
+          raise "Workflow config not found!
+          ROBOT_ROOT = #{ROBOT_ROOT}
+          expecting to find workflow_config_file in #{@workflow_config_file}
+          "
+        end
+      end
+
+      def workflow_collection
+        return @workflow_name + "_" + @collection_name
+      end
+
+      def workflow_id
+        return @workflow_name
+      end
+    
+      # Which repository are we operating against? 
+      # Should return either "dor" or "sdr"
+      def repository
+        return @workflow_config['repository']
+      end
+      
+      # Construct the fully qualified filename and see if
+      # a file exists there. If it doesn't exist or isn't
+      # a file, raise an error. 
+      def workflow_process_xml_filename
+        file = File.join(@workflow_config_dir, @workflow_name + '.xml')
+        if File.file? file
+          return file
+        else
+          raise "#{file} is not a file"
+        end
+      end
+
+      # Return the contents of the file at workflow_process_xml_filename
+      def workflow_process_xml
+        return IO.read(workflow_process_xml_filename)
+      end
+
+      def object_template_filepath
+        Dir.foreach(@collection_config_dir) do |file| 
+          if file.match(/ObjectTemplate.xml$/)
+            return File.join(@collection_config_dir, file)
+          end
+        end
+        Dir.foreach(@workflow_config_dir) do |file| 
+          if file.match(/ObjectTemplate.xml$/)
+            return File.join(@workflow_config_dir, file)
+          end
+        end
+        raise "Object Template not found"
+      end
+      
+      # receives a workflow step and returns 
+      def queue(workflow_step)
+        return WorkQueue.new(self, workflow_step)
+      end
+    end
+  end
+end

data/lib/lyber_core/robots/workspace.rb

@@ -0,0 +1,77 @@
+require 'fileutils'
+module LyberCore
+  module Robots
+    class Workspace
+
+      attr_reader :workflow_name
+      attr_reader :collection_name
+      attr_reader :workspace_base
+
+      def initialize(workflow_name, collection_name=nil)
+        @workflow_name = workflow_name
+        @collection_name = collection_name
+        @workspace_base = set_workspace_base
+        ensure_workspace_exists(@workspace_base)
+      end
+      
+      # Usually WORKSPACE_HOME is set in your environment config file,
+      # but you can override the default location of workspace files
+      # by setting a WORKSPACE_HOME environment variable
+      def set_workspace_home
+        begin
+          if not (workspace_home = ENV['WORKSPACE_HOME'] )
+            workspace_home = WORKSPACE_HOME
+          end
+        rescue NameError => e
+          LyberCore::Log.fatal("WORKSPACE_HOME is undefined. Do you need to set it in your config file?")
+          raise e
+        end
+      end
+      
+      def set_workspace_base
+        workspace_home = set_workspace_home
+        if (@collection_name)
+          @workspace_base = File.join(workspace_home, @workflow_name, @collection_name)
+        else
+          @workspace_base = File.join(workspace_home, @workflow_name)
+        end
+      end
+      
+      def ensure_workspace_exists(workspace)
+        begin
+          FileUtils.mkdir_p(workspace) unless File.directory?(workspace)
+        rescue
+          LyberCore::Log.fatal("Can't create workspace_base #{workspace}")
+          raise
+        end
+      end
+              
+      # Remove the first part of the druid
+      # @param [String] druid
+      # @return [String]
+      def normalized_druid(druid)
+        druid.sub(/druid:/, '')
+      end
+
+      def object_dir(dir_type, druid)
+        dir_name = File.join(@workspace_base, dir_type, normalized_druid(druid))
+        ensure_workspace_exists(dir_name)
+        return dir_name
+      end
+
+      # The place where the original tar file from google is stored
+      def original_dir(druid)
+        object_dir('original', druid)
+      end
+
+      def content_dir(druid)
+        return object_dir('content', druid)
+      end
+
+      def metadata_dir(druid)
+        return object_dir('metadata', druid)
+      end
+
+    end
+  end
+end

data/lib/lyber_core/utils.rb

@@ -0,0 +1,4 @@
+
+require 'lyber_core/utils/file_utilities'
+require 'lyber_core/utils/checksum_validate'
+require 'lyber_core/utils/bagit_bag'

data/lib/lyber_core/utils/bagit_bag.rb

@@ -0,0 +1,100 @@
+require 'find'
+require 'fileutils'
+require 'bagit'     # http://github.com/flazz/bagit
+require 'dor_service'
+
+module LyberCore
+  module Utils
+    class BagitBag
+
+      def initialize(bag_dir)
+        @bag_dir = bag_dir
+        if (File.exist?(@bag_dir))
+          FileUtils.rm_r(@bag_dir)
+        end
+        @bag = BagIt::Bag.new @bag_dir
+      end
+
+      def add_content_files(source_dir, use_links)
+        data_content_dir = File.join(@bag_dir, 'data', 'content')
+        copy_dir(source_dir,data_content_dir, use_links)
+      end
+
+      def copy_dir(source_dir, target_dir, use_links)
+        FileUtils.mkdir_p(target_dir)
+        Dir.foreach(source_dir)  do  |file|
+          unless (file == '.' or file == '..')
+            source_file = File.join(source_dir, file)
+            target_file = File.join(target_dir, file)
+            if File.directory?(source_file)
+              copy_dir(source_file, target_file, use_links)
+            elsif (use_links)
+              File.link(source_file, target_file)
+            else
+              File.copy(source_file, target_file)
+            end
+          end
+        end
+      end
+
+      def add_metadata_file_from_string( metadata_string, file_name)
+        if (not metadata_string.nil? )
+          data_file_path = "metadata/#{file_name}"
+          @bag.add_file(data_file_path) do |io|
+            io.puts metadata_string
+          end
+        end
+      end
+
+      def write_metadata_info(md_hash)
+        payload = bag_payload()
+        bag_info_hash = {
+          'Bag-Size' =>  bag_size_human(payload[0]),
+          'Payload-Oxum' =>  "#{payload[0]}.#{payload[1]}",
+        }
+        @bag.write_bag_info(md_hash.merge(bag_info_hash))
+        File.rename(@bag.bag_info_txt_file, File.join(@bag.bag_dir,'bag-info.txt'))
+      end
+
+      def bag_payload()
+        bytes = 0
+        files = 0
+        Find.find(@bag.data_dir) do |filepath|
+          if (not File.directory?(filepath))
+            bytes += File.size(filepath)
+            files += 1
+          end
+        end
+        return [bytes, files]
+      end
+
+      def bag_size_human(bytes)
+        count = 0
+        size = bytes
+        while ( size >= 1000 and count < 4 )
+          size /= 1000.0
+          count += 1
+        end
+        if (count == 0)
+          return sprintf("%d B", size)
+        else
+          return sprintf("%.2f %s", size, %w[B KB MB GB TB][count] )
+        end
+      end
+
+      def write_manifests()
+        @bag.manifest!
+        @bag.tagmanifest!
+      end
+
+      def validate()
+        if not @bag.valid?
+         raise "bag not valid: #{@bag_dir}"
+       end
+
+      end
+
+    end
+
+  end
+end