lyber-core 1.3.0 → 3.2.2

data/lib/lyber_core/robots/workflow.rb

@@ -1,104 +0,0 @@
-module LyberCore
-  module Robots
-    class Workflow
-
-      attr_reader :workflow_name
-      attr_reader :workflow_config_dir
-      attr_reader :workflow_config_file
-      attr_reader :workflow_config
-      
-      attr_reader :collection_name
-      attr_reader :collection_config_dir
-      
-
-      # @param [String] workflow_name name of the workflow
-      # @param [Hash] options a hash of optional arguments
-      # @return [LyberCore::Robots::Workflow] a workflow object
-      # @example Create a new workflow object with a collection_name
-      #   @wf = LyberCore::Robots::Workflow.new(workflow_name, {:collection_name => collection})
-      def initialize(workflow_name, options = {})
-        # ROBOT_ROOT must be set before invoking a robot
-        raise "ROBOT_ROOT isn't set. Please set it to point to where your config files live." unless defined? ROBOT_ROOT
-        
-        @workflow_name = workflow_name
-        @collection_name = options[:collection_name]
-        self.load_workflow_config
-      end
-      
-      def load_workflow_config
-        # # can override the default location of workflow config files
-        # # by setting WORKFLOW_CONFIG_HOME environmental variable
-        unless ROBOT_ROOT        
-          if not (config_home = ENV['WORKFLOW_CONFIG_HOME'] )
-            config_home = File.join(File.dirname(__FILE__), "..", "..", "config")
-          end
-        else
-          config_home = File.join(ROBOT_ROOT, "config", "workflows")
-        end
-        
-        @workflow_config_dir = File.join(config_home, @workflow_name )
-        LyberCore::Log.debug("@workflow_config_dir = #{@workflow_config_dir}")
-        @collection_config_dir = File.join(@workflow_config_dir, @collection_name ) if(@collection_name)
-        @workflow_config_file = File.join(@workflow_config_dir, 'workflow-config.yaml')
-        if (File.exist?(@workflow_config_file))
-          @workflow_config = YAML.load_file(workflow_config_file)
-        else
-          raise "Workflow config not found!
-          ROBOT_ROOT = #{ROBOT_ROOT}
-          expecting to find workflow_config_file in #{@workflow_config_file}
-          "
-        end
-      end
-
-      def workflow_collection
-        return @workflow_name + "_" + @collection_name
-      end
-
-      def workflow_id
-        return @workflow_name
-      end
-    
-      # Which repository are we operating against? 
-      # Should return either "dor" or "sdr"
-      def repository
-        return @workflow_config['repository']
-      end
-      
-      # Construct the fully qualified filename and see if
-      # a file exists there. If it doesn't exist or isn't
-      # a file, raise an error. 
-      def workflow_process_xml_filename
-        file = File.join(@workflow_config_dir, @workflow_name + '.xml')
-        if File.file? file
-          return file
-        else
-          raise "#{file} is not a file"
-        end
-      end
-
-      # Return the contents of the file at workflow_process_xml_filename
-      def workflow_process_xml
-        return IO.read(workflow_process_xml_filename)
-      end
-
-      def object_template_filepath
-        Dir.foreach(@collection_config_dir) do |file| 
-          if file.match(/ObjectTemplate.xml$/)
-            return File.join(@collection_config_dir, file)
-          end
-        end
-        Dir.foreach(@workflow_config_dir) do |file| 
-          if file.match(/ObjectTemplate.xml$/)
-            return File.join(@workflow_config_dir, file)
-          end
-        end
-        raise "Object Template not found"
-      end
-      
-      # receives a workflow step and returns 
-      def queue(workflow_step)
-        return WorkQueue.new(self, workflow_step)
-      end
-    end
-  end
-end

data/lib/lyber_core/robots/workspace.rb

@@ -1,77 +0,0 @@
-require 'fileutils'
-module LyberCore
-  module Robots
-    class Workspace
-      
-      attr_reader :workflow_name
-      attr_reader :collection_name
-      attr_reader :workspace_base
-
-      def initialize(workflow_name, collection_name=nil)
-        @workflow_name = workflow_name
-        @collection_name = collection_name
-        @workspace_base = set_workspace_base
-        ensure_workspace_exists(@workspace_base)
-      end
-      
-      # Usually WORKSPACE_HOME is set in your environment config file,
-      # but you can override the default location of workspace files
-      # by setting a WORKSPACE_HOME environment variable
-      def set_workspace_home
-        begin
-          if not (workspace_home = ENV['WORKSPACE_HOME'] )
-            workspace_home = Dor::Config.robots.workspace
-          end
-        rescue NameError => e
-          LyberCore::Log.fatal("WORKSPACE_HOME is undefined. Do you need to set it in your config file?")
-          raise e
-        end
-      end
-      
-      def set_workspace_base
-        workspace_home = set_workspace_home
-        if (@collection_name)
-          @workspace_base = File.join(workspace_home, @workflow_name, @collection_name)
-        else
-          @workspace_base = File.join(workspace_home, @workflow_name)
-        end
-      end
-      
-      def ensure_workspace_exists(workspace)
-        begin
-          FileUtils.mkdir_p(workspace) unless File.directory?(workspace)
-        rescue
-          LyberCore::Log.fatal("Can't create workspace_base #{workspace}")
-          raise
-        end
-      end
-              
-      # Remove the first part of the druid
-      # @param [String] druid
-      # @return [String]
-      def normalized_druid(druid)
-        druid.sub(/druid:/, '')
-      end
-
-      def object_dir(dir_type, druid)
-        dir_name = File.join(@workspace_base, dir_type, normalized_druid(druid))
-        ensure_workspace_exists(dir_name)
-        return dir_name
-      end
-
-      # The place where the original tar file from google is stored
-      def original_dir(druid)
-        object_dir('original', druid)
-      end
-
-      def content_dir(druid)
-        return object_dir('content', druid)
-      end
-
-      def metadata_dir(druid)
-        return object_dir('metadata', druid)
-      end
-
-    end
-  end
-end

data/lib/lyber_core/utils.rb

@@ -1,4 +0,0 @@
-
-require 'lyber_core/utils/file_utilities'
-require 'lyber_core/utils/checksum_validate'
-require 'lyber_core/utils/bagit_bag'

data/lib/lyber_core/utils/bagit_bag.rb

@@ -1,100 +0,0 @@
-require 'find'
-require 'fileutils'
-require 'bagit'     # http://github.com/flazz/bagit
-require 'dor_service'
-
-module LyberCore
-  module Utils
-    class BagitBag
-
-      def initialize(bag_dir)
-        @bag_dir = bag_dir
-        if (File.exist?(@bag_dir))
-          FileUtils.rm_r(@bag_dir)
-        end
-        @bag = BagIt::Bag.new @bag_dir
-      end
-
-      def add_content_files(source_dir, use_links)
-        data_content_dir = File.join(@bag_dir, 'data', 'content')
-        copy_dir(source_dir,data_content_dir, use_links)
-      end
-
-      def copy_dir(source_dir, target_dir, use_links)
-        FileUtils.mkdir_p(target_dir)
-        Dir.foreach(source_dir)  do  |file|
-          unless (file == '.' or file == '..')
-            source_file = File.join(source_dir, file)
-            target_file = File.join(target_dir, file)
-            if File.directory?(source_file)
-              copy_dir(source_file, target_file, use_links)
-            elsif (use_links)
-              File.link(source_file, target_file)
-            else
-              File.copy(source_file, target_file)
-            end
-          end
-        end
-      end
-
-      def add_metadata_file_from_string( metadata_string, file_name)
-        if (not metadata_string.nil? )
-          data_file_path = "metadata/#{file_name}"
-          @bag.add_file(data_file_path) do |io|
-            io.puts metadata_string
-          end
-        end
-      end
-
-      def write_metadata_info(md_hash)
-        payload = bag_payload()
-        bag_info_hash = {
-          'Bag-Size' =>  bag_size_human(payload[0]),
-          'Payload-Oxum' =>  "#{payload[0]}.#{payload[1]}",
-        }
-        @bag.write_bag_info(md_hash.merge(bag_info_hash))
-        File.rename(@bag.bag_info_txt_file, File.join(@bag.bag_dir,'bag-info.txt'))
-      end
-
-      def bag_payload()
-        bytes = 0
-        files = 0
-        Find.find(@bag.data_dir) do |filepath|
-          if (not File.directory?(filepath))
-            bytes += File.size(filepath)
-            files += 1
-          end
-        end
-        return [bytes, files]
-      end
-
-      def bag_size_human(bytes)
-        count = 0
-        size = bytes
-        while ( size >= 1000 and count < 4 )
-          size /= 1000.0
-          count += 1
-        end
-        if (count == 0)
-          return sprintf("%d B", size)
-        else
-          return sprintf("%.2f %s", size, %w[B KB MB GB TB][count] )
-        end
-      end
-
-      def write_manifests()
-        @bag.manifest!
-        @bag.tagmanifest!
-      end
-
-      def validate()
-        if not @bag.valid?
-         raise "bag not valid: #{@bag_dir}"
-       end
-
-      end
-
-    end
-
-  end
-end

data/lib/lyber_core/utils/checksum_validate.rb

@@ -1,65 +0,0 @@
-require 'nokogiri'
-require 'active_support'
-
-module LyberCore
-  module Utils
-    class ChecksumValidate
-      #Code here
-
-      def self.compare_hashes(hash1, hash2)
-           return (hash1 == hash2)
-      end
-
-      def self.get_hash_differences(hash1, hash2)
-         return hash1.diff(hash2)
-      end
-
-      def self.md5_hash_from_md5sum(md5sum)
-        checksum_hash = {}
-        md5sum.each do |line|
-          line.chomp!
-          digest,filename = line.split(/[ *]{2}/)
-          checksum_hash[filename] = digest.downcase
-        end
-        return checksum_hash
-      end
-
-      def self.md5_hash_from_mets(mets)
-        mets_checksum_hash = {}
-        doc = Nokogiri::XML(mets)
-        doc.xpath('/mets:mets/mets:fileSec//mets:file', {'mets' => 'http://www.loc.gov/METS/'}).each do |filenode|
-           digest = filenode.attribute('CHECKSUM')
-           if (digest)
-             flocat = filenode.xpath('mets:FLocat', {'mets' => 'http://www.loc.gov/METS/'}).first
-             if (flocat)
-               filename = flocat.attribute_with_ns('href', 'http://www.w3.org/1999/xlink')
-               if (filename)
-                 mets_checksum_hash[filename.text] = digest.text.downcase
-               end
-             end
-           end
-        end
-        return mets_checksum_hash
-      end
-
-      def self.md5_hash_from_content_metadata(content_md)
-        content_md_checksum_hash = {}
-        doc = Nokogiri::XML(content_md)
-        doc.xpath('/contentMetadata/resource[@type="page"]/file').each do |filenode|
-           filename = filenode.attribute('id')
-           if (filename)
-             md5_element = filenode.xpath('checksum[@type="MD5"]').first
-             if (md5_element)
-               digest = md5_element.text
-               if (digest)
-                 content_md_checksum_hash[filename.text] = digest.downcase
-               end
-             end
-           end
-        end
-        return content_md_checksum_hash
-
-      end
-    end
-  end
-end

data/lib/lyber_core/utils/file_utilities.rb

@@ -1,168 +0,0 @@
-require 'fileutils'
-require 'systemu'
-
-# File Utilities for use in transferring filesystem objects,
-# decrypting a file, unpacking a targz archive, and validating checksums
-# @author rnanders@stanford.edu
-module LyberCore
-  module Utils
-    class FileUtilities
-
-
-      # Executes a system command in a subprocess. 
-      # The method will return stdout from the command if execution was successful.
-      # The method will raise an exception if if execution fails. 
-      # The exception's message will contain the explaination of the failure.
-      # @param [String] command the command to be executed
-      # @return [String] stdout from the command if execution was successful
-      def FileUtilities.execute(command)
-        status, stdout, stderr = systemu(command)
-        if (status.exitstatus != 0)
-          raise stderr
-        end
-        return stdout
-      rescue
-        msg = "Command failed to execute: [#{command}] caused by <STDERR = #{stderr.split($/).join('; ')}>"
-        msg << " STDOUT = #{stdout.split($/).join('; ')}" if (stdout && (stdout.length > 0))
-        raise msg
-
-      end
-
-      # Generates a dirname for storing or retrieving a file in
-      # "pair tree" hierachical structure, where the path is derived
-      # from segments of a barcode string
-      #
-      # = Input:
-      # * barcode = barcode string
-      #
-      # = Return value:
-      # * A string containing a slash-delimited dirname derived from the barcode
-      def FileUtilities.pair_tree_from_barcode(barcode)
-        if (barcode.class != String)
-          raise "Barcode must be a String"
-        end
-        # figure out if this is a SUL barcode or from coordinate library
-        library_prefix=barcode[0..4]
-        if ( library_prefix == '36105' )
-          pair_tree=barcode[5..10].gsub(/(..)/, '\1/')
-        else
-          library_prefix=barcode[0..2]
-          pair_tree=barcode[3..8].gsub(/(..)/, '\1/')
-        end
-        return "#{library_prefix}/#{pair_tree}"
-      end
-
-      # Transfers a filesystem object (file or directory)
-      # from a source to a target location. Uses rsync in "archive" mode
-      # over an ssh connection.
-      #
-      # = Inputs:
-      # * filename = basename of the filesystem object to be transferred
-      # * source_dir = dirname of the source location from which the object is read
-      # * dest_dir = dirname of the target location to which the object is written
-      # If one of the locations is on a remote server, then the dirname should be
-      # prefixed with  user@hosthame:
-      #
-      # = Return value:
-      # * The method will return true if the transfer is successful.
-      # * The method will raise an exception if either the rsync command fails,
-      # or a test for the existence of the transferred object fails.
-      # The exception's message will contain the explaination of the failure
-      #
-      # Network transfers will only succeed if the appropriate public key
-      # authentication has been previously set up.
-      def FileUtilities.transfer_object(filename, source_dir, dest_dir)
-        source_path=File.join(source_dir, filename)
-        rsync='rsync -a -e ssh '
-        rsync_cmd = rsync + "'" + source_path + "' " +  dest_dir
-        LyberCore::Log.debug("rsync command is: #{rsync_cmd}")
-        self.execute(rsync_cmd)
-        if not File.exists?(File.join(dest_dir, filename))
-          raise "#{filename} is not found in  #{dest_dir}"
-        end
-        return true
-      end
-
-      # Decrypts a GPG encrypted file using the "gpg" command
-      #
-      # = Inputs:
-      # * workspace_dir = dirname containing the file
-      # * targzgpg = the filename of the GPG encrypted file
-      # * targz = the filename of the unencrypted file
-      # * passphrase = the string used to decrypt the file
-      #
-      # = Return value:
-      # * The method will return true if the decryption is successful.
-      # * The method will raise an exception if either the decryption command fails,
-      # or a test for the existence of the decrypted file fails.
-      # The exception's message will contain the explaination of the failure
-      def FileUtilities.gpgdecrypt(workspace_dir, targzgpg, targz, passphrase)
-        LyberCore::Log.debug("decrypting #{targzgpg}")
-        gpg_cmd="/usr/bin/gpg --passphrase '#{passphrase}'  "  +
-                 "--batch --no-mdc-warning --no-secmem-warning " +
-                 " --output " + File.join(workspace_dir, targz)  +
-                " --decrypt " + File.join(workspace_dir, targzgpg)
-        self.execute(gpg_cmd)
-        if not File.exists?(File.join(workspace_dir, targz))
-          raise "#{targz} was not created in  #{workspace_dir}"
-        end
-        return true
-      end
-
-      # Unpacks a TAR-ed, GZipped archive using a "tar -xzf" command
-      #
-      # = Inputs:
-      # * original_dir = dirname containing the archive file
-      # * targz = the filename of the archive file
-      # * destination_dir = the target directory into which the contents are written
-      #
-      # = Return value:
-      # * The method will return true if the unpacking is successful.
-      # * The method will raise an exception if either the unpack command fails,
-      # or a test for the existence of files in the target directory fails.
-      # The exception's message will contain the explaination of the failure.
-      def FileUtilities.unpack(original_dir, targz, destination_dir)
-        LyberCore::Log.debug("unpacking #{targz}")
-        FileUtils.mkdir_p(destination_dir)
-        dir_save = Dir.pwd
-        Dir.chdir(destination_dir)
-        unpack_cmd="tar -xzf " + File.join(original_dir, targz)
-        self.execute(unpack_cmd)
-        if not (Dir.entries(destination_dir).length > 0)
-          raise "#{destination_dir} is empty"
-        end
-        return true
-      ensure
-        Dir.chdir(dir_save)
-      end
-
-      # Verifies MD5 checksums for the files in a directory
-      # against the checksum values in the supplied file
-      # (Uses md5sum command)
-      #
-      # = Inputs:
-      # * directory = dirname containing the file to be checked
-      # * checksum_file = the name of the file containing the expected checksums
-      #
-      # = Return value:
-      # * The method will return true if the verification is successful.
-      # * The method will raise an exception if either the md5sum command fails,
-      # or a test of the md5sum output indicates a checksum mismatch.
-      # The exception's message will contain the explaination of the failure.
-      def FileUtilities.verify_checksums(directory, checksum_file)
-        LyberCore::Log.debug("verifying checksums in #{directory}")
-        dir_save = Dir.pwd
-        Dir.chdir(directory)
-        checksum_cmd = 'md5sum -c ' + checksum_file + ' | grep -v OK | wc -l'
-        badcount = self.execute(checksum_cmd).to_i
-        if not (badcount==0)
-          raise "#{badcount} files had bad checksums"
-        end
-        return true
-      ensure
-        Dir.chdir(dir_save)
-      end
-    end
-
-  end
-end