sdr-replication 0.1.0

data/lib/replication/command_consumer.rb

@@ -0,0 +1,55 @@
+require "rubygems"
+require "amqp"
+require "amqp/extensions/rabbitmq"
+require "yaml"
+
+EventMachine.run do
+
+
+	 connection_params =                                        {
+		 :host      => "127.0.0.1",
+		 :port      => 5672,
+		 :user      => "guest",
+		 :pass      => "guest",
+		 :vhost     => "/",
+		 :ssl       => false,
+		 :frame_max => 131072
+	 }
+
+	 connection = AMQP.connect(connection_params)
+	 channel    = AMQP::Channel.new(connection, :auto_recovery => true)
+
+
+	 channel.prefetch(1)
+
+# Acknowledgements are good for letting the server know
+# that the task is finished. If the consumer doesn't send
+# the acknowledgement, then the task is considered to be unfinished
+# and will be requeued when consumer closes AMQP connection (because of a crash, for example).
+
+	 channel.queue("sdr.replication.notification", :durable => true, :auto_delete => false).subscribe(:ack => true) do |metadata, payload|
+		  case metadata.type
+			   when "sdr_replication_notification"
+					data = YAML.load(payload)
+					puts "sdr_replication_notification request with #{data.inspect}"
+
+					message = "'" + payload + "'"
+					header = "'" + metadata.attributes.to_yaml + "'"
+
+
+					(pid = fork) ? Process.detach(pid) : exec("./dpn_check_rep.rb #{message} #{header}")
+					#shellout = "./dpn_check_rep.rb '#{data[:name]}' #{data[:size]}"
+					#puts "dpn_check_rep.rb Executing #{shellout}"; system(shellout)
+					puts "Call to SDR DPN replication request done"
+					puts
+
+			   when "dpn_replication_notification"
+			   else
+					puts "[commands] Unknown command: #{metadata.type}"
+		  end
+
+		  # message is processed, acknowledge it so that broker discards it
+		  metadata.ack
+	 end
+	 Signal.trap("INT") { connection.close { EventMachine.stop } }
+end

data/lib/replication/command_producer.rb

@@ -0,0 +1,105 @@
+require "amqp"
+require "amqp/extensions/rabbitmq"
+require "yaml"
+require "json"
+require "securerandom"
+require "time"
+
+
+connection_params =   {
+	:host      => "127.0.0.1",
+	:port      => 5672,
+	:user      => "guest",
+	:pass      => "guest",
+	:vhost     => "/",
+	:ssl       => false,
+	:frame_max => 131072
+}
+
+#usually send one message only, rely on message broker to recover
+doit = 1
+
+EventMachine.run do
+	 sleep(0.5)
+
+	 connection = AMQP.connect(connection_params)
+	 channel    = AMQP::Channel.new(connection)
+
+
+# publish new commands every 2 seconds, count + one times
+	 count = 0
+	 EventMachine.add_periodic_timer(2.0) do
+		  count += 1
+		  if count >= doit then
+			   EM.stop_event_loop
+		  end
+		  puts "count = #{count}"
+		  puts "Messaging that sdr has content for replication, parameters are name and size."
+		  payload = { :name => "12345.tar", :size => 4096 }.to_yaml
+
+		  correlation_id = SecureRandom.random_number(10000)
+		  utctime = Time.now.utc
+		  ttl = 20   #time for transaction to live
+
+		  channel.default_exchange.publish(payload,
+										   :arguments => { "x-message-ttl" => 10 },
+										   :type        => "sdr_replication_notification",
+										   :routing_key => "sdr.replication.notification",
+										   :reply_to 	=> "dpn.replication.reply",
+										   :correlation_id => correlation_id,
+										   :timestamp  => utctime.to_i
+		  )
+
+	 end
+
+	 puts "Sending notification message"
+	 Signal.trap("INT") { connection.close { EventMachine.stop } }
+end
+
+
+
+exit
+
+#listen for response to previous message
+EventMachine.run do
+	 sleep(0.5)
+
+	 connection = AMQP.connect(connection_params)
+	 channel    = AMQP::Channel.new(connection, :auto_recovery => true)
+	 channel.prefetch(1)
+
+	 count = 0
+	 EventMachine.add_periodic_timer(10.0) do
+		  count += 1
+		  puts "waiting....."
+		  if count > doit+1 then
+			   EM.stop_event_loop
+		  end
+	 end
+
+	 channel.queue("dpn.replication.notification", :durable => true, :auto_delete => false).subscribe(:ack => true) do |metadata, payload|
+		  case metadata.type
+			   when "dpn_replication_ack"
+					message = YAML.load(payload)
+					if message[:message] == "dpn_available_reply" && message[:acknak] == 'ack'  then
+						 puts "[sdr dpn says to go ahead and replicate content to temp area."
+						 # (pid = fork) ? Process.detach(pid) : exec("start_replication_workflow.rb
+						 puts "Start workflow to copy content to sdr dpn for replication. "
+					else
+						 puts "no replication possible, DPN say's no!"
+						 # (pid = fork) ? Process.detach(pid) : exec("replication_denied_workflow.rb
+					end
+
+			   when "dpn_replication_notification"
+					puts "DPN wants to replicate"
+			   else
+					puts "[commands] Unknown command: #{metadata.type}"
+		  end
+
+		  # message is processed, acknowledge it so that broker discards it
+		  metadata.ack
+	 end
+	 puts "Waiting for 'ack' notification message"
+	 Signal.trap("INT") { connection.close { EventMachine.stop } }
+
+end

data/lib/replication/dpn_check_rep.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+$: << "./"
+require 'securerandom'
+require "rubygems"
+require 'time'
+require 'date'
+require "eventmachine"
+require "amqp"
+require "amqp/extensions/rabbitmq"
+require "yaml"
+
+
+connection_params =   {
+	:host      => "127.0.0.1",
+	:port      => 5672,
+	:user      => "guest",
+	:pass      => "guest",
+	:vhost     => "/",
+	:ssl       => false,
+	:frame_max => 131072
+}
+
+#arguments passed are json strings. payload, header.
+puts "*************************************"
+puts "In dpn_check_rep.rb"
+if ARGV.length != 2 then
+	 exit 5001 #should not get here, but this is an error condition
+end
+
+message = YAML.load(ARGV[0])
+header  = YAML.load(ARGV[1])
+
+if header[:type] !=  "sdr_replication_notification"  then
+	 exit 5002 #bad message type
+end
+
+puts "sdr_replication_notification message #{message.inspect}"
+puts "sdr_replication_notification header #{header.inspect}"
+
+puts "checking to see if DPN can replicate content from SDR"
+
+#
+#Do checking!
+			 #
+
+sleep 4.5      #simulate checking
+
+
+exit
+
+EventMachine.run do
+	 sleep(0.5)
+	 connection = AMQP.connect(connection_params)
+	 channel    = AMQP::Channel.new(connection)
+
+	 count = 0
+	 EventMachine.add_periodic_timer(2.0) do
+		  count += 1
+		  if count > 2 then
+			   EM.stop_event_loop
+		  end
+	 end
+
+	 EM.add_timer(1.0) do
+		  puts "Working: #{Time.now}"
+		  STDOUT.flush
+	 end
+
+	 puts "Acking that content can be replicated"
+	 payload = { :message => "dpn_available_reply", :acknak => 'ack' }.to_yaml
+
+	 channel.default_exchange.publish(payload,
+									  :type        => "dpn_replication_ack",
+									  :routing_key => header[:reply_to],
+									  :reply_to 	=> "dpn.err.notification",
+									  :correlation_id => header[:correlation_id],
+									  :timestamp  => header[:timestamp])
+
+	 puts "Sending ack notification message"
+	 Signal.trap("INT") { connection.close { EventMachine.stop } }
+
+end

data/lib/replication/file_fixity.rb

@@ -0,0 +1,98 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'sdr_replication'
+
+module Replication
+
+  # The fixity properties of a file, used to determine file content equivalence.
+  # Placing this data in a class by itself facilitates using the MD5, SHA1, etc checksums (and optionally the file size)
+  # as a single key when doing comparisons against other file instances.  The design assumes that this file fixity
+  # is sufficiently unique to act as a comparator for determining file equality or verifying checksum manifests.
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class FileFixity
+
+    # @param [Hash<Symbol,Object>] options Key,Value pairs specifying initial values of attributes
+    def initialize(options=nil)
+      @checksums=Hash.new
+      options = {} if options.nil?
+      options.each do |key,value|
+        #instance_variable_set("@#{key}", value)
+        send "#{key}=", value
+      end
+    end
+
+    # @return [String] The name of the file, relative to its base directory
+    #   (for payload files, path relative to the data folder.  For tag files, path relative to the bag home folder)
+    attr_accessor :file_id
+
+    # @return [Integer] The size of the file in bytes
+    attr_accessor :bytes
+
+    # @return [Hash<Symbol,String>] The MD5, SHA1, SHA256, etc checksum values of the file
+    attr_accessor :checksums
+
+    # @param [Symbol,String] type The type of checksum (e.g. :md5, :sha1, :sha256)
+    # @return [String] The value of the file digest
+    def get_checksum(type)
+      checksum_type = type.to_s.downcase.to_sym
+      self.checksums[checksum_type]
+    end
+
+    # @param type [Symbol,String] The type of checksum
+    # @param value [String] value of the file digest
+    # @return [void] Set the value for the specified checksum type in the checksum hash
+    def set_checksum(type,value)
+      checksum_type = type.to_s.downcase.to_sym
+      Fixity.validate_checksum_types(checksum_type)
+      self.checksums[checksum_type] = value
+    end
+
+    # @param other [FileFixity] The other file fixity being compared to this fixity
+    # @return [Boolean] Returns true if self and other have comparable fixity data.
+    def eql?(other)
+      matching_checksum_types = self.checksums.keys & other.checksums.keys
+      return false if matching_checksum_types.size == 0
+      matching_checksum_types.each do |type|
+        return false if self.checksums[type] != other.checksums[type]
+      end
+      true
+    end
+
+    # (see #eql?)
+    def ==(other)
+      eql?(other)
+    end
+
+    # @return [Fixnum] Compute a hash-code for the fixity value array.
+    #   Two file instances with the same content will have the same hash code (and will compare using eql?).
+    # @note The hash and eql? methods override the methods inherited from Object.
+    #   These methods ensure that instances of this class can be used as Hash keys.  See
+    #   * {http://www.paulbutcher.com/2007/10/navigating-the-equality-maze/}
+    #   * {http://techbot.me/2011/05/ruby-basics-equality-operators-ruby/}
+    #   Also overriden is {#==} so that equality tests in other contexts will also return the expected result.
+    def hash
+      [self.file_id].hash
+    end
+
+    # @param [FileFixity] other The other FileFixity object being compared to this one
+    # @param [String] left The label to use for values from this base FileFixity object
+    # @param [String] right he label to use for values from the other FileFixity object
+    # @return [Hash<symbol,Hash<String,String>] details of the checksum differences between fixity objects
+    def diff(other,left='base',right='other')
+      diff_hash = Hash.new
+      matching_checksum_types = (self.checksums.keys & other.checksums.keys)
+      matching_checksum_types = (self.checksums.keys | other.checksums.keys) if matching_checksum_types.empty?
+      matching_checksum_types.each do |type|
+        base_checksum = self.checksums[type]
+        other_checksum = other.checksums[type]
+        if base_checksum != other_checksum
+          diff_hash[type] = {left => base_checksum, right => other_checksum }
+        end
+      end
+      return diff_hash.size > 0 ? diff_hash : nil
+    end
+
+  end
+
+end

data/lib/replication/fixity.rb

@@ -0,0 +1,155 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'sdr_replication'
+
+module Replication
+
+  # A Struct to hold properties of a given checksum digest type
+  ChecksumType = Struct.new(:id, :hex_length, :names)
+
+  # A helper class that facilites the generation and processing of checksums
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class Fixity
+
+    @@default_checksum_types = [:sha1, :sha256]
+
+    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
+    def Fixity.default_checksum_types
+      @@default_checksum_types
+    end
+
+    # @param [Array<Symbol>] types The list of checksum types to be used when generating fixity data
+    # @return [Void] Set the list of checksum types to be used when generating fixity data
+    def Fixity.default_checksum_types=(*types)
+      @@default_checksum_types = Fixity.validate_checksum_types(*types)
+    end
+
+    @@valid_checksum_types = [
+        ChecksumType.new(:md5, 32, ['MD5']),
+        ChecksumType.new(:sha1, 40, ['SHA-1', 'SHA1']),
+        ChecksumType.new(:sha256, 64, ['SHA-256', 'SHA256']),
+        ChecksumType.new(:sha384, 96, ['SHA-384', 'SHA384']),
+        ChecksumType.new(:sha512, 128, ['SHA-512', 'SHA512'])
+    ]
+
+    # @return [Array<ChecksumType>] The list of allowed ChecksumType structs containing the type's properties
+    def Fixity.valid_checksum_types
+      @@valid_checksum_types
+    end
+
+    # @return [Array<Symbol>] The list of allowed checksum types
+    def Fixity.valid_checksum_ids
+      @@valid_checksum_types.map { |type| type.id }
+    end
+
+    # @param [Array<Symbol>] types The list of checksum types being specified by the caller
+    # @return [Object] The list of specified checksum types after being checked for validity
+    def Fixity.validate_checksum_types(*types)
+      checksum_types = types.flatten
+      invalid_types = checksum_types - valid_checksum_ids
+      raise "Invalid digest type specified: #{invalid_types.inspect}" unless invalid_types.empty?
+      checksum_types
+    end
+
+    # @param [Array<Symbol>] checksum_types The list of checksum types being specified by the caller
+    # @return [Array<Digest::Class>] The list of digest implementation objects that will generate the checksums
+    def Fixity.get_digesters(checksum_types=@@default_checksum_types)
+      checksum_types.inject(Hash.new) do |digesters, checksum_type|
+        case checksum_type
+          when :md5
+            digesters[checksum_type] = Digest::MD5.new
+          when :sha1
+            digesters[checksum_type] = Digest::SHA1.new
+          when :sha256
+            digesters[checksum_type] = Digest::SHA2.new(256)
+          when :sha384
+            digesters[checksum_type] = Digest::SHA2.new(384)
+          when :sha512
+            digesters[checksum_type] = Digest::SHA2.new(512)
+          else
+            raise "Unrecognized checksum type: #{checksum_type}"
+        end
+        digesters
+      end
+    end
+
+    # @param pathname [Pathname] The location of the file to be digested
+    # @param [Object] base_pathname The base directory from which relative paths (file IDS) will be derived
+    # @param [Object] checksum_types The list of checksum types being specified by the caller (or default list)
+    # @return [FileFixity] Generate a FileFixity instance containing fixity properties measured from of a physical file
+    def Fixity.fixity_from_file(pathname, base_pathname, checksum_types=@@default_checksum_types)
+      file_fixity = FileFixity.new
+      file_fixity.file_id = pathname.relative_path_from(base_pathname).to_s
+      file_fixity.bytes = pathname.size
+      digesters = Fixity.get_digesters(checksum_types)
+      pathname.open("r") do |stream|
+        while buffer = stream.read(8192)
+          digesters.values.each { |digest| digest.update(buffer) }
+        end
+      end
+      digesters.each { |checksum_type, digest| file_fixity.checksums[checksum_type] = digest.hexdigest }
+      file_fixity
+    end
+
+     # @param [Pathname] base_pathname The directory path used as the base for deriving relative paths (file IDs)
+    # @param [Array<Pathname>] path_list The list of pathnames for files whose fixity will be generated
+    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the actual files
+    def Fixity.generate_checksums(base_pathname, path_list, checksum_types=@@default_checksum_types)
+      path_list = base_pathname.find  if path_list.nil?
+      file_fixity_hash = Hash.new
+      path_list.select{|pathname| pathname.file?}.each  do |file|
+        file_fixity = Fixity.fixity_from_file(file, base_pathname)
+        file_fixity_hash[file_fixity.file_id] = file_fixity
+      end
+      file_fixity_hash
+    end
+
+    # @param [Integer] length The length of the checksum value in hex format
+    # @return [ChecksumType] The ChecksumType struct that contains the properties of the matching checksum type
+    def Fixity.type_for_length(length)
+      @@valid_checksum_types.select {|type| type.hex_length == length}.first
+    end
+
+    # @param [Object] file_id The filename or relative path of the file from its base directory
+    # @param [Object] checksum_values The digest values of the file
+    # @return [FileFixity] Generate a FileFixity instance containing fixity properties supplied by the caller
+    def Fixity.fixity_from_checksum_values(file_id, checksum_values)
+      file_fixity = FileFixity.new
+      file_fixity.file_id = file_id
+      checksum_values.each do |digest|
+        checksum_type = Fixity.type_for_length(digest.length)
+        file_fixity.checksums[checksum_type.id] = digest
+      end
+      file_fixity
+    end
+
+    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @return [Hash<String,Hash<Symbol,String] A hash containing file ids and checksum data derived from the file_fixity_hash
+    def Fixity.file_checksum_hash(file_fixity_hash)
+      checksum_hash = Hash.new
+      file_fixity_hash.values.each{|file| checksum_hash[file.file_id] = file.checksums}
+      checksum_hash
+    end
+
+    # @param [Symbol,String] checksum_type The type of checksum digest to be generated
+    # @param [Pathname,String] file_pathname The location of the file to digest
+    # @return [String] The operating system shell command that will generate the checksum digest value
+    def Fixity.openssl_digest_command(checksum_type,file_pathname)
+      command = "openssl dgst -#{checksum_type} #{file_pathname}"
+      command
+    end
+
+    # @param [Symbol,String] checksum_type The type of checksum digest to be generated
+    # @param [Pathname,String] file_pathname The location of the file to digest
+    # @return [String] The checksum digest value for the file
+    def Fixity.openssl_digest(checksum_type,file_pathname)
+      command = openssl_digest_command(checksum_type,file_pathname)
+      stdout = OperatingSystem.execute(command)
+      checksum = stdout.scan(/[A-Za-z0-9]+/).last
+      checksum
+    end
+
+  end
+
+end