sdr-replication 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e99a0814e4383ec6287dec7df41825d786e65919
-  data.tar.gz: 4890c2758dd820f22ce8aa9015621e1b33dec9ae
+  metadata.gz: 477696a5ac274697dfb1f337e562a859965695e8
+  data.tar.gz: ee4274bb597963bb1faec8783a0a41875abbd369
 SHA512:
-  metadata.gz: deb3400e53fcbdf16cf8263ffe1deed70be83889633b4fecfa0de58e41a076e7f502de61d22f079b09ff6b81ddb29586bb6e898782f22e45010ecfd46d93828d
-  data.tar.gz: a0533d23addcc264e7aeca97f985bdad1d70e8081230e09ab5337a03314b058374541b1587f47a5f89dd2f558115677190d49b320b892a77e7031bb15ac9c3ff
+  metadata.gz: 094e30cbc6e5e8c490e5560506a34b69cff7ab463cb4215e34d269b3f9abc9754fece858ac734a56448e1e4f0bd0332f6e7b0aee6d72e9d9436f5722afa6bfed
+  data.tar.gz: 1b1593dc47c1190a7bc1ea60b6f2bdb6635f21fc1170a7ef3cd18bb3be481d223b1b30435b72e327226ac33a81f63129a53405821cb4f7f04bbcc0c01ae64f54

data/lib/replication/sdr_object.rb

@@ -0,0 +1,18 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'sdr_replication'
+
+module Replication
+
+  class SdrObject < Moab::StorageObject
+
+    def initialize(druid)
+      @digital_object_id = druid
+      storage_object = StorageServices.find_storage_object(druid, include_deposit=true)
+      @object_pathname = storage_object.object_pathname
+      @storage_root = storage_object.storage_root
+    end
+
+
+  end
+
+end

data/lib/replication/sdr_object_version.rb

@@ -6,40 +6,139 @@ module Replication
   #
   # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
   #   All rights reserved.  See {file:LICENSE.rdoc} for details.
-  class SdrObjectVersion
+  class SdrObjectVersion < Moab::StorageObjectVersion
 
-    # @return [Moab::StorageObjectVersion] Represents the object version's storage location
-    attr_accessor :moab_object_version
+    def digital_object_id
+      storage_object.digital_object_id
+    end
 
-    # @param [Moab::StorageObjectVersion] object_version Represents the object version's storage location
-    # @return [SdrObjectVersion] Initialize a new SdrObjectVersion object
-    def initialize(object_version)
-      @moab_object_version = object_version
+    # @return [Moab::FileInventory] The moab version manifest for the version
+    def version_inventory
+      @version_inventory ||= file_inventory('version')
     end
 
-    # @return [String] The digital object identifier (druid)
-    def sdr_object_id
-      @sdr_object_id ||=  moab_object_version.storage_object.digital_object_id
+    # @return [Moab::FileInventory] The moab version manifest for the version
+    def version_additions
+      @version_additions ||= file_inventory('additions')
     end
 
-    # @return [Integer] The digital object version number
-    def sdr_version_id
-      @sdr_version_id ||= moab_object_version.version_id
+    # @return [Hash] The contents of the versionMetadata file
+    def parse_version_metadata
+      metadata = Hash.new
+      pathname = find_filepath('metadata', 'versionMetadata.xml')
+      if pathname.exist?
+        doc = Nokogiri::XML(pathname.read)
+        nodeset = doc.xpath("/versionMetadata/version")
+        metadata[:version_id] = nodeset.last['versionId'].to_i unless nodeset.empty?
+      end
+      metadata
     end
 
-    # @return [Moab::FileInventory] The moab version manifest for the version
-    def version_inventory
-      @version_inventory ||= moab_object_version.file_inventory('version')
+    # @return [Hash] The contents of the identityMetadata file
+    def parse_identity_metadata
+      metadata = Hash.new
+      pathname = find_filepath('metadata', 'identityMetadata.xml')
+      if pathname.exist?
+        doc = Nokogiri::XML(pathname.read)
+        nodeset = doc.xpath("/identityMetadata/objectType")
+        metadata[:object_type] = nodeset.first.text unless nodeset.empty?
+        nodeset = doc.xpath("/identityMetadata/objectLabel")
+        metadata[:object_label] = nodeset.first.text unless nodeset.empty?
+      end
+      metadata
     end
 
-    # @return [Moab::FileInventory] The moab version manifest for the version
-    def version_additions
-      @version_additions ||= moab_object_version.file_inventory('additions')
+    # @return [Hash] The contents of the relationshipMetadata file
+    def parse_relationship_metadata
+      metadata = Hash.new
+      pathname = find_filepath('metadata', 'relationshipMetadata.xml')
+      if pathname.exist?
+        doc = Nokogiri::XML(pathname.read)
+        nodeset = doc.xpath("//hydra:isGovernedBy", 'hydra' => 'http://projecthydra.org/ns/relations#')
+        unless nodeset.empty?
+          apo_id = nodeset.first.attribute_with_ns('resource', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')
+          if apo_id
+            metadata[:governed_by] = apo_id.text.split('/')[-1]
+          end
+        end
+      end
+      metadata
+    end
+
+    def update_object_data
+
+      identity_metadata = parse_identity_metadata
+      relationship_metadata = parse_relationship_metadata
+
+      digital_object_data = {
+          :digital_object_id => digital_object_id,
+          :home_repository => 'sdr'
+      }
+
+      sdr_object_data = {
+          :sdr_object_id => digital_object_id,
+          :object_type => identity_metadata[:object_type],
+          :object_label => identity_metadata[:object_label],
+          :governing_object => relationship_metadata[:governed_by],
+          :latest_version => storage_object.current_version_id
+      }
+
+      if version_id == 1
+        ArchiveCatalog.find_or_create_item(:digital_objects,digital_object_data)
+        ArchiveCatalog.find_or_create_item(:sdr_objects,sdr_object_data)
+      else
+        ArchiveCatalog.update_item(:sdr_objects, digital_object_id, sdr_object_data)
+      end
+
+    end
+
+    def update_version_data
+
+      sdr_object_version_data = {
+          :sdr_object_id => digital_object_id,
+          :sdr_version_id => version_id,
+          :replica_id => composite_key,
+          :ingest_date => version_inventory.inventory_datetime
+      }
+      ArchiveCatalog.find_or_create_item(:sdr_object_versions, sdr_object_version_data)
+
+
+      content = version_inventory.group('content')
+      metadata = version_inventory.group('metadata')
+      sdr_version_full = {
+          :sdr_object_id => digital_object_id,
+          :sdr_version_id => version_id,
+          :inventory_type => 'full',
+          :content_files => content.file_count,
+          :content_bytes => content.byte_count,
+          :content_blocks => content.block_count,
+          :metadata_files => metadata.file_count,
+          :metadata_bytes => metadata.byte_count,
+          :metadata_blocks => metadata.block_count
+      }
+      ArchiveCatalog.find_or_create_item(:sdr_version_stats, sdr_version_full)
+
+      content = version_additions.group('content')
+      metadata = version_additions.group('metadata')
+      sdr_version_delta = {
+          :sdr_object_id => digital_object_id,
+          :sdr_version_id => version_id,
+          :inventory_type => 'delta',
+          :content_files => content.file_count,
+          :content_bytes => content.byte_count,
+          :content_blocks => content.block_count,
+          :metadata_files => metadata.file_count,
+          :metadata_bytes => metadata.byte_count,
+          :metadata_blocks => metadata.block_count
+      }
+      ArchiveCatalog.find_or_create_item(:sdr_version_stats, sdr_version_delta)
+
     end
 
+
     # @return [String] The unique identifier for the digital object replica
     def replica_id
-      @replica_id ||= "#{sdr_object_id.split(':').last}-#{moab_object_version.version_name}"
+      @replica_id ||= "#{storage_object.digital_object_id.split(':').last}-#{version_name}"
     end
 
     # @return [Replica] The Replica of the object version that is archived to tape, etc
@@ -52,7 +151,7 @@ module Replication
       bag_dir = replica.replica_pathname
       bag = BagitBag.create_bag(bag_dir)
       bag.bag_checksum_types = [:sha256]
-      bag.add_payload_tarfile("#{replica_id}.tar",moab_object_version.version_pathname, moab_object_version.storage_object.object_pathname.parent)
+      bag.add_payload_tarfile("#{replica_id}.tar",version_pathname, storage_object.object_pathname.parent)
       bag.write_bag_info_txt
       bag.write_manifest_checksums('tagmanifest', bag.generate_tagfile_checksums)
       bag

data/lib/sdr_replication.rb

@@ -20,6 +20,7 @@ require 'replication/file_fixity'
 require 'replication/fixity'
 require 'replication/operating_system'
 require 'replication/replica'
+require 'replication/sdr_object'
 require 'replication/sdr_object_version'
 require 'replication/tarfile'
 include Replication

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sdr-replication
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Richard Anderson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-24 00:00:00.000000000 Z
+date: 2014-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json_pure
@@ -26,6 +26,20 @@ dependencies:
         version: '0'
 - !ruby/object:Gem::Dependency
   name: moab-versioning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: systemu
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: systemu
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,13 +67,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rest-client
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -161,13 +175,11 @@ files:
 - lib/libdir.rb
 - lib/replication/archive_catalog.rb
 - lib/replication/bagit_bag.rb
-- lib/replication/command_consumer.rb
-- lib/replication/command_producer.rb
-- lib/replication/dpn_check_rep.rb
 - lib/replication/file_fixity.rb
 - lib/replication/fixity.rb
 - lib/replication/operating_system.rb
 - lib/replication/replica.rb
+- lib/replication/sdr_object.rb
 - lib/replication/sdr_object_version.rb
 - lib/replication/tarfile.rb
 - lib/sdr_replication.rb
@@ -190,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 2.2.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 2.3.0
 signing_key: 
 specification_version: 4
 summary: Core methods for support of SDR Preservation Core replication

data/lib/replication/command_consumer.rb

@@ -1,55 +0,0 @@
-require "rubygems"
-require "amqp"
-require "amqp/extensions/rabbitmq"
-require "yaml"
-
-EventMachine.run do
-
-
-	 connection_params =                                        {
-		 :host      => "127.0.0.1",
-		 :port      => 5672,
-		 :user      => "guest",
-		 :pass      => "guest",
-		 :vhost     => "/",
-		 :ssl       => false,
-		 :frame_max => 131072
-	 }
-
-	 connection = AMQP.connect(connection_params)
-	 channel    = AMQP::Channel.new(connection, :auto_recovery => true)
-
-
-	 channel.prefetch(1)
-
-# Acknowledgements are good for letting the server know
-# that the task is finished. If the consumer doesn't send
-# the acknowledgement, then the task is considered to be unfinished
-# and will be requeued when consumer closes AMQP connection (because of a crash, for example).
-
-	 channel.queue("sdr.replication.notification", :durable => true, :auto_delete => false).subscribe(:ack => true) do |metadata, payload|
-		  case metadata.type
-			   when "sdr_replication_notification"
-					data = YAML.load(payload)
-					puts "sdr_replication_notification request with #{data.inspect}"
-
-					message = "'" + payload + "'"
-					header = "'" + metadata.attributes.to_yaml + "'"
-
-
-					(pid = fork) ? Process.detach(pid) : exec("./dpn_check_rep.rb #{message} #{header}")
-					#shellout = "./dpn_check_rep.rb '#{data[:name]}' #{data[:size]}"
-					#puts "dpn_check_rep.rb Executing #{shellout}"; system(shellout)
-					puts "Call to SDR DPN replication request done"
-					puts
-
-			   when "dpn_replication_notification"
-			   else
-					puts "[commands] Unknown command: #{metadata.type}"
-		  end
-
-		  # message is processed, acknowledge it so that broker discards it
-		  metadata.ack
-	 end
-	 Signal.trap("INT") { connection.close { EventMachine.stop } }
-end

data/lib/replication/command_producer.rb

@@ -1,105 +0,0 @@
-require "amqp"
-require "amqp/extensions/rabbitmq"
-require "yaml"
-require "json"
-require "securerandom"
-require "time"
-
-
-connection_params =   {
-	:host      => "127.0.0.1",
-	:port      => 5672,
-	:user      => "guest",
-	:pass      => "guest",
-	:vhost     => "/",
-	:ssl       => false,
-	:frame_max => 131072
-}
-
-#usually send one message only, rely on message broker to recover
-doit = 1
-
-EventMachine.run do
-	 sleep(0.5)
-
-	 connection = AMQP.connect(connection_params)
-	 channel    = AMQP::Channel.new(connection)
-
-
-# publish new commands every 2 seconds, count + one times
-	 count = 0
-	 EventMachine.add_periodic_timer(2.0) do
-		  count += 1
-		  if count >= doit then
-			   EM.stop_event_loop
-		  end
-		  puts "count = #{count}"
-		  puts "Messaging that sdr has content for replication, parameters are name and size."
-		  payload = { :name => "12345.tar", :size => 4096 }.to_yaml
-
-		  correlation_id = SecureRandom.random_number(10000)
-		  utctime = Time.now.utc
-		  ttl = 20   #time for transaction to live
-
-		  channel.default_exchange.publish(payload,
-										   :arguments => { "x-message-ttl" => 10 },
-										   :type        => "sdr_replication_notification",
-										   :routing_key => "sdr.replication.notification",
-										   :reply_to 	=> "dpn.replication.reply",
-										   :correlation_id => correlation_id,
-										   :timestamp  => utctime.to_i
-		  )
-
-	 end
-
-	 puts "Sending notification message"
-	 Signal.trap("INT") { connection.close { EventMachine.stop } }
-end
-
-
-
-exit
-
-#listen for response to previous message
-EventMachine.run do
-	 sleep(0.5)
-
-	 connection = AMQP.connect(connection_params)
-	 channel    = AMQP::Channel.new(connection, :auto_recovery => true)
-	 channel.prefetch(1)
-
-	 count = 0
-	 EventMachine.add_periodic_timer(10.0) do
-		  count += 1
-		  puts "waiting....."
-		  if count > doit+1 then
-			   EM.stop_event_loop
-		  end
-	 end
-
-	 channel.queue("dpn.replication.notification", :durable => true, :auto_delete => false).subscribe(:ack => true) do |metadata, payload|
-		  case metadata.type
-			   when "dpn_replication_ack"
-					message = YAML.load(payload)
-					if message[:message] == "dpn_available_reply" && message[:acknak] == 'ack'  then
-						 puts "[sdr dpn says to go ahead and replicate content to temp area."
-						 # (pid = fork) ? Process.detach(pid) : exec("start_replication_workflow.rb
-						 puts "Start workflow to copy content to sdr dpn for replication. "
-					else
-						 puts "no replication possible, DPN say's no!"
-						 # (pid = fork) ? Process.detach(pid) : exec("replication_denied_workflow.rb
-					end
-
-			   when "dpn_replication_notification"
-					puts "DPN wants to replicate"
-			   else
-					puts "[commands] Unknown command: #{metadata.type}"
-		  end
-
-		  # message is processed, acknowledge it so that broker discards it
-		  metadata.ack
-	 end
-	 puts "Waiting for 'ack' notification message"
-	 Signal.trap("INT") { connection.close { EventMachine.stop } }
-
-end

data/lib/replication/dpn_check_rep.rb

@@ -1,83 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: utf-8
-$: << "./"
-require 'securerandom'
-require "rubygems"
-require 'time'
-require 'date'
-require "eventmachine"
-require "amqp"
-require "amqp/extensions/rabbitmq"
-require "yaml"
-
-
-connection_params =   {
-	:host      => "127.0.0.1",
-	:port      => 5672,
-	:user      => "guest",
-	:pass      => "guest",
-	:vhost     => "/",
-	:ssl       => false,
-	:frame_max => 131072
-}
-
-#arguments passed are json strings. payload, header.
-puts "*************************************"
-puts "In dpn_check_rep.rb"
-if ARGV.length != 2 then
-	 exit 5001 #should not get here, but this is an error condition
-end
-
-message = YAML.load(ARGV[0])
-header  = YAML.load(ARGV[1])
-
-if header[:type] !=  "sdr_replication_notification"  then
-	 exit 5002 #bad message type
-end
-
-puts "sdr_replication_notification message #{message.inspect}"
-puts "sdr_replication_notification header #{header.inspect}"
-
-puts "checking to see if DPN can replicate content from SDR"
-
-#
-#Do checking!
-			 #
-
-sleep 4.5      #simulate checking
-
-
-exit
-
-EventMachine.run do
-	 sleep(0.5)
-	 connection = AMQP.connect(connection_params)
-	 channel    = AMQP::Channel.new(connection)
-
-	 count = 0
-	 EventMachine.add_periodic_timer(2.0) do
-		  count += 1
-		  if count > 2 then
-			   EM.stop_event_loop
-		  end
-	 end
-
-	 EM.add_timer(1.0) do
-		  puts "Working: #{Time.now}"
-		  STDOUT.flush
-	 end
-
-	 puts "Acking that content can be replicated"
-	 payload = { :message => "dpn_available_reply", :acknak => 'ack' }.to_yaml
-
-	 channel.default_exchange.publish(payload,
-									  :type        => "dpn_replication_ack",
-									  :routing_key => header[:reply_to],
-									  :reply_to 	=> "dpn.err.notification",
-									  :correlation_id => header[:correlation_id],
-									  :timestamp  => header[:timestamp])
-
-	 puts "Sending ack notification message"
-	 Signal.trap("INT") { connection.close { EventMachine.stop } }
-
-end