fedora-migrate 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1c6f75dc48d80e784199985d948420c2c449de4
-  data.tar.gz: 0002dcb8c3e0508510359bc427c70cd2cf0b1f76
+  metadata.gz: ce0bad9581c3b5b080d0a73a26792ee7942ce1b3
+  data.tar.gz: 79932ebfe5726d0ef5dd67b1a5598847ed056175
 SHA512:
-  metadata.gz: cc62419d71580ecda28950eea55ada46b397d517730eff12f3d3bdbfb3b184565a1499f8ec2e219d24f61395b7be42ea2f357b18ec772d4887b6706786a97301
-  data.tar.gz: e2adbf133dc201a4176958a53b83fe9dfc32351dfd59a72651e73ac527b4be01adf8ee762e76ac6233bb118b88daca67af1091ac4e09291fb2b888d28a56d4f6
+  metadata.gz: 79d5f57909a5a92ea9155b6b09b2e85733df39212e62e9c227e9732d5c5eeb8c286e1c8905c52fb40d511e8c5641568c763327405593bf1c2583ef97e0a81222
+  data.tar.gz: 5a57b9afc653f14a33dcffaad44fc15bd5c9cde314b785e866b2786d8af92336dcfc8bcb212e24362cc6f7162e599bd9b7b41f0cd43761d6f26f23da0effab01

data/.gitignore

@@ -15,3 +15,4 @@ mkmf.log
 jetty
 .ruby-version
 .rvmrc
+report.json

data/lib/fedora-migrate.rb

@@ -19,6 +19,7 @@ module FedoraMigrate
   autoload :Hooks
   autoload :Logger
   autoload :MigrationOptions
+  autoload :MigrationReport
   autoload :Mover
   autoload :ObjectMover
   autoload :Permissions
@@ -33,17 +34,13 @@ module FedoraMigrate
   autoload :TripleConverter
 
   class << self
-    attr_reader :fedora_config, :config_options, :source
+    attr_reader :fedora_config, :source
     attr_accessor :configurator
 
     def fedora_config
       @fedora_config ||= ActiveFedora::Config.new(configurator.fedora3_config)
     end
 
-    def config_options
-      @config_options ||= "comming soon!"
-    end
-
     def source
       @source ||= FedoraMigrate::RubydoraConnection.new(fedora_config.credentials)
     end
@@ -56,6 +53,7 @@ module FedoraMigrate
       migrator = FedoraMigrate::RepositoryMigrator.new(args[:namespace], args[:options])
       migrator.migrate_objects
       migrator.migrate_relationships
+      migrator
     end
 
   end

data/lib/fedora_migrate/content_mover.rb

@@ -1,24 +1,44 @@
 module FedoraMigrate
   class ContentMover < Mover
 
+    include DatastreamVerification
+
+    class Report
+      attr_accessor :name, :mime_type, :original_date, :error
+      def success?
+        error.nil?
+      end
+    end
+
     def migrate
-      return nil_content_message if source.content.nil?
+      return report if nil_source
       move_content
+      report_results
       insert_date_created_by_application
+      super
+    end
+
+    def results_report
+      Report.new
     end
 
     def move_content
       target.content = source.content
       target.original_name = source.label.try(:gsub, /"/, '\"')
       target.mime_type = source.mimeType
-      Logger.info "#{target.inspect}"
       save
+      report.error = "Failed checksum" unless valid?
+    end
+
+    def report_results
+      report.name = target.original_name
+      report.mime_type = target.mime_type
     end
 
     def insert_date_created_by_application
-      result = perform_sparql_insert
-      return true if result.status == 204
-      raise FedoraMigrate::Errors::MigrationError, "problem with sparql #{result.status} #{result.body}"
+      result = perform_sparql_insert 
+      report.original_date = source.createDate.iso8601
+      report.error = "There was a problem with sparql #{result.status} #{result.body}" unless result.status == 204
     end
 
     def sparql_insert
@@ -35,15 +55,17 @@ EOF
 
     private
 
-    def nil_content_message
-      Logger.info "datastream '#{source.dsid}' is nil. It's probably defined in the target but not present in the source"
-      true
-    end
-
     def perform_sparql_insert
       ActiveFedora.fedora.connection.patch(target.metadata.metadata_uri, sparql_insert, "Content-Type" => "application/sparql-update")
     end
 
+    def nil_source
+      if source.content.nil?
+        report.error = "Nil source -- it's probably defined in the target but not present in the source"
+        true
+      end
+    end
+
   end
 
 end

data/lib/fedora_migrate/datastream_mover.rb

@@ -1,8 +1,6 @@
 module FedoraMigrate
   class DatastreamMover < Mover
 
-    include DatastreamVerification
-
     attr_accessor :versionable
 
     def post_initialize
@@ -25,6 +23,7 @@ module FedoraMigrate
       before_datastream_migration
       migrate_datastream
       after_datastream_migration
+      super
     end
 
     private
@@ -40,22 +39,20 @@ module FedoraMigrate
     # Reload the target, otherwise the checksum is nil
     def migrate_current
       migrate_content
-      target.reload
-      valid?
+      target.reload if report.last.success?
     end
 
     # Rubydora stores the versions array as the most recent first. We explicitly sort them according to createDate
     def migrate_versions
       source.versions.sort { |a,b| a.createDate <=> b.createDate }.each do |version|
         migrate_content(version)
-        target.create_version unless application_creates_versions?
-        valid?(version)
+        target.create_version if report.last.success? && !application_creates_versions?
       end
     end
 
     def migrate_content datastream=nil
       datastream ||= source
-      FedoraMigrate::ContentMover.new(datastream, target).migrate
+      report << FedoraMigrate::ContentMover.new(datastream, target).migrate
     end
 
   end

data/lib/fedora_migrate/datastream_verification.rb

@@ -4,9 +4,7 @@ module FedoraMigrate::DatastreamVerification
 
   def valid? datastream=nil
     @datastream = datastream || @source
-    check = has_matching_checksums? || has_matching_nokogiri_checksums?
-    FedoraMigrate::Logger.warn "#{@datastream.pid} datastream #{@datastream.dsid} validation failed" unless check
-    check
+    has_matching_checksums? || has_matching_nokogiri_checksums?
   end
 
   def has_matching_checksums?

data/lib/fedora_migrate/dates_mover.rb

@@ -1,13 +1,26 @@
 module FedoraMigrate
   class DatesMover < Mover
 
+    Report = Struct.new(:uploaded, :modified)
+
     def migrate
-      if source.respond_to?(:createdDate) && target.respond_to?(:date_uploaded)
-        target.date_uploaded = source.createdDate
-      end
-      if source.respond_to?(:lastModifiedDate) && target.respond_to?(:date_modified)
-        target.date_modified = source.lastModifiedDate
-      end
+      migrate_date_uploaded if source.respond_to?(:createdDate) && target.respond_to?(:date_uploaded)
+      migrate_date_modified if source.respond_to?(:lastModifiedDate) && target.respond_to?(:date_modified)
+      super
+    end
+
+    def results_report
+      Report.new
+    end
+
+    def migrate_date_uploaded
+      target.date_uploaded = source.createdDate
+      report.uploaded = source.createdDate
+    end
+
+    def migrate_date_modified
+      target.date_modified = source.lastModifiedDate
+      report.modified = source.lastModifiedDate
     end
 
   end

data/lib/fedora_migrate/migration_report.rb

@@ -0,0 +1,40 @@
+module FedoraMigrate
+  class MigrationReport
+
+    attr_accessor :results
+
+    def initialize report=nil
+      @results = report.nil? ? Hash.new : JSON.parse(File.read(report))
+    end
+
+    def empty?
+      results.empty?
+    end
+
+    def failed_objects
+      results.keys.map { |k| k unless results[k]["status"] }.compact
+    end
+
+    def failures
+      failed_objects.count
+    end
+
+    def total_objects
+      results.keys.count
+    end
+
+    def report_failures output = String.new
+      failed_objects.each do |k|
+        output << "#{k}:\n\tobject: #{results[k]["object"]}\n\trelationships: #{results[k]["relationships"]}\n\n"
+      end
+      output
+    end
+
+    def save path=nil
+      json = JSON.load(results.to_json)
+      file = path.nil? ? "report.json" : File.join(path,"report.json")
+      File.write(file, JSON.pretty_generate(json))
+    end
+
+  end
+end

data/lib/fedora_migrate/mover.rb

@@ -4,24 +4,29 @@ module FedoraMigrate
     include MigrationOptions
     include Hooks
 
-    attr_accessor :target, :source
+    attr_accessor :target, :source, :report
 
     def initialize *args
       @source = args[0]
       @target = args[1]
       @options = args[2]
+      @report = results_report
       post_initialize
     end
 
     def post_initialize
     end
 
+    def results_report
+      []
+    end
+
+    def migrate
+      report
+    end
+
     def save
-      if target.save
-        Logger.info "success for target UID #{target_description}"
-      else
-        raise FedoraMigrate::Errors::MigrationError, "Failed to save target: #{target_errors}"
-      end
+      raise FedoraMigrate::Errors::MigrationError, "Failed to save target: #{target_errors}" unless target.save
     end
 
     def target_errors
@@ -32,14 +37,6 @@ module FedoraMigrate
       end
     end
 
-    def target_description
-      if target.respond_to?(:id)
-        target.id
-      else
-        target.inspect
-      end
-    end
-
     def id_component object=nil
       object ||= source
       raise FedoraMigrate::Errors::MigrationError, "can't get the id component without an object" if object.nil?

data/lib/fedora_migrate/object_mover.rb

@@ -3,13 +3,16 @@ module FedoraMigrate
 
     RIGHTS_DATASTREAM = "rightsMetadata".freeze
 
+    ContentDatastreamReport = Struct.new(:ds, :versions)
+    RDFDatastreamReport = Struct.new(:ds, :status)
+    Report = Struct.new(:id, :class, :content_datastreams, :rdf_datastreams, :permissions, :dates)
+
     def migrate
       prepare_target
       conversions.collect { |ds| convert_rdf_datastream(ds) }
-      migrate_content_datastreams
-      migrate_permissions
-      migrate_dates
+      migrate_datastreams
       complete_target
+      super
     end
 
     def post_initialize
@@ -17,19 +20,32 @@ module FedoraMigrate
       create_target_model if target.nil?
     end
 
+    def results_report
+      Report.new.tap do |report|
+        report.content_datastreams = []
+        report.rdf_datastreams = []
+      end
+    end
+
     def prepare_target
-      Logger.info "running before_object_migration hooks"
+      report.class = target.class.to_s
+      report.id = target.id
       before_object_migration
     end
 
     def complete_target
-      Logger.info "running after_object_migration hooks"
       after_object_migration
       save
     end
 
     private
 
+    def migrate_datastreams
+      migrate_content_datastreams
+      migrate_permissions
+      migrate_dates
+    end
+
     # We have to call save before migrating content datastreams, otherwise versions aren't recorded
     # TODO: this will fail if required fields are defined in a descMetadata datastream that is not
     # converted to RDF (issue #8)
@@ -37,14 +53,14 @@ module FedoraMigrate
       save
       target.attached_files.keys.each do |ds|
         mover = FedoraMigrate::DatastreamMover.new(source.datastreams[ds.to_s], target.attached_files[ds.to_s], options)
-        mover.migrate
+        report.content_datastreams << ContentDatastreamReport.new(ds, mover.migrate)
       end
     end
 
     def convert_rdf_datastream ds
       if source.datastreams.key?(ds)
         mover = FedoraMigrate::RDFDatastreamMover.new(datastream_content(ds), target)
-        mover.migrate
+        report.rdf_datastreams << RDFDatastreamReport.new(ds, mover.migrate)
       end
     end
 
@@ -55,12 +71,12 @@ module FedoraMigrate
     def migrate_permissions
       if source.datastreams.keys.include?(RIGHTS_DATASTREAM) && target.respond_to?(:permissions)
         mover = FedoraMigrate::PermissionsMover.new(source.datastreams[RIGHTS_DATASTREAM], target)
-        mover.migrate
+        report.permissions = mover.migrate
       end
     end
 
     def migrate_dates
-      FedoraMigrate::DatesMover.new(source, target).migrate
+      report.dates = FedoraMigrate::DatesMover.new(source, target).migrate
     end
 
     def create_target_model

data/lib/fedora_migrate/permissions_mover.rb

@@ -13,13 +13,13 @@ module FedoraMigrate
 
     def migrate
       FedoraMigrate::Permissions.instance_methods.each do |permission|
-        Logger.info "setting #{permission} to #{self.send(permission)}"
+        report << "#{permission} = #{self.send(permission)}"
         target.send(permission.to_s+"=", self.send(permission))
       end
       save
+      super
     end
 
-
     private
 
     def datastream_from_content ds = FedoraMigrate::RightsMetadata.new

data/lib/fedora_migrate/rdf_datastream_mover.rb

@@ -4,11 +4,11 @@ module FedoraMigrate
   class RDFDatastreamMover < Mover
 
     def migrate
-      Logger.info "converting datastream '#{source.dsid}' to RDF"
       before_rdf_datastream_migration
       migrate_rdf_triples
       after_rdf_datastream_migration
       save
+      super
     end
 
     def migrate_rdf_triples

data/lib/fedora_migrate/rels_ext_datastream_mover.rb

@@ -7,6 +7,7 @@ module FedoraMigrate
       migrate_statements
       target.ldp_source.update
       update_index
+      super
     end
 
     def post_initialize
@@ -19,7 +20,9 @@ module FedoraMigrate
 
     def migrate_statements
       statements.each do |statement|
-        target.ldp_source.graph << [target.rdf_subject, migrate_predicate(statement.predicate), migrate_object(statement.object)]
+        triple = [target.rdf_subject, migrate_predicate(statement.predicate), migrate_object(statement.object)]
+        target.ldp_source.graph << triple
+        report << triple.join("--")
       end
     end
 
@@ -43,7 +46,7 @@ module FedoraMigrate
 
     def has_missing_object?(statement)
       return false if ActiveFedora::Base.exists?(id_component(statement.object))
-      Logger.warn "#{source.pid} could not migrate relationship #{statement.predicate} because #{statement.object} doesn't exist in Fedora 4"
+      report << "could not migrate relationship #{statement.predicate} because #{statement.object} doesn't exist in Fedora 4"
       true
     end