fedora-migrate 0.0.1 → 0.1.0

data/lib/fedora_migrate/datastream_verification.rb

@@ -0,0 +1,36 @@
+module FedoraMigrate::DatastreamVerification
+  
+  attr_accessor :datastream
+
+  def valid? datastream=nil
+    @datastream = datastream || @source
+    check = has_matching_checksums? || has_matching_nokogiri_checksums?
+    FedoraMigrate::Logger.warn "#{@datastream.pid} datastream #{@datastream.dsid} validation failed" unless check
+    check
+  end
+
+  def has_matching_checksums?
+    datastream.checksum == target_checksum || checksum(datastream.content) == target_checksum
+  end
+
+  def has_matching_nokogiri_checksums?
+    return false unless datastream.mimeType == "text/xml"
+    checksum(Nokogiri::XML(datastream.content).to_xml) == checksum(Nokogiri::XML(target_content).to_xml)
+  end
+
+  private 
+
+  def target_checksum
+    target.digest.first.to_s.split(/:/).last
+  end
+
+  # In some cases, the data is in ldp_source but target.content is empty, so we check both places
+  def target_content
+    target.content.empty? ? target.ldp_source.content : target.content
+  end
+
+  def checksum content
+    Digest::SHA1.hexdigest(content)
+  end
+
+end

data/lib/fedora_migrate/dates_mover.rb

@@ -0,0 +1,14 @@
+module FedoraMigrate
+  class DatesMover < Mover
+
+    def migrate
+      if source.respond_to?(:createdDate) && target.respond_to?(:date_uploaded)
+        target.date_uploaded = source.createdDate
+      end
+      if source.respond_to?(:lastModifiedDate) && target.respond_to?(:date_modified)
+        target.date_modified = source.lastModifiedDate
+      end
+    end
+
+  end
+end

data/lib/fedora_migrate/hooks.rb

@@ -1,11 +1,34 @@
+# Override this methods to perform additional actions before and after
+# migation of objects and datastreams.
+#
+# To do so, simply define a FedoraMigrate::Hooks module anywhere in
+# you application and substitute methods for the ones listed below
 module FedoraMigrate
   module Hooks
 
+    # Called from FedoraMigrate::ObjectMover
     def before_object_migration
     end
 
+    # Called from FedoraMigrate::ObjectMover
     def after_object_migration
     end
 
+    # Called from FedoraMigrate::RDFDatastreamMover
+    def before_rdf_datastream_migration
+    end
+
+    # Called from FedoraMigrate::RDFDatastreamMover
+    def after_rdf_datastream_migration
+    end
+
+    # Called from FedoraMigrate::DatastreamMover
+    def before_datastream_migration
+    end
+
+    # Called from FedoraMigrate::DatastreamMover
+    def after_datastream_migration
+    end
+
   end
 end

data/lib/fedora_migrate/migration_options.rb

@@ -7,5 +7,23 @@ module FedoraMigrate
       self.conversions = options.nil? ? [] : [options[:convert]].flatten      
     end
 
+    def forced?
+      option_true?(:force)
+    end
+
+    def not_forced?
+      !forced?
+    end
+
+    def application_creates_versions?
+      option_true?(:application_creates_versions)
+    end
+    
+    private
+    
+    def option_true?(name)
+      !!(options && options[name])
+    end
+
   end
 end

data/lib/fedora_migrate/mover.rb

@@ -39,6 +39,18 @@ module FedoraMigrate
         target.inspect
       end
     end
+
+    def id_component object=nil
+      object ||= source
+      raise FedoraMigrate::Errors::MigrationError, "can't get the id component without an object" if object.nil?
+      self.class.id_component(object)
+    end
+
+    def self.id_component object
+      return object.pid.split(/:/).last if object.kind_of?(Rubydora::DigitalObject)
+      return object.to_s.split(/:/).last if object.respond_to?(:to_s)
+      nil
+    end
     
   end
 end

data/lib/fedora_migrate/object_mover.rb

@@ -5,9 +5,10 @@ module FedoraMigrate
 
     def migrate
       prepare_target
-      migrate_content_datastreams
       conversions.collect { |ds| convert_rdf_datastream(ds) }
+      migrate_content_datastreams
       migrate_permissions
+      migrate_dates
       complete_target
     end
 
@@ -19,7 +20,6 @@ module FedoraMigrate
     def prepare_target
       Logger.info "running before_object_migration hooks"
       before_object_migration
-      save
     end
 
     def complete_target
@@ -30,20 +30,28 @@ module FedoraMigrate
 
     private
 
+    # We have to call save before migrating content datastreams, otherwise versions aren't recorded
+    # TODO: this will fail if required fields are defined in a descMetadata datastream that is not
+    # converted to RDF (issue #8)
     def migrate_content_datastreams
+      save
       target.attached_files.keys.each do |ds|
-        mover = FedoraMigrate::DatastreamMover.new(source.datastreams[ds.to_s], target.attached_files[ds.to_s])
+        mover = FedoraMigrate::DatastreamMover.new(source.datastreams[ds.to_s], target.attached_files[ds.to_s], options)
         mover.migrate
       end
     end
 
     def convert_rdf_datastream ds
-      if source.datastreams.keys.include?(ds)
-        mover = FedoraMigrate::RDFDatastreamMover.new(source.datastreams[ds.to_s], target)
+      if source.datastreams.key?(ds)
+        mover = FedoraMigrate::RDFDatastreamMover.new(datastream_content(ds), target)
         mover.migrate
       end
     end
 
+    def datastream_content(dsid)
+      source.datastreams[dsid.to_s]
+    end
+
     def migrate_permissions
       if source.datastreams.keys.include?(RIGHTS_DATASTREAM) && target.respond_to?(:permissions)
         mover = FedoraMigrate::PermissionsMover.new(source.datastreams[RIGHTS_DATASTREAM], target)
@@ -51,12 +59,15 @@ module FedoraMigrate
       end
     end
 
+    def migrate_dates
+      FedoraMigrate::DatesMover.new(source, target).migrate
+    end
+
     def create_target_model
-      afmodel = source.models.map { |m| m if m.match(/afmodel/) }.compact.first.split(/:/).last
-      Logger.info "found #{afmodel} in source object #{source.pid}"
-      @target = afmodel.constantize.new(id: source.pid.split(/:/).last)
+      builder = FedoraMigrate::TargetConstructor.new(source.models).build
+      raise FedoraMigrate::Errors::MigrationError, "No qualified targets found in #{source.pid}" if builder.target.nil?
+      @target = builder.target.new(id: id_component)
     end
 
   end
-  
 end

data/lib/fedora_migrate/rdf_datastream_mover.rb

@@ -1,28 +1,45 @@
+require 'rchardet'
+
 module FedoraMigrate
   class RDFDatastreamMover < Mover
 
     def migrate
       Logger.info "converting datastream '#{source.dsid}' to RDF"
-      parse_rdf_triples
-      force_attribute_change
+      before_rdf_datastream_migration
+      migrate_rdf_triples
+      after_rdf_datastream_migration
       save
     end
 
-    def parse_rdf_triples
-      parser = FedoraMigrate::RDFDatastreamParser.new(target.uri, source.content)
-      parser.parse
-      parser.statements.each do |statement|
-        target.resource << statement
-      end
+    def migrate_rdf_triples
+      target.resource << updated_graph
     end
 
-    # See projecthydra/active_fedora#540
-    # Forcibly setting each attribute's changed status to true
-    def force_attribute_change
-      target.class.delegated_attributes.keys.each do |term|
-        target.send(term+"_will_change!")
+    private
+
+      def updated_graph
+        reader.new(updated_datastream_content)
+      end
+
+      def updated_datastream_content
+        correct_encoding(datastream_content).gsub(/<.+#{source.pid}>/,"<#{target.uri}>")
       end
-    end
 
+      def datastream_content
+        source.content
+      end
+
+      # Scholarsphere has some ISO-8859 encoded data, which violates the NTriples spec.
+      # Here we correct that.
+      def correct_encoding(input)
+        input.encode!(Encoding::UTF_8)
+      rescue Encoding::UndefinedConversionError
+        cd = ::CharDet.detect(input)
+        input.force_encoding(Encoding.find(cd["encoding"].upcase)).encode!(Encoding::UTF_8)
+      end
+
+      def reader
+        RDF::Reader.for(:ntriples)
+      end
   end
 end

data/lib/fedora_migrate/rels_ext_datastream_mover.rb

@@ -1,89 +1,55 @@
-require 'rubydora'
 module FedoraMigrate
   class RelsExtDatastreamMover < Mover
 
-    attr_accessor :relationships, :ng_xml, :subject
-
-    RELS_EXT = Rubydora::RelationshipsMixin::RELS_EXT
     RELS_EXT_DATASTREAM = "RELS-EXT".freeze
 
-    def post_initialize
-      retrieve_subject
-      @relationships ||= {}
-      @ng_xml = Nokogiri::XML(source.datastreams[RELS_EXT_DATASTREAM].content)
-      parse_relationships if has_relationships?
-    end
-
-    def has_relationships?
-      source.datastreams.keys.include?(RELS_EXT_DATASTREAM)
+    def migrate
+      migrate_statements
+      target.ldp_source.update
+      update_index
     end
 
-    def migrate
-      relationships.each do |predicate, objects|
-        unless objects.empty?
-          if is_singular?(predicate.to_s)
-            objects.collect { |object| migrate_incomming_relationship(predicate, object) }
-          else
-            migrate_outgoing_relationship(predicate, objects)
-          end
-        end
-      end
+    def post_initialize
+      @target ||= ActiveFedora::Base.find(id_component)
+    rescue ActiveFedora::ObjectNotFoundError
+      raise FedoraMigrate::Errors::MigrationError, "Target object was not found in Fedora 4. Did you migrate it?"
     end
 
     private
 
-    # because of projecthydra/rubydora#90 
-    def parse_relationships
-      RELS_EXT.keys.each do |key|
-        query = "//ns0:"+RELS_EXT[key].split(/#/).last
-        relationships[key.to_sym] = query_results(query)
+    def migrate_statements
+      statements.each do |statement|
+        target.ldp_source.graph << [target.rdf_subject, migrate_predicate(statement.predicate), migrate_object(statement.object)]
       end
     end
 
-    def query_results query, results = Array.new
-      ng_xml.xpath(query).each do |predicate|
-        results << retrieve_object(predicate.attribute("resource").text.split(/:/).last)
-      end
-      return results
+    def update_index
+      target.reload
+      target.update_index
     end
 
-    def retrieve_subject
-      @subject = ActiveFedora::Base.find(source.pid.split(/:/).last)
-    rescue ActiveFedora::ObjectNotFoundError
-      raise FedoraMigrate::Errors::MigrationError, "Source was not found in Fedora4. Did you migrated it?"
+    def graph
+      @graph ||= RDF::Graph.new { |g| g.from_rdfxml(source.datastreams[RELS_EXT_DATASTREAM].content) }
     end
 
-    def retrieve_object id
-      object = ActiveFedora::Base.find(id)
-    rescue ActiveFedora::ObjectNotFoundError
-      raise FedoraMigrate::Errors::MigrationError, "Could not find object with id #{id}"
+    # Override this if any predicate transformation is needed
+    def migrate_predicate(fc3_uri)
+      fc3_uri
     end
 
-    # TODO: This is problematic and may not work in all situations (issue #7)
-    def migrate_incomming_relationship predicate, object
-      Logger.info "adding #{subject.id} to #{object.id} with predicate #{predicate.to_s}"
-      object.reflections.each do |key, association|
-        unless association.predicate.to_s.split(/#/).empty?
-          if association.predicate.to_s.split(/#/).last.gsub(/is/,"").underscore == predicate.to_s
-            object.send(key.to_s) << subject
-          end
-        end
-      end
+    def migrate_object(fc3_uri)
+      RDF::URI.new(ActiveFedora::Base.id_to_uri(id_component(fc3_uri)))
     end
 
-    # TODO: Very stinky... needs a different approach (issue #7)
-    def migrate_outgoing_relationship predicate, objects
-      Logger.info "adding #{objects.count.to_s} members to #{subject.id} with predicate #{predicate.to_s}"
-      subject.reflections.each do |key, association|
-        if key.to_s.match(/_ids$/)
-          subject.send(key.to_s+"=", objects.collect { |o| o.id })
-          subject.save
-        end
-      end
+    def has_missing_object?(statement)
+      return false if ActiveFedora::Base.exists?(id_component(statement.object))
+      Logger.warn "#{source.pid} could not migrate relationship #{statement.predicate} because #{statement.object} doesn't exist in Fedora 4"
+      true
     end
 
-    def is_singular?(str)
-      str.singularize == str
+    # All the graph statements except hasModel and those with missing objects
+    def statements
+      graph.statements.reject { |stmt| stmt.predicate == ActiveFedora::RDF::Fcrepo::Model.hasModel || has_missing_object?(stmt) }
     end
 
   end

data/lib/fedora_migrate/repository_migrator.rb

@@ -3,50 +3,55 @@ module FedoraMigrate
 
     include MigrationOptions
 
-    attr_accessor :source_objects, :results, :namespace
+    attr_accessor :source_objects, :namespace, :failed
 
     def initialize namespace = nil, options = {}
       @namespace = namespace || repository_namespace
       @options = options
+      @failed = 0
       @source_objects = get_source_objects
-      @results = []
       conversion_options
     end
 
+    # TODO: need a reporting mechanism for results (issue #4)
     def migrate_objects
-      source_objects.each do |source|
-        Logger.info "Migrating source object #{source.pid}"
-        begin
-          results << { source.pid => [FedoraMigrate::ObjectMover.new(source, nil, options).migrate] }
-        rescue NameError => e
-          results << { source.pid => e.to_s }
-        rescue FedoraMigrate::Errors::MigrationError => e
-          results << { source.pid => e.to_s }
-        end
-      end
+      source_objects.each { |source| migrate_object(source) }
+      @failed == 0
     end
 
     # TODO: need a reporting mechanism for results (issue #4)
     def migrate_relationships
-      source_objects.each do |source|
-        Logger.info "Migrating relationships for source object #{source.pid}"
-        begin
-          FedoraMigrate::RelsExtDatastreamMover.new(source).migrate
-        rescue FedoraMigrate::Errors::MigrationError => e
-          results << { source.pid => e.to_s }
-        rescue ActiveFedora::AssociationTypeMismatch => e
-          results << { source.pid => e.to_s }
-        end
-      end
-    end
-
-    # TODO: page through all the objects (issue #6)
+      return "Relationship migration halted because #{failed.to_s} objects didn't migrate successfully." if failed > 0 && not_forced?
+      source_objects.each { |source| migrate_relationship(source) }
+      @failed == 0
+    end
+
     def get_source_objects
       FedoraMigrate.source.connection.search(nil).collect { |o| qualifying_object(o) }.compact
     end
 
     private
 
+    def migrate_object source
+      Logger.info "Migrating source object #{source.pid}"
+      FedoraMigrate::ObjectMover.new(source, nil, options).migrate
+    rescue StandardError => e
+      Logger.warn "#{source.pid} failed.\n#{error_message(e)}"
+      @failed = @failed + 1
+    end
+
+    def migrate_relationship source
+      Logger.info "Migrating relationships for source object #{source.pid}"
+      FedoraMigrate::RelsExtDatastreamMover.new(source).migrate
+    rescue StandardError => e
+      Logger.warn "#{source.pid} relationship migration failed.\n#{error_message(e)}"
+      @failed = @failed + 1
+    end
+
+    def error_message e
+      [e.inspect, e.backtrace.join("\n\t")].join("\n\t")
+    end
+
     def repository_namespace
       FedoraMigrate.source.connection.repository_profile["repositoryPID"]["repositoryPID"].split(/:/).first.strip
     end