fedora-migrate 0.0.1 → 0.1.0

data/spec/integration/content_versions_spec.rb

@@ -9,6 +9,20 @@ describe "Versioned content" do
     )
   end
 
+  let(:application_mover) do
+    FedoraMigrate::DatastreamMover.new(
+      FedoraMigrate.source.connection.find("sufia:rb68xc089").datastreams["content"], 
+      ExampleModel::VersionedContent.create.attached_files["content"],
+      { application_creates_versions: true }
+    )
+  end
+
+  it "calls the before and after hooks when migrating" do
+    expect(mover).to receive(:before_datastream_migration)
+    expect(mover).to receive(:after_datastream_migration)
+    mover.migrate
+  end
+
   context "with migrating versions" do
     subject do
       mover.migrate
@@ -21,7 +35,16 @@ describe "Versioned content" do
       expect(subject.mime_type).to eql "image/png"
       expect(subject.original_name).to eql "world.png"
     end
-  end
+    context "and the application creates the versions" do
+      subject do
+        application_mover.migrate
+        application_mover.target
+      end
+      it "FedoraMigrate creates no versions" do
+        expect(subject.versions.count).to eql 0
+      end
+    end
+  end 
 
   context "without migrating versions" do
     subject do

data/spec/integration/missing_relationships_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe "Collections with missing files" do
+
+  let(:collection)    { "x346dj04v" }
+  let(:files)         { ["x346dj06d", "x346dj08z"] }
+  let(:missing_file)  { "x346dj07p" }
+
+  before do
+    FedoraMigrate::ObjectMover.new(FedoraMigrate.find("scholarsphere:#{collection}"), ExampleModel::Collection.new(collection)).migrate
+    files.each { |f| FedoraMigrate::ObjectMover.new(FedoraMigrate.find("scholarsphere:#{f}"), ExampleModel::MigrationObject.new(f)).migrate }
+  end
+
+  context "when migrating relationships" do
+
+    let(:migrated_collection) { ExampleModel::Collection.first }
+    let(:error_message) do
+      "scholarsphere:#{collection} could not migrate relationship info:fedora/fedora-system:def/relations-external#hasCollectionMember because info:fedora/scholarsphere:#{missing_file} doesn't exist in Fedora 4"
+    end 
+
+    it "should only migrate existing relationships and log failed ones" do
+      expect(FedoraMigrate::Logger).to receive(:warn).with(error_message)
+      FedoraMigrate::RelsExtDatastreamMover.new(FedoraMigrate.find("scholarsphere:#{collection}")).migrate
+      expect(migrated_collection.members.count).to eql 2
+      expect(migrated_collection.member_ids).to_not include(missing_file)
+    end
+
+  end
+
+end

data/spec/integration/object_migration_spec.rb

@@ -8,7 +8,7 @@ describe "Migrating an object" do
   context "when the target model is provided" do
 
     let(:mover) { FedoraMigrate::ObjectMover.new source, ExampleModel::MigrationObject.new }
-    
+
     subject do
       mover.migrate
       mover.target
@@ -27,6 +27,18 @@ describe "Migrating an object" do
       expect(subject.edit_users).to include("jilluser@example.com")
     end
 
+    describe "objects with Om datastreams" do
+      let(:mover) { FedoraMigrate::ObjectMover.new(source, ExampleModel::OmDatastreamExample.new) }
+      subject do
+        mover.migrate
+        mover.target
+      end
+      it "should migrate the object without warnings" do
+        expect(FedoraMigrate::Logger).to_not receive(:warn)
+        expect(subject.characterization.ng_xml).to be_equivalent_to(fits_xml)
+      end
+    end
+
   end
 
   context "when we have to determine the model" do
@@ -64,7 +76,7 @@ describe "Migrating an object" do
         Object.send(:remove_const, :GenericFile) if defined?(GenericFile)
       end
       it "should fail" do
-        expect{mover.migrate}.to raise_error(NameError)
+        expect{mover.migrate}.to raise_error(FedoraMigrate::Errors::MigrationError)
       end
     end
 
@@ -74,7 +86,7 @@ describe "Migrating an object" do
 
     context "and we want to convert it to a provided model" do
       let(:mover) { FedoraMigrate::ObjectMover.new(source, ExampleModel::RDFObject.new, {convert: "descMetadata"}) }
-    
+
       subject do
         mover.migrate
         mover.target
@@ -90,13 +102,32 @@ describe "Migrating an object" do
         expect(subject.title).to eql(["world.png"])
       end
 
+      it "should migrate the createdDate and lastModifiedDate" do
+        # The value of lastModifiedDate will depend on when you loaded your test fixtures
+        expect(subject.date_modified).to eq source.lastModifiedDate
+        expect(subject.date_uploaded).to eq '2014-10-15T03:50:37.063Z'
+      end
+    end
+
+    context "with ISO-8859-1 characters" do
+      let(:problem_source) { FedoraMigrate.source.connection.find("scholarsphere:5712mc568") }
+      let(:mover) { FedoraMigrate::ObjectMover.new(problem_source, ExampleModel::RDFObject.new, {convert: "descMetadata"}) }
+      subject do
+        mover.migrate
+        mover.target
+      end
+
+      it "should migrate the content" do
+        expect(subject.description.first).to match(/^The relationship between school administrators and music teachers/)
+      end
+
     end
 
     context "and we want to convert multiple datastreas" do
 
-      # Need a fixture with two different datastreams for this test to be more effective      
+      # Need a fixture with two different datastreams for this test to be more effective
       let(:mover) { FedoraMigrate::ObjectMover.new(source, ExampleModel::RDFObject.new, {convert: ["descMetadata", "descMetadata"]}) }
-    
+
       subject do
         mover.migrate
         mover.target
@@ -107,6 +138,19 @@ describe "Migrating an object" do
       end
     end
 
+    context "with RDF errors" do
+      let(:problem_source) { FedoraMigrate.source.connection.find("scholarsphere:sf2686078") }
+      let(:mover) { FedoraMigrate::ObjectMover.new(problem_source, ExampleModel::RDFObject.new, {convert: "descMetadata"}) }
+      subject do
+        mover.migrate
+        mover.target
+      end
+
+      it "should migrate the content" do
+        expect(subject.title).to eql([" The \"Value Added\" in Editorial Acquisitions.pdf"])
+      end
+    end
+
   end
 
 end

data/spec/integration/rdf_migration_spec.rb

@@ -1,22 +1,47 @@
 require 'spec_helper'
 
 describe "Mirating RDF terms" do
+  let(:mover) { FedoraMigrate::RDFDatastreamMover.new(file, ExampleModel::RDFProperties.new) }
 
-  let(:mover) do
-    FedoraMigrate::RDFDatastreamMover.new(
-      FedoraMigrate.source.connection.find("sufia:xp68km39w").datastreams["descMetadata"], 
-      ExampleModel::RDFProperties.new
-    )
-  end
+  context "normal record" do
 
-  subject do
-    mover.migrate
-    mover.target
-  end
+    let(:file) { FedoraMigrate.source.connection.find("sufia:xp68km39w").datastreams["descMetadata"] }
+
+    it "should call the before and after hooks when migrating" do
+      expect(mover).to receive(:before_rdf_datastream_migration)
+      expect(mover).to receive(:after_rdf_datastream_migration)
+      mover.migrate
+    end
 
-  it "adds each triple to a new Fedora4 resource" do
-    expect(subject.title.first).to eql "Sample Migration Object A"
-    expect(subject.creator.first).to eql "Adam Wead"
+    describe "using triples" do
+      subject do
+        mover.migrate
+        mover.target
+      end
+
+      it "adds each triple to a new Fedora4 resource" do
+        expect(subject.title.first).to eql "Sample Migration Object A"
+        expect(subject.creator.first).to eql "Adam Wead"
+      end
+    end
   end
 
+  context "record with UTF8 chracters" do
+
+    let(:file) { FedoraMigrate.source.connection.find("scholarsphere:7d279232g").datastreams["descMetadata"] }
+
+    describe "using triples" do
+      subject do
+        mover.migrate
+        mover.target
+      end
+
+      it "adds each triple to a new Fedora4 resource" do
+        expect(subject.title.first).to eql "Emerging Role of Genomic Profiling of Advanced Tumors to Aid in Treatment Selection: What Nurses Should Know"
+        expect(subject.creator.first).to eql "Eileen Bannon, RN, MSN, OCN, CBCN"
+        expect(subject.description.first). to eql "Objectives:\r\n•  Explain the role of a new genomic assay (Target Now™) in guiding oncology treatment plans.\r\n•  Describe the Target Now™ assay.\r\n•  Present a case study where Target Now™ was instrumental in the patient’s treatment plan."
+      end
+    end
+
+  end
 end

data/spec/integration/relationship_migration_spec.rb

@@ -4,7 +4,7 @@ describe "Migrating objects with relationships" do
 
   before :all do
     class GenericFile < ActiveFedora::Base
-      belongs_to :batch, predicate: ActiveFedora::RDF::RelsExt.isPartOf
+      belongs_to :batch, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
       property :title, predicate: ::RDF::DC.title do |index|
         index.as :stored_searchable, :facetable
       end
@@ -14,14 +14,14 @@ describe "Migrating objects with relationships" do
     end
 
     class Batch < ActiveFedora::Base
-      has_many :generic_files, predicate: ActiveFedora::RDF::RelsExt.isPartOf
+      has_many :generic_files, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
     end
   end
 
   after :all do
     Object.send(:remove_const, :GenericFile) if defined?(GenericFile)
   end
-  
+
   let(:parent_source) { FedoraMigrate.source.connection.find("sufia:rb68xc09k") }
   let(:child_source)  { FedoraMigrate.source.connection.find("sufia:rb68xc11m") }
 
@@ -32,18 +32,19 @@ describe "Migrating objects with relationships" do
       FedoraMigrate::ObjectMover.new(child_source).migrate
     end
 
-    describe "migrating the parent object's relationships" do
+    subject { Batch.find("rb68xc09k") }
 
+    describe "migrating the parent object's relationships" do
       before do
         FedoraMigrate::RelsExtDatastreamMover.new(child_source).migrate
       end
-
-      subject { Batch.find("rb68xc09k") }
-
-      specify "you can migrate the parent object's relationships" do
+      specify "should connect the child to the parent" do
+        expect(subject.generic_files.count).to eql 1
+      end
+      specify "is repeatable" do
+        FedoraMigrate::RelsExtDatastreamMover.new(child_source).migrate
         expect(subject.generic_files.count).to eql 1
       end
-
     end
 
   end

data/spec/integration/repository_migration_spec.rb

@@ -7,23 +7,21 @@ describe "Migrating the repository" do
     before do
       Object.send(:remove_const, :GenericFile) if defined?(GenericFile)
       Object.send(:remove_const, :Batch) if defined?(Batch)
+      Object.send(:remove_const, :Collection) if defined?(Collection)
     end
 
-    subject do
-      results = FedoraMigrate.migrate_repository(namespace: "sufia", options: {convert: "descMetadata"})
-      results.map { |e| e.values }.flatten
+    it "should log warnings" do
+      expect(FedoraMigrate::Logger).to receive(:warn).at_least(6).times
+      FedoraMigrate.migrate_repository(namespace: "sufia", options: {convert: "descMetadata"})
     end
-    it { is_expected.to include("uninitialized constant GenericFile")}
-    it { is_expected.to include("uninitialized constant Batch")}
-    it { is_expected.to include("Source was not found in Fedora4. Did you migrated it?")}
   end
 
-  context "with all target objects are defined" do
+  context "when all target objects are defined" do
 
     before do
       Object.send(:remove_const, :GenericFile) if defined?(GenericFile)
       class GenericFile < ExampleModel::MigrationObject
-        belongs_to :batch, predicate: ActiveFedora::RDF::RelsExt.isPartOf
+        belongs_to :batch, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
         property :title, predicate: ::RDF::DC.title do |index|
           index.as :stored_searchable, :facetable
         end
@@ -32,28 +30,57 @@ describe "Migrating the repository" do
         end
       end
 
-      Object.send(:remove_const, :Batch) if defined?(Batch)      
+      Object.send(:remove_const, :Batch) if defined?(Batch)
       class Batch < ActiveFedora::Base
-        has_many :generic_files, predicate: ActiveFedora::RDF::RelsExt.isPartOf
+        has_many :generic_files, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
+      end
+
+      Object.send(:remove_const, :Collection) if defined?(Collection)
+      class Collection < ExampleModel::Collection
       end
     end
 
     after do
       Object.send(:remove_const, :GenericFile) if defined?(GenericFile)
       Object.send(:remove_const, :Batch) if defined?(Batch)
+      Object.send(:remove_const, :Collection) if defined?(Collection)
     end
 
-    it "should move every object" do
-      results = FedoraMigrate.migrate_repository(namespace: "sufia", options: {convert: "descMetadata"})
-      expect(results.collect { |r| r.values}.flatten.uniq).to eql [true]
-      expect(GenericFile.find("rb68xc089").title).to eql(["world.png"])
-      expect(GenericFile.find("xp68km39w").title).to eql(["Sample Migration Object A"])
-      expect(GenericFile.find("xp68km39w").creator).to eql(["Adam Wead"])
-      expect(GenericFile.all.count).to eql 4
-      expect(Batch.all.count).to eql 1
-      expect(Batch.first.generic_files.count).to eql 2
+    let(:file1) { GenericFile.find("rb68xc089") }
+    let(:file2) { GenericFile.find("xp68km39w") }
+    let(:versions_report) { GenericFile.all.map { |f| f.content.versions.count }.uniq }
+
+    context "by default" do
+      before do
+        FedoraMigrate.migrate_repository(namespace: "sufia", options: {convert: "descMetadata"})
+      end
+      it "should move every object and its versions" do
+        expect(file1.title).to eql ["world.png"]
+        expect(file2.title).to eql ["Sample Migration Object A"]
+        expect(file2.creator).to eql ["Adam Wead"]
+        expect(GenericFile.all.count).to eql 6
+        expect(Batch.all.count).to eql 2
+        expect(Batch.all.first.generic_files.count).to eql 2
+        expect(Batch.all.last.generic_files.count).to eql 2
+        expect(Collection.all.count).to eql 1
+        expect(Collection.first.members.count).to eql 2
+        expect(versions_report).to match_array [0,3,9]
+      end
+    end
+
+    context "and the application will create versions" do
+      before do
+        FedoraMigrate.migrate_repository(namespace: "sufia", 
+          options: {convert: "descMetadata", application_creates_versions: true}
+        )
+      end
+      it "should move every object but not its versions" do
+        expect(file1.title).to eql ["world.png"]
+        expect(versions_report).to eql [0]
+      end
     end
 
   end
 
 end
+

data/spec/integration/versions_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe "Versions" do
+
+  let(:source) { FedoraMigrate.source.connection.find("sufia:rb68xc089") }
+  let(:mover)  { FedoraMigrate::ObjectMover.new source, ExampleModel::MigrationObject.new }
+
+  subject do
+    mover.migrate
+    mover.target
+  end
+
+  # Query the metadata node for a given version and return its hasDateCreatedByApplication expressed as an integer
+  def date_created_by_application version
+    uri = subject.content.versions.with_label(version).uri
+    response = ActiveFedora.fedora.connection.get(uri+"/fcr:metadata")
+    graph = ::RDF::Graph.new << ::RDF::Reader.for(:ttl).new(response.body)
+    value = graph.query(predicate: RDF::URI("http://www.loc.gov/premis/rdf/v1#hasDateCreatedByApplication")).first.object.to_s
+    DateTime.iso8601(value).to_i
+  end
+
+  def desc_metadata_source_versions
+    source.datastreams["descMetadata"].versions.sort { |a,b| a.createDate <=> b.createDate }
+  end
+
+  it "should be migrated in the order they were created with their original creation dates" do
+    expect(desc_metadata_source_versions[0].createDate.to_i).to eql date_created_by_application("version1")
+    expect(desc_metadata_source_versions[1].createDate.to_i).to eql date_created_by_application("version2")
+    expect(desc_metadata_source_versions[2].createDate.to_i).to eql date_created_by_application("version3")
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,13 +1,20 @@
 require 'byebug' unless ENV['TRAVIS']
+ENV['RAILS_ENV'] = "test"
+
 require 'fedora-migrate'
 require 'equivalent-xml/rspec_matchers'
 require 'support/example_model'
 require 'active_fedora/cleaner'
-ENV['environment'] = "test"
 
+require 'http_logger'
 ActiveFedora::Base.logger = Logger.new(STDERR)
 ActiveFedora::Base.logger.level = Logger::WARN
 
+# HttpLogger.logger = Logger.new(STDOUT)
+# HttpLogger.ignore = [/(127\.0\.0\.1|localhost):8983\/solr/]
+# HttpLogger.colorize = false
+# HttpLogger.log_headers = true
+
 RSpec.configure do |config|
   config.expect_with :rspec do |expectations|
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true

data/spec/support/example_model.rb

@@ -1,3 +1,34 @@
+# Shenanigans because we're not in a Rails environment and we need to load bits of 
+# code that depend on Rails in order to test migrating objects.
+Hydra::Engine.config.autoload_paths.each { |path| $LOAD_PATH.unshift path }
+# in gem version 2.4, .find_by_name isn't pulling up gems given in the Gemfile
+# as opposed to those in the gemspec file.
+# This is a workaround:
+Gem::Specification.all.each do |g|
+  HAC_DIR = g.gem_dir if g.name.match("hydra-access-controls")
+  HCL_DIR = g.gem_dir if g.name.match("hydra-collections")
+  HCR_DIR = g.gem_dir if g.name.match("hydra-core")
+end
+
+# Load Rails-specific bits of hydra-access-controls
+require HAC_DIR+'/app/vocabularies/acl'
+require HAC_DIR+'/app/vocabularies/hydra/acl'
+require HAC_DIR+'/app/models/role_mapper'
+require HAC_DIR+'/app/models/ability'
+require HAC_DIR+'/app/models/hydra/access_controls/access_control_list'
+require HAC_DIR+'/app/models/hydra/access_controls/permission'
+require HAC_DIR+'/app/models/hydra/access_controls/embargo'
+require HAC_DIR+'/app/models/hydra/access_controls/lease'
+require HAC_DIR+'/app/models/concerns/hydra/with_depositor'
+require HAC_DIR+'/app/services/hydra/lease_service'
+require HAC_DIR+'/app/services/hydra/embargo_service'
+require HAC_DIR+'/app/validators/hydra/future_date_validator'
+
+# Loading hydra-collections
+require 'hydra-collections'
+require HCR_DIR+'/app/models/concerns/hydra/model_methods'
+require HCL_DIR+'/app/models/concerns/hydra/collection'
+
 module ExampleModel
 
   class RDFProperties < ActiveFedora::Base
@@ -7,6 +38,10 @@ module ExampleModel
     property :creator, predicate: ::RDF::DC.creator do |index|
       index.as :stored_searchable, :facetable
     end
+    property :description, predicate: ::RDF::DC.description do |index|
+      index.type :text
+      index.as :stored_searchable
+    end
   end
 
   class VersionedDatastream < ActiveFedora::File
@@ -24,13 +59,34 @@ module ExampleModel
     contains "characterization", class_name: "ActiveFedora::File"
   end
 
+  class OmDatastreamExample < ActiveFedora::Base
+    contains "characterization", class_name: "ActiveFedora::OmDatastream"
+  end
+
   class RDFObject < ActiveFedora::Base
     property :title, predicate: ::RDF::DC.title do |index|
       index.as :stored_searchable, :facetable
     end
+
+    property :description, predicate: ::RDF::DC.description
+
+    property :date_uploaded, predicate: ::RDF::DC.dateSubmitted, multiple: false do |index|
+      index.type :date
+      index.as :stored_sortable
+    end
+
+    property :date_modified, predicate: ::RDF::DC.modified, multiple: false do |index|
+      index.type :date
+      index.as :stored_sortable
+    end
+
     contains "content", class_name: "ExampleModel::VersionedDatastream"
     contains "thumbnail", class_name: "ActiveFedora::File"
     contains "characterization", class_name: "ActiveFedora::File"
   end
 
+  class Collection < ActiveFedora::Base
+    include Hydra::Collection
+  end
+
 end