fedora-migrate 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ce0bad9581c3b5b080d0a73a26792ee7942ce1b3
-  data.tar.gz: 79932ebfe5726d0ef5dd67b1a5598847ed056175
+  metadata.gz: 127fba59aedee21753a2ab76236f32f9fd405dc3
+  data.tar.gz: 18b7cafa6247b840afcf79924565859333d99c2c
 SHA512:
-  metadata.gz: 79d5f57909a5a92ea9155b6b09b2e85733df39212e62e9c227e9732d5c5eeb8c286e1c8905c52fb40d511e8c5641568c763327405593bf1c2583ef97e0a81222
-  data.tar.gz: 5a57b9afc653f14a33dcffaad44fc15bd5c9cde314b785e866b2786d8af92336dcfc8bcb212e24362cc6f7162e599bd9b7b41f0cd43761d6f26f23da0effab01
+  metadata.gz: 468b7a0174d44e48b7c8b4d55c8ed6036e835a73e55cf3076f378806477e940c1cee5ebe96476cb6316a6582746d8dd5ace72f51c03813dcb56ad4cc500e9683
+  data.tar.gz: 0d961a07e3c4dc51691b5caa0c28065d6900b4c8dbe15c8d4e97ca1a8fe6f2a770d366894ec8b6c0b80e4cf225daccd568e94633f13cef6faa1f95d93c76f10d

data/.gitignore

@@ -15,4 +15,4 @@ mkmf.log
 jetty
 .ruby-version
 .rvmrc
-report.json
+migration_report

data/.travis.yml

@@ -1,19 +1,17 @@
 language: ruby
+cache: bundler
+sudo: false
 rvm:
-  - 2.1
-  - 2.0
+  - 2.2
 env:
   global:
-  - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
+    - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
 notifications:
-  email:
-    recipients:
-      - "ul-dlt-hydra@lists.psu.edu"
-    on_success: "change"
-    on_failure: "always"
   irc:
     channels:
       - "irc.freenode.org#scholarsphere"
       - "irc.freenode.org#projecthydra"
     template:
       - "%{repository}//%{branch}@%{commit} by %{author}: %{message} - %{build_url}"
+before_script:
+  - jdk_switcher use oraclejdk8

data/README.md

@@ -52,11 +52,13 @@ as an example:
 ``` ruby
 desc "Migrate all my objects"
 task migrate: :environment do
-  results = FedoraMigrate.migrate_repository(namespace: "mynamespace")
+  results = FedoraMigrate.migrate_repository("mynamespace")
   puts results
 end
 ```
 
+Where `mynamespace` is your Fedora 3 pid namespace.
+
 Run the task
 
     $ bundle exec rake migrate

data/fedora-migrate.gemspec

@@ -18,13 +18,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "hydra-head", "~> 9.0"
+  spec.add_dependency "hydra-head", "~> 9.5"
   spec.add_dependency "rubydora", "~> 1.8"
   spec.add_dependency "rchardet"
   
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "equivalent-xml"
-  spec.add_development_dependency "hydra-collections", "~> 4.0"
+  spec.add_development_dependency "hydra-collections", "~> 5.0"
   spec.add_development_dependency "jettywrapper"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"

data/lib/fedora_migrate/file_configurator.rb

@@ -8,7 +8,7 @@ module FedoraMigrate
 
     def load_fedora3_config
       return @fedora_config unless @fedora_config.empty?
-      @fedora_config_path = get_config_path(:fedora3)
+      @fedora_config_path = config_path(:fedora3)
       Logger.info("loading fedora config from #{::File.expand_path(@fedora_config_path)}")
 
       begin

data/lib/fedora_migrate/migration_options.rb

@@ -18,6 +18,11 @@ module FedoraMigrate
     def application_creates_versions?
       option_true?(:application_creates_versions)
     end
+
+    def blacklist
+      return [] if options.nil?
+      options.fetch(:blacklist, [])
+    end
     
     private
     

data/lib/fedora_migrate/migration_report.rb

@@ -1,10 +1,18 @@
 module FedoraMigrate
   class MigrationReport
 
-    attr_accessor :results
+    attr_accessor :path, :results
 
-    def initialize report=nil
-      @results = report.nil? ? Hash.new : JSON.parse(File.read(report))
+    DEFAULT_PATH = "migration_report".freeze
+
+    def initialize path=nil
+      @path = path.nil? ? DEFAULT_PATH : path
+      FileUtils::mkdir_p(@path)
+      reload
+    end
+
+    def reload
+      @results = load_results_from_directory
     end
 
     def empty?
@@ -30,11 +38,29 @@ module FedoraMigrate
       output
     end
 
-    def save path=nil
-      json = JSON.load(results.to_json)
-      file = path.nil? ? "report.json" : File.join(path,"report.json")
+    # Receives and individual report and writes it to the MigrationReport directory
+    def save pid, report
+      file = File.join(path,file_from_pid(pid))
+      json = JSON.load(report.to_json)
       File.write(file, JSON.pretty_generate(json))
     end
 
+    private
+
+    def load_results_from_directory assembled = Hash.new
+      Dir.glob(File.join(path,"*.json")).each do |file|
+        assembled[pid_from_file(file)] = JSON.parse(File.read(file))
+      end
+      assembled
+    end
+
+    def pid_from_file file
+      File.basename(file, ".*").gsub(/_/,":")
+    end
+
+    def file_from_pid pid
+      pid.gsub(/:/,"_")+".json"
+    end
+
   end
 end

data/lib/fedora_migrate/repository_migrator.rb

@@ -3,7 +3,7 @@ module FedoraMigrate
 
     include MigrationOptions
 
-    attr_accessor :source_objects, :namespace, :report
+    attr_accessor :source_objects, :namespace, :report, :source, :result
 
     SingleObjectReport = Struct.new(:status, :object, :relationships)
 
@@ -16,20 +16,36 @@ module FedoraMigrate
     end
 
     def migrate_objects
-      source_objects.each { |source| migrate_object(source) }
+      source_objects.each do |object|
+        @source = object
+        migrate_current_object
+      end
+      report.reload
+    end
+
+    def migrate_current_object
+      return unless migration_required?
+      initialize_report
+      migrate_object
+    end
+
+    def initialize_report
+      @result = SingleObjectReport.new
+      @result.status = false
+      report.save(source.pid, @result)
     end
 
     def migrate_relationships
       return "Relationship migration halted because #{failures.to_s} objects didn't migrate successfully." if failures > 0 && not_forced?
-      source_objects.each { |source| migrate_relationship(source) }
+      source_objects.each do |object|
+        @source = object
+        @result = find_or_create_single_object_report
+        migrate_relationship unless blacklist.include?(source.pid)
+      end
     end
 
     def get_source_objects
-      if report.empty?
-        FedoraMigrate.source.connection.search(nil).collect { |o| qualifying_object(o) }.compact
-      else
-        report.failed_objects.map { |o| FedoraMigrate.source.connection.find(o) }
-      end
+      FedoraMigrate.source.connection.search(nil).collect { |o| qualifying_object(o) }.compact
     end
 
     def failures
@@ -38,28 +54,24 @@ module FedoraMigrate
 
     private
 
-    def migrate_object source
-      object_report = SingleObjectReport.new
-      begin
-        object_report.object = FedoraMigrate::ObjectMover.new(source, nil, options).migrate
-        object_report.status = true
-      rescue StandardError => e
-        object_report.object = e.inspect
-        object_report.status = false
-      end
-      report.results[source.pid] = object_report
+    def migrate_object
+      result.object = FedoraMigrate::ObjectMover.new(source, nil, options).migrate
+      result.status = true
+    rescue StandardError => e
+      result.object = e.inspect
+      result.status = false
+    ensure
+      report.save(source.pid, result)
     end
 
-    def migrate_relationship source
-      relationship_report = find_or_create_single_object_report(source)
-      begin
-        relationship_report.relationships = FedoraMigrate::RelsExtDatastreamMover.new(source).migrate
-        relationship_report.status = true
-      rescue StandardError => e
-        relationship_report.relationships = e.inspect
-        relationship_report.status = false
-      end
-      report.results[source.pid] = relationship_report
+    def migrate_relationship
+      result.relationships = FedoraMigrate::RelsExtDatastreamMover.new(source).migrate
+      result.status = true
+    rescue StandardError => e
+      result.relationships = e.inspect
+      result.status = false
+    ensure
+      report.save(source.pid, result)
     end
 
     def repository_namespace
@@ -71,8 +83,19 @@ module FedoraMigrate
       return object if name.match(namespace)
     end
 
-    def find_or_create_single_object_report source
-      report.results[source.pid] || SingleObjectReport.new
+    def migration_required?
+      return false if blacklist.include?(source.pid)
+      return true if report.results[source.pid].nil?
+      !report.results[source.pid]["status"]
     end
+
+    def find_or_create_single_object_report
+      if report.results[source.pid].nil?
+        SingleObjectReport.new
+      else
+        SingleObjectReport.new(report.results[source.pid]["status"],report.results[source.pid]["object"],report.results[source.pid]["relationships"])
+      end
+    end
+
   end
 end

data/lib/fedora_migrate/version.rb

@@ -1,3 +1,3 @@
 module FedoraMigrate
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/spec/fixtures/reports/failed/sufia_5m60qr94g.json

@@ -0,0 +1,7 @@
+{
+  "status": false,
+  "object": "Sample object migration failure",
+  "relationships": [
+
+  ]
+}

data/spec/fixtures/reports/failed/sufia_5m60qr95r.json

@@ -0,0 +1,58 @@
+{
+  "status": true,
+  "object": {
+    "id": "5m60qr95r",
+    "class": "GenericFile",
+    "content_datastreams": [
+      {
+        "ds": "content",
+        "versions": [
+          {
+            "name": "file2.txt",
+            "mime_type": "text/plain",
+            "original_date": "2015-01-19T21:32:47Z"
+          }
+        ]
+      },
+      {
+        "ds": "thumbnail",
+        "versions": [
+          {
+            "error": "Nil source -- it's probably defined in the target but not present in the source"
+          }
+        ]
+      },
+      {
+        "ds": "characterization",
+        "versions": [
+          {
+            "error": "Nil source -- it's probably defined in the target but not present in the source"
+          }
+        ]
+      }
+    ],
+    "rdf_datastreams": [
+      {
+        "ds": "descMetadata",
+        "status": [
+
+        ]
+      }
+    ],
+    "permissions": [
+      "read_groups = []",
+      "edit_groups = []",
+      "discover_groups = []",
+      "read_users = []",
+      "edit_users = [\"awead@psu.edu\"]",
+      "discover_users = []"
+    ],
+    "dates": {
+      "uploaded": null,
+      "modified": null
+    }
+  },
+  "relationships": [
+    "http://localhost:8983/fedora/rest/test/5m60qr95r--info:fedora/fedora-system:def/relations-external#isPartOf--http://localhost:8983/fedora/rest/test/5m60qr94g"
+  ]
+}

data/spec/fixtures/reports/failed/sufia_5m60qr961.json

@@ -0,0 +1,58 @@
+{
+  "status": true,
+  "object": {
+    "id": "5m60qr961",
+    "class": "GenericFile",
+    "content_datastreams": [
+      {
+        "ds": "content",
+        "versions": [
+          {
+            "name": "file1.txt",
+            "mime_type": "text/plain",
+            "original_date": "2015-01-19T21:32:49Z"
+          }
+        ]
+      },
+      {
+        "ds": "thumbnail",
+        "versions": [
+          {
+            "error": "Nil source -- it's probably defined in the target but not present in the source"
+          }
+        ]
+      },
+      {
+        "ds": "characterization",
+        "versions": [
+          {
+            "error": "Nil source -- it's probably defined in the target but not present in the source"
+          }
+        ]
+      }
+    ],
+    "rdf_datastreams": [
+      {
+        "ds": "descMetadata",
+        "status": [
+
+        ]
+      }
+    ],
+    "permissions": [
+      "read_groups = []",
+      "edit_groups = []",
+      "discover_groups = []",
+      "read_users = []",
+      "edit_users = [\"awead@psu.edu\"]",
+      "discover_users = []"
+    ],
+    "dates": {
+      "uploaded": null,
+      "modified": null
+    }
+  },
+  "relationships": [
+    "http://localhost:8983/fedora/rest/test/5m60qr961--info:fedora/fedora-system:def/relations-external#isPartOf--http://localhost:8983/fedora/rest/test/5m60qr94g"
+  ]
+}

data/spec/fixtures/reports/failed/sufia_5m60qr979.json

@@ -0,0 +1,34 @@
+{
+  "status": true,
+  "object": {
+    "id": "5m60qr979",
+    "class": "Collection",
+    "content_datastreams": [
+
+    ],
+    "rdf_datastreams": [
+      {
+        "ds": "descMetadata",
+        "status": [
+
+        ]
+      }
+    ],
+    "permissions": [
+      "read_groups = [\"public\"]",
+      "edit_groups = []",
+      "discover_groups = []",
+      "read_users = []",
+      "edit_users = [\"awead@psu.edu\"]",
+      "discover_users = []"
+    ],
+    "dates": {
+      "uploaded": "2015-01-19T21:34:23.805Z",
+      "modified": "2015-03-03T18:48:51.16Z"
+    }
+  },
+  "relationships": [
+    "http://localhost:8983/fedora/rest/test/5m60qr979--info:fedora/fedora-system:def/relations-external#hasCollectionMember--http://localhost:8983/fedora/rest/test/5m60qr95r",
+    "http://localhost:8983/fedora/rest/test/5m60qr979--info:fedora/fedora-system:def/relations-external#hasCollectionMember--http://localhost:8983/fedora/rest/test/5m60qr961"
+  ]
+}

data/spec/fixtures/reports/failed/sufia_rb68xc089.json

@@ -0,0 +1,7 @@
+{
+  "status": false,
+  "object": "Sample object migration failure",
+  "relationships": [
+
+  ]
+}