fedora-migrate 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1cd495240ef6ef07dafbfa15720b1c5eca3a4146
+  data.tar.gz: 4d59a16932745f6082133cf3a5492a024e499f84
+SHA512:
+  metadata.gz: 3a906945035f3966d5c5ecb24668081da701e52a5cec8f881b0b75d3e938ee72a53354008315911e1dd3914793311f9efb79991aeb78199ef210f659e6b9c111
+  data.tar.gz: 18a47d1a58619bd9a57f4489537f79a35655f184eac5f3a47e2424f5feef646946e407d1bdc6c52504cd529d42f7f61bfa98935dab8f8e98c83e881178660ff6

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+jetty
+.ruby-version

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+rvm:
+  - 2.1
+  - 2.0
+env:
+  global:
+  - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
+notifications:
+  email:
+    recipients:
+      - "ul-dlt-hydra@lists.psu.edu"
+    on_success: "change"
+    on_failure: "always"
+  irc:
+    channels:
+      - "irc.freenode.org#scholarsphere"
+      - "irc.freenode.org#projecthydra"
+    template:
+      - "%{repository}//%{branch}@%{commit} by %{author}: %{message} - %{build_url}"

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fedora-migrate.gemspec
+gemspec
+
+gem 'byebug'
+gem 'pry'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Adam Wead
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# FedoraMigrate
+
+Migrates content from a Fedora3 repository to a Fedora4 one.
+
+## Status
+
+Very alpha. This has been tested against Penn State's existing Scholarsohere applications, as well
+as generic Sufia applications.
+
+## TODOs
+
+See the list of issues.
+
+## Contributing
+
+### Hydra Developers
+
+For Hydra developers, or anyone with a signed CLA, please clone the repo and submit PRs via
+topic branches. If you don't have rights to projecthydra-labs and do have a signed
+CLA, please send a note to hydra-tech@googlegroups.com.
+
+1. Clone it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+### Non-Hydra Developers
+
+Anyone is welcome to use this software and report issues.
+In order to merge any work contributed, you'll need to sign a contributor license agreement.

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+
+Dir.glob('tasks/*.rake').each { |r| import r }
+
+task default: :ci

data/config/fedora.yml

@@ -0,0 +1,14 @@
+development:
+  user: fedoraAdmin
+  password: fedoraAdmin
+  url: http://127.0.0.1:8983/fedora/rest
+  base_path: /dev
+test:
+  user: fedoraAdmin
+  password: fedoraAdmin
+  url: http://localhost:8983/fedora/rest
+  base_path: /test
+production:
+  user: fedoraAdmin
+  password: fedoraAdmin
+  url: http://127.0.0.1:8983/fedora/rest

data/config/fedora3.yml

@@ -0,0 +1,12 @@
+development:
+  user: fedoraAdmin
+  password: fedoraAdmin
+  url: http://localhost:8983/fedora3
+test:
+  user: fedoraAdmin
+  password: fedoraAdmin
+  url: http://localhost:8983/fedora3
+production:
+  user: fedoraAdmin
+  password: fedoraAdmin
+  url: http://localhost:8983/fedora3

data/config/jetty.yml

@@ -0,0 +1,6 @@
+default:
+  startup_wait: 90
+  jetty_port: 8983
+  java_opts:
+    - "-Xmx256m" 
+    - "-XX:MaxPermSize=128m"

data/config/solr.yml

@@ -0,0 +1,15 @@
+development:
+  default:
+    url: http://localhost:8983/solr/development
+  full_text:
+    url: http://localhost:8983/solr/development
+test: 
+  default:
+    url: http://localhost:8983/solr/test
+  full_text:
+    url: http://localhost:8983/solr/test
+production:
+  default:
+    url: http://localhost:8080/solr/production 
+  full_text:
+    url: http://localhost:8080/solr/production

data/fedora-migrate.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fedora_migrate/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fedora-migrate"
+  spec.version       = FedoraMigrate::VERSION
+  spec.authors       = ["Adam Wead"]
+  spec.email         = ["amsterdamos@gmail.com"]
+  spec.summary       = %q{Migrate Hydra-based repository data from Fedora3 to Fedora4}
+  spec.description   = %q{Migrates data (models, datastreams, content) from a Fedora3 repository to Fedora4}
+  spec.homepage      = ""
+  spec.license       = "APACHE2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rubydora", "~> 1.8" 
+  spec.add_dependency "hydra-head", "~> 9.0.0.beta1"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "jettywrapper"
+  spec.add_development_dependency "equivalent-xml"
+  spec.add_development_dependency "sufia", "~> 6.0.0.beta1"
+end

data/lib/fedora-migrate.rb

@@ -0,0 +1,82 @@
+require "fedora_migrate/version"
+require "active_support"
+require "active_fedora"
+require "hydra-core"
+
+# Loads rake tasks
+Dir[File.expand_path(File.join(File.dirname(__FILE__),"tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
+
+# Shenanigans because we're not in a Rails environment and we need
+# Hydra::AccessControls
+Hydra::Engine.config.autoload_paths.each { |path| $LOAD_PATH.unshift path }
+# in gem version 2.4, .find_by_name isn't pulling up gems given in the Gemfile
+# as opposed to those in the gemspec file.
+# This is a workaround:
+Gem::Specification.all.each do |g|
+  HAC_DIR = g.gem_dir if g.name.match("hydra-access-controls")
+end
+require HAC_DIR+'/app/vocabularies/acl'
+require HAC_DIR+'/app/vocabularies/hydra/acl'
+require HAC_DIR+'/app/models/role_mapper'
+require HAC_DIR+'/app/models/ability'
+require HAC_DIR+'/app/models/hydra/access_controls/access_control_list'
+require HAC_DIR+'/app/models/hydra/access_controls/permission'
+require HAC_DIR+'/app/models/hydra/access_controls/embargo'
+require HAC_DIR+'/app/models/hydra/access_controls/lease'
+require HAC_DIR+'/app/services/hydra/lease_service'
+require HAC_DIR+'/app/services/hydra/embargo_service'
+require HAC_DIR+'/app/validators/hydra/future_date_validator'
+
+module FedoraMigrate
+  extend ActiveSupport::Autoload
+
+  autoload :DatastreamMover
+  autoload :Errors
+  autoload :FileConfigurator
+  autoload :Hooks
+  autoload :Logger
+  autoload :MigrationOptions
+  autoload :Mover
+  autoload :ObjectMover
+  autoload :Permissions
+  autoload :PermissionsMover
+  autoload :RDFDatastreamMover
+  autoload :RDFDatastreamParser
+  autoload :RelsExtDatastreamMover
+  autoload :RepositoryMigrator
+  autoload :RightsMetadata
+  autoload :RubydoraConnection
+  autoload :TripleConverter
+
+  class << self
+    attr_reader :fedora_config, :config_options, :source
+    attr_accessor :configurator
+
+    def fedora_config
+      @fedora_config ||= ActiveFedora::Config.new(configurator.fedora3_config)
+    end
+
+    def config_options
+      @config_options ||= "comming soon!"
+    end
+
+    def source
+      @source ||= FedoraMigrate::RubydoraConnection.new(fedora_config.credentials)
+    end 
+
+    def find id
+      FedoraMigrate.source.connection.find(id)
+    end
+
+    def migrate_repository args
+      migrator = FedoraMigrate::RepositoryMigrator.new(args[:namespace], args[:options])
+      migrator.migrate_objects
+      migrator.migrate_relationships
+      migrator.results
+    end
+
+  end
+
+  self.configurator ||= FedoraMigrate::FileConfigurator.new
+
+end

data/lib/fedora_migrate/datastream_mover.rb

@@ -0,0 +1,78 @@
+module FedoraMigrate
+  class DatastreamMover < Mover
+
+    attr_accessor :versionable
+
+    def post_initialize
+      raise FedoraMigrate::Errors::MigrationError, "You must supply a target" if target.nil?
+    end
+
+    def versionable?
+      versionable.nil? ? target_versionable? : versionable
+    end
+
+    def target_versionable?
+      if target.respond_to?(:versionable?)
+        target.versionable?
+      else 
+        false
+      end
+    end
+    
+    def migrate
+      if versionable?
+        migrate_versions
+      else
+        migrate_current
+      end
+    end
+
+    private
+
+    # Reloading the target, otherwise #get_checksum is nil
+    def migrate_current
+      migrate_content
+      target.reload
+      verify
+    end
+
+    def migrate_versions
+      source.versions.each do |version|
+        migrate_content(version)
+        target.create_version
+        verify(version)
+      end
+    end
+
+    # TODO: lastModified isn't the right place for the original creation date (issue #1)
+    def migrate_content datastream=nil
+      datastream ||= source
+      if datastream.content.nil?
+        Logger.info "datastream '#{datastream.dsid}' is nil. It's probably defined in the target but not present in the source"
+        return true
+      end
+      target.content = datastream.content
+      target.original_name = datastream.label
+      target.mime_type = datastream.mimeType
+      target.last_modified = datastream.createDate
+      Logger.info "#{target.inspect}"
+      save
+    end
+
+    # TODO: Reporting mechanism? If there isn't a checksum it defaults to "none" (issue #4)
+    def verify datastream=nil
+      datastream ||= source
+      target_checksum = get_checksum
+      return true if datastream.checksum == "none"
+      unless datastream.checksum == target_checksum.split(/:/).last
+        Logger.fatal "expected #{datastream.dsid} #{datastream.checksumType} #{datastream.checksum} to match #{target_checksum}"
+      end
+    end
+
+    def get_checksum
+      target.digest.first.to_s
+    end
+
+  end
+
+end

data/lib/fedora_migrate/errors.rb

@@ -0,0 +1,7 @@
+module FedoraMigrate
+  module Errors
+    # raise this error when moving or mirgating content
+    class MigrationError < StandardError
+    end
+  end
+end

data/lib/fedora_migrate/file_configurator.rb

@@ -0,0 +1,34 @@
+module FedoraMigrate
+  class FileConfigurator < ActiveFedora::FileConfigurator
+
+    def fedora3_config
+      load_fedora3_config
+      @fedora_config
+    end
+
+    def load_fedora3_config
+      return @fedora_config unless @fedora_config.empty?
+      @fedora_config_path = get_config_path(:fedora3)
+      Logger.info("loading fedora config from #{::File.expand_path(@fedora_config_path)}")
+
+      begin
+        config_erb = ERB.new(IO.read(@fedora_config_path)).result(binding)
+      rescue Exception => e
+        raise("fedora.yml was found, but could not be parsed with ERB. \n#{$!.inspect}")
+      end
+
+      begin
+        fedora_yml = YAML.load(config_erb)
+      rescue Psych::SyntaxError => e
+        raise "fedora.yml was found, but could not be parsed. " \
+              "Error #{e.message}"
+      end
+
+      config = fedora_yml.symbolize_keys
+
+      cfg = config[ActiveFedora.environment.to_sym] || {}
+      @fedora_config = cfg.kind_of?(Array) ? cfg.map(&:symbolize_keys) : cfg.symbolize_keys
+    end
+
+  end
+end

data/lib/fedora_migrate/hooks.rb

@@ -0,0 +1,11 @@
+module FedoraMigrate
+  module Hooks
+
+    def before_object_migration
+    end
+
+    def after_object_migration
+    end
+
+  end
+end

data/lib/fedora_migrate/logger.rb

@@ -0,0 +1,36 @@
+module FedoraMigrate
+  class Logger
+
+    class << self
+
+      def method_missing method_name, *arguments, &block
+        logger.send(method_name, *arguments, &block)
+      rescue
+        super
+      end
+
+      def respond_to?(method_name, include_private = false)
+        logger.respond_to? method_name
+      end
+
+      def info msg
+        super("FedoraMigrate INFO: ##{caller_locations(1,1)[0].label} " + msg)
+      end
+
+      def warn msg
+        super("FedoraMigrate WARN: ##{caller_locations(1,1)[0].label} " + msg)
+      end
+
+      def fatal msg
+        super("FedoraMigrate FATAL: ##{caller_locations(1,1)[0].label} " + msg)
+      end
+
+      private
+
+      def logger
+        ActiveFedora::Base.logger || ::Logger.new(STDOUT)
+      end
+
+    end
+  end
+end