fedora-migrate 0.0.1

data/lib/fedora_migrate/migration_options.rb

@@ -0,0 +1,11 @@
+module FedoraMigrate
+  module MigrationOptions
+
+    attr_accessor :options, :conversions
+
+    def conversion_options
+      self.conversions = options.nil? ? [] : [options[:convert]].flatten      
+    end
+
+  end
+end

data/lib/fedora_migrate/mover.rb

@@ -0,0 +1,44 @@
+module FedoraMigrate
+  class Mover
+
+    include MigrationOptions
+    include Hooks
+
+    attr_accessor :target, :source
+
+    def initialize *args
+      @source = args[0]
+      @target = args[1]
+      @options = args[2]
+      post_initialize
+    end
+
+    def post_initialize
+    end
+
+    def save
+      if target.save
+        Logger.info "success for target UID #{target_description}"
+      else
+        raise FedoraMigrate::Errors::MigrationError, "Failed to save target: #{target_errors}"
+      end
+    end
+
+    def target_errors
+      if target.respond_to?(:errors)
+        target.errors.full_messages.join(" -- ")
+      else
+        target.inspect
+      end
+    end
+
+    def target_description
+      if target.respond_to?(:id)
+        target.id
+      else
+        target.inspect
+      end
+    end
+    
+  end
+end

data/lib/fedora_migrate/object_mover.rb

@@ -0,0 +1,62 @@
+module FedoraMigrate
+  class ObjectMover < Mover
+
+    RIGHTS_DATASTREAM = "rightsMetadata".freeze
+
+    def migrate
+      prepare_target
+      migrate_content_datastreams
+      conversions.collect { |ds| convert_rdf_datastream(ds) }
+      migrate_permissions
+      complete_target
+    end
+
+    def post_initialize
+      conversion_options
+      create_target_model if target.nil?
+    end
+
+    def prepare_target
+      Logger.info "running before_object_migration hooks"
+      before_object_migration
+      save
+    end
+
+    def complete_target
+      Logger.info "running after_object_migration hooks"
+      after_object_migration
+      save
+    end
+
+    private
+
+    def migrate_content_datastreams
+      target.attached_files.keys.each do |ds|
+        mover = FedoraMigrate::DatastreamMover.new(source.datastreams[ds.to_s], target.attached_files[ds.to_s])
+        mover.migrate
+      end
+    end
+
+    def convert_rdf_datastream ds
+      if source.datastreams.keys.include?(ds)
+        mover = FedoraMigrate::RDFDatastreamMover.new(source.datastreams[ds.to_s], target)
+        mover.migrate
+      end
+    end
+
+    def migrate_permissions
+      if source.datastreams.keys.include?(RIGHTS_DATASTREAM) && target.respond_to?(:permissions)
+        mover = FedoraMigrate::PermissionsMover.new(source.datastreams[RIGHTS_DATASTREAM], target)
+        mover.migrate
+      end
+    end
+
+    def create_target_model
+      afmodel = source.models.map { |m| m if m.match(/afmodel/) }.compact.first.split(/:/).last
+      Logger.info "found #{afmodel} in source object #{source.pid}"
+      @target = afmodel.constantize.new(id: source.pid.split(/:/).last)
+    end
+
+  end
+  
+end

data/lib/fedora_migrate/permissions.rb

@@ -0,0 +1,32 @@
+module FedoraMigrate::Permissions
+
+  # Taken from Hydra::AccessControls::Permissions under version 7.2.2
+  #
+  # We need the reader methods to get permissions from the Fedora3
+  # rightsMetadata datastreams
+
+  def read_groups
+    rightsMetadata.groups.map {|k, v| k if v == 'read'}.compact
+  end
+  
+  def edit_groups
+    rightsMetadata.groups.map {|k, v| k if v == 'edit'}.compact
+  end
+        
+  def discover_groups
+    rightsMetadata.groups.map {|k, v| k if v == 'discover'}.compact
+  end
+
+  def read_users
+    rightsMetadata.users.map {|k, v| k if v == 'read'}.compact
+  end
+
+  def edit_users
+    rightsMetadata.users.map {|k, v| k if v == 'edit'}.compact
+  end
+
+  def discover_users
+    rightsMetadata.users.map {|k, v| k if v == 'discover'}.compact
+  end
+
+end

data/lib/fedora_migrate/permissions_mover.rb

@@ -0,0 +1,31 @@
+module FedoraMigrate
+  class PermissionsMover < Mover
+
+    include FedoraMigrate::Permissions
+
+    attr_accessor :rightsMetadata
+
+    def post_initialize
+      if source.respond_to?(:content)
+        @rightsMetadata = datastream_from_content
+      end
+    end
+
+    def migrate
+      FedoraMigrate::Permissions.instance_methods.each do |permission|
+        Logger.info "setting #{permission} to #{self.send(permission)}"
+        target.send(permission.to_s+"=", self.send(permission))
+      end
+      save
+    end
+
+
+    private
+
+    def datastream_from_content ds = FedoraMigrate::RightsMetadata.new
+      ds.ng_xml = source.content
+      ds
+    end
+
+  end
+end

data/lib/fedora_migrate/rdf_datastream_mover.rb

@@ -0,0 +1,28 @@
+module FedoraMigrate
+  class RDFDatastreamMover < Mover
+
+    def migrate
+      Logger.info "converting datastream '#{source.dsid}' to RDF"
+      parse_rdf_triples
+      force_attribute_change
+      save
+    end
+
+    def parse_rdf_triples
+      parser = FedoraMigrate::RDFDatastreamParser.new(target.uri, source.content)
+      parser.parse
+      parser.statements.each do |statement|
+        target.resource << statement
+      end
+    end
+
+    # See projecthydra/active_fedora#540
+    # Forcibly setting each attribute's changed status to true
+    def force_attribute_change
+      target.class.delegated_attributes.keys.each do |term|
+        target.send(term+"_will_change!")
+      end
+    end
+
+  end
+end

data/lib/fedora_migrate/rdf_datastream_parser.rb

@@ -0,0 +1,29 @@
+module FedoraMigrate
+  class RDFDatastreamParser
+
+    attr_accessor :subject, :datastream, :statements
+
+    def initialize subject, content
+      @subject = subject
+      @datastream = content
+      @statements = []
+    end
+
+    def parse
+      datastream.split(/\n/).each do |s|
+        triple = FedoraMigrate::TripleConverter.new(s)
+        build_statement(triple) unless triple.predicate.nil?
+      end
+    end
+
+    private
+
+    def build_statement triple
+      statement = RDF::Statement(RDF::URI.new(subject), triple.predicate, triple.object)
+      Logger.info "using converted rdf triple #{statement}"
+      statements << statement
+    end
+
+  end
+
+end

data/lib/fedora_migrate/rels_ext_datastream_mover.rb

@@ -0,0 +1,90 @@
+require 'rubydora'
+module FedoraMigrate
+  class RelsExtDatastreamMover < Mover
+
+    attr_accessor :relationships, :ng_xml, :subject
+
+    RELS_EXT = Rubydora::RelationshipsMixin::RELS_EXT
+    RELS_EXT_DATASTREAM = "RELS-EXT".freeze
+
+    def post_initialize
+      retrieve_subject
+      @relationships ||= {}
+      @ng_xml = Nokogiri::XML(source.datastreams[RELS_EXT_DATASTREAM].content)
+      parse_relationships if has_relationships?
+    end
+
+    def has_relationships?
+      source.datastreams.keys.include?(RELS_EXT_DATASTREAM)
+    end
+
+    def migrate
+      relationships.each do |predicate, objects|
+        unless objects.empty?
+          if is_singular?(predicate.to_s)
+            objects.collect { |object| migrate_incomming_relationship(predicate, object) }
+          else
+            migrate_outgoing_relationship(predicate, objects)
+          end
+        end
+      end
+    end
+
+    private
+
+    # because of projecthydra/rubydora#90 
+    def parse_relationships
+      RELS_EXT.keys.each do |key|
+        query = "//ns0:"+RELS_EXT[key].split(/#/).last
+        relationships[key.to_sym] = query_results(query)
+      end
+    end
+
+    def query_results query, results = Array.new
+      ng_xml.xpath(query).each do |predicate|
+        results << retrieve_object(predicate.attribute("resource").text.split(/:/).last)
+      end
+      return results
+    end
+
+    def retrieve_subject
+      @subject = ActiveFedora::Base.find(source.pid.split(/:/).last)
+    rescue ActiveFedora::ObjectNotFoundError
+      raise FedoraMigrate::Errors::MigrationError, "Source was not found in Fedora4. Did you migrated it?"
+    end
+
+    def retrieve_object id
+      object = ActiveFedora::Base.find(id)
+    rescue ActiveFedora::ObjectNotFoundError
+      raise FedoraMigrate::Errors::MigrationError, "Could not find object with id #{id}"
+    end
+
+    # TODO: This is problematic and may not work in all situations (issue #7)
+    def migrate_incomming_relationship predicate, object
+      Logger.info "adding #{subject.id} to #{object.id} with predicate #{predicate.to_s}"
+      object.reflections.each do |key, association|
+        unless association.predicate.to_s.split(/#/).empty?
+          if association.predicate.to_s.split(/#/).last.gsub(/is/,"").underscore == predicate.to_s
+            object.send(key.to_s) << subject
+          end
+        end
+      end
+    end
+
+    # TODO: Very stinky... needs a different approach (issue #7)
+    def migrate_outgoing_relationship predicate, objects
+      Logger.info "adding #{objects.count.to_s} members to #{subject.id} with predicate #{predicate.to_s}"
+      subject.reflections.each do |key, association|
+        if key.to_s.match(/_ids$/)
+          subject.send(key.to_s+"=", objects.collect { |o| o.id })
+          subject.save
+        end
+      end
+    end
+
+    def is_singular?(str)
+      str.singularize == str
+    end
+
+  end
+end

data/lib/fedora_migrate/repository_migrator.rb

@@ -0,0 +1,60 @@
+module FedoraMigrate
+  class RepositoryMigrator
+
+    include MigrationOptions
+
+    attr_accessor :source_objects, :results, :namespace
+
+    def initialize namespace = nil, options = {}
+      @namespace = namespace || repository_namespace
+      @options = options
+      @source_objects = get_source_objects
+      @results = []
+      conversion_options
+    end
+
+    def migrate_objects
+      source_objects.each do |source|
+        Logger.info "Migrating source object #{source.pid}"
+        begin
+          results << { source.pid => [FedoraMigrate::ObjectMover.new(source, nil, options).migrate] }
+        rescue NameError => e
+          results << { source.pid => e.to_s }
+        rescue FedoraMigrate::Errors::MigrationError => e
+          results << { source.pid => e.to_s }
+        end
+      end
+    end
+
+    # TODO: need a reporting mechanism for results (issue #4)
+    def migrate_relationships
+      source_objects.each do |source|
+        Logger.info "Migrating relationships for source object #{source.pid}"
+        begin
+          FedoraMigrate::RelsExtDatastreamMover.new(source).migrate
+        rescue FedoraMigrate::Errors::MigrationError => e
+          results << { source.pid => e.to_s }
+        rescue ActiveFedora::AssociationTypeMismatch => e
+          results << { source.pid => e.to_s }
+        end
+      end
+    end
+
+    # TODO: page through all the objects (issue #6)
+    def get_source_objects
+      FedoraMigrate.source.connection.search(nil).collect { |o| qualifying_object(o) }.compact
+    end
+
+    private
+
+    def repository_namespace
+      FedoraMigrate.source.connection.repository_profile["repositoryPID"]["repositoryPID"].split(/:/).first.strip
+    end
+
+    def qualifying_object object
+      name = object.pid.split(/:/).first
+      return object if name.match(namespace)
+    end
+
+  end
+end

data/lib/fedora_migrate/rights_metadata.rb

@@ -0,0 +1,281 @@
+require 'active_support/core_ext/string'
+
+module FedoraMigrate
+  # Implements Hydra RightsMetadata XML terminology for asserting access permissions
+  class RightsMetadata < ActiveFedora::OmDatastream       
+    extend Deprecation
+    
+    set_terminology do |t|
+      t.root(:path=>"rightsMetadata", :xmlns=>"http://hydra-collab.stanford.edu/schemas/rightsMetadata/v1", :schema=>"http://github.com/projecthydra/schemas/tree/v1/rightsMetadata.xsd") 
+      t.copyright {
+        ## BEGIN possible delete, justin 2012-06-22
+        t.machine {
+          t.cclicense   
+          t.license     
+        }
+        t.human_readable(:path=>"human")
+        t.license(:proxy=>[:machine, :license ])            
+        t.cclicense(:proxy=>[:machine, :cclicense ])                  
+        ## END possible delete
+
+        t.title(:path=>'human', :attributes=>{:type=>'title'})
+        t.description(:path=>'human', :attributes=>{:type=>'description'})
+        t.url(:path=>'machine', :attributes=>{:type=>'uri'})
+      }
+      t.access do
+        t.human_readable(:path=>"human")
+        t.machine {
+          t.group
+          t.person
+        }
+        t.person(:proxy=>[:machine, :person])
+        t.group(:proxy=>[:machine, :group])
+        # accessor :access_person, :term=>[:access, :machine, :person]
+      end
+      t.discover_access(:ref=>[:access], :attributes=>{:type=>"discover"})
+      t.read_access(:ref=>[:access], :attributes=>{:type=>"read"})
+      t.edit_access(:ref=>[:access], :attributes=>{:type=>"edit"})
+      # A bug in OM prevnts us from declaring proxy terms at the root of a Terminology
+      # t.access_person(:proxy=>[:access,:machine,:person])
+      # t.access_group(:proxy=>[:access,:machine,:group])
+      
+      t.embargo {
+        t.human_readable(path: "human")
+        t.machine{
+          t.date(type: :time, attributes: {type: "release"})
+          t.date_deactivated(type: "deactivated")
+          t.visibility_during(path: "visibility", attributes: {scope: 'during'})
+          t.visibility_after(path: "visibility", attributes: {scope: 'after'})
+        }
+      }
+
+      t.lease {
+        t.human_readable(path: "human")
+        t.machine{
+          t.date(type: :time, attributes: {type: "expire"})
+          t.date_deactivated(type: :time, attributes: {type: "deactivated"})
+          t.visibility_during(path: "visibility", attributes: {scope: 'during'})
+          t.visibility_after(path: "visibility", attributes: {scope: 'after'})
+        }
+      }
+
+      t.license(:ref=>[:copyright])
+
+      t.visibility_during_embargo proxy: [:embargo, :machine, :visibility_during]
+      t.visibility_after_embargo proxy: [:embargo, :machine, :visibility_after]
+      t.visibility_during_lease proxy: [:lease, :machine, :visibility_during]
+      t.visibility_after_lease proxy: [:lease, :machine, :visibility_after]
+      t.embargo_history proxy: [:embargo, :human_readable]
+      t.lease_history proxy: [:lease, :human_readable]
+      t.embargo_release_date proxy: [:embargo, :machine, :date], type: :time
+      t.embargo_deactivation_date proxy: [:embargo, :machine, :date_deactivated]
+      t.lease_expiration_date proxy: [:lease, :machine, :date], type: :time
+      t.lease_deactivation_date proxy: [:lease, :machine, :date_deactivated]
+
+    end
+
+    # Generates an empty Mods Article (used when you call ModsArticle.new without passing in existing xml)
+    def self.xml_template
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.rightsMetadata(:version=>"0.1", "xmlns"=>"http://hydra-collab.stanford.edu/schemas/rightsMetadata/v1") {
+          xml.copyright {
+            xml.human(:type=>'title')
+            xml.human(:type=>'description')
+            xml.machine(:type=>'uri')
+            
+          }
+          xml.access(:type=>"discover") {
+            xml.human
+            xml.machine
+          }
+          xml.access(:type=>"read") {
+            xml.human 
+            xml.machine
+          }
+          xml.access(:type=>"edit") {
+            xml.human
+            xml.machine
+          }
+          xml.embargo{
+            xml.machine
+          }
+          xml.lease{
+            xml.machine
+          }
+        }
+      end
+      return builder.doc
+    end
+      
+    # Returns the permissions for the selected person/group
+    # If new_access_level is provided, updates the selected person/group access_level to the one specified 
+    # A new_access_level of "none" will remove all access_levels for the selected person/group
+    # @param [Hash] selector hash in format {type => identifier}
+    # @param new_access_level (default nil)
+    # @return Hash in format {type => access_level}.  
+    # 
+    # ie. 
+    # permissions({:person=>"person123"})
+    # => {"person123"=>"edit"}
+    # permissions({:person=>"person123"}, "read")
+    # => {"person123"=>"read"}
+    # permissions({:person=>"person123"})
+    # => {"person123"=>"read"}
+    def permissions(selector, new_access_level=nil)
+      type = selector.keys.first.to_sym
+      actor = selector.values.first
+      if new_access_level.nil?
+        xpath = xpath(type, actor)
+        nodeset = self.find_by_terms(xpath)
+        if nodeset.empty?
+          return "none"
+        else
+          return nodeset.first.ancestors("access").first.attributes["type"].text
+        end
+      else
+        remove_all_permissions(selector)
+        if new_access_level == "none" 
+          self.content = self.to_xml
+        else
+          access_type_symbol = "#{new_access_level}_access".to_sym
+          current_values = term_values(access_type_symbol, type)
+          self.update_values([access_type_symbol, type] => current_values + [actor] )
+        end
+        return new_access_level
+      end
+        
+    end
+    
+    # Reports on which groups have which permissions
+    # @return Hash in format {group_name => group_permissions, group_name => group_permissions}
+    def groups
+      return quick_search_by_type(:group)
+    end
+    
+    def individuals
+      Deprecation.warn(RightsMetadata, "The method `individuals' is deprecated and will be removed from Hydra::Datastream::RightsMetadata in hydra-head 8.0.  Use `users' instead.", caller)
+      users
+    end
+
+    # Reports on which users have which permissions
+    # @return Hash in format {user_name => user_permissions, user_name => user_permissions}
+    def users
+      return quick_search_by_type(:person)
+    end
+    
+    # Updates permissions for all of the persons and groups in a hash
+    # @param params ex. {"group"=>{"group1"=>"discover","group2"=>"edit"}, "person"=>{"person1"=>"read","person2"=>"discover"}}
+    # Currently restricts actor type to group or person.  Any others will be ignored
+    def update_permissions(params)
+      params.fetch("group", {}).each_pair {|group_id, access_level| self.permissions({"group"=>group_id}, access_level)}
+      params.fetch("person", {}).each_pair {|person_id, access_level| self.permissions({"person"=>person_id}, access_level)}
+    end
+
+    # Updates all permissions
+    # @param params ex. {"group"=>{"group1"=>"discover","group2"=>"edit"}, "person"=>{"person1"=>"read","person2"=>"discover"}}
+    # Restricts actor type to group or person.  Any others will be ignored
+    def permissions= (params)
+      groups_for_update = params['group'] ? params['group'].keys : []
+      group_ids = groups.keys | groups_for_update
+      group_ids.each {|group_id| self.permissions({"group"=>group_id}, params['group'].fetch(group_id, 'none'))}
+      users_for_update = params['person'] ? params['person'].keys : []
+      user_ids = users.keys | users_for_update
+      user_ids.each {|person_id| self.permissions({"person"=>person_id}, params['person'].fetch(person_id, 'none'))}
+    end
+    
+    # @param [Symbol] type (either :group or :person)
+    # @return 
+    # This method limits the response to known access levels.  Probably runs a bit faster than .permissions().
+    def quick_search_by_type(type)
+      result = {}
+      [{:discover_access=>"discover"},{:read_access=>"read"},{:edit_access=>"edit"}].each do |access_levels_hash|
+        access_level = access_levels_hash.keys.first
+        access_level_name = access_levels_hash.values.first
+        self.find_by_terms(*[access_level, type]).each do |entry|
+          result[entry.text] = access_level_name
+        end
+      end
+      return result
+    end
+
+    def under_embargo?
+      (embargo_release_date.present? && Date.today < embargo_release_date.first) ? true : false
+    end
+
+    def active_lease?
+      lease_expiration_date.present? && Date.today < lease_expiration_date.first
+    end
+
+    def to_solr(solr_doc=Hash.new)
+      [:discover, :read, :edit].each do |access|
+        vals = send("#{access}_access").machine.group
+        solr_doc[Hydra.config.permissions[access].group] = vals unless vals.empty?
+        vals = send("#{access}_access").machine.person
+        solr_doc[Hydra.config.permissions[access].individual] = vals unless vals.empty?
+      end
+      if embargo_release_date.present?
+        key = Hydra.config.permissions.embargo.release_date.sub(/_[^_]+$/, '') #Strip off the suffix
+        ::Solrizer.insert_field(solr_doc, key, embargo_release_date, :stored_sortable)
+      end
+      if lease_expiration_date.present?
+        key = Hydra.config.permissions.lease.expiration_date.sub(/_[^_]+$/, '') #Strip off the suffix
+        ::Solrizer.insert_field(solr_doc, key, lease_expiration_date, :stored_sortable)
+      end
+      solr_doc[::Solrizer.solr_name("visibility_during_embargo", :symbol)] = visibility_during_embargo unless visibility_during_embargo.nil?
+      solr_doc[::Solrizer.solr_name("visibility_after_embargo", :symbol)] = visibility_after_embargo unless visibility_after_embargo.nil?
+      solr_doc[::Solrizer.solr_name("visibility_during_lease", :symbol)] = visibility_during_lease unless visibility_during_lease.nil?
+      solr_doc[::Solrizer.solr_name("visibility_after_lease", :symbol)] = visibility_after_lease unless visibility_after_lease.nil?
+      solr_doc[::Solrizer.solr_name("embargo_history", :symbol)] = embargo_history unless embargo_history.nil?
+      solr_doc[::Solrizer.solr_name("lease_history", :symbol)] = lease_history unless lease_history.nil?
+      solr_doc
+    end
+
+    def indexer
+      self.class.indexer
+    end
+
+    def self.indexer
+      @indexer ||= Solrizer::Descriptor.new(:string, :stored, :indexed, :multivalued)
+    end
+
+    def date_indexer
+      self.class.date_indexer
+    end
+
+    def self.date_indexer
+      @date_indexer ||= Solrizer::Descriptor.new(:date, :stored, :indexed)
+    end
+
+    # Completely clear the permissions
+    def clear_permissions!
+      remove_all_permissions({:person=>true})
+      remove_all_permissions({:group=>true})
+    end
+
+
+    
+    private
+    # Purge all access given group/person 
+    def remove_all_permissions(selector)
+      return unless ng_xml
+      type = selector.keys.first.to_sym
+      actor = selector.values.first
+      xpath = xpath(type, actor)
+      nodes_to_purge = self.find_by_terms(xpath)
+      nodes_to_purge.each {|node| node.remove}
+    end
+
+    # @param [Symbol] type (:group, :person)
+    # @param [String,TrueClass] actor the user we want to find. If actor is true, then don't query.
+    def xpath(type, actor)
+      raise ArgumentError, "Type must either be ':group' or ':person'. You provided: '#{type.inspect}'" unless [:group, :person].include?(type)
+      path = "//oxns:access/oxns:machine/oxns:#{type}"
+      if actor.is_a? String
+        clean_actor = actor.gsub("'", '')
+        path += "[text() = '#{clean_actor}']" 
+      end
+      path
+    end
+    
+  end
+end