curation_concerns 0.1.0

data/curation_concerns-models/app/actors/curation_concerns/generic_file_actor.rb

@@ -0,0 +1,150 @@
+module CurationConcerns
+  # Actions are decoupled from controller logic so that they may be called from a controller or a background job.
+  class GenericFileActor
+    include CurationConcerns::ManagesEmbargoesActor
+
+    attr_reader :generic_file, :user, :attributes, :curation_concern
+
+    def initialize(generic_file, user)
+      # we're setting attributes and curation_concern to bridge the difference
+      # between CurationConcerns::GenericFileActor and ManagesEmbargoesActor
+      @curation_concern = generic_file
+      @generic_file = generic_file
+      @user = user
+    end
+
+    # Adds the appropriate metadata, visibility and relationships to generic_file
+    #
+    # *Note*: In past versions of Sufia this method did not perform a save because it is mainly used in conjunction with
+    # create_content, which also performs a save.  However, due to the relationship between Hydra::PCDM objects,
+    # we have to save both the parent work and the generic_file in order to record the "metadata" relationship
+    # between them.
+    # @param [String] batch_id id of the batch that the file was uploaded within
+    # @param [String] work_id id of the parent work that will contain the generic_file.
+    # @param [Hash] generic_file_params specifying the visibility, lease and/or embargo of the generic file.  If you don't provide at least one of visibility, embargo_release_date or lease_expiration_date, visibility will be copied from the parent.
+
+    def create_metadata(batch_id, work_id, generic_file_params={})
+      generic_file.apply_depositor_metadata(user)
+      time_in_utc = DateTime.now.new_offset(0)
+      generic_file.date_uploaded = time_in_utc
+      generic_file.date_modified = time_in_utc
+      generic_file.creator = [user.name]
+      # TODO: Remove this? see https://github.com/projecthydra-labs/curation_concerns/issues/27
+      if batch_id && generic_file.respond_to?(:batch_id=)
+        generic_file.batch_id = batch_id
+      else
+        ActiveFedora::Base.logger.warn "unable to find batch to attach to"
+      end
+
+      unless work_id.blank?
+        work = ActiveFedora::Base.find(work_id)
+
+        if !((generic_file_params || {}).keys  & ["visibility", "embargo_release_date", "lease_expiration_date"]).empty?
+          interpret_visibility generic_file_params
+        else
+          copy_visibility(work, generic_file)
+        end
+        work.generic_files << generic_file
+        # Save the work so the association between the work and the generic_file is persisted (head_id)
+        work.save
+      end
+      yield(generic_file) if block_given?
+    end
+
+    def create_content(file)
+      # Tell UploadFileToGenericFile service to skip versioning because versions will be minted by VersionCommitter (called by save_characterize_and_record_committer) when necessary
+      Hydra::Works::UploadFileToGenericFile.call(generic_file, file, versioning: false)
+      generic_file.label ||= file.original_filename
+      generic_file.title = [generic_file.label] if generic_file.title.blank?
+      save_characterize_and_record_committer do
+        if CurationConcerns.config.respond_to?(:after_create_content)
+          CurationConcerns.config.after_create_content.call(generic_file, user)
+        end
+      end
+    end
+
+    def revert_content(revision_id)
+      generic_file.original_file.restore_version(revision_id)
+      save_characterize_and_record_committer do
+        if CurationConcerns.config.respond_to?(:after_revert_content)
+          CurationConcerns.config.after_revert_content.call(generic_file, user, revision_id)
+        end
+      end
+    end
+
+    def update_content(file)
+      # Tell UploadFileToGenericFile service to skip versioning because versions will be minted by VersionCommitter (called by save_characterize_and_record_committer) when necessary
+      Hydra::Works::UploadFileToGenericFile.call(generic_file, file, versioning: false)
+      save_characterize_and_record_committer do
+        if CurationConcerns.config.respond_to?(:after_update_content)
+          CurationConcerns.config.after_update_content.call(generic_file, user)
+        end
+      end
+    end
+
+    def update_metadata(model_attributes, all_attributes)
+      update_visibility(all_attributes)
+      model_attributes.delete(:visibility)  # Applying this attribute is handled by update_visibility
+      generic_file.attributes = model_attributes
+      generic_file.date_modified = DateTime.now
+      save do
+        if CurationConcerns.config.respond_to?(:after_update_metadata)
+          CurationConcerns.config.after_update_metadata.call(generic_file, user)
+        end
+      end
+    end
+
+    def destroy
+      generic_file.destroy
+      if CurationConcerns.config.respond_to?(:after_destroy)
+        CurationConcerns.config.after_destroy.call(generic_file.id, user)
+      end
+    end
+
+    # Saves the generic file, queues a job to characterize it, and records the committer.
+    # Takes a block which is run if the save was successful.
+    def save_characterize_and_record_committer
+      save do
+        push_characterize_job
+        CurationConcerns::VersioningService.create(generic_file.original_file, user)
+        yield if block_given?
+      end
+    end
+
+    # Takes an optional block and executes the block if the save was successful.
+    # returns false if the save was unsuccessful
+    def save
+      save_tries = 0
+      begin
+        return false unless generic_file.save
+      rescue RSolr::Error::Http => error
+        ActiveFedora::Base.logger.warn "CurationConcerns::GenericFileActor#save Caught RSOLR error #{error.inspect}"
+        save_tries+=1
+        # fail for good if the tries is greater than 3
+        raise error if save_tries >=3
+        sleep 0.01
+        retry
+      end
+      yield if block_given?
+      true
+    end
+
+    def push_characterize_job
+      CurationConcerns.queue.push(CharacterizeJob.new(@generic_file.id))
+    end
+
+    protected
+
+    # This method can be overridden in case there is a custom approach for visibility (e.g. embargo)
+    def update_visibility(attributes)
+      interpret_visibility(attributes)  # relies on CurationConcerns::ManagesEmbargoesActor to interpret and apply visibility
+    end
+
+    private
+
+    # copy visibility from source_concern to destination_concern
+    def copy_visibility(source_concern, destination_concern)
+      destination_concern.visibility =  source_concern.visibility
+    end
+  end
+end

data/curation_concerns-models/app/actors/curation_concerns/work_actor_behavior.rb

@@ -0,0 +1,88 @@
+
+module CurationConcerns::WorkActorBehavior
+  include CurationConcerns::ManagesEmbargoesActor
+
+  def create
+    # set the @files ivar then remove the files attribute so it isn't set by default.
+    files && attributes.delete(:files)
+    # Files must be attached before saving in order to persist their relationship to the work
+    assign_pid && interpret_visibility && attach_files && super && assign_representative && copy_visibility
+  end
+
+  def update
+    add_to_collections(attributes.delete(:collection_ids)) &&
+        interpret_visibility && super && attach_files && copy_visibility
+  end
+
+  delegate :visibility_changed?, to: :curation_concern
+
+  protected
+
+  # Is this here to ensure that the curation_concern has a pid set before any of the other methods are executed?
+  def assign_pid
+    curation_concern.send(:assign_id)
+  end
+
+  def files
+    return @files if defined?(@files)
+    @files = [attributes[:files]].flatten.compact
+  end
+
+  def attach_files
+    files.all? do |file|
+      attach_file(file)
+    end
+  end
+
+  # The default behavior of active_fedora's aggregates association,
+  # when assigning the id accessor (e.g. collection_ids = ['foo:1']) is to add
+  # to new collections, but not remove from old collections.
+  # This method ensures it's removed from the old collections.
+  def add_to_collections(new_collection_ids)
+    return true unless new_collection_ids
+    # remove from old collections
+    # TODO Implement parent_collection_ids https://github.com/projecthydra-labs/hydra-pcdm/issues/157
+    (curation_concern.parent_collections.map(&:id) - new_collection_ids).each do |old_id|
+      Collection.find(old_id).members.delete(curation_concern)
+    end
+
+    #add to new
+    new_collection_ids.each do |coll_id|
+      collection = Collection.find(coll_id)
+      collection.members << curation_concern
+      collection.save
+    end
+    true
+  end
+
+  def assign_representative
+    @generic_files ||= []
+    unless curation_concern.representative
+      curation_concern.representative = @generic_files.first.id unless @generic_files.empty?
+    end
+    curation_concern.save
+  end
+
+  private
+
+  def attach_file(file)
+    generic_file = ::GenericFile.new
+    generic_file_actor = CurationConcerns::GenericFileActor.new(generic_file, user)
+    #TODO we're passing an ID rather than an object. This means the actor does an unnecessary lookup
+    generic_file_actor.create_metadata(curation_concern.id, curation_concern.id)
+    generic_file.visibility = visibility
+    generic_file_actor.create_content(file)
+    @generic_files ||= []
+    @generic_files << generic_file # This is so that other methods like assign_representative can access the generic_files wihtout reloading them from fedora
+    curation_concern.generic_files << generic_file
+  end
+
+  def valid_file?(file_path)
+    return file_path.present? && File.exists?(file_path) && !File.zero?(file_path)
+  end
+
+  # The path of the fedora node where we store the file data
+  def file_path
+    'content'
+  end
+end

data/curation_concerns-models/app/jobs/active_fedora_id_based_job.rb

@@ -0,0 +1,22 @@
+class ActiveFedoraIdBasedJob
+  def queue_name
+    :id_based
+  end
+
+  attr_accessor :id
+
+  def initialize(id)
+    self.id = id
+  end
+
+  def object
+    @object ||= ActiveFedora::Base.find(id)
+  end
+
+  alias_method :generic_file, :object
+  alias_method :generic_file_id, :id
+
+  def run
+    raise RuntimeError, "Define #run in a subclass"
+  end
+end

data/curation_concerns-models/app/jobs/active_fedora_pid_based_job.rb

@@ -0,0 +1,6 @@
+class ActiveFedoraPidBasedJob < ActiveFedoraIdBasedJob
+  extend Deprecation
+  def self.extended(document)
+    Deprecation.warn ActiveFedoraPidBasedJob, "ActiveFedoraPidBasedJob is deprecated; use ActiveFedoraIdBasedJob instead."
+  end
+end

data/curation_concerns-models/app/jobs/audit_job.rb

@@ -0,0 +1,58 @@
+class AuditJob < ActiveFedoraIdBasedJob
+  def queue_name
+    :audit
+  end
+
+  attr_accessor :uri, :id, :file_id
+
+  # URI of the resource to audit.
+  # This URI could include the actual resource (e.g. content) and the version to audit:
+  #     http://localhost:8983/fedora/rest/test/a/b/c/abcxyz/content/fcr:versions/version1
+  # but it could also just be:
+  #     http://localhost:8983/fedora/rest/test/a/b/c/abcxyz/content
+  # @param [String] id of the parent object
+  # @param [String] file_id used to find the file within its parent object (usually "original_file")
+  # @param [String] uri of the specific file/version to be audited
+  def initialize(id, file_id, uri)
+    super(id)
+    self.file_id = file_id
+    self.uri = uri
+  end
+
+  def run
+    fixity_ok = false
+    log = run_audit
+    fixity_ok = (log.pass == 1)
+    unless fixity_ok
+      if CurationConcerns.config.respond_to?(:after_audit_failure)
+        login = generic_file.depositor
+        user = User.find_by_user_key(login)
+        CurationConcerns.config.after_audit_failure.call(generic_file, user, log.created_at)
+      end
+    end
+    fixity_ok
+  end
+
+  protected
+
+    def run_audit
+      begin
+        fixity_ok = ActiveFedora::FixityService.new(uri).check
+      rescue Ldp::NotFound
+        error_msg = "resource not found"
+      end
+
+      if fixity_ok
+        passing = 1
+        ChecksumAuditLog.prune_history(id, file_id)
+      else
+        logger.warn "***AUDIT*** Audit failed for #{uri} #{error_msg}"
+        passing = 0
+      end
+      ChecksumAuditLog.create!(pass: passing, generic_file_id: id, version: uri, file_id: file_id)
+    end
+
+    def logger
+      ActiveFedora::Base.logger
+    end
+end

data/curation_concerns-models/app/jobs/characterize_job.rb

@@ -0,0 +1,11 @@
+class CharacterizeJob < ActiveFedoraIdBasedJob
+  def queue_name
+    :characterize
+  end
+
+  def run
+    CurationConcerns::CharacterizationService.run(generic_file)
+    generic_file.save
+    CurationConcerns.queue.push(CreateDerivativesJob.new(generic_file.id))
+  end
+end

data/curation_concerns-models/app/jobs/copy_permissions_job.rb

@@ -0,0 +1,24 @@
+class CopyPermissionsJob
+  def queue_name
+    :permissions
+  end
+
+  attr_accessor :id
+
+  def initialize(id)
+    self.id = id
+  end
+
+  def run
+    # work = ActiveFedora::Base.load_instance_from_solr(id)
+    work = ActiveFedora::Base.find(id)  # Hydra::Works::GenericWork is not compatible with load_instance_from_solr.  Using (slower) ActiveFedora::Base.find instead.
+    if work.respond_to?(:generic_files)
+      work.generic_files.each do |file|
+
+        work.permissions.each {|perm| file.permissions << Hydra::AccessControls::Permission.new(perm.to_hash)}
+        new_permissions = file.permissions.uniq! { |item| item.to_hash}
+        file.permissions = new_permissions unless new_permissions.blank?
+      end
+    end
+  end
+end

data/curation_concerns-models/app/jobs/create_derivatives_job.rb

@@ -0,0 +1,15 @@
+class CreateDerivativesJob < ActiveFedoraIdBasedJob
+  def queue_name
+    :derivatives
+  end
+
+  def run
+    return unless generic_file.original_file.has_content?
+    if generic_file.video?
+      return unless CurationConcerns.config.enable_ffmpeg
+    end
+    
+    generic_file.create_derivatives
+    generic_file.save
+  end
+end

data/curation_concerns-models/app/jobs/import_url_job.rb

@@ -0,0 +1,56 @@
+require 'net/https'
+require 'uri'
+require 'tempfile'
+
+class ImportUrlJob < ActiveFedoraIdBasedJob
+
+  def queue_name
+    :import_url
+  end
+
+  def run
+    user = User.find_by_user_key(generic_file.depositor)
+
+    Tempfile.open(id.gsub('/', '_')) do |f|
+      copy_remote_file(generic_file.import_url, f)
+      # attach downloaded file to generic file stubbed out
+      if CurationConcerns::GenericFileActor.new(generic_file, user).create_content(f)
+
+        # send message to user on download success
+        if CurationConcerns.config.respond_to?(:after_import_url_success)
+          CurationConcerns.config.after_import_url_success.call(generic_file, user)
+        end
+      else
+
+        # send message to user on download failure
+        if CurationConcerns.config.respond_to?(:after_import_url_failure)
+          CurationConcerns.config.after_import_url_failure.call(generic_file, user)
+        end
+      end
+    end
+  end
+
+  def copy_remote_file(import_url, f)
+    f.binmode
+    # download file from url
+    uri = URI(generic_file.import_url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == "https"  # enable SSL/TLS
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    mime_type = nil
+
+    http.start do
+      http.request_get(uri.request_uri) do |resp|
+        mime_type = resp.content_type
+        resp.read_body do |segment|
+          f.write(segment)
+        end
+      end
+    end
+    f.rewind
+  end
+
+  def job_user
+    User.batchuser
+  end
+end

data/curation_concerns-models/app/jobs/ingest_local_file_job.rb

@@ -0,0 +1,48 @@
+class IngestLocalFileJob
+  attr_accessor :directory, :filename, :user_key, :generic_file_id
+
+  def queue_name
+    :ingest
+  end
+
+  def initialize(generic_file_id, directory, filename, user_key)
+    self.generic_file_id = generic_file_id
+    self.directory = directory
+    self.filename = filename
+    self.user_key = user_key
+  end
+
+  def run
+    user = User.find_by_user_key(user_key)
+    raise "Unable to find user for #{user_key}" unless user
+    generic_file = GenericFile.find(generic_file_id)
+    generic_file.label ||= filename
+    path = File.join(directory, filename)
+
+    actor = CurationConcerns::GenericFileActor.new(generic_file, user)
+
+    if actor.create_content(File.open(path))
+      FileUtils.rm(path)
+
+      # send message to user on import success
+      if CurationConcerns.config.respond_to?(:after_import_local_file_success)
+        CurationConcerns.config.after_import_local_file_success.call(generic_file, user, filename)
+      end
+    else
+
+      # send message to user on import failure
+      if CurationConcerns.config.respond_to?(:after_import_local_file_failure)
+        CurationConcerns.config.after_import_local_file_failure.call(generic_file, user, filename)
+      end
+    end
+  end
+
+  def job_user
+    User.batchuser
+  end
+
+  def mime_type(file_name)
+    mime_types = MIME::Types.of(file_name)
+    mime_types.empty? ? "application/octet-stream" : mime_types.first.content_type
+  end
+end

data/curation_concerns-models/app/jobs/resolrize_job.rb

@@ -0,0 +1,9 @@
+class ResolrizeJob
+  def queue_name
+    :resolrize
+  end
+
+  def run
+    ActiveFedora::Base.reindex_everything
+  end
+end

data/curation_concerns-models/app/models/checksum_audit_log.rb

@@ -0,0 +1,21 @@
+class ChecksumAuditLog < ActiveRecord::Base
+
+  def self.get_audit_log(id, path, version_uri)
+    ChecksumAuditLog.find_or_create_by(generic_file_id: id, file_id: path, version: version_uri)
+  end
+
+  # Check to see if there are previous passing logs that we can delete
+  # we want to keep the first passing event after a failure, the most current passing event,
+  # and all failures so that this table doesn't grow too large
+  # Simple way (a little naieve): if the last 2 were passing, delete the first one
+  def self.prune_history(id, path)
+    list = logs_for(id, path).limit(2)
+    if list.size > 1 && (list[0].pass == 1) && (list[1].pass == 1)
+      list[0].destroy
+    end
+  end
+
+  def self.logs_for(id, path)
+    ChecksumAuditLog.where(generic_file_id: id, file_id: path).order('created_at desc, id desc')
+  end
+end

data/curation_concerns-models/app/models/concerns/curation_concerns/ability.rb

@@ -0,0 +1,34 @@
+module CurationConcerns
+  module Ability
+    extend ActiveSupport::Concern
+    included do
+      self.ability_logic += [:curation_concerns_permissions]
+    end
+
+    def curation_concerns_permissions
+
+      unless current_user.new_record?
+        can :create, CurationConcerns::ClassifyConcern
+        # TODO: Shouldn't this be in `everyone_can_create_curation_concerns` ?
+        can :create, ::GenericFile
+      end
+
+      if user_groups.include? 'admin'
+       can [:create, :discover, :show, :read, :edit, :update, :destroy], :all
+      end
+
+      can :collect, :all
+
+    end
+
+    # Add this to your ability_logic if you want all logged in users to be able
+    # to submit content
+    def everyone_can_create_curation_concerns
+      unless current_user.new_record?
+        can :create, [ CurationConcerns.configuration.curation_concerns ]
+        can :create, ::Collection
+      end
+    end
+
+  end
+end

data/curation_concerns-models/app/models/concerns/curation_concerns/basic_metadata.rb

@@ -0,0 +1,87 @@
+module CurationConcerns
+  module BasicMetadata
+    extend ActiveSupport::Concern
+
+    included do
+
+      property :label, predicate: ActiveFedora::RDF::Fcrepo::Model.downloadFilename, multiple: false
+
+      property :depositor, predicate: ::RDF::URI.new("http://id.loc.gov/vocabulary/relators/dpt"), multiple: false do |index|
+        index.as :symbol, :stored_searchable
+      end
+
+      property :relative_path, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#relativePath'), multiple: false
+
+      property :import_url, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#importUrl'), multiple: false do |index|
+        index.as :symbol
+      end
+
+      property :part_of, predicate: ::RDF::DC.isPartOf
+      property :resource_type, predicate: ::RDF::DC.type do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :title, predicate: ::RDF::DC.title do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :creator, predicate: ::RDF::DC.creator do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :contributor, predicate: ::RDF::DC.contributor do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :description, predicate: ::RDF::DC.description do |index|
+        index.type :text
+        index.as :stored_searchable
+      end
+      property :tag, predicate: ::RDF::DC.relation do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :rights, predicate: ::RDF::DC.rights do |index|
+        index.as :stored_searchable
+      end
+      property :publisher, predicate: ::RDF::DC.publisher do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :date_created, predicate: ::RDF::DC.created do |index|
+        index.as :stored_searchable
+      end
+
+      # We reserve date_uploaded for the original creation date of the record.
+      # For example, when migrating data from a fedora3 repo to fedora4,
+      # fedora's system created date will reflect the date when the record
+      # was created in fedora4, but the date_uploaded will preserve the
+      # original creation date from the old repository.
+      property :date_uploaded, predicate: ::RDF::DC.dateSubmitted, multiple: false do |index|
+        index.type :date
+        index.as :stored_sortable
+      end
+
+      property :date_modified, predicate: ::RDF::DC.modified, multiple: false do |index|
+        index.type :date
+        index.as :stored_sortable
+      end
+      property :subject, predicate: ::RDF::DC.subject do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :language, predicate: ::RDF::DC.language do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :identifier, predicate: ::RDF::DC.identifier do |index|
+        index.as :stored_searchable
+      end
+      property :based_near, predicate: ::RDF::FOAF.based_near do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :related_url, predicate: ::RDF::RDFS.seeAlso do |index|
+        index.as :stored_searchable
+      end
+      property :bibliographic_citation, predicate: ::RDF::DC.bibliographicCitation do |index|
+        index.as :stored_searchable
+      end
+      property :source, predicate: ::RDF::DC.source do |index|
+        index.as :stored_searchable
+      end
+    end
+
+  end
+end

data/curation_concerns-models/app/models/concerns/curation_concerns/collection_behavior.rb

@@ -0,0 +1,47 @@
+module CurationConcerns
+  module CollectionBehavior
+    extend ActiveSupport::Concern
+
+    include Hydra::AccessControls::WithAccessRight
+    include Hydra::Collection
+    include CurationConcerns::Noid
+    include CurationConcerns::HumanReadableType
+    include CurationConcerns::HasRepresentative
+    include CurationConcerns::Permissions
+
+    included do
+      validates :title, presence: true
+    end
+
+    def add_member(collectible)
+      if can_add_to_members?(collectible)
+        self.members << collectible
+        save
+      end
+    end
+
+    def to_s
+      title.present? ? title : "No Title"
+    end
+
+    def bytes
+      members.reduce(0) { |sum, gf| sum + gf.content.size.to_i }
+    end
+
+    def to_solr(solr_doc={})
+      super(solr_doc).tap do |solr_doc|
+        Solrizer.set_field(solr_doc, 'generic_type', human_readable_type, :facetable)
+      end
+    end
+
+    def can_be_member_of_collection?(collection)
+      collection == self ? false : true
+    end
+
+    private
+
+    def can_add_to_members?(collectible)
+      collectible.try(:can_be_member_of_collection?, self)
+    end
+  end
+end