curation_concerns 0.12.0.pre1 → 0.12.0.pre2

data/app/indexers/curation_concerns/work_indexer.rb

@@ -0,0 +1,15 @@
+module CurationConcerns
+  class WorkIndexer < ActiveFedora::IndexingService
+    include IndexesThumbnails
+    def generate_solr_document
+      super.tap do |solr_doc|
+        # We know that all the members of GenericWorks are FileSets so we can use
+        # member_ids which requires fewer Fedora API calls than file_set_ids.
+        # file_set_ids requires loading all the members from Fedora but member_ids
+        # looks just at solr
+        solr_doc[Solrizer.solr_name('member_ids', :symbol)] = object.member_ids
+        Solrizer.set_field(solr_doc, 'generic_type', 'Work', :facetable)
+      end
+    end
+  end
+end

data/app/jobs/audit_job.rb

@@ -0,0 +1,49 @@
+class AuditJob < ActiveJob::Base
+  queue_as :audit
+
+  # URI of the resource to audit.
+  # This URI could include the actual resource (e.g. content) and the version to audit:
+  #     http://localhost:8983/fedora/rest/test/a/b/c/abcxyz/content/fcr:versions/version1
+  # but it could also just be:
+  #     http://localhost:8983/fedora/rest/test/a/b/c/abcxyz/content
+  # @param [FileSet] the parent object
+  # @param [String] file_id used to find the file within its parent object (usually "original_file")
+  # @param [String] uri of the specific file/version to be audited
+  def perform(file_set, file_id, uri)
+    log = run_audit(file_set, file_id, uri)
+    fixity_ok = log.pass == 1
+    unless fixity_ok
+      if CurationConcerns.config.callback.set?(:after_audit_failure)
+        login = file_set.depositor
+        user = User.find_by_user_key(login)
+        CurationConcerns.config.callback.run(:after_audit_failure, file_set, user, log.created_at)
+      end
+    end
+    fixity_ok
+  end
+
+  protected
+
+    def run_audit(file_set, file_id, uri)
+      begin
+        fixity_ok = ActiveFedora::FixityService.new(uri).check
+      rescue Ldp::NotFound
+        error_msg = 'resource not found'
+      end
+
+      if fixity_ok
+        passing = 1
+        ChecksumAuditLog.prune_history(file_set.id, file_id)
+      else
+        logger.warn "***AUDIT*** Audit failed for #{uri} #{error_msg}"
+        passing = 0
+      end
+      ChecksumAuditLog.create!(pass: passing, file_set_id: file_set.id, version: uri, file_id: file_id)
+    end
+
+  private
+
+    def logger
+      ActiveFedora::Base.logger || CurationConcerns::NullLogger.new
+    end
+end

data/app/jobs/characterize_job.rb

@@ -0,0 +1,11 @@
+class CharacterizeJob < ActiveJob::Base
+  queue_as :characterize
+
+  # @param [FileSet] file_set
+  # @param [String] filename a local path for the file to characterize. By using this, we don't have to pull a copy out of fedora.
+  def perform(file_set, filename)
+    Hydra::Works::CharacterizationService.run(file_set, filename)
+    file_set.save!
+    CreateDerivativesJob.perform_later(file_set, filename)
+  end
+end

data/app/jobs/create_derivatives_job.rb

@@ -0,0 +1,21 @@
+class CreateDerivativesJob < ActiveJob::Base
+  queue_as :derivatives
+
+  # @param [FileSet] file_set
+  # @param [String] file_name
+  def perform(file_set, file_name)
+    return if file_set.video? && !CurationConcerns.config.enable_ffmpeg
+
+    file_set.create_derivatives(file_name)
+    # The thumbnail is indexed in the solr document, so reindex
+    file_set.update_index
+    file_set.parent.update_index if parent_needs_reindex?(file_set)
+  end
+
+  # If this file_set is the thumbnail for the parent work,
+  # then the parent also needs to be reindexed.
+  def parent_needs_reindex?(file_set)
+    return false unless file_set.parent
+    file_set.parent.thumbnail_id == file_set.id
+  end
+end

data/app/jobs/import_url_job.rb

@@ -0,0 +1,48 @@
+require 'net/https'
+require 'uri'
+require 'tempfile'
+
+class ImportUrlJob < ActiveJob::Base
+  queue_as :import_url
+
+  def perform(file_set)
+    user = User.find_by_user_key(file_set.depositor)
+
+    Tempfile.open(file_set.id.tr('/', '_')) do |f|
+      copy_remote_file(file_set, f)
+
+      # reload the generic file once the data is copied since this is a long running task
+      file_set.reload
+
+      # attach downloaded file to generic file stubbed out
+      if CurationConcerns::FileSetActor.new(file_set, user).create_content(f)
+        # send message to user on download success
+        CurationConcerns.config.callback.run(:after_import_url_success, file_set, user)
+      else
+        CurationConcerns.config.callback.run(:after_import_url_failure, file_set, user)
+      end
+    end
+  end
+
+  protected
+
+    def copy_remote_file(file_set, f)
+      f.binmode
+      # download file from url
+      uri = URI(file_set.import_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == 'https' # enable SSL/TLS
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      mime_type = nil
+
+      http.start do
+        http.request_get(uri.request_uri) do |resp|
+          mime_type = resp.content_type
+          resp.read_body do |segment|
+            f.write(segment)
+          end
+        end
+      end
+      f.rewind
+    end
+end

data/app/jobs/ingest_file_job.rb

@@ -0,0 +1,30 @@
+class IngestFileJob < ActiveJob::Base
+  queue_as :ingest
+
+  # @param [FileSet] file_set
+  # @param [String] filename
+  # @param [String,NilClass] mime_type
+  # @param [String] user_key
+  # @param [String] relation ('original_file')
+  def perform(file_set, filename, mime_type, user_key, relation = 'original_file')
+    file = File.open(filename, "rb")
+    # If mime-type is known, wrap in an IO decorator
+    # Otherwise allow Hydra::Works service to determine mime_type
+    if mime_type
+      file = Hydra::Derivatives::IoDecorator.new(file)
+      file.mime_type = mime_type
+      file.original_name = File.basename(filename)
+    end
+
+    # Tell AddFileToFileSet service to skip versioning because versions will be minted by VersionCommitter (called by save_characterize_and_record_committer) when necessary
+    Hydra::Works::AddFileToFileSet.call(file_set, file, relation.to_sym, versioning: false)
+
+    # Persist changes to the file_set
+    file_set.save!
+
+    # Do post file ingest actions
+    user = User.find_by_user_key(user_key)
+    CurationConcerns::VersioningService.create(file_set.send(relation.to_sym), user)
+    CurationConcerns.config.callback.run(:after_create_content, file_set, user)
+  end
+end

data/app/jobs/ingest_local_file_job.rb

@@ -0,0 +1,20 @@
+class IngestLocalFileJob < ActiveJob::Base
+  queue_as :ingest_local
+
+  def perform(file_set_id, directory, filename, user_key)
+    user = User.find_by_user_key(user_key)
+    fail "Unable to find user for #{user_key}" unless user
+    file_set = FileSet.find(file_set_id)
+    file_set.label ||= filename
+    path = File.join(directory, filename)
+
+    actor = CurationConcerns::FileSetActor.new(file_set, user)
+
+    if actor.create_content(File.open(path))
+      FileUtils.rm(path)
+      CurationConcerns.config.callback.run(:after_import_local_file_success, file_set, user, filename)
+    else
+      CurationConcerns.config.callback.run(:after_import_local_file_failure, file_set, user, filename)
+    end
+  end
+end

data/app/jobs/resolrize_job.rb

@@ -0,0 +1,7 @@
+class ResolrizeJob < ActiveJob::Base
+  queue_as :resolrize
+
+  def perform
+    ActiveFedora::Base.reindex_everything
+  end
+end

data/app/models/checksum_audit_log.rb

@@ -0,0 +1,20 @@
+class ChecksumAuditLog < ActiveRecord::Base
+  def self.get_audit_log(id, path, version_uri)
+    ChecksumAuditLog.find_or_create_by(file_set_id: id, file_id: path, version: version_uri)
+  end
+
+  # Check to see if there are previous passing logs that we can delete
+  # we want to keep the first passing event after a failure, the most current passing event,
+  # and all failures so that this table doesn't grow too large
+  # Simple way (a little naieve): if the last 2 were passing, delete the first one
+  def self.prune_history(id, path)
+    list = logs_for(id, path).limit(2)
+    if list.size > 1 && (list[0].pass == 1) && (list[1].pass == 1)
+      list[0].destroy
+    end
+  end
+
+  def self.logs_for(id, path)
+    ChecksumAuditLog.where(file_set_id: id, file_id: path).order('created_at desc, id desc')
+  end
+end

data/app/models/collection.rb

@@ -0,0 +1,6 @@
+# Generated by curation_concerns:models:install
+class Collection < ActiveFedora::Base
+  include ::CurationConcerns::CollectionBehavior
+  # You can replace these metadata if they're not suitable
+  include CurationConcerns::BasicMetadata
+end

data/app/models/concerns/curation_concerns/ability.rb

@@ -0,0 +1,49 @@
+module CurationConcerns
+  module Ability
+    extend ActiveSupport::Concern
+    included do
+      self.ability_logic += [:curation_concerns_permissions, :add_to_collection]
+    end
+
+    def curation_concerns_permissions
+      unless current_user.new_record?
+        can :create, CurationConcerns::ClassifyConcern
+      end
+
+      # user can version if they can edit
+      alias_action :versions, to: :update
+
+      if admin?
+        admin_permissions
+      else
+        cannot :index, Hydra::AccessControls::Embargo
+        cannot :index, Hydra::AccessControls::Lease
+      end
+    end
+
+    def admin_permissions
+      can [:create, :discover, :show, :read, :edit, :update, :destroy], :all
+    end
+
+    def admin?
+      user_groups.include? 'admin'
+    end
+
+    def add_to_collection
+      return if current_user.new_record?
+      can :collect, :all
+    end
+
+    def registered_user?
+      user_groups.include? 'registered'
+    end
+
+    # Add this to your ability_logic if you want all logged in users to be able
+    # to submit content
+    def everyone_can_create_curation_concerns
+      return unless registered_user?
+      can :create, [::FileSet, ::Collection]
+      can :create, [CurationConcerns.config.curation_concerns]
+    end
+  end
+end

data/app/models/concerns/curation_concerns/basic_metadata.rb

@@ -0,0 +1,64 @@
+module CurationConcerns
+  module BasicMetadata
+    extend ActiveSupport::Concern
+
+    included do
+      property :label, predicate: ActiveFedora::RDF::Fcrepo::Model.downloadFilename, multiple: false
+
+      property :relative_path, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#relativePath'), multiple: false
+
+      property :import_url, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#importUrl'), multiple: false do |index|
+        index.as :symbol
+      end
+
+      property :part_of, predicate: ::RDF::Vocab::DC.isPartOf
+      property :resource_type, predicate: ::RDF::Vocab::DC.type do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :creator, predicate: ::RDF::Vocab::DC11.creator do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :contributor, predicate: ::RDF::Vocab::DC11.contributor do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :description, predicate: ::RDF::Vocab::DC11.description do |index|
+        index.type :text
+        index.as :stored_searchable
+      end
+      property :tag, predicate: ::RDF::Vocab::DC11.relation do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :rights, predicate: ::RDF::Vocab::DC.rights do |index|
+        index.as :stored_searchable
+      end
+      property :publisher, predicate: ::RDF::Vocab::DC11.publisher do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :date_created, predicate: ::RDF::Vocab::DC.created do |index|
+        index.as :stored_searchable
+      end
+
+      property :subject, predicate: ::RDF::Vocab::DC11.subject do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :language, predicate: ::RDF::Vocab::DC11.language do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :identifier, predicate: ::RDF::Vocab::DC.identifier do |index|
+        index.as :stored_searchable
+      end
+      property :based_near, predicate: ::RDF::Vocab::FOAF.based_near do |index|
+        index.as :stored_searchable, :facetable
+      end
+      property :related_url, predicate: ::RDF::RDFS.seeAlso do |index|
+        index.as :stored_searchable
+      end
+      property :bibliographic_citation, predicate: ::RDF::Vocab::DC.bibliographicCitation do |index|
+        index.as :stored_searchable
+      end
+      property :source, predicate: ::RDF::Vocab::DC.source do |index|
+        index.as :stored_searchable
+      end
+    end
+  end
+end

data/app/models/concerns/curation_concerns/collection.rb

@@ -0,0 +1,16 @@
+module CurationConcerns
+  module Collection
+    extend ActiveSupport::Concern
+    extend Deprecation
+    include Hydra::Works::CollectionBehavior
+    include Hydra::WithDepositor # for access to apply_depositor_metadata
+    include Hydra::AccessControls::Permissions
+    include CurationConcerns::RequiredMetadata
+    include Hydra::Works::CollectionBehavior
+
+    def add_members(new_member_ids)
+      return if new_member_ids.nil? || new_member_ids.empty?
+      members << ActiveFedora::Base.find(new_member_ids)
+    end
+  end
+end

data/app/models/concerns/curation_concerns/collection_behavior.rb

@@ -0,0 +1,62 @@
+module CurationConcerns
+  module CollectionBehavior
+    extend ActiveSupport::Concern
+
+    include Hydra::AccessControls::WithAccessRight
+    include CurationConcerns::Collection
+    include CurationConcerns::Noid
+    include CurationConcerns::HumanReadableType
+    include CurationConcerns::HasRepresentative
+    include CurationConcerns::Permissions
+
+    included do
+      validates_with HasOneTitleValidator
+    end
+
+    def to_s
+      title.present? ? title : 'No Title'
+    end
+
+    module ClassMethods
+      def indexer
+        CurationConcerns::CollectionIndexer
+      end
+    end
+
+    # Compute the sum of each file in the collection using Solr to
+    # avoid having to access Fedora
+    #
+    # @return [Fixnum] size of collection in bytes
+    # @raise [RuntimeError] unsaved record does not exist in solr
+    def bytes
+      return 0 if member_ids.count == 0
+
+      raise "Collection must be saved to query for bytes" if new_record?
+
+      # One query per member_id because Solr is not a relational database
+      sizes = member_ids.collect do |work_id|
+        query = ActiveFedora::SolrQueryBuilder.construct_query_for_rel(has_model: ::FileSet.to_class_uri)
+        argz = { fl: "id, #{file_size_field}",
+                 fq: "{!join from=#{member_ids_field} to=id}id:#{work_id}"
+        }
+        files = ActiveFedora::SolrService.query(query, argz)
+        files.reduce(0) { |sum, f| sum + f[file_size_field].to_i }
+      end
+
+      sizes.reduce(0, :+)
+    end
+
+    private
+
+      # Field name to look up when locating the size of each file in Solr.
+      # Override for your own installation if using something different
+      def file_size_field
+        Solrizer.solr_name(:file_size, CurationConcerns::FileSetIndexer::STORED_INTEGER)
+      end
+
+      # Solr field name collections and works use to index member ids
+      def member_ids_field
+        Solrizer.solr_name('member_ids', :symbol)
+      end
+  end
+end

data/app/models/concerns/curation_concerns/file_set/belongs_to_works.rb

@@ -0,0 +1,47 @@
+module CurationConcerns
+  module FileSet
+    module BelongsToWorks
+      extend ActiveSupport::Concern
+
+      included do
+        before_destroy :remove_representative_relationship
+      end
+
+      def generic_works
+        in_objects # in_objects is provided by Hydra::PCDM::ObjectBehavior
+      end
+
+      # OPTIMIZE: We can load this from Solr much faster than loading the objects
+      def generic_work_ids
+        generic_works.map(&:id)
+      end
+
+      # Returns the first parent object
+      # This is a hack to handle things like FileSets inheriting access controls from their parent.  (see CurationConcerns::ParentContainer in app/controllers/concerns/curation_concers/parent_container.rb)
+      def parent
+        in_objects.first
+      end
+
+      # Returns the id of first parent object
+      # This is a hack to handle things like FileSets inheriting access controls from their parent.  (see CurationConcerns::ParentContainer in app/controllers/concerns/curation_concers/parent_container.rb)
+      delegate :id, to: :parent, prefix: true
+
+      # Files with sibling relationships
+      # Returns all FileSets aggregated by any of the GenericWorks that aggregate the current object
+      def related_files
+        generic_works = self.generic_works
+        return [] if generic_works.empty?
+        generic_works.flat_map { |work| work.file_sets.select { |file_set| file_set.id != id } }
+      end
+
+      # If any parent works are pointing at this object as their representative, remove that pointer.
+      def remove_representative_relationship
+        generic_works = self.generic_works
+        return if generic_works.empty?
+        generic_works.each do |work|
+          work.update(representative_id: nil) if work.representative_id == id
+        end
+      end
+    end
+  end
+end

data/app/models/concerns/curation_concerns/file_set/derivatives.rb

@@ -0,0 +1,65 @@
+module CurationConcerns
+  module FileSet
+    module Derivatives
+      extend ActiveSupport::Concern
+
+      included do
+        Hydra::Derivatives.source_file_service = CurationConcerns::LocalFileService
+        Hydra::Derivatives.output_file_service = CurationConcerns::PersistDerivatives
+        Hydra::Derivatives::FullTextExtract.output_file_service = CurationConcerns::PersistDirectlyContainedOutputFileService
+        after_destroy :cleanup_derivatives
+      end
+
+      # This completely overrides the version in Hydra::Works so that we
+      # read and write to a local file. It's important that characterization runs
+      # before derivatives so that we have a credible mime_type field to work with.
+      def create_derivatives(filename)
+        case mime_type
+        when *self.class.pdf_mime_types
+          Hydra::Derivatives::PdfDerivatives.create(filename,
+                                                    outputs: [{ label: :thumbnail, format: 'jpg', size: '338x493', url: derivative_url('thumbnail') }])
+          Hydra::Derivatives::FullTextExtract.create(filename,
+                                                     outputs: [{ url: uri, container: "extracted_text" }])
+        when *self.class.office_document_mime_types
+          Hydra::Derivatives::DocumentDerivatives.create(filename,
+                                                         outputs: [{ label: :thumbnail, format: 'jpg',
+                                                                     size: '200x150>',
+                                                                     url: derivative_url('thumbnail') }])
+          Hydra::Derivatives::FullTextExtract.create(filename,
+                                                     outputs: [{ url: uri, container: "extracted_text" }])
+        when *self.class.audio_mime_types
+          Hydra::Derivatives::AudioDerivatives.create(filename,
+                                                      outputs: [{ label: 'mp3', format: 'mp3', url: derivative_url('mp3') },
+                                                                { label: 'ogg', format: 'ogg', url: derivative_url('ogg') }])
+        when *self.class.video_mime_types
+          Hydra::Derivatives::VideoDerivatives.create(filename,
+                                                      outputs: [{ label: :thumbnail, format: 'jpg', url: derivative_url('thumbnail') },
+                                                                { label: 'webm', format: 'webm', url: derivative_url('webm') },
+                                                                { label: 'mp4', format: 'mp4', url: derivative_url('mp4') }])
+        when *self.class.image_mime_types
+          Hydra::Derivatives::ImageDerivatives.create(filename,
+                                                      outputs: [{ label: :thumbnail, format: 'jpg', size: '200x150>', url: derivative_url('thumbnail') }])
+        end
+      end
+
+      private
+
+        # The destination_name parameter has to match up with the file parameter
+        # passed to the DownloadsController
+        def derivative_url(destination_name)
+          path = derivative_path_factory.derivative_path_for_reference(self, destination_name)
+          URI("file://#{path}").to_s
+        end
+
+        def cleanup_derivatives
+          derivative_path_factory.derivatives_for_reference(self).each do |path|
+            FileUtils.rm_f(path)
+          end
+        end
+
+        def derivative_path_factory
+          DerivativePath
+        end
+    end
+  end
+end

data/app/models/concerns/curation_concerns/file_set/full_text_indexing.rb

@@ -0,0 +1,11 @@
+module CurationConcerns
+  module FileSet
+    module FullTextIndexing
+      extend ActiveSupport::Concern
+
+      included do
+        contains 'full_text'
+      end
+    end
+  end
+end

data/app/models/concerns/curation_concerns/file_set/indexing.rb

@@ -0,0 +1,14 @@
+module CurationConcerns
+  module FileSet
+    module Indexing
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        # override the default indexing service
+        def indexer
+          CurationConcerns::FileSetIndexer
+        end
+      end
+    end
+  end
+end

data/app/models/concerns/curation_concerns/file_set/querying.rb

@@ -0,0 +1,17 @@
+module CurationConcerns
+  module FileSet
+    module Querying
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def where_digest_is(digest_string)
+          where Solrizer.solr_name('digest', :symbol) => urnify(digest_string)
+        end
+
+        def urnify(digest_string)
+          "urn:sha1:#{digest_string}"
+        end
+      end
+    end
+  end
+end

data/app/models/concerns/curation_concerns/file_set_behavior.rb

@@ -0,0 +1,36 @@
+module CurationConcerns
+  module FileSetBehavior
+    extend ActiveSupport::Concern
+    include Hydra::Works::FileSetBehavior
+    include Hydra::Works::VirusCheck
+    include Hydra::Works::Characterization
+    include Hydra::WithDepositor
+    include CurationConcerns::Serializers
+    include CurationConcerns::Noid
+    include CurationConcerns::FileSet::Derivatives
+    include CurationConcerns::Permissions
+    include CurationConcerns::BasicMetadata
+    include CurationConcerns::FileSet::FullTextIndexing
+    include CurationConcerns::FileSet::Indexing
+    include CurationConcerns::FileSet::BelongsToWorks
+    include CurationConcerns::FileSet::Querying
+    include CurationConcerns::HumanReadableType
+    include CurationConcerns::RequiredMetadata
+    include CurationConcerns::Naming
+    include Hydra::AccessControls::Embargoable
+    include GlobalID::Identification
+
+    included do
+      attr_accessor :file
+      self.human_readable_type = 'File'
+    end
+
+    def representative_id
+      to_param
+    end
+
+    def thumbnail_id
+      to_param
+    end
+  end
+end

data/app/models/concerns/curation_concerns/has_representative.rb

@@ -0,0 +1,13 @@
+module CurationConcerns::HasRepresentative
+  extend ActiveSupport::Concern
+
+  included do
+    belongs_to :representative,
+               predicate: ::RDF::Vocab::EBUCore.hasRelatedMediaFragment,
+               class_name: 'ActiveFedora::Base'
+
+    belongs_to :thumbnail,
+               predicate: ::RDF::Vocab::EBUCore.hasRelatedImage,
+               class_name: 'ActiveFedora::Base'
+  end
+end

data/app/models/concerns/curation_concerns/human_readable_type.rb

@@ -0,0 +1,17 @@
+module CurationConcerns
+  module HumanReadableType
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :human_readable_type
+      self.human_readable_type = name.demodulize.titleize
+    end
+
+    def to_solr(solr_doc = {})
+      super(solr_doc).tap do |doc|
+        doc[Solrizer.solr_name('human_readable_type', :facetable)] = human_readable_type
+        doc[Solrizer.solr_name('human_readable_type', :stored_searchable)] = human_readable_type
+      end
+    end
+  end
+end