sufia 5.0.0 → 6.0.0.beta1

data/sufia-models/app/jobs/batch_update_job.rb

@@ -10,7 +10,7 @@ class BatchUpdateJob
 
   def initialize(login, params)
     self.login = login
-    self.title = params[:title]
+    self.title = params[:title] || {}
     self.file_attributes = params[:generic_file]
     self.visibility = params[:visibility]
     self.batch_id = params[:id]
@@ -25,7 +25,9 @@ class BatchUpdateJob
     batch.generic_files.each do |gf|
       update_file(gf, user)
     end
-    batch.update_attributes({status:["Complete"]})
+
+    batch.update(status: ["Complete"])
+
     if denied.empty?
       send_user_success_message(user, batch) unless saved.empty?
     else
@@ -35,12 +37,12 @@ class BatchUpdateJob
 
   def update_file(gf, user)
     unless user.can? :edit, gf
-      ActiveFedora::Base.logger.error "User #{user.user_key} DENIED access to #{gf.pid}!"
+      ActiveFedora::Base.logger.error "User #{user.user_key} DENIED access to #{gf.id}!"
       denied << gf
       return
     end
-    gf.title = title[gf.pid] if title[gf.pid] rescue gf.label
-    gf.attributes=file_attributes
+    gf.title = title[gf.id] if title[gf.id]
+    gf.attributes = file_attributes
     gf.visibility= visibility
 
     save_tries = 0
@@ -48,13 +50,13 @@ class BatchUpdateJob
       gf.save!
     rescue RSolr::Error::Http => error
       save_tries += 1
-      ActiveFedora::Base.logger.warn "BatchUpdateJob caught RSOLR error on #{gf.pid}: #{error.inspect}"
+      ActiveFedora::Base.logger.warn "BatchUpdateJob caught RSOLR error on #{gf.id}: #{error.inspect}"
       # fail for good if the tries is greater than 3
       raise error if save_tries >=3
       sleep 0.01
       retry
     end #
-    Sufia.queue.push(ContentUpdateEventJob.new(gf.pid, login))
+    Sufia.queue.push(ContentUpdateEventJob.new(gf.id, login))
     saved << gf
   end
 
@@ -67,5 +69,4 @@ class BatchUpdateJob
     message = denied.count > 1 ? multiple_failure(batch.noid, denied) : single_failure(batch.noid, denied.first)
     User.batchuser.send_message(user, message, failure_subject, sanitize_text = false)
   end
-  
 end

data/sufia-models/app/jobs/import_url_job.rb

@@ -11,12 +11,12 @@ class ImportUrlJob < ActiveFedoraPidBasedJob
   def run
     user = User.find_by_user_key(generic_file.depositor)
 
-    Tempfile.open(self.pid) do |f|
+    Tempfile.open(pid.gsub('/', '_')) do |f|
       path = copy_remote_file(generic_file.import_url, f)
       # attach downloaded file to generic file stubbed out
       if Sufia::GenericFile::Actor.new(generic_file, user).create_content(f, path, 'content')
         # add message to user for downloaded file
-        message = "The file (#{generic_file.content.label}) was successfully imported."
+        message = "The file (#{generic_file.label}) was successfully imported."
         job_user.send_message(user, message, 'File Import')
       else
         job_user.send_message(user, generic_file.errors.full_messages.join(', '), 'File Import Error')

data/sufia-models/app/models/batch.rb

@@ -3,24 +3,23 @@ class Batch < ActiveFedora::Base
   include Sufia::ModelMethods
   include Sufia::Noid
 
-  has_metadata name: "descMetadata", type: BatchRdfDatastream
+  has_many :generic_files, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
 
-  belongs_to :user, property: "creator"
-  has_many :generic_files, property: :is_part_of
+  property :creator, predicate: ::RDF::DC.creator
+  property :title, predicate: ::RDF::DC.title
+  property :status, predicate: ::RDF::DC.type
 
-  has_attributes :title, :creator, :part, :status, datastream: :descMetadata, multiple: true
-
-  def self.find_or_create(pid)
+  def self.find_or_create(id)
     begin
-      Batch.find(pid)
+      Batch.find(id)
     rescue ActiveFedora::ObjectNotFoundError
-      Batch.create({pid: pid})
+      Batch.create(id: id)
     end
   end
 
-  def to_solr(solr_doc={}, opts={})
-    solr_doc = super(solr_doc, opts)
-    solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile.noid_indexer)] = noid
-    return solr_doc
+  def to_solr(solr_doc={})
+    super.tap do |solr_doc|
+      solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile::Indexing.noid_indexer)] = noid
+    end
   end
 end

data/sufia-models/app/models/checksum_audit_log.rb

@@ -1,19 +1,21 @@
 class ChecksumAuditLog < ActiveRecord::Base
 
-  def ChecksumAuditLog.get_audit_log(version)
-    ChecksumAuditLog.find_or_create_by_pid_and_dsid_and_version(pid: version.pid,
-                                                                dsid: version.dsid,
-                                                                version: version.versionID)
+  def self.get_audit_log(id, path, version_uri)
+    ChecksumAuditLog.find_or_create_by(pid: id, dsid: path, version: version_uri)
   end
 
-  def ChecksumAuditLog.prune_history(version)
-    ## Check to see if there are previous passing logs that we can delete
-    # we want to keep the first passing event after a failure, the most current passing event, and all failures so that this table doesn't grow too large
-    # Simple way (a little naieve): if the last 2 were passing, delete the first one
-    logs = GenericFile.load_instance_from_solr(version.pid).logs(version.dsid)
-    list = logs.limit(2)
+  # Check to see if there are previous passing logs that we can delete
+  # we want to keep the first passing event after a failure, the most current passing event,
+  # and all failures so that this table doesn't grow too large
+  # Simple way (a little naieve): if the last 2 were passing, delete the first one
+  def self.prune_history(id, path)
+    list = logs_for(id, path).limit(2)
     if list.size > 1 && (list[0].pass == 1) && (list[1].pass == 1)
       list[0].destroy
     end
   end
+
+  def self.logs_for(id, path)
+    ChecksumAuditLog.where(pid: id, dsid: path).order('created_at desc, id desc')
+  end
 end

data/sufia-models/app/models/concerns/sufia/ability.rb

@@ -15,8 +15,8 @@ module Sufia
     end
 
     def proxy_deposit_abilities
-      can :transfer, String do |pid|
-        get_depositor_from_pid(pid) == current_user.user_key
+      can :transfer, String do |id|
+        depositor_for_document(id) == current_user.user_key
       end
       can :create, ProxyDepositRequest if user_groups.include? 'registered'
       can :accept, ProxyDepositRequest, receiving_user_id: current_user.id, status: 'pending'
@@ -47,10 +47,8 @@ module Sufia
 
     private
 
-    def get_depositor_from_pid(pid)
-      ::GenericFile.load_instance_from_solr(pid).depositor
-    rescue
-      nil
+    def depositor_for_document(document_id)
+      ::GenericFile.load_instance_from_solr(document_id).depositor
     end
   end
 end

data/sufia-models/app/models/concerns/sufia/collection.rb

@@ -11,7 +11,6 @@ module Sufia
       before_save :update_permissions
       validates :title, presence: true
 
-      has_metadata "properties", type: PropertiesDatastream
     end
 
     def terms_for_display
@@ -35,10 +34,10 @@ module Sufia
       noid
     end
 
-    def to_solr(solr_doc={}, opts={})
-      super(solr_doc, opts)
-      solr_doc[Solrizer.solr_name("noid", Sufia::GenericFile.noid_indexer)] = noid
-      return solr_doc
+    def to_solr(solr_doc={})
+      super.tap do |solr_doc|
+        solr_doc[Solrizer.solr_name("noid", Sufia::GenericFile::Indexing.noid_indexer)] = noid
+      end
     end
 
     def update_permissions

data/sufia-models/app/models/concerns/sufia/generic_file.rb

@@ -8,102 +8,19 @@ module Sufia
     include Sufia::GenericFile::MimeTypes
     include Sufia::GenericFile::Export
     include Sufia::GenericFile::Characterization
-    include Sufia::GenericFile::Audit
     include Sufia::GenericFile::Permissions
     include Sufia::GenericFile::WebForm
     include Sufia::GenericFile::Derivatives
     include Sufia::GenericFile::Trophies
     include Sufia::GenericFile::Featured
     include Sufia::GenericFile::Metadata
+    include Sufia::GenericFile::Content
     include Sufia::GenericFile::Versions
     include Sufia::GenericFile::VirusCheck
-    include Sufia::GenericFile::ReloadOnSave
     include Sufia::GenericFile::FullTextIndexing
     include Sufia::GenericFile::ProxyDeposit
     include Hydra::Collections::Collectible
-
-    included do
-      belongs_to :batch, property: :is_part_of
-
-      around_save :retry_warming
-
-      attr_accessible *(terms_for_display + [:part_of, :permissions])
-    end
-
-    def persistent_url
-      "#{Sufia.config.persistent_hostpath}#{noid}"
-    end
-
-    def retry_warming
-      save_tries = 0
-      conflict_tries = 0
-      begin
-        yield
-      rescue RSolr::Error::Http => error
-        save_tries += 1
-        logger.warn "Retry Solr caught RSOLR error on #{self.pid}: #{error.inspect}"
-        # fail for good if the tries is greater than 3
-        raise if save_tries >=3
-        sleep 0.01
-        retry
-      rescue  ActiveResource::ResourceConflict => error
-        conflict_tries += 1
-        logger.warn "Retry caught Active Resource Conflict #{self.pid}: #{error.inspect}"
-        raise if conflict_tries >=10
-        sleep 0.01
-        retry
-      rescue => error
-        if error.to_s.downcase.include? "conflict"
-          conflict_tries += 1
-          logger.warn "Retry caught Active Resource Conflict #{self.pid}: #{error.inspect}"
-          raise if conflict_tries >=10
-          sleep 0.01
-          retry
-        else
-          raise
-        end
-      end
-    end
-
-    # Get the files with a sibling relationship (belongs_to :batch)
-    # The batch id is minted when visiting the upload screen and attached
-    # to each file when it is done uploading.  The Batch object is not created
-    # until all objects are done uploading and the user is redirected to
-    # BatchController#edit.  Therefore, we must handle the case where
-    # self.batch_id is set but self.batch returns nil.
-    def related_files
-      return [] if batch.nil?
-      batch.generic_files.reject { |sibling| sibling.id == id }
-    end
-
-    # Unstemmed, searchable, stored
-    def self.noid_indexer
-      @noid_indexer ||= Solrizer::Descriptor.new(:text, :indexed, :stored)
-    end
-
-    def to_solr(solr_doc={}, opts={})
-      super(solr_doc, opts).tap do |solr_doc|
-        solr_doc[Solrizer.solr_name('label')] = self.label
-        solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile.noid_indexer)] = noid
-        solr_doc[Solrizer.solr_name('file_format')] = file_format
-        solr_doc[Solrizer.solr_name('file_format', :facetable)] = file_format
-        solr_doc['all_text_timv'] = full_text.content
-        solr_doc = index_collection_pids(solr_doc)
-      end
-    end
-
-    def label=(new_label)
-      @inner_object.label = new_label
-      if self.title.empty?
-        self.title = [new_label].compact
-      end
-    end
-
-    # Is this file in the middle of being processed by a batch?
-    def processing?
-       return false if self.batch.blank?
-       return false if !self.batch.methods.include? :status
-       return (!self.batch.status.empty?) && (self.batch.status.count == 1) && (self.batch.status[0] == "processing")
-    end
+    include Sufia::GenericFile::Batches
+    include Sufia::GenericFile::Indexing
   end
 end

data/sufia-models/app/models/concerns/sufia/generic_file/batches.rb

@@ -0,0 +1,29 @@
+module Sufia
+  module GenericFile
+    module Batches
+      extend ActiveSupport::Concern
+      included do
+        belongs_to :batch, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
+      end
+
+      # Get the files with a sibling relationship (belongs_to :batch)
+      # The batch id is minted when visiting the upload screen and attached
+      # to each file when it is done uploading.  The Batch object is not created
+      # until all objects are done uploading and the user is redirected to
+      # BatchController#edit.  Therefore, we must handle the case where
+      # self.batch_id is set but self.batch returns nil.
+      def related_files
+        return [] if batch.nil?
+        batch.generic_files.reject { |sibling| sibling.id == id }
+      end
+
+      # Is this file in the middle of being processed by a batch?
+      def processing?
+         return false if self.batch.blank? || !self.batch.methods.include?(:status)
+         (!self.batch.status.empty?) && (self.batch.status.count == 1) && (self.batch.status[0] == "processing")
+      end
+
+    end
+  end
+end
+

data/sufia-models/app/models/concerns/sufia/generic_file/characterization.rb

@@ -3,7 +3,7 @@ module Sufia
     module Characterization
       extend ActiveSupport::Concern
       included do
-        has_metadata "characterization", type: FitsDatastream
+        contains "characterization", class_name: 'FitsDatastream'
         has_attributes :mime_type, datastream: :characterization, multiple: false
         has_attributes :format_label, :file_size, :last_modified,
                                         :filename, :original_checksum, :rights_basis,
@@ -45,11 +45,11 @@ module Sufia
         metadata = content.extract_metadata
         characterization.ng_xml = metadata if metadata.present?
         append_metadata
-        self.filename = [self.label]
+        self.filename = [content.original_name]
         save
       end
 
-      # Populate descMetadata with fields from FITS (e.g. Author from pdfs)
+      # Populate GenericFile's properties with fields from FITS (e.g. Author from pdfs)
       def append_metadata
         terms = self.characterization_terms
         Sufia.config.fits_to_desc_mapping.each_pair do |k, v|

data/sufia-models/app/models/concerns/sufia/generic_file/content.rb

@@ -0,0 +1,13 @@
+module Sufia
+  module GenericFile
+    module Content
+      extend ActiveSupport::Concern
+
+      included do
+        contains "content", class_name: 'FileContentDatastream'
+        contains "thumbnail"
+      end
+
+    end
+  end
+end

data/sufia-models/app/models/concerns/sufia/generic_file/derivatives.rb

@@ -9,15 +9,15 @@ module Sufia
         makes_derivatives do |obj|
           case obj.mime_type
           when *pdf_mime_types
-            obj.transform_datastream :content, { thumbnail: { format: 'jpg', size: '338x493', datastream: 'thumbnail' } }
+            obj.transform_file :content, { thumbnail: { format: 'jpg', size: '338x493', datastream: 'thumbnail' } }
           when *office_document_mime_types
-            obj.transform_datastream :content, { thumbnail: { format: 'jpg', size: '200x150>', datastream: 'thumbnail' } }, processor: :document
+            obj.transform_file :content, { thumbnail: { format: 'jpg', size: '200x150>', datastream: 'thumbnail' } }, processor: :document
           when *audio_mime_types
-            obj.transform_datastream :content, { mp3: { format: 'mp3', datastream: 'mp3' }, ogg: { format: 'ogg', datastream: 'ogg' } }, processor: :audio
+            obj.transform_file :content, { mp3: { format: 'mp3', datastream: 'mp3' }, ogg: { format: 'ogg', datastream: 'ogg' } }, processor: :audio
           when *video_mime_types
-            obj.transform_datastream :content, { webm: { format: 'webm', datastream: 'webm' }, mp4: { format: 'mp4', datastream: 'mp4' }, thumbnail: { format: 'jpg', datastream: 'thumbnail' } }, processor: :video
+            obj.transform_file :content, { webm: { format: 'webm', datastream: 'webm' }, mp4: { format: 'mp4', datastream: 'mp4' }, thumbnail: { format: 'jpg', datastream: 'thumbnail' } }, processor: :video
           when *image_mime_types
-            obj.transform_datastream :content, { thumbnail: { format: 'jpg', size: '200x150>', datastream: 'thumbnail' } }
+            obj.transform_file :content, { thumbnail: { format: 'jpg', size: '200x150>', datastream: 'thumbnail' } }
           end
         end
       end

data/sufia-models/app/models/concerns/sufia/generic_file/export.rb

@@ -41,6 +41,10 @@ module Sufia
         return text.join("\n")
       end
 
+      def persistent_url
+        "#{Sufia.config.persistent_hostpath}#{noid}"
+      end
+
       # MIME type: 'application/x-openurl-ctx-kev'
       def export_as_openurl_ctx_kev
         export_text = []

data/sufia-models/app/models/concerns/sufia/generic_file/full_text_indexing.rb

@@ -4,7 +4,7 @@ module Sufia
       extend ActiveSupport::Concern
 
       included do
-        has_file_datastream 'full_text', versionable: false
+        contains 'full_text'
       end
 
       def append_metadata
@@ -27,7 +27,7 @@ module Sufia
           extracted_text = JSON.parse(resp.body)[''].rstrip
           full_text.content = extracted_text if extracted_text.present?
         rescue => e
-          logger.error("Error extracting content from #{self.pid}: #{e.inspect}")
+          logger.error("Error extracting content from #{self.id}: #{e.inspect}")
         end
     end
   end

data/sufia-models/app/models/concerns/sufia/generic_file/indexing.rb

@@ -0,0 +1,23 @@
+module Sufia
+  module GenericFile
+    module Indexing
+      extend ActiveSupport::Concern
+
+      # Unstemmed, searchable, stored
+      def self.noid_indexer
+        @noid_indexer ||= Solrizer::Descriptor.new(:text, :indexed, :stored)
+      end
+
+      def to_solr(solr_doc={})
+        super.tap do |solr_doc|
+          solr_doc[Solrizer.solr_name('label')] = label
+          solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile::Indexing.noid_indexer)] = noid
+          solr_doc[Solrizer.solr_name('file_format')] = file_format
+          solr_doc[Solrizer.solr_name('file_format', :facetable)] = file_format
+          solr_doc['all_text_timv'] = full_text.content
+          solr_doc = index_collection_ids(solr_doc)
+        end
+      end
+    end
+  end
+end

data/sufia-models/app/models/concerns/sufia/generic_file/metadata.rb

@@ -4,17 +4,86 @@ module Sufia
       extend ActiveSupport::Concern
 
       included do
-        has_metadata "descMetadata", type: GenericFileRdfDatastream
-        has_metadata "properties", type: PropertiesDatastream
-        has_file_datastream "content", type: FileContentDatastream
-        has_file_datastream "thumbnail"
-
-        has_attributes :relative_path, :depositor, :import_url, datastream: :properties, multiple: false
-        has_attributes :date_uploaded, :date_modified, datastream: :descMetadata, multiple: false
-        has_attributes :related_url, :based_near, :part_of, :creator,
-                                    :contributor, :title, :tag, :description, :rights,
-                                    :publisher, :date_created, :subject,
-                                    :resource_type, :identifier, :language, datastream: :descMetadata, multiple: true
+
+        property :label, predicate: ::RDF::DC.title, multiple: false
+
+        property :depositor, predicate: ::RDF::URI.new("http://id.loc.gov/vocabulary/relators/dpt"), multiple: false do |index|
+          index.as :symbol, :stored_searchable
+        end
+
+        property :relative_path, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#relativePath'), multiple: false
+
+        property :import_url, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#importUrl'), multiple: false do |index|
+          index.as :symbol
+        end
+
+        property :part_of, predicate: ::RDF::DC.isPartOf
+        property :resource_type, predicate: ::RDF::DC.type do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :title, predicate: ::RDF::DC.title do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :creator, predicate: ::RDF::DC.creator do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :contributor, predicate: ::RDF::DC.contributor do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :description, predicate: ::RDF::DC.description do |index|
+          index.type :text
+          index.as :stored_searchable
+        end
+        property :tag, predicate: ::RDF::DC.relation do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :rights, predicate: ::RDF::DC.rights do |index|
+          index.as :stored_searchable
+        end
+        property :publisher, predicate: ::RDF::DC.publisher do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :date_created, predicate: ::RDF::DC.created do |index|
+          index.as :stored_searchable
+        end
+        property :date_uploaded, predicate: ::RDF::DC.dateSubmitted, multiple: false do |index|
+          index.type :date
+          index.as :stored_sortable
+        end
+        property :date_modified, predicate: ::RDF::DC.modified, multiple: false do |index|
+          index.type :date
+          index.as :stored_sortable
+        end
+        property :subject, predicate: ::RDF::DC.subject do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :language, predicate: ::RDF::DC.language do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :identifier, predicate: ::RDF::DC.identifier do |index|
+          index.as :stored_searchable
+        end
+        property :based_near, predicate: ::RDF::FOAF.based_near do |index|
+          index.as :stored_searchable, :facetable
+        end
+        property :related_url, predicate: ::RDF::RDFS.seeAlso do |index|
+          index.as :stored_searchable
+        end
+        property :bibliographic_citation, predicate: ::RDF::DC.bibliographicCitation do |index|
+          index.as :stored_searchable
+        end
+        property :source, predicate: ::RDF::DC.source do |index|
+          index.as :stored_searchable
+        end
+
+        # TODO: Move this somewhere more appropriate
+        begin
+          LocalAuthority.register_vocabulary(self, "subject", "lc_subjects")
+          LocalAuthority.register_vocabulary(self, "language", "lexvo_languages")
+          LocalAuthority.register_vocabulary(self, "tag", "lc_genres")
+        rescue
+          puts "tables for vocabularies missing"
+        end
       end
 
       # Add a schema.org itemtype