sufia-models 5.0.0 → 6.0.0.beta1

data/app/models/concerns/sufia/generic_file/versions.rb

@@ -1,14 +1,14 @@
 module Sufia
   module GenericFile
     module Versions
+      @@count = 0
       def record_version_committer(user)
         version = content.latest_version
         # content datastream not (yet?) present
         return if version.nil?
-        VersionCommitter.create(obj_id: version.pid,
-                                datastream_id: version.dsid,
-                                version_id: version.versionID,
-                                committer_login: user.user_key)
+        @@count += 1
+        # raise "Recording #{@@count} #{version.uri} for #{user.user_key}" if @@count == 3
+        VersionCommitter.create(version_id: version.uri, committer_login: user.user_key)
       end
 
     end

data/app/models/concerns/sufia/generic_file/web_form.rb

@@ -5,19 +5,27 @@ module Sufia
       include Sufia::GenericFile::AccessibleAttributes
       included do
         before_save :remove_blank_assertions
+        attr_accessible *(terms_for_display + [:part_of, :permissions_attributes])
       end
 
       def remove_blank_assertions
         terms_for_editing.each do |key|
-          self[key] = nil if self[key] == ['']
+          if self[key] == ['']
+            self[key] = []
+            changed_attributes.delete(key) if attribute_was(key) == []
+          end
         end
       end
 
       # override this method if you need to initialize more complex RDF assertions (b-nodes)
       def initialize_fields
-        terms_for_editing.each do |key|
+        terms_for_editing.select { |key| self[key].blank? }.each do |key|
           # if value is empty, we create an one element array to loop over for output
-          self[key] = [''] if self[key].empty?
+          if self.class.multiple?(key)
+            self[key] = ['']
+          else
+            self[key] = ''
+          end
         end
       end
 
@@ -40,10 +48,10 @@ module Sufia
 
       def to_jq_upload
         return {
-          "name" => self.title,
-          "size" => self.file_size,
+          "name" => title,
+          "size" => file_size,
           "url" => "/files/#{noid}",
-          "thumbnail_url" => self.pid,
+          "thumbnail_url" => id,
           "delete_url" => "deleteme", # generic_file_path(id: id),
           "delete_type" => "DELETE"
         }

data/app/models/concerns/sufia/generic_file.rb

@@ -8,102 +8,19 @@ module Sufia
     include Sufia::GenericFile::MimeTypes
     include Sufia::GenericFile::Export
     include Sufia::GenericFile::Characterization
-    include Sufia::GenericFile::Audit
     include Sufia::GenericFile::Permissions
     include Sufia::GenericFile::WebForm
     include Sufia::GenericFile::Derivatives
     include Sufia::GenericFile::Trophies
     include Sufia::GenericFile::Featured
     include Sufia::GenericFile::Metadata
+    include Sufia::GenericFile::Content
     include Sufia::GenericFile::Versions
     include Sufia::GenericFile::VirusCheck
-    include Sufia::GenericFile::ReloadOnSave
     include Sufia::GenericFile::FullTextIndexing
     include Sufia::GenericFile::ProxyDeposit
     include Hydra::Collections::Collectible
-
-    included do
-      belongs_to :batch, property: :is_part_of
-
-      around_save :retry_warming
-
-      attr_accessible *(terms_for_display + [:part_of, :permissions])
-    end
-
-    def persistent_url
-      "#{Sufia.config.persistent_hostpath}#{noid}"
-    end
-
-    def retry_warming
-      save_tries = 0
-      conflict_tries = 0
-      begin
-        yield
-      rescue RSolr::Error::Http => error
-        save_tries += 1
-        logger.warn "Retry Solr caught RSOLR error on #{self.pid}: #{error.inspect}"
-        # fail for good if the tries is greater than 3
-        raise if save_tries >=3
-        sleep 0.01
-        retry
-      rescue  ActiveResource::ResourceConflict => error
-        conflict_tries += 1
-        logger.warn "Retry caught Active Resource Conflict #{self.pid}: #{error.inspect}"
-        raise if conflict_tries >=10
-        sleep 0.01
-        retry
-      rescue => error
-        if error.to_s.downcase.include? "conflict"
-          conflict_tries += 1
-          logger.warn "Retry caught Active Resource Conflict #{self.pid}: #{error.inspect}"
-          raise if conflict_tries >=10
-          sleep 0.01
-          retry
-        else
-          raise
-        end
-      end
-    end
-
-    # Get the files with a sibling relationship (belongs_to :batch)
-    # The batch id is minted when visiting the upload screen and attached
-    # to each file when it is done uploading.  The Batch object is not created
-    # until all objects are done uploading and the user is redirected to
-    # BatchController#edit.  Therefore, we must handle the case where
-    # self.batch_id is set but self.batch returns nil.
-    def related_files
-      return [] if batch.nil?
-      batch.generic_files.reject { |sibling| sibling.id == id }
-    end
-
-    # Unstemmed, searchable, stored
-    def self.noid_indexer
-      @noid_indexer ||= Solrizer::Descriptor.new(:text, :indexed, :stored)
-    end
-
-    def to_solr(solr_doc={}, opts={})
-      super(solr_doc, opts).tap do |solr_doc|
-        solr_doc[Solrizer.solr_name('label')] = self.label
-        solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile.noid_indexer)] = noid
-        solr_doc[Solrizer.solr_name('file_format')] = file_format
-        solr_doc[Solrizer.solr_name('file_format', :facetable)] = file_format
-        solr_doc['all_text_timv'] = full_text.content
-        solr_doc = index_collection_pids(solr_doc)
-      end
-    end
-
-    def label=(new_label)
-      @inner_object.label = new_label
-      if self.title.empty?
-        self.title = [new_label].compact
-      end
-    end
-
-    # Is this file in the middle of being processed by a batch?
-    def processing?
-       return false if self.batch.blank?
-       return false if !self.batch.methods.include? :status
-       return (!self.batch.status.empty?) && (self.batch.status.count == 1) && (self.batch.status[0] == "processing")
-    end
+    include Sufia::GenericFile::Batches
+    include Sufia::GenericFile::Indexing
   end
 end

data/app/models/concerns/sufia/model_methods.rb

@@ -8,23 +8,25 @@ module Sufia
 
     # OVERRIDE to support Hydra::Datastream::Properties which does not
     #   respond to :depositor_values but :depositor
-    # Adds metadata about the depositor to the asset
-    # Most important behavior: if the asset has a rightsMetadata datastream, this method will add +depositor_id+ to its individual edit permissions.
-
+    # Adds metadata about the depositor to the asset and ads +depositor_id+ to
+    # its individual edit permissions.
     def apply_depositor_metadata(depositor)
-      rights_ds = self.datastreams["rightsMetadata"]
-      prop_ds = self.datastreams["properties"]
       depositor_id = depositor.respond_to?(:user_key) ? depositor.user_key : depositor
 
-      rights_ds.update_indexed_attributes([:edit_access, :person]=>depositor_id) unless rights_ds.nil?
-      prop_ds.depositor = depositor_id unless prop_ds.nil?
+      self.edit_users += [depositor_id]
+      self.depositor = depositor_id
 
       return true
     end
 
     def to_s
-      return Array(title).join(" | ") if title.present?
-      label || "No Title"
+      if title.present?
+        Array(title).join(" | ")
+      elsif label.present?
+        Array(label).join(" | ")
+      else
+        "No Title"
+      end
     end
 
   end

data/app/models/concerns/sufia/user.rb

@@ -18,55 +18,33 @@ module Sufia::User
     # Users should be followable
     acts_as_followable
 
-    # Set up proxy-related relationships
+    # Setup accessible (or protected) attributes for your model
     has_many :proxy_deposit_requests, foreign_key: 'receiving_user_id'
+
     has_many :deposit_rights_given, foreign_key: 'grantor_id', class_name: 'ProxyDepositRights', dependent: :destroy
     has_many :can_receive_deposits_from, through: :deposit_rights_given, source: :grantee
+
     has_many :deposit_rights_received, foreign_key: 'grantee_id', class_name: 'ProxyDepositRights', dependent: :destroy
     has_many :can_make_deposits_for, through: :deposit_rights_received, source: :grantor
 
-    # Validate and normalize ORCIDs
-    validates_with OrcidValidator
-    after_validation :normalize_orcid
-
-    # Set up user profile avatars
     mount_uploader :avatar, AvatarUploader, mount_on: :avatar_file_name
     validates_with AvatarValidator
-
     has_many :trophies
     attr_accessor :update_directory
   end
 
-  # Coerce the ORCID into URL format
-  def normalize_orcid
-    # Skip normalization if:
-    #   1. validation has already flagged the ORCID as invalid
-    #   2. the orcid field is blank
-    #   3. the orcid is already in its normalized form
-    return if self.errors[:orcid].first.present? || self.orcid.blank? || self.orcid.starts_with?('http://orcid.org/')
-    bare_orcid = /\d{4}-\d{4}-\d{4}-\d{4}/.match(self.orcid).string
-    self.orcid = "http://orcid.org/#{bare_orcid}"
-  end
-
   # Format the json for select2 which requires just an id and a field called text.
   # If we need an alternate format we should probably look at a json template gem
   def as_json(opts = nil)
-    { id: user_key, text: display_name ? "#{display_name} (#{user_key})" : user_key }
-  end
-
-  # Populate user instance with attributes from remote system (e.g., LDAP)
-  # There is no default implementation -- override this in your application
-  def populate_attributes
+    {id: user_key, text: display_name ? "#{display_name} (#{user_key})" : user_key}
   end
 
   def email_address
-    self.email
+    return self.email
   end
 
   def name
-    self.display_name.titleize || raise
-  rescue
-    self.user_key
+    return self.display_name.titleize || self.user_key rescue self.user_key
   end
 
   # Redefine this for more intuitive keys in Redis
@@ -77,13 +55,13 @@ module Sufia::User
 
   def trophy_files
     trophies.map do |t|
-      ::GenericFile.load_instance_from_solr(Sufia::Noid.namespaceize(t.generic_file_id))
+      ::GenericFile.load_instance_from_solr(t.generic_file_id)
     end
   end
 
   # method needed for messaging
   def mailboxer_email(obj=nil)
-    nil
+    return nil
   end
 
   # The basic groups method, override or will fallback to Sufia::Ldap::User
@@ -107,9 +85,7 @@ module Sufia::User
       [:email, :login, :display_name, :address, :admin_area,
         :department, :title, :office, :chat_id, :website, :affiliation,
         :telephone, :avatar, :group_list, :groups_last_update, :facebook_handle,
-        :twitter_handle, :googleplus_handle, :linkedin_handle, :remove_avatar,
-        :orcid
-      ]
+        :twitter_handle, :googleplus_handle, :linkedin_handle, :remove_avatar]
     end
 
     def current
@@ -143,5 +119,7 @@ module Sufia::User
     def from_url_component(component)
       User.find_by_user_key(component.gsub(/-dot-/, '.'))
     end
+
   end
+
 end

data/app/models/datastreams/file_content_datastream.rb

@@ -1,4 +1,4 @@
-class FileContentDatastream < ActiveFedora::Datastream
+class FileContentDatastream < ActiveFedora::File
   include Hydra::Derivatives::ExtractMetadata
   include Sufia::FileContent::Versions
 end

data/app/models/datastreams/fits_datastream.rb

@@ -1,7 +1,7 @@
 class FitsDatastream < ActiveFedora::OmDatastream
   include OM::XML::Document
 
-  def prefix
+  def prefix(_)
     ""
   end
 

data/app/models/file_usage.rb

@@ -10,7 +10,7 @@ class FileUsage
     self.id = id
     self.path = Sufia::Engine.routes.url_helpers.generic_file_path(Sufia::Noid.noidify(id))
     earliest = Sufia.config.analytic_start_date
-    self.created = DateTime.parse(file.create_date)
+    self.created = ::GenericFile.find(id).create_date
     self.created = earliest > created ? earliest : created unless earliest.blank?
     self.downloads = FileDownloadStat.to_flots FileDownloadStat.statistics(id, created, user_id)
     self.pageviews = FileViewStat.to_flots FileViewStat.statistics(id, created, user_id)
@@ -23,8 +23,8 @@ class FileUsage
   def total_pageviews
     self.pageviews.reduce(0) { |total, result| total + result[1].to_i }
   end
-  
-  # Package data for visualization using JQuery Flot 
+
+  # Package data for visualization using JQuery Flot
   def to_flot
     [
       { label: "Pageviews",  data: pageviews },

data/app/models/local_authority.rb

@@ -9,10 +9,10 @@ class LocalAuthority < ActiveRecord::Base
     return unless self.where(name: name).empty?
     authority = self.create(name: name)
     format = opts.fetch(:format, :ntriples)
-    predicate = opts.fetch(:predicate, RDF::SKOS.prefLabel)
+    predicate = opts.fetch(:predicate, ::RDF::SKOS.prefLabel)
     entries = []
     sources.each do |uri|
-      RDF::Reader.open(uri, format: format) do |reader|
+      ::RDF::Reader.open(uri, format: format) do |reader|
         reader.each_statement do |statement|
           if statement.predicate == predicate
             entries << LocalAuthorityEntry.new(local_authority: authority,

data/app/models/proxy_deposit_request.rb

@@ -78,7 +78,7 @@ class ProxyDepositRequest < ActiveRecord::Base
 
   def title
     return 'file not found' if deleted_file?
-    query = ActiveFedora::SolrService.construct_query_for_pids([pid])
+    query = ActiveFedora::SolrQueryBuilder.construct_query_for_ids([pid])
     solr_response = ActiveFedora::SolrService.query(query, raw: true)
     SolrDocument.new(solr_response['response']['docs'].first, solr_response).title
   end

data/app/services/sufia/generic_file_audit_service.rb

@@ -0,0 +1,83 @@
+module Sufia
+  class GenericFileAuditService
+    attr_reader :generic_file
+    def initialize(file)
+      @generic_file = file
+    end
+
+    NO_RUNS = 999
+
+    # provides a human readable version of the audit status
+    def human_readable_audit_status
+      stat = audit_stat
+      case stat
+        when 0
+          'failing'
+        when 1
+          'passing'
+        else
+          stat
+      end
+    end
+
+    # TODO: Run audits on all attached files. We're only auditing "content" at tht moment
+    # Pushes an AuditJob for each version of content if it hasn't been audited recently
+    # Returns the set of most recent audit status for each version of the content file
+    def audit
+      audit_content([])
+    end
+
+
+    private
+      def audit_content(log)
+        if generic_file.content.has_versions?
+          audit_file_versions("content", log)
+        else
+          log << audit_file("content", generic_file.content.uri)
+        end
+      end
+
+      def audit_file_versions file, log
+        generic_file.attached_files[file].versions.all.each do |version|
+          log << audit_file(file, version.uri,  version.label)
+        end
+        log
+      end
+
+      def audit_stat
+        audit_results = audit.collect { |result| result["pass"] }
+
+        # check how many non runs we had
+        non_runs = audit_results.reduce(0) { |sum, value| value == NO_RUNS ? sum += 1 : sum }
+        if non_runs == 0
+          audit_results.reduce(true) { |sum, value| sum && value }
+        elsif non_runs < audit_results.length
+          result = audit_results.reduce(true) { |sum, value| value == NO_RUNS ? sum : sum && value }
+          "Some audits have not been run, but the ones run were #{result ? 'passing' : 'failing'}."
+        else
+          'Audits have not yet been run on this file.'
+        end
+      end
+
+      def audit_file(file, uri, label = nil)
+        latest_audit = ChecksumAuditLog.logs_for(generic_file.id, file).first
+        return latest_audit unless needs_audit?(latest_audit)
+        Sufia.queue.push(AuditJob.new(generic_file.id, file, uri))
+        latest_audit || ChecksumAuditLog.new(pass: NO_RUNS, pid: generic_file.id, dsid: file, version: label)
+      end
+
+      def needs_audit?(latest_audit)
+        return true unless latest_audit
+        unless latest_audit.updated_at
+          logger.warn "***AUDIT*** problem with audit log! Latest Audit is not nil, but updated_at is not set #{latest_audit}"
+          return true
+        end
+        days_since_last_audit(latest_audit) >= Sufia.config.max_days_between_audits
+      end
+
+      def days_since_last_audit(latest_audit)
+        (DateTime.now - latest_audit.updated_at.to_date).to_i
+      end
+
+  end
+end

data/app/services/sufia/id_service.rb

@@ -9,11 +9,11 @@ module Sufia
 
     @minter = ::Noid::Minter.new(template: noid_template)
     @pid = $$
-    @namespace = Sufia.config.id_namespace
+    #@namespace = Sufia.config.id_namespace
     @semaphore = Mutex.new
     def self.valid?(identifier)
       # remove the fedora namespace since it's not part of the noid
-      noid = identifier.split(":").last
+      noid = identifier#.split(":").last
       return @minter.valid? noid
     end
     def self.mint
@@ -28,20 +28,20 @@ module Sufia
     protected
 
     def self.next_id
-      pid = ''
+      id = ''
       File.open(Sufia.config.minter_statefile, File::RDWR|File::CREAT, 0644) do |f|
         f.flock(File::LOCK_EX)
         yaml = YAML::load(f.read)
         yaml = {template: noid_template} unless yaml
         minter = ::Noid::Minter.new(yaml)
-        pid = "#{@namespace}:#{minter.mint}"
+        id = minter.mint
         f.rewind
         yaml = YAML::dump(minter.dump)
         f.write yaml
         f.flush
         f.truncate(f.pos)
       end
-      return pid
+      id
     end
   end
 end

data/app/services/sufia/noid.rb

@@ -2,16 +2,14 @@ module Sufia
   module Noid
     extend ActiveSupport::Concern
 
-    module ClassMethods
-      ## This overrides the default behavior, which is to ask Fedora for a pid
-      # @see ActiveFedora::Sharding.assign_pid
-      def assign_pid(_)
-        Sufia::IdService.mint
-      end
+    ## This overrides the default behavior, which is to ask Fedora for a pid
+    # @see ActiveFedora::Sharding.assign_pid
+    def assign_pid
+      Sufia::IdService.mint
     end
 
     def noid
-      Noid.noidify(self.pid)
+      Noid.noidify(id)
     end
 
     # Redefine this for more intuitive keys in Redis
@@ -24,6 +22,11 @@ module Sufia
         String(identifier).split(":").last
       end
 
+      # Create a pairtree like path for the given identifier
+      def treeify(identifier)
+        (identifier.scan(/..?/).first(4) + [identifier]).join('/')
+      end
+
       def namespaceize(identifier)
         return identifier if identifier.include?(':')
         "#{namespace}:#{identifier}"

data/app/services/sufia/repository_audit_service.rb

@@ -0,0 +1,9 @@
+module Sufia
+  class RepositoryAuditService
+    def self.audit_everything
+      ::GenericFile.find_each do |gf|
+        gf.audit
+      end
+    end
+  end
+end

data/lib/generators/sufia/models/cached_stats_generator.rb

@@ -1,12 +1,32 @@
-require_relative 'abstract_migration_generator'
+# -*- encoding : utf-8 -*-
+require 'rails/generators'
+require 'rails/generators/migration'
+
+class Sufia::Models::CachedStatsGenerator < Rails::Generators::Base
+  include Rails::Generators::Migration
 
-class Sufia::Models::CachedStatsGenerator < Sufia::Models::AbstractMigrationGenerator
   source_root File.expand_path('../templates', __FILE__)
+  argument :model_name, type: :string , default: "user"
 
   desc """
 This generator adds the ability to cache usage stats to your application:
  1. Creates several database migrations if they do not exist in /db/migrate
+ 2. Adds stats methods to the user model
        """
+  # Implement the required interface for Rails::Generators::Migration.
+  # taken from http://github.com/rails/rails/blob/master/activerecord/lib/generators/active_record.rb
+  def self.next_migration_number(path)
+    if @prev_migration_nr
+      @prev_migration_nr += 1
+    else
+      if last_migration = Dir[File.join(path, '*.rb')].sort.last
+        @prev_migration_nr = last_migration.sub(File.join(path, '/'), '').to_i + 1
+      else
+        @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+      end
+    end
+    @prev_migration_nr.to_s
+  end
 
   def banner
     say_status("warning", "ADDING STATS CACHING-RELATED SUFIA MODELS", :yellow)
@@ -14,11 +34,35 @@ This generator adds the ability to cache usage stats to your application:
 
   # Setup the database migrations
   def copy_migrations
+    # Can't get this any more DRY, because we need this order.
     [
       'create_file_view_stats.rb',
-      'create_file_download_stats.rb'
+      'create_file_download_stats.rb',
+      'create_user_stats.rb'
     ].each do |file|
       better_migration_template file
     end
   end
+
+  def add_stats_mixin_to_user_model
+    file_path = "app/models/#{model_name.underscore}.rb"
+
+    if File.exists?(file_path)
+      inject_into_file file_path, after: /include Sufia\:\:User.*$/ do
+        "\n include Sufia::UserUsageStats"
+      end
+    else
+      puts "     \e[31mFailure\e[0m  Sufia requires a user object. This generators assumes that the model is defined in the file #{file_path}, which does not exist.  If you used a different name, please re-run the generator and provide that name as an argument. Such as \b  rails -g sufia client"
+    end
+  end
+
+  private
+
+  def better_migration_template(file)
+    begin
+      migration_template "migrations/#{file}", "db/migrate/#{file}"
+    rescue Rails::Generators::Error => e
+      say_status("warning", e.message, :yellow)
+    end
+  end
 end

data/lib/generators/sufia/models/install_generator.rb

@@ -1,7 +1,12 @@
-require_relative 'abstract_migration_generator'
+# -*- encoding : utf-8 -*-
+require 'rails/generators'
+require 'rails/generators/migration'
+
+class Sufia::Models::InstallGenerator < Rails::Generators::Base
+  include Rails::Generators::Migration
 
-class Sufia::Models::InstallGenerator < Sufia::Models::AbstractMigrationGenerator
   source_root File.expand_path('../templates', __FILE__)
+
   argument     :model_name, type: :string , default: "user"
   desc """
 This generator makes the following changes to your application:
@@ -14,15 +19,30 @@ This generator makes the following changes to your application:
  7. Runs full-text generator
  8. Runs proxies generator
  9. Runs cached stats generator
-10. Runs ORCID field generator
-11. Runs user stats generator
        """
+
+  # Implement the required interface for Rails::Generators::Migration.
+  # taken from http://github.com/rails/rails/blob/master/activerecord/lib/generators/active_record.rb
+  def self.next_migration_number(path)
+    if @prev_migration_nr
+      @prev_migration_nr += 1
+    else
+      if last_migration = Dir[File.join(path, '*.rb')].sort.last
+        @prev_migration_nr = last_migration.sub(File.join(path, '/'), '').to_i + 1
+      else
+        @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+      end
+    end
+    @prev_migration_nr.to_s
+  end
+
   def banner
     say_status("warning", "GENERATING SUFIA MODELS", :yellow)
   end
 
   # Setup the database migrations
   def copy_migrations
+    # Can't get this any more DRY, because we need this order.
     [
       "acts_as_follower_migration.rb",
       "add_social_to_users.rb",
@@ -94,13 +114,13 @@ This generator makes the following changes to your application:
     generate 'sufia:models:cached_stats'
   end
 
-  # Adds orcid field to user model
-  def orcid_field
-    generate 'sufia:models:orcid_field'
-  end
+  private
 
-  # Adds user stats-related migration & methods
-  def user_stats
-    generate 'sufia:models:user_stats'
+  def better_migration_template(file)
+    begin
+      migration_template "migrations/#{file}", "db/migrate/#{file}"
+    rescue Rails::Generators::Error => e
+      say_status("warning", e.message, :yellow)
+    end
   end
 end