sufia-models 6.0.0 → 6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c12a4f1425665634659dfd88233a2c365594589d
-  data.tar.gz: 06d1405c0b4f1537c071700faadc20a0264f34e8
+  metadata.gz: 88115b1a7a645491dcff7e47680ae4f2f4699d5f
+  data.tar.gz: c6348bf5e986f9b0666b5f8c3494fc8c864e74ae
 SHA512:
-  metadata.gz: 07a66fa4a7b7c950459a91143ac460859ce38edc9a7d336badb586bbaed916c168fe468dcdcad36405fa8d653774db8ad1acc9a9da07ab42b06e066ba18a6c1e
-  data.tar.gz: ae73b9db8e0607bfb3dcfcab6aef808a8b88be53b8447c66db279a6071ec8433f024dfdac404a5c5167865a6df153f22b03854e625df929175fe07f98cb6a93b
+  metadata.gz: 43f9323fcfa7cd7d73ce861dcb42c4fd2df22f7b09c3975d4fe18a654a66d39a49c07990399ef97106cfe4c9738420faafec1f0f7b3ef8c0281da8277204e3b9
+  data.tar.gz: bab9cc33d1e179f8b6e590429710d328c2571626a56dd4884e37ce0deb883ec2327cee36d502291f1a9c3f80d9342b1d6833eabb1a666140af81616edbbb67db

data/app/jobs/audit_job.rb

@@ -20,7 +20,6 @@ class AuditJob < ActiveFedoraIdBasedJob
   end
 
   def run
-    fixity_ok = false
     log = run_audit
     fixity_ok = (log.pass == 1)
     unless fixity_ok

data/app/jobs/batch_update_job.rb

@@ -62,11 +62,11 @@ class BatchUpdateJob
 
   def send_user_success_message user, batch
     message = saved.count > 1 ? multiple_success(batch.id, saved) : single_success(batch.id, saved.first)
-    User.batchuser.send_message(user, message, success_subject, sanitize_text = false)
+    User.batchuser.send_message(user, message, success_subject, false)
   end
 
   def send_user_failure_message user, batch
     message = denied.count > 1 ? multiple_failure(batch.id, denied) : single_failure(batch.id, denied.first)
-    User.batchuser.send_message(user, message, failure_subject, sanitize_text = false)
+    User.batchuser.send_message(user, message, failure_subject, false)
   end
 end

data/app/models/batch.rb

@@ -26,7 +26,7 @@ class Batch < ActiveFedora::Base
     def self.safe_create(id)
       begin      
         Batch.create(id: id)
-      rescue ActiveFedora::IllegalOperation => ex
+      rescue ActiveFedora::IllegalOperation
         # This is the exception thrown by LDP when we attempt to 
         # create a duplicate object. If we can find the object
         # then we are good to go.

data/app/models/concerns/sufia/ability.rb

@@ -19,7 +19,7 @@ module Sufia
       can :transfer, String do |id|
         depositor_for_document(id) == current_user.user_key
       end
-      can :create, ProxyDepositRequest if user_groups.include? 'registered'
+      can :create, ProxyDepositRequest if registered_user?
       can :accept, ProxyDepositRequest, receiving_user_id: current_user.id, status: 'pending'
       can :reject, ProxyDepositRequest, receiving_user_id: current_user.id, status: 'pending'
       # a user who sent a proxy deposit request can cancel it if it's pending.
@@ -31,16 +31,16 @@ module Sufia
     end
 
     def featured_work_abilities
-      can [:create, :destroy, :update], FeaturedWork if user_groups.include? 'admin'
+      can [:create, :destroy, :update], FeaturedWork if admin_user?
     end
 
     def generic_file_abilities
       can :view_share_work, [GenericFile]
-      can :create, [GenericFile, Collection] if user_groups.include? 'registered'
+      can :create, [GenericFile, Collection] if registered_user?
     end
 
     def editor_abilities
-      if user_groups.include? 'admin'
+      if admin_user?
         can :create, TinymceAsset
         can [:create, :update], ContentBlock
       end
@@ -53,9 +53,16 @@ module Sufia
 
     private
 
-    def depositor_for_document(document_id)
-      ::GenericFile.load_instance_from_solr(document_id).depositor
-    end
+      def depositor_for_document(document_id)
+        ::GenericFile.load_instance_from_solr(document_id).depositor
+      end
 
+      def registered_user?
+        user_groups.include? 'registered'
+      end
+
+      def admin_user?
+        user_groups.include? 'admin'
+      end
   end
 end

data/app/models/concerns/sufia/collection_behavior.rb

@@ -15,10 +15,43 @@ module Sufia
       self.visibility = "open"
     end
 
-    # Compute the sum of each file in the collection
-    # Return an integer of the result
+    # Compute the sum of each file in the collection using Solr to
+    # avoid having to hit Fedora 
+    #
+    # @return [Fixnum] size of collection in bytes
+    # @raise [RuntimeError] unsaved record does not exist in solr
     def bytes
-      members.reduce(0) { |sum, gf| sum + gf.content.size.to_i }
+      rows = members.count
+      return 0 if rows == 0
+
+      raise "Collection must be saved to query for bytes" if new_record?
+
+      query = ActiveFedora::SolrQueryBuilder.construct_query_for_rel(has_model: file_model)
+      args = {
+        fq: "{!join from=hasCollectionMember_ssim to=id}id:#{id}",
+        fl: "id, #{file_size_field}",
+        rows: rows
+      }
+
+      files = ActiveFedora::SolrService.query(query, args)
+      files.reduce(0) { |sum, f| sum += f[file_size_field].to_i }
     end
+
+    protected
+      # Field to look up when locating the size of each file in Solr.
+      # Override for your own installation if using something different
+      def file_size_field
+        Solrizer.solr_name('file_size', stored_integer_descriptor)
+      end
+
+      # Override if you are storing your file size in a different way
+      def stored_integer_descriptor
+        Sufia::GenericFileIndexingService::STORED_INTEGER
+      end
+
+      # Override if not using GenericFiles
+      def file_model
+        ::GenericFile.to_class_uri
+      end
   end
 end

data/app/models/concerns/sufia/generic_file/characterization.rb

@@ -4,23 +4,52 @@ module Sufia
       extend ActiveSupport::Concern
       included do
         contains "characterization", class_name: 'FitsDatastream'
-        has_attributes :mime_type, datastream: :characterization, multiple: false
-        has_attributes :format_label, :file_size, :last_modified,
-                                        :filename, :original_checksum, :rights_basis,
-                                        :copyright_basis, :copyright_note,
-                                        :well_formed, :valid, :status_message,
-                                        :file_title, :file_author, :page_count,
-                                        :file_language, :word_count, :character_count,
-                                        :paragraph_count, :line_count, :table_count,
-                                        :graphics_count, :byte_order, :compression,
-                                        :color_space, :profile_name,
-                                        :profile_version, :orientation, :color_map,
-                                        :image_producer, :capture_device,
-                                        :scanning_software, :exif_version,
-                                        :gps_timestamp, :latitude, :longitude,
-                                        :character_set, :markup_basis,
-                                        :markup_language, :bit_depth,
-                                        :channels, :data_format, :offset, :frame_rate, datastream: :characterization, multiple: true
+        property :mime_type,         delegate_to: 'characterization', multiple: false do |index|
+          index.as :stored_searchable
+        end
+        property :format_label,      delegate_to: 'characterization'
+        property :file_size,         delegate_to: 'characterization'
+        property :last_modified,     delegate_to: 'characterization'
+        property :filename,          delegate_to: 'characterization'
+        property :original_checksum, delegate_to: 'characterization'
+        property :rights_basis,      delegate_to: 'characterization'
+        property :copyright_basis,   delegate_to: 'characterization'
+        property :copyright_note,    delegate_to: 'characterization'
+        property :well_formed,       delegate_to: 'characterization'
+        property :valid,             delegate_to: 'characterization'
+        property :status_message,    delegate_to: 'characterization'
+        property :file_title,        delegate_to: 'characterization'
+        property :file_author,       delegate_to: 'characterization'
+        property :page_count,        delegate_to: 'characterization'
+        property :file_language,     delegate_to: 'characterization'
+        property :word_count,        delegate_to: 'characterization'
+        property :character_count,   delegate_to: 'characterization'
+        property :paragraph_count,   delegate_to: 'characterization'
+        property :line_count,        delegate_to: 'characterization'
+        property :table_count,       delegate_to: 'characterization'
+        property :graphics_count,    delegate_to: 'characterization'
+        property :byte_order,        delegate_to: 'characterization'
+        property :compression,       delegate_to: 'characterization'
+        property :color_space,       delegate_to: 'characterization'
+        property :profile_name,      delegate_to: 'characterization'
+        property :profile_version,   delegate_to: 'characterization'
+        property :orientation,       delegate_to: 'characterization'
+        property :color_map,         delegate_to: 'characterization'
+        property :image_producer,    delegate_to: 'characterization'
+        property :capture_device,    delegate_to: 'characterization'
+        property :scanning_software, delegate_to: 'characterization'
+        property :exif_version,      delegate_to: 'characterization'
+        property :gps_timestamp,     delegate_to: 'characterization'
+        property :latitude,          delegate_to: 'characterization'
+        property :longitude,         delegate_to: 'characterization'
+        property :character_set,     delegate_to: 'characterization'
+        property :markup_basis,      delegate_to: 'characterization'
+        property :markup_language,   delegate_to: 'characterization'
+        property :bit_depth,         delegate_to: 'characterization'
+        property :channels,          delegate_to: 'characterization'
+        property :data_format,       delegate_to: 'characterization'
+        property :offset,            delegate_to: 'characterization'
+        property :frame_rate,        delegate_to: 'characterization'
 
       end
 
@@ -71,7 +100,7 @@ module Sufia
 
       def characterization_terms
         h = {}
-        self.characterization.class.terminology.terms.each_pair do |k, v|
+        FitsDatastream.terminology.terms.each_pair do |k, v|
           next unless v.respond_to? :proxied_term
           term = v.proxied_term
           begin

data/app/models/concerns/sufia/generic_file/metadata.rb

@@ -11,6 +11,8 @@ module Sufia
           index.as :symbol, :stored_searchable
         end
 
+        property :arkivo_checksum, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#arkivoChecksum'), multiple: false
+
         property :relative_path, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#relativePath'), multiple: false
 
         property :import_url, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#importUrl'), multiple: false do |index|
@@ -91,6 +93,7 @@ module Sufia
         rescue
           puts "tables for vocabularies missing"
         end
+        type ::RDF::URI.new('http://pcdm.org/models#Object')
       end
 
     end

data/app/models/concerns/sufia/user.rb

@@ -1,3 +1,5 @@
+require 'oauth'
+
 module Sufia::User
   extend ActiveSupport::Concern
 
@@ -33,10 +35,37 @@ module Sufia::User
     mount_uploader :avatar, AvatarUploader, mount_on: :avatar_file_name
     validates_with AvatarValidator
 
+    # Add token to authenticate Arkivo API calls
+    after_initialize :set_arkivo_token, unless: :persisted? if Sufia.config.arkivo_api
+
     has_many :trophies
     attr_accessor :update_directory
   end
 
+  def zotero_token
+    self[:zotero_token].blank? ? nil : Marshal::load(self[:zotero_token])
+  end
+
+  def zotero_token=(value)
+    if value.blank?
+      # Resetting the token
+      self[:zotero_token] = value
+    else
+      self[:zotero_token] = Marshal::dump(value)
+    end
+  end
+
+  def set_arkivo_token
+    self.arkivo_token ||= token_algorithm
+  end
+
+  def token_algorithm
+    loop do
+      token = SecureRandom.base64(24)
+      return token if User.find_by(arkivo_token: token).nil?
+    end
+  end
+
   # Coerce the ORCID into URL format
   def normalize_orcid
     # Skip normalization if:
@@ -77,8 +106,13 @@ module Sufia::User
 
   def trophy_files
     trophies.map do |t|
-      ::GenericFile.load_instance_from_solr(t.generic_file_id)
-    end
+      begin
+        ::GenericFile.load_instance_from_solr(t.generic_file_id)
+      rescue ActiveFedora::ObjectNotFoundError
+        logger.error("Invalid trophy for user #{user_key} (generic file id #{t.generic_file_id})")
+        nil
+      end
+    end.compact
   end
 
   # method needed for messaging

data/app/models/datastreams/fits_datastream.rb

@@ -1,10 +1,6 @@
 class FitsDatastream < ActiveFedora::OmDatastream
   include OM::XML::Document
 
-  def prefix(_)
-    ""
-  end
-
   set_terminology do |t|
     t.root(path: "fits",
            xmlns: "http://hul.harvard.edu/ois/xml/ns/fits/fits_output",
@@ -12,7 +8,7 @@ class FitsDatastream < ActiveFedora::OmDatastream
     t.identification {
       t.identity {
         t.format_label(path: {attribute: "format"})
-        t.mime_type(path: {attribute: "mimetype"}, index_as: [:stored_searchable])
+        t.mime_type(path: {attribute: "mimetype"})
       }
     }
     t.fileinfo {

data/app/models/file_usage.rb

@@ -9,13 +9,30 @@ class FileUsage
 
     self.id = id
     self.path = Sufia::Engine.routes.url_helpers.generic_file_path(id)
-    earliest = Sufia.config.analytic_start_date
-    self.created = ::GenericFile.find(id).create_date
-    self.created = earliest > created ? earliest : created unless earliest.blank?
+    self.created = date_for_analytics(file)
     self.downloads = FileDownloadStat.to_flots FileDownloadStat.statistics(id, created, user_id)
     self.pageviews = FileViewStat.to_flots FileViewStat.statistics(id, created, user_id)
   end
 
+  # file.date_uploaded reflects the date the file was uploaded by the user 
+  # and therefore (if available) the date that we want to use for the stats 
+  # file.create_date reflects the date the file was added to Fedora. On data 
+  # migrated from one repository to another the created_date can be later 
+  # than the date the file was uploaded.
+  def date_for_analytics(file) 
+    earliest = Sufia.config.analytic_start_date
+    date_uploaded = string_to_date file.date_uploaded
+    date_analytics = date_uploaded ? date_uploaded : file.create_date
+    return date_analytics if earliest.blank?
+    earliest > date_analytics ? earliest : date_analytics 
+  end
+
+  def string_to_date(date_str)
+    return DateTime.parse(date_str) 
+  rescue ArgumentError, TypeError
+    return nil
+  end
+
   def total_downloads
     self.downloads.reduce(0) { |total, result| total + result[1].to_i }
   end

data/app/services/sufia/generic_file_audit_service.rb

@@ -45,6 +45,17 @@ module Sufia
       end
 
       def audit_stat
+        # Only access version if we can (file was loaded from fedora)
+        if generic_file.content.respond_to? :has_versions?
+          audit_stat_by_version
+
+        # file loaded from solr
+        else
+          audit_stat_by_id
+        end
+      end
+
+      def audit_stat_by_version
         audit_results = audit.collect { |result| result["pass"] }
 
         # check how many non runs we had
@@ -59,6 +70,19 @@ module Sufia
         end
       end
 
+      # Check the file by only what is in the audit log.
+      # Do not try to access the versions if we do not have access to them.
+      # This occurs when a file is loaded from solr instead of fedora
+      def audit_stat_by_id
+        audit_results = ChecksumAuditLog.logs_for(generic_file.id, "content").collect { |result| result["pass"] }
+
+        if audit_results.length >  0
+          audit_results.reduce(true) { |sum, value| sum && value }
+        else
+          'Audits have not yet been run on this file.'
+        end
+      end
+
       def audit_file(file, uri, label = nil)
         latest_audit = ChecksumAuditLog.logs_for(generic_file.id, file).first
         return latest_audit unless needs_audit?(latest_audit)

data/app/services/sufia/generic_file_indexing_service.rb

@@ -1,11 +1,14 @@
 module Sufia
   class GenericFileIndexingService < ActiveFedora::IndexingService
+    STORED_INTEGER = Solrizer::Descriptor.new(:integer, :stored)
+
     def generate_solr_document
       super.tap do |solr_doc|
         solr_doc[Solrizer.solr_name('label')] = object.label
         solr_doc[Solrizer.solr_name('file_format')] = object.file_format
         solr_doc[Solrizer.solr_name('file_format', :facetable)] = object.file_format
         solr_doc['all_text_timv'] = object.full_text.content
+        solr_doc[Solrizer.solr_name('file_size', STORED_INTEGER)] = object.content.size.to_i
       end
     end
   end

data/app/services/sufia/noid.rb

@@ -1,3 +1,5 @@
+require 'active_fedora/noid'
+
 module Sufia
   module Noid
     extend ActiveSupport::Concern
@@ -5,18 +7,17 @@ module Sufia
     ## This overrides the default behavior, which is to ask Fedora for an id
     # @see ActiveFedora::Persistence.assign_id
     def assign_id
-      Sufia::IdService.mint if Sufia.config.enable_noids
+      service.mint if Sufia.config.enable_noids
     end
 
     def to_param
       id
     end
 
-    class << self
-      # Create a pairtree like path for the given identifier
-      def treeify(identifier)
-        (identifier.scan(/..?/).first(4) + [identifier]).join('/')
+    private
+
+      def service
+        @service ||= ActiveFedora::Noid::Service.new
       end
-    end
   end
 end

data/lib/generators/sufia/models/abstract_migration_generator.rb

@@ -25,6 +25,6 @@ class Sufia::Models::AbstractMigrationGenerator < Rails::Generators::Base
   def better_migration_template(file)
     migration_template "migrations/#{file}", "db/migrate/#{file}"
   rescue Rails::Generators::Error => e
-    say_status("warning", e.message, :yellow)
+    say_status("error", e.message, :red)
   end
 end

data/lib/generators/sufia/models/arkivo_api_generator.rb

@@ -0,0 +1,39 @@
+require_relative 'abstract_migration_generator'
+
+class Sufia::Models::ArkivoApiGenerator < Sufia::Models::AbstractMigrationGenerator
+  source_root File.expand_path('../templates', __FILE__)
+
+  desc """
+This generator sets up Zotero/Arkivo API integration for your application:
+       """
+
+  def banner
+    say_status("info", "ADDING ZOTERO/ARKIVO API INTEGRATION", :blue)
+  end
+
+  # Turn on the feature set in Sufia's config
+  def inject_arkivo_config
+    inject_into_file 'config/initializers/sufia.rb', after: /^Sufia\.config do.*$/ do
+      "\n  # Sufia can integrate with Zotero's Arkivo service for automatic deposit\n" +
+        "  # of Zotero-managed research items.\n" +
+        "  # Defaults to false.  See README for more info\n" +
+        "  config.arkivo_api = true\n"
+    end
+  end
+
+  # Copy the routing constraint over
+  def copy_routing_constraint
+    copy_file 'config/arkivo_constraint.rb', 'config/initializers/arkivo_constraint.rb'
+  end
+
+  # Copy the database migration
+  def copy_migration
+    better_migration_template 'add_arkivo_to_users.rb'
+  end
+
+  # Copy the config files for Zotero and Arkivo
+  def copy_config_files
+    copy_file 'config/arkivo.yml', 'config/arkivo.yml'
+    copy_file 'config/zotero.yml', 'config/zotero.yml'
+  end
+end

data/lib/generators/sufia/models/cached_stats_generator.rb

@@ -9,7 +9,7 @@ This generator adds the ability to cache usage stats to your application:
        """
 
   def banner
-    say_status("warning", "ADDING STATS CACHING-RELATED SUFIA MODELS", :yellow)
+    say_status("info", "ADDING STATS CACHING-RELATED SUFIA MODELS", :blue)
   end
 
   # Setup the database migrations

data/lib/generators/sufia/models/clamav_generator.rb

@@ -0,0 +1,19 @@
+# -*- encoding : utf-8 -*-
+require 'rails/generators'
+
+class Sufia::Models::ClamavGenerator < Rails::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+
+  desc """
+This generator makes the following changes to your application:
+ 1. Generates clamav initializer
+       """
+
+  def banner
+    say_status("info", "Generating clamav initializers", :blue)
+  end
+
+  def create_initializer_file
+    copy_file 'config/clamav.rb', 'config/initializers/clamav.rb'
+  end
+end

data/lib/generators/sufia/models/fulltext_generator.rb

@@ -11,7 +11,7 @@ This generator makes the following changes to your application:
        """
 
   def banner
-    say_status("warning", "GENERATING SUFIA FULL-TEXT", :yellow)
+    say_status("info", "GENERATING SUFIA FULL-TEXT", :blue)
   end
 
   # Copy Sufia's solrconfig into the dir from which the jetty:config task pulls

data/lib/generators/sufia/models/install_generator.rb

@@ -17,7 +17,7 @@ This generator makes the following changes to your application:
 10. Runs user stats generator
        """
   def banner
-    say_status("warning", "GENERATING SUFIA MODELS", :yellow)
+    say_status("info", "GENERATING SUFIA MODELS", :blue)
   end
 
   # Setup the database migrations
@@ -65,6 +65,7 @@ This generator makes the following changes to your application:
     copy_file 'config/redis_config.rb', 'config/initializers/redis_config.rb'
     copy_file 'config/resque_admin.rb', 'config/initializers/resque_admin.rb'
     copy_file 'config/resque_config.rb', 'config/initializers/resque_config.rb'
+    copy_file 'config/resque.rake', 'lib/tasks/resque.rake'
   end
 
   def create_collection
@@ -103,4 +104,9 @@ This generator makes the following changes to your application:
   def user_stats
     generate 'sufia:models:user_stats'
   end
+
+  # Adds clamav initializtion
+  def clamav
+    generate 'sufia:models:clamav'
+  end
 end

data/lib/generators/sufia/models/orcid_field_generator.rb

@@ -9,7 +9,7 @@ This generator adds a field to hold users' ORCIDs to your application:
        """
 
   def banner
-    say_status("warning", "ADDING ORCID FIELD TO USER MODEL", :yellow)
+    say_status("info", "ADDING ORCID FIELD TO USER MODEL", :blue)
   end
 
   # Setup the database migration

data/lib/generators/sufia/models/proxies_generator.rb

@@ -9,7 +9,7 @@ This generator adds proxies and transfers to your application:
        """
 
   def banner
-    say_status("warning", "ADDING PROXY/TRANSFER-RELATED SUFIA MODELS", :yellow)
+    say_status("info", "ADDING PROXY/TRANSFER-RELATED SUFIA MODELS", :blue)
   end
 
   # Setup the database migrations

data/lib/generators/sufia/models/templates/config/arkivo.yml

@@ -0,0 +1,6 @@
+development:
+  url: http://localhost:8888/
+test:
+  url: http://localhost:8888/
+production:
+  url: http://production.host/

data/lib/generators/sufia/models/templates/config/arkivo_constraint.rb

@@ -0,0 +1,11 @@
+module Sufia
+  class ArkivoConstraint
+    def self.matches?(request)
+      # Add your own logic here to authorize trusted connections to
+      # the API e.g., if your installation of Arkivo runs on a host
+      # with the 10.0.0.3 IP address, you could use:
+      # request.remote_ip == '10.0.0.3'
+      false
+    end
+  end
+end

data/lib/generators/sufia/models/templates/config/redis_config.rb

@@ -11,6 +11,7 @@ if defined?(PhusionPassenger)
       $redis = Redis.new(host: config[:host], port: config[:port], thread_safe: true) rescue nil
       Resque.redis = $redis
       Resque.redis.client.reconnect if Resque.redis
+      Resque.redis.namespace = "#{Sufia.config.redis_namespace}:#{Rails.env}"
     end
   end
 else
@@ -18,7 +19,6 @@ else
   $redis = Redis.new(host: config[:host], port: config[:port], thread_safe: true) rescue nil
 end
 
-
 # Code borrowed from Obie's Redis patterns talk at RailsConf'12
 Nest.class_eval do
   def initialize(key, redis=$redis)

data/lib/generators/sufia/models/templates/config/resque.rake

@@ -0,0 +1,16 @@
+# Load the Rails app all the time.
+# See https://github.com/resque/resque/wiki/FAQ for more details
+require 'resque/pool'
+require 'resque/pool/tasks'
+
+# This provides access to the Rails env within all Resque workers
+task "resque:setup" => :environment
+
+task 'resque:pool:setup' do
+  ActiveRecord::Base.connection.disconnect!
+
+  Resque::Pool.after_prefork do |j|
+    ActiveRecord::Base.establish_connection
+    Resque.redis.client.reconnect
+  end
+end

data/lib/generators/sufia/models/templates/config/sufia.rb

@@ -1,7 +1,5 @@
-# Returns an array containing the vhost 'CoSign service' value and URL
 Sufia.config do |config|
-
-  config.fits_to_desc_mapping= {
+  config.fits_to_desc_mapping = {
     file_title: :title,
     file_author: :creator
   }
@@ -118,15 +116,14 @@ Sufia.config do |config|
   # Specify how many seconds back from the current time that we should show by default of the user's activity on the user's dashboard
   # config.activity_to_show_default_seconds_since_now = 24*60*60
 
+  # Sufia can integrate with Zotero's Arkivo service for automatic deposit
+  # of Zotero-managed research items.
+  # config.arkivo_api = false
+
   # Specify a date you wish to start collecting Google Analytic statistics for.
   # Leaving it blank will set the start date to when ever the file was uploaded by
   # NOTE: if you have always sent analytics to GA for downloads and page views leave this commented out
   # config.analytic_start_date = DateTime.new(2014,9,10)
-  #
-  # Method of converting ids into URIs for storage in Fedora
-  # config.translate_uri_to_id = lambda { |uri| uri.to_s.split('/')[-1] }
-  # config.translate_id_to_uri = lambda { |id|
-  #      "#{ActiveFedora.fedora.host}#{ActiveFedora.fedora.base_path}/#{Sufia::Noid.treeify(id)}" }
 
   # If browse-everything has been configured, load the configs.  Otherwise, set to nil.
   begin

data/lib/generators/sufia/models/templates/config/zotero.yml

@@ -0,0 +1,6 @@
+#
+# To integrate your app with Zotero, set ENV variables with your API info
+#
+zotero:
+  client_key: <%= ENV["ZOTERO_CLIENT_KEY"] %>
+  client_secret: <%= ENV["ZOTERO_CLIENT_SECRET"] %>

data/lib/generators/sufia/models/templates/migrations/add_arkivo_to_users.rb

@@ -0,0 +1,8 @@
+class AddArkivoToUsers < ActiveRecord::Migration
+  def change
+    add_column :users, :arkivo_token, :string
+    add_column :users, :arkivo_subscription, :string
+    add_column :users, :zotero_token, :binary
+    add_column :users, :zotero_userid, :string
+  end
+end

data/lib/generators/sufia/models/update_content_blocks_generator.rb

@@ -8,7 +8,7 @@ This generator creates a database migration to add an external_key column to the
        """
 
   def banner
-    say_status("warning", "CREATING MIGRATION FILE", :yellow)
+    say_status("info", "CREATING MIGRATION FILE", :blue)
   end
 
   def copy_migrations

data/lib/generators/sufia/models/upgrade400_generator.rb

@@ -12,7 +12,7 @@ This generator for upgrading sufia-models from 3.7.2 to 4.0 makes the following
        """
 
   def banner
-    say_status("warning", "UPGRADING SUFIA MODELS", :yellow)
+    say_status("info", "UPGRADING SUFIA MODELS", :blue)
   end
 
   # Setup the database migrations
@@ -44,7 +44,7 @@ This generator for upgrading sufia-models from 3.7.2 to 4.0 makes the following
         "  # Defaults to FALSE\n" +
         "  # Requires a Google Analytics id and OAuth2 keyfile.  See README for more info\n" +
         "  #config.analytics = false\n"
-      end
+    end
   end
 
   # Sets up full-text indexing (Solr config + jars)

data/lib/generators/sufia/models/usagestats_generator.rb

@@ -10,7 +10,7 @@ This generator makes the following changes to your application:
        """
 
   def banner
-    say_status("warning", "GENERATING SUFIA USAGE STATS", :yellow)
+    say_status("info", "GENERATING SUFIA USAGE STATS", :blue)
   end
 
   def create_configuration_file

data/lib/generators/sufia/models/user_stats_generator.rb

@@ -9,7 +9,7 @@ This generator adds usage stats methods to the user model in your application:
        """
 
   def banner
-    say_status("warning", "ADDING USER STATS-RELATED ABILITIES TO SUFIA MODELS", :yellow)
+    say_status("info", "ADDING USER STATS-RELATED ABILITIES TO SUFIA MODELS", :blue)
   end
 
   # Setup the database migrations

data/lib/sufia/models/engine.rb

@@ -14,11 +14,11 @@ module Sufia
       # Set some configuration defaults
       config.persistent_hostpath = "http://localhost/files/"
       config.enable_ffmpeg = false
-      config.noid_template = '.reeddeeddk'
       config.ffmpeg_path = 'ffmpeg'
       config.fits_message_length = 5
       config.temp_file_base = nil
       config.enable_noids = true
+      config.noid_template = '.reeddeeddk'
       config.minter_statefile = '/tmp/minter-state'
       config.redis_namespace = "sufia"
       config.fits_path = "fits.sh"
@@ -29,14 +29,12 @@ module Sufia
       config.queue = Sufia::Resque::Queue
       config.max_notifications_for_dashboard = 5
       config.activity_to_show_default_seconds_since_now = 24*60*60
+      config.arkivo_api = false
+      config.retry_unless_sleep = 0.0
 
-      # Defaulting analytic start date to when ever the file was uploaded by leaving it blank
+      # Defaulting analytic start date to whenever the file was uploaded by leaving it blank
       config.analytic_start_date = nil
 
-      config.translate_uri_to_id = lambda { |uri| uri.to_s.split('/')[-1] }
-      config.translate_id_to_uri = lambda { |id|
-        "#{ActiveFedora.fedora.host}#{ActiveFedora.fedora.base_path}/#{Sufia::Noid.treeify(id)}" }
-
       config.autoload_paths += %W(
         #{config.root}/app/models/datastreams
       )
@@ -53,11 +51,11 @@ module Sufia
       initializer 'requires' do
         require 'activerecord-import'
         require 'hydra/derivatives'
+        require 'active_fedora/noid'
         require 'sufia/models/file_content'
         require 'sufia/models/file_content/versions'
         require 'sufia/models/user_local_directory_behavior'
         require 'sufia/noid'
-        require 'sufia/id_service'
         require 'sufia/analytics'
         require 'sufia/pageview'
         require 'sufia/download'
@@ -70,8 +68,10 @@ module Sufia
           Hydra::Derivatives.fits_path      = c.fits_path
           Hydra::Derivatives.enable_ffmpeg  = c.enable_ffmpeg
 
-          ActiveFedora::Base.translate_uri_to_id = c.translate_uri_to_id
-          ActiveFedora::Base.translate_id_to_uri = c.translate_id_to_uri
+          ActiveFedora::Base.translate_uri_to_id = ActiveFedora::Noid.config.translate_uri_to_id
+          ActiveFedora::Base.translate_id_to_uri = ActiveFedora::Noid.config.translate_id_to_uri
+          ActiveFedora::Noid.config.template = c.noid_template
+          ActiveFedora::Noid.config.statefile = c.minter_statefile
         end
       end
     end

data/lib/sufia/models/file_content/versions.rb

@@ -1,11 +1,6 @@
 module Sufia
   module FileContent
     module Versions
-      extend ActiveSupport::Concern
-
-      included do
-        has_many_versions
-      end
 
       def latest_version
         versions.last unless versions.empty?

data/lib/sufia/models/stats/user_stat_importer.rb

@@ -7,6 +7,7 @@ module Sufia
       @verbose = options[:verbose]
       @logging = options[:logging]
       @delay_secs = options[:delay_secs].to_f
+      @number_of_retries = options[:number_of_retries].to_i
     end
 
     def import
@@ -14,14 +15,17 @@ module Sufia
       sorted_users.each do |user|
         start_date = date_since_last_cache(user)
 
+        # this user has already been processed today continue without delay
+        next if start_date.to_date >= Date.today
+
         stats = {}
         file_ids_for_user(user).each do |file_id|
-          view_stats = FileViewStat.statistics(file_id, start_date, user.id)
-          stats = tally_results(view_stats, :views, stats)
+          view_stats = rescue_and_retry("Retried FileViewStat on #{user} for file #{file_id} too many times.") { FileViewStat.statistics(file_id, start_date, user.id) }
+          stats = tally_results(view_stats, :views, stats) unless view_stats.blank?
           delay
 
-          dl_stats = FileDownloadStat.statistics(file_id, start_date, user.id)
-          stats = tally_results(dl_stats, :downloads, stats)
+          dl_stats = rescue_and_retry("Retried FileDownloadStat on #{user} for file #{file_id} too many times.") { FileDownloadStat.statistics(file_id, start_date, user.id) }
+          stats = tally_results(dl_stats, :downloads, stats) unless dl_stats.blank?
           delay
         end
 
@@ -47,6 +51,22 @@ private
       sleep @delay_secs
     end
 
+    def rescue_and_retry(fail_message)
+      retry_count = 0
+      begin
+        return yield
+      rescue StandardError => e
+        retry_count += 1
+        if retry_count < @number_of_retries
+          delay
+          retry
+        else
+          log_message fail_message
+          log_message "Last exception #{e}"
+        end
+      end
+    end
+
     def date_since_last_cache(user)
       last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last
 

data/lib/sufia/models/utils.rb

@@ -1,12 +1,28 @@
+# This module includes utility methods
 module Sufia
   module Utils
     extend ActiveSupport::Concern
 
+    # retry the block if the conditional call is true unless we hit the maximum tries
+    #
+    # @param number_of_tries [enumerator] maximum number of times to retry the block (eg. 7.times)
+    # @param condition [#call] conditional to call and see if we SHOULD retry
+    # @yeild block [] code you want to run and retry
+    #
+    # @return result of the block call
     def retry_unless(number_of_tries, condition, &block)
       self.class.retry_unless(number_of_tries, condition, &block)
     end
 
     module ClassMethods
+
+      # retry the block if the conditional call is true unless we hit the maximum tries
+      #
+      # @param number_of_tries [enumerator] maximum number of times to retry the block
+      # @param condition [#call] conditional to call and see if we SHOULD retry
+      # @yeild block [] code you want to run and retry
+      #
+      # @return result of the block call
       def retry_unless(number_of_tries, condition, &block)
         raise ArgumentError, "First argument must be an enumerator" unless number_of_tries.is_a? Enumerator
         raise ArgumentError, "Second argument must be a lambda" unless condition.respond_to? :call
@@ -14,6 +30,7 @@ module Sufia
         number_of_tries.each do
           result = block.call
           return result unless condition.call
+          sleep(Sufia.config.retry_unless_sleep) if Sufia.config.retry_unless_sleep > 0
         end
         raise RuntimeError, "retry_unless could not complete successfully. Try upping the # of tries?"
       end

data/lib/sufia/models/version.rb

@@ -1,5 +1,5 @@
 module Sufia
   module Models
-    VERSION = "6.0.0"
+    VERSION = "6.1.0"
   end
 end

data/lib/sufia/permissions/writable.rb

@@ -13,7 +13,7 @@ module Sufia
 
       def paranoid_permissions
         valid = true
-        VALIDATIONS.each do |validation|
+        paranoid_edit_permissions.each do |validation|
           if validation[:condition].call(self)
             errors[validation[:key]] ||= []
             errors[validation[:key]] << validation[:message]
@@ -23,12 +23,13 @@ module Sufia
         return valid
       end
 
-      VALIDATIONS = [
-        {key: :edit_users, message: 'Depositor must have edit access', condition: lambda { |obj| !obj.edit_users.include?(obj.depositor) }},
-        {key: :edit_groups, message: 'Public cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('public') }},
-        {key: :edit_groups, message: 'Registered cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('registered') }}
-      ]
-
+      def paranoid_edit_permissions
+        [
+          {key: :edit_users, message: 'Depositor must have edit access', condition: lambda { |obj| !obj.edit_users.include?(obj.depositor) }},
+          {key: :edit_groups, message: 'Public cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('public') }},
+          {key: :edit_groups, message: 'Registered cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('registered') }}
+        ]
+      end
 
       def clear_permissions!
         self.permissions = []

data/lib/tasks/migrate.rake

@@ -1,4 +1,4 @@
- namespace :sufia do
+namespace :sufia do
   namespace :migrate do
 
     desc "Migrate proxy deposits"

data/lib/tasks/sufia-models_tasks.rake

@@ -8,6 +8,20 @@ namespace :solr do
 end
 
 namespace :sufia do
+  namespace :user do
+    desc 'Populate user tokens'
+    task tokens: :environment do
+      unless Sufia.config.arkivo_api
+        puts "Zotero integration is not enabled"
+        next
+      end
+      User.where(arkivo_token: nil).each do |user|
+        user.set_arkivo_token
+        user.save
+      end
+    end
+  end
+
   namespace :jetty do
 
     FULLTEXT_JARS = %w(
@@ -66,8 +80,10 @@ end
 
 def download_from_maven url, dst
   full_url = '/remotecontent?filepath=' + url
-  file = File.open(dst, "w")
-  Net::HTTP.start("search.maven.org") do |http|
+  file = File.open(dst, "wb")
+  endpoint = Net::HTTP.new('search.maven.org', 443)
+  endpoint.use_ssl = true
+  endpoint.start do |http|
     puts "Fetching #{full_url}"
     begin
       http.request_get(full_url) do |resp|

data/sufia-models.gemspec

@@ -30,13 +30,13 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activeresource', "~> 4.0" # No longer a dependency of rails 4.0
 
   spec.add_dependency "hydra-head", "~> 9.0"
-  spec.add_dependency "active-fedora", "~> 9.0"
+  spec.add_dependency "active-fedora", "~> 9.1", ">= 9.1.1"
   spec.add_dependency "hydra-collections", [">= 5.0.2", "< 6.0"]
   spec.add_dependency 'hydra-derivatives', '~> 1.0'
+  spec.add_dependency 'active_fedora-noid', '~> 0.1'
   spec.add_dependency 'nest', '~> 1.1'
   spec.add_dependency 'resque', '~> 1.23'
   spec.add_dependency 'resque-pool', '~> 0.3'
-  spec.add_dependency 'noid', '~> 0.6'
   spec.add_dependency 'mailboxer', '~> 0.12'
   spec.add_dependency 'acts_as_follower', '>= 0.1.1', '< 0.3'
   spec.add_dependency 'carrierwave', '~> 0.9'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sufia-models
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.1.0
 platform: ruby
 authors:
 - Jeremy Friesen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-27 00:00:00.000000000 Z
+date: 2015-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,14 +86,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '9.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 9.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '9.0'
+        version: '9.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 9.1.1
 - !ruby/object:Gem::Dependency
   name: hydra-collections
   requirement: !ruby/object:Gem::Requirement
@@ -129,61 +135,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: nest
+  name: active_fedora-noid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: resque
+  name: nest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.23'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.23'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: resque-pool
+  name: resque
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.23'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.23'
 - !ruby/object:Gem::Dependency
-  name: noid
+  name: resque-pool
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: mailboxer
   requirement: !ruby/object:Gem::Requirement
@@ -377,30 +383,35 @@ files:
 - app/services/sufia/analytics.rb
 - app/services/sufia/generic_file_audit_service.rb
 - app/services/sufia/generic_file_indexing_service.rb
-- app/services/sufia/id_service.rb
 - app/services/sufia/noid.rb
 - app/services/sufia/repository_audit_service.rb
 - config/locales/sufia.en.yml
 - lib/generators/sufia/models/abstract_migration_generator.rb
+- lib/generators/sufia/models/arkivo_api_generator.rb
 - lib/generators/sufia/models/cached_stats_generator.rb
+- lib/generators/sufia/models/clamav_generator.rb
 - lib/generators/sufia/models/fulltext_generator.rb
 - lib/generators/sufia/models/install_generator.rb
 - lib/generators/sufia/models/orcid_field_generator.rb
 - lib/generators/sufia/models/proxies_generator.rb
 - lib/generators/sufia/models/templates/app/models/collection.rb
 - lib/generators/sufia/models/templates/config/analytics.yml
+- lib/generators/sufia/models/templates/config/arkivo.yml
+- lib/generators/sufia/models/templates/config/arkivo_constraint.rb
 - lib/generators/sufia/models/templates/config/clamav.rb
-- lib/generators/sufia/models/templates/config/mailboxer.rb
 - lib/generators/sufia/models/templates/config/mime_types.rb
 - lib/generators/sufia/models/templates/config/redis.yml
 - lib/generators/sufia/models/templates/config/redis_config.rb
 - lib/generators/sufia/models/templates/config/resque-pool.yml
+- lib/generators/sufia/models/templates/config/resque.rake
 - lib/generators/sufia/models/templates/config/resque_admin.rb
 - lib/generators/sufia/models/templates/config/resque_config.rb
 - lib/generators/sufia/models/templates/config/setup_mail.rb
 - lib/generators/sufia/models/templates/config/solrconfig.xml
 - lib/generators/sufia/models/templates/config/sufia.rb
+- lib/generators/sufia/models/templates/config/zotero.yml
 - lib/generators/sufia/models/templates/migrations/acts_as_follower_migration.rb
+- lib/generators/sufia/models/templates/migrations/add_arkivo_to_users.rb
 - lib/generators/sufia/models/templates/migrations/add_avatars_to_users.rb
 - lib/generators/sufia/models/templates/migrations/add_external_key_to_content_blocks.rb
 - lib/generators/sufia/models/templates/migrations/add_groups_to_users.rb
@@ -470,7 +481,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Models and services for sufia

data/app/services/sufia/id_service.rb

@@ -1,45 +0,0 @@
-require 'noid'
-
-module Sufia
-  module IdService
-
-    def self.noid_template
-      Sufia.config.noid_template
-    end
-
-    @minter = ::Noid::Minter.new(template: noid_template)
-    @semaphore = Mutex.new
-    def self.valid?(identifier)
-      # remove the fedora namespace since it's not part of the noid
-      noid = identifier#.split(":").last
-      @minter.valid? noid
-    end
-    def self.mint
-      @semaphore.synchronize do
-        while true
-          id = self.next_id
-          return id unless ActiveFedora::Base.exists?(id)
-        end
-      end
-    end
-
-    protected
-
-    def self.next_id
-      id = ''
-      File.open(Sufia.config.minter_statefile, File::RDWR|File::CREAT, 0644) do |f|
-        f.flock(File::LOCK_EX)
-        yaml = YAML::load(f.read)
-        yaml = {template: noid_template} unless yaml
-        minter = ::Noid::Minter.new(yaml)
-        id = minter.mint
-        f.rewind
-        yaml = YAML::dump(minter.dump)
-        f.write yaml
-        f.flush
-        f.truncate(f.pos)
-      end
-      id
-    end
-  end
-end

data/lib/generators/sufia/models/templates/config/mailboxer.rb

@@ -1,17 +0,0 @@
-Mailboxer.setup do |config|
-
-  #Configures if you applications uses or no the email sending for Notifications and Messages
-  config.uses_emails = true
-
-  #Configures the default from for the email sent for Messages and Notifications of Mailboxer
-  config.default_from = "no-reply@mailboxer.com"
-
-  #Configures the methods needed by mailboxer
-  config.email_method = :mailboxer_email
-  config.name_method = :name
-
-  #Configures if you use or not a search engine and wich one are you using
-  #Supported enignes: [:solr,:sphinx]
-  config.search_enabled = false
-  config.search_engine = :solr
-end