sufia 6.0.0 → 6.1.0

data/sufia-models/lib/sufia/models/file_content.rb

@@ -1,6 +0,0 @@
-module Sufia
-  module FileContent
-    extend ActiveSupport::Autoload
-    autoload :Versions, 'sufia/models/file_content/versions'
-  end
-end

data/sufia-models/lib/sufia/models/file_content/versions.rb

@@ -1,21 +0,0 @@
-module Sufia
-  module FileContent
-    module Versions
-      extend ActiveSupport::Concern
-
-      included do
-        has_many_versions
-      end
-
-      def latest_version
-        versions.last unless versions.empty?
-      end
-
-      def save
-        super.tap do |passing|
-          create_version if passing
-        end
-      end
-    end
-  end
-end

data/sufia-models/lib/sufia/models/resque.rb

@@ -1,36 +0,0 @@
-# Borrowed from:
-# https://github.com/jeremy/resque-rails/blob/master/lib/resque/rails/queue.rb
-module Sufia
-  module Resque
-    class Queue
-      attr_reader :default_queue_name
-
-      def initialize(default_queue_name)
-        @default_queue_name = default_queue_name
-      end
-
-      def push(job)
-        push_tries = 0
-        queue = job.respond_to?(:queue_name) ? job.queue_name : default_queue_name
-        begin
-          ::Resque.enqueue_to queue, MarshaledJob, Base64.encode64(Marshal.dump(job))
-        rescue Redis::CannotConnectError
-          ActiveFedora::Base.logger.error "Redis is down!"
-        rescue Redis::TimeoutError => error
-          ActiveFedora::Base.logger.warn "Redis Timed out.  Trying again! #{job.inspect}"
-          push_tries+=1
-          # fail for good if the tries is greater than 3
-          raise error if push_tries >=3
-          sleep 0.01
-          retry
-        end
-      end
-    end
-
-    class MarshaledJob
-      def self.perform(marshaled_job)
-        Marshal.load(Base64.decode64(marshaled_job)).run
-      end
-    end
-  end
-end

data/sufia-models/lib/sufia/models/stats/user_stat_importer.rb

@@ -1,108 +0,0 @@
-module Sufia
-  class UserStatImporter
-
-    UserRecord = Struct.new("UserRecord", :id, :user_key, :last_stats_update)
-
-    def initialize(options={})
-      @verbose = options[:verbose]
-      @logging = options[:logging]
-      @delay_secs = options[:delay_secs].to_f
-    end
-
-    def import
-      log_message('Begin import of User stats.')
-      sorted_users.each do |user|
-        start_date = date_since_last_cache(user)
-
-        stats = {}
-        file_ids_for_user(user).each do |file_id|
-          view_stats = FileViewStat.statistics(file_id, start_date, user.id)
-          stats = tally_results(view_stats, :views, stats)
-          delay
-
-          dl_stats = FileDownloadStat.statistics(file_id, start_date, user.id)
-          stats = tally_results(dl_stats, :downloads, stats)
-          delay
-        end
-
-        create_or_update_user_stats(stats, user)
-      end
-      log_message('User stats import complete.')
-    end
-
-    # Returns an array of users sorted by the date of their last
-    # stats update. Users that have not been recently updated 
-    # will be at the top of the array.
-    def sorted_users
-      users = []
-      ::User.find_each do |user|
-        users.push(UserRecord.new(user.id, user.user_key, date_since_last_cache(user)))
-      end
-      users.sort! {|a, b| a.last_stats_update <=> b.last_stats_update}
-    end
-
-private
-
-    def delay 
-      sleep @delay_secs
-    end
-
-    def date_since_last_cache(user)
-      last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last
-
-      if last_cached_stat
-        last_cached_stat.date + 1.day
-      else
-        Sufia.config.analytic_start_date
-      end
-    end
-
-    def file_ids_for_user(user)
-      ids = []
-      ::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl:"id") do |group|
-        ids.concat group.map { |doc| doc["id"] }
-      end
-      ids
-    end
-
-    # For each date, add the view and download counts for this
-    # file to the view & download sub-totals for that day.
-    # The resulting hash will look something like this:
-    # {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
-    #  "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
-    def tally_results(file_stats, stat_name, total_stats)
-      file_stats.each do |stats|
-        # Exclude the stats from today since it will only be a partial day's worth of data
-        break if stats.date == Date.today
-
-        date_key = stats.date.to_s
-        old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
-        new_count = old_count + stats.method(stat_name).call
-
-        old_values = total_stats[date_key] || {}
-        total_stats.store(date_key, old_values)
-        total_stats[date_key].store(stat_name, new_count)
-      end
-      total_stats
-    end
-
-    def create_or_update_user_stats(stats, user)
-      stats.each do |date_string, data|
-        date = Time.zone.parse(date_string)
-
-        user_stat = UserStat.where(user_id: user.id).where(date: date).first
-        user_stat ||= UserStat.new(user_id: user.id, date: date)
-
-        user_stat.file_views = data[:views] || 0
-        user_stat.file_downloads = data[:downloads] || 0
-        user_stat.save!
-      end
-    end
-
-    def log_message(message)
-      puts message if @verbose
-      Rails.logger.info "#{self.class}: #{message}" if @logging
-    end
-
-  end
-end

data/sufia-models/lib/sufia/models/user_local_directory_behavior.rb

@@ -1,29 +0,0 @@
-# To enable local file ingest, 
-# - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate. 
-# - Include this module in your User model, or define a .files() method that behaves the same
-# - Set Sufia.config.enable_local_ingest to true
-# 
-module Sufia::UserLocalDirectoryBehavior
-  
-  # You can use this validator in your User model.
-  # Ensures that a string defining the path to the user's directory has been provided 
-  # and corresponds to a real directory on the server.
-  # @example
-  #   validate :directory_must_exist
-  def directory_must_exist
-    unless directory.blank? || File.directory?(directory)
-      errors.add(:directory, "must be an existing directory")
-    end
-  end
-
-  # List the contents of the user's directory on the server
-  # Indicates whether each item is a directory or not.
-  def files
-    return [] unless directory.present? && File.directory?(directory)
-    Dir[File.join(directory, '*')].inject([]) do |accum, val|
-      accum << { name: File.basename(val), directory: File.directory?(val)}
-      accum
-    end
-  end
-  
-end

data/sufia-models/lib/sufia/models/utils.rb

@@ -1,22 +0,0 @@
-module Sufia
-  module Utils
-    extend ActiveSupport::Concern
-
-    def retry_unless(number_of_tries, condition, &block)
-      self.class.retry_unless(number_of_tries, condition, &block)
-    end
-
-    module ClassMethods
-      def retry_unless(number_of_tries, condition, &block)
-        raise ArgumentError, "First argument must be an enumerator" unless number_of_tries.is_a? Enumerator
-        raise ArgumentError, "Second argument must be a lambda" unless condition.respond_to? :call
-        raise ArgumentError, "Must pass a block of code to retry" unless block_given?
-        number_of_tries.each do
-          result = block.call
-          return result unless condition.call
-        end
-        raise RuntimeError, "retry_unless could not complete successfully. Try upping the # of tries?"
-      end
-    end
-  end
-end

data/sufia-models/lib/sufia/models/version.rb

@@ -1,5 +0,0 @@
-module Sufia
-  module Models
-    VERSION = "6.0.0"
-  end
-end

data/sufia-models/lib/sufia/models/virus_found_error.rb

@@ -1,4 +0,0 @@
-module Sufia
-  class VirusFoundError < StandardError
-  end
-end

data/sufia-models/lib/sufia/permissions.rb

@@ -1,9 +0,0 @@
-module Sufia
-  module Permissions
-    extend ActiveSupport::Autoload
-
-    autoload :Writable
-    autoload :Readable
-
-  end
-end

data/sufia-models/lib/sufia/permissions/readable.rb

@@ -1,20 +0,0 @@
-module Sufia
-  module Permissions
-    module Readable
-      extend ActiveSupport::Concern
-
-      def public?
-        read_groups.include?('public')
-      end
-
-      def registered?
-        read_groups.include?('registered')
-      end
-
-      def private?
-        !(public? || registered?)
-      end
-
-    end
-  end
-end

data/sufia-models/lib/sufia/permissions/writable.rb

@@ -1,74 +0,0 @@
-module Sufia
-  module Permissions
-    module Writable
-      extend ActiveSupport::Concern
-
-      #we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
-      include Hydra::AccessControls::Permissions
-      include Hydra::AccessControls::Visibility
-
-      included do
-        validate :paranoid_permissions
-      end
-
-      def paranoid_permissions
-        valid = true
-        VALIDATIONS.each do |validation|
-          if validation[:condition].call(self)
-            errors[validation[:key]] ||= []
-            errors[validation[:key]] << validation[:message]
-            valid = false
-          end
-        end
-        return valid
-      end
-
-      VALIDATIONS = [
-        {key: :edit_users, message: 'Depositor must have edit access', condition: lambda { |obj| !obj.edit_users.include?(obj.depositor) }},
-        {key: :edit_groups, message: 'Public cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('public') }},
-        {key: :edit_groups, message: 'Registered cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('registered') }}
-      ]
-
-
-      def clear_permissions!
-        self.permissions = []
-      end
-
-      ## Updates those permissions that are provided to it. Does not replace any permissions unless they are provided
-      # def permissions=(params)
-      #   raise "Fixme #{params}"
-      #   perm_hash = permission_hash
-      #   params[:new_user_name].each { |name, access| perm_hash['person'][name] = access } if params[:new_user_name].present?
-      #   params[:new_group_name].each { |name, access| perm_hash['group'][name] = access } if params[:new_group_name].present?
-
-      #   params[:user].each { |name, access| perm_hash['person'][name] = access} if params[:user]
-      #   params[:group].each { |name, access| perm_hash['group'][name] = access if ['read', 'edit'].include?(access)} if params[:group]
-
-      #   # rightsMetadata.update_permissions(perm_hash)
-      # end
-
-      # def permissions
-      #   raise "Fixme "
-      #   perms = super
-      #   perms.map {|p| { name: p.name, access: p.access, type:p.type } }
-      # end
-
-      private
-
-      def permission_hash
-        old_perms = self.permissions
-        user_perms =  {}
-        old_perms.select{|r| r[:type] == 'user'}.each do |r|
-          user_perms[r[:name]] = r[:access]
-        end
-        user_perms
-        group_perms =  {}
-        old_perms.select{|r| r[:type] == 'group'}.each do |r|
-          group_perms[r[:name]] = r[:access]
-        end
-        {'person'=>user_perms, 'group'=>group_perms}
-      end
-
-    end
-  end
-end

data/sufia-models/lib/tasks/batch_cleanup.rake

@@ -1,19 +0,0 @@
-namespace :sufia do
-
-  desc "Reports on and optionally removes empty batches that contain no associated files"
-  task :empty_batches, [:remove] => :environment do |t, args|
-    option = args.to_hash.fetch(:remove, "keep")
-    Batch.all.each do |batch|
-      if batch.generic_files.empty?
-        print "#{batch.id} contains no files - "
-        if option == "remove"
-          batch.destroy
-          puts "deleted"
-        else
-          puts "to delete, rerun with the remove option: rake sufia:empty_batches[remove]"
-        end
-      end
-    end
-  end
-
-end

data/sufia-models/lib/tasks/migrate.rake

@@ -1,21 +0,0 @@
- namespace :sufia do
-  namespace :migrate do
-
-    desc "Migrate proxy deposits"
-    task proxy_deposits: :environment do
-      ProxyDepositRequest.all.each do |pd|
-        pd.generic_file_id = pd.generic_file_id.delete "#{Sufia.config.redis_namespace}:"
-        pd.save
-      end
-    end
-
-    desc "Migrate audit logs"
-    task audit_logs: :environment do
-      ChecksumAuditLog.all.each do |cs|
-        cs.generic_file_id = cs.generic_file_id.delete "#{Sufia.config.redis_namespace}:"
-        cs.save
-      end
-    end
-
-  end
-end

data/sufia-models/lib/tasks/resque.rake

@@ -1,13 +0,0 @@
-require 'resque/pool/tasks'
-
-# This provides access to the Rails env within all Resque workers
-task 'resque:setup' => :environment
-
-# Set up resque-pool
-task 'resque:pool:setup' do
-  ActiveRecord::Base.connection.disconnect!
-  Resque::Pool.after_prefork do |job|
-    ActiveRecord::Base.establish_connection
-    Resque.redis.client.reconnect
-  end
-end

data/sufia-models/lib/tasks/stats_tasks.rake

@@ -1,12 +0,0 @@
-namespace :sufia do
-  namespace :stats do
-
-    desc "Cache file view & download stats for all users"
-    task :user_stats => :environment do
-      require 'sufia/models/stats/user_stat_importer'
-      importer = Sufia::UserStatImporter.new(verbose: true, logging: true)
-      importer.import
-    end
-
-  end
-end

data/sufia-models/lib/tasks/sufia-models_tasks.rake

@@ -1,80 +0,0 @@
-require 'net/http'
-
-namespace :solr do
-  desc "Enqueue a job to resolrize the repository objects"
-  task reindex: :environment do
-    Sufia.queue.push(ResolrizeJob.new)
-  end
-end
-
-namespace :sufia do
-  namespace :jetty do
-
-    FULLTEXT_JARS = %w(
-        org/apache/james/apache-mime4j-core/0.7.2/apache-mime4j-core-0.7.2.jar
-        org/apache/james/apache-mime4j-dom/0.7.2/apache-mime4j-dom-0.7.2.jar
-        org/apache/solr/solr-cell/4.0.0/solr-cell-4.0.0.jar
-        org/bouncycastle/bcmail-jdk15/1.45/bcmail-jdk15-1.45.jar
-        org/bouncycastle/bcprov-jdk15/1.45/bcprov-jdk15-1.45.jar
-        de/l3s/boilerpipe/boilerpipe/1.1.0/boilerpipe-1.1.0.jar
-        org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar
-        dom4j/dom4j/1.6.1/dom4j-1.6.1.jar
-        org/apache/pdfbox/fontbox/1.7.0/fontbox-1.7.0.jar
-        com/ibm/icu/icu4j/49.1/icu4j-49.1.jar
-        com/googlecode/mp4parser/isoparser/1.0-RC-1/isoparser-1.0-RC-1.jar
-        jdom/jdom/1.0/jdom-1.0.jar
-        org/apache/pdfbox/jempbox/1.7.0/jempbox-1.7.0.jar
-        com/googlecode/juniversalchardet/juniversalchardet/1.0.3/juniversalchardet-1.0.3.jar
-        com/drewnoakes/metadata-extractor/2.4.0-beta-1/metadata-extractor-2.4.0-beta-1.jar
-        edu/ucar/netcdf/4.2-min/netcdf-4.2-min.jar
-        org/apache/pdfbox/pdfbox/1.7.0/pdfbox-1.7.0.jar
-        org/apache/poi/poi/3.8/poi-3.8.jar
-        org/apache/poi/poi-ooxml/3.8/poi-ooxml-3.8.jar
-        org/apache/poi/poi-ooxml-schemas/3.8/poi-ooxml-schemas-3.8.jar
-        org/apache/poi/poi-scratchpad/3.8/poi-scratchpad-3.8.jar
-        rome/rome/0.9/rome-0.9.jar
-        org/ccil/cowan/tagsoup/tagsoup/1.2.1/tagsoup-1.2.1.jar
-        org/apache/tika/tika-core/1.2/tika-core-1.2.jar
-        org/apache/tika/tika-parsers/1.2/tika-parsers-1.2.jar
-        org/gagravarr/vorbis-java-core/0.1/vorbis-java-core-0.1.jar
-        org/gagravarr/vorbis-java-tika/0.1/vorbis-java-tika-0.1.jar
-        xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar
-        org/apache/xmlbeans/xmlbeans/2.3.0/xmlbeans-2.3.0.jar
-        org/tukaani/xz/1.0/xz-1.0.jar
-        org/aspectj/aspectjrt/1.8.5/aspectjrt-1.8.5.jar
-      )
-
-    desc 'Configure jetty with full-text indexing'
-    task config: :download_jars do
-      Rake::Task['jetty:config'].invoke
-    end
-
-    desc 'Download Solr full-text extraction jars'
-    task :download_jars do
-      puts "Downloading full-text jars from maven.org ..."
-      fulltext_dir = 'jetty/solr/lib/contrib/extraction/lib'
-      FileUtils.mkdir_p(fulltext_dir) unless File.directory?(fulltext_dir)
-      Dir.chdir(fulltext_dir) do
-        FULLTEXT_JARS.each do |jar|
-          destination = jar.split('/').last
-          download_from_maven(jar, destination) unless File.exists?(destination)
-        end
-      end
-    end
-  end
-end
-
-def download_from_maven url, dst
-  full_url = '/remotecontent?filepath=' + url
-  file = File.open(dst, "w")
-  Net::HTTP.start("search.maven.org") do |http|
-    puts "Fetching #{full_url}"
-    begin
-      http.request_get(full_url) do |resp|
-        resp.read_body { |segment| file.write(segment) }
-      end
-    ensure
-      file.close
-    end
-  end
-end