curation_concerns 0.1.0

data/curation_concerns-models/app/services/curation_concerns/characterization_service.rb

@@ -0,0 +1,71 @@
+module CurationConcerns
+  # Run FITS to gather technical metadata about the content and the full text.
+  # Store this extracted metadata in the characterization datastream.
+  class CharacterizationService
+    include Hydra::Derivatives::ExtractMetadata
+
+    delegate :mime_type, :uri, to: :@generic_file
+    attr_reader :generic_file
+
+    def self.run(generic_file)
+      new(generic_file).characterize
+    end
+
+    def initialize(generic_file)
+      @generic_file = generic_file
+    end
+
+    ## Extract the metadata from the content datastream and record it in the characterization datastream
+    def characterize
+      store_metadata(extract_metadata)
+      store_fulltext(extract_fulltext)
+      generic_file.filename = [generic_file.original_file.original_name]
+    end
+
+    protected
+
+      def store_fulltext(extracted_text)
+        if extracted_text.present?
+          extracted_text_file = generic_file.build_extracted_text
+          extracted_text_file.content = extracted_text
+        end
+      end
+
+      def extract_fulltext
+        FullTextExtractionService.run(generic_file)
+      end
+
+      def store_metadata(metadata)
+        generic_file.characterization.ng_xml = metadata if metadata.present?
+        append_metadata
+      end
+
+      def extract_metadata
+        return unless generic_file.original_file.has_content?
+        Hydra::FileCharacterization.characterize(generic_file.original_file.content, filename_for_characterization.join, :fits) do |config|
+          config[:fits] = Hydra::Derivatives.fits_path
+        end
+      end
+
+      # Populate GenericFile's properties with fields from FITS (e.g. Author from pdfs)
+      def append_metadata
+        terms = generic_file.characterization_terms
+        CurationConcerns.config.fits_to_desc_mapping.each_pair do |k, v|
+          if terms.has_key?(k)
+            # coerce to array to remove a conditional
+            terms[k] = [terms[k]] unless terms[k].is_a? Array
+            terms[k].each do |term_value|
+              proxy_term = generic_file.send(v)
+              if proxy_term.kind_of?(Array)
+                proxy_term << term_value unless proxy_term.include?(term_value)
+              else
+                # these are single-valued terms which cannot be appended to
+                generic_file.send("#{v}=", term_value)
+              end
+            end
+          end
+        end
+      end
+
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/full_text_extraction_service.rb

@@ -0,0 +1,38 @@
+module CurationConcerns
+  # Extract the full text from the content using Solr's extract handler
+  class FullTextExtractionService
+    def self.run(generic_file)
+      new(generic_file).extract
+    end
+
+    delegate :original_file, :logger, :mime_type, :id, to: :@generic_file
+
+    def initialize(generic_file)
+      @generic_file = generic_file
+    end
+
+    def extract
+      uri = URI("#{connection_url}/update/extract?extractOnly=true&wt=json&extractFormat=text")
+      req = Net::HTTP.new(uri.host, uri.port)
+      resp = req.post(uri.to_s, original_file.content, {
+          'Content-type' => "#{mime_type};charset=utf-8",
+          'Content-Length' => original_file.content.size.to_s
+        })
+      raise "URL '#{uri}' returned code #{resp.code}" unless resp.code == "200"
+      original_file.content.rewind if original_file.content.respond_to?(:rewind)
+      JSON.parse(resp.body)[''].rstrip
+    rescue => e
+      logger.error("Error extracting content from #{id}: #{e.inspect}")
+      return nil
+    end
+
+    def connection_url
+      case
+        when Blacklight.connection_config[:url] then Blacklight.connection_config[:url]
+        when Blacklight.connection_config["url"] then Blacklight.connection_config["url"]
+        when Blacklight.connection_config[:fulltext] then Blacklight.connection_config[:fulltext]["url"]
+        else Blacklight.connection_config[:default]["url"]
+      end
+    end
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/generic_file_audit_service.rb

@@ -0,0 +1,85 @@
+module CurationConcerns
+  class GenericFileAuditService
+    attr_reader :generic_file
+    def initialize(file)
+      @generic_file = file
+    end
+
+    NO_RUNS = 999
+
+    # provides a human readable version of the audit status
+    def human_readable_audit_status(stat)
+      case stat
+        when 0
+          'failing'
+        when 1
+          'passing'
+        else
+          stat
+      end
+    end
+
+    # Audits each version of each file if it hasn't been audited recently
+    # Returns the set of most recent audit status for each version of the content file
+    # @param [Hash] log container for messages, mapping file ids to status
+    def audit(log={})
+      generic_file.files.each { |f| log[f.id] = audit_file(f) }
+      log
+    end
+
+    private
+
+      # Retrieve or generate the audit check for a file (all versions are checked for versioned files)
+      # @param [ActiveFedora::File] file to audit
+      # @param [Array] log container for messages
+      def audit_file(file, log=[])
+        versions = file.has_versions? ? file.versions.all : file
+        versions.each { |v| log << audit_file_version(file.id, v.uri) }
+        log
+      end
+
+      # Retrieve or generate the audit check for a file and provide a human-readable status message.
+      # @param [ActiveFedora::File] file to audit
+      def audit_stat(file)
+        audit_results = audit_file(file).collect { |result| result["pass"] }
+        # check how many non runs we had
+        non_runs = audit_results.reduce(0) { |sum, value| value == NO_RUNS ? sum += 1 : sum }
+        if non_runs == 0
+          audit_results.reduce(true) { |sum, value| sum && value }
+        elsif non_runs < audit_results.length
+          result = audit_results.reduce(true) { |sum, value| value == NO_RUNS ? sum : sum && value }
+          "Some audits have not been run, but the ones run were #{human_readable_audit_status(result)}."
+        else
+          'Audits have not yet been run on this file.'
+        end
+      end
+
+      # Retrieve or generate the audit check for a specific version of a file
+      # @param [String] file_id used to find the file within its parent object (usually "original_file")
+      # @param [String] version_uri the version to be audited (or the file uri for non-versioned files)
+      def audit_file_version(file_id, version_uri)
+        latest_audit = ChecksumAuditLog.logs_for(generic_file.id, file_id).first
+        return latest_audit unless needs_audit?(latest_audit)
+        CurationConcerns.queue.push(AuditJob.new(generic_file.id, file_id, version_uri))
+        latest_audit || ChecksumAuditLog.new(pass: NO_RUNS, generic_file_id: generic_file.id, file_id: file_id, version: version_uri)
+      end
+
+      # Check if time since the last audit is greater than the maximum days allowed between audits
+      # @param [ChecksumAuditLog] latest_audit the most recent audit event
+      def needs_audit?(latest_audit)
+        return true unless latest_audit
+        unless latest_audit.updated_at
+          logger.warn "***AUDIT*** problem with audit log! Latest Audit is not nil, but updated_at is not set #{latest_audit}"
+          return true
+        end
+        days_since_last_audit(latest_audit) >= CurationConcerns.config.max_days_between_audits
+      end
+
+      # Return the number of days since the latest audit event
+      # @param [ChecksumAuditLog] latest_audit the most recent audit event
+      def days_since_last_audit(latest_audit)
+        (DateTime.now - latest_audit.updated_at.to_date).to_i
+      end
+
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/generic_file_indexing_service.rb

@@ -0,0 +1,14 @@
+module CurationConcerns
+  class GenericFileIndexingService < ActiveFedora::IndexingService
+    def generate_solr_document
+      super.tap do |solr_doc|
+        solr_doc[Solrizer.solr_name('label')] = object.label
+        solr_doc[Solrizer.solr_name('file_format')] = object.file_format
+        solr_doc[Solrizer.solr_name('file_format', :facetable)] = object.file_format
+        solr_doc[Solrizer.solr_name(:file_size, :symbol)] = object.file_size[0]
+        solr_doc['all_text_timv'] = object.full_text.content
+        solr_doc[Solrizer.solr_name('generic_work_ids', :symbol)] = object.generic_work_ids unless object.generic_work_ids.empty?
+      end
+    end
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/generic_work_indexing_service.rb

@@ -0,0 +1,16 @@
+module CurationConcerns
+  class GenericWorkIndexingService < ActiveFedora::IndexingService
+
+    def generate_solr_document
+      super.tap do |solr_doc|
+        # We know that all the members of GenericWorks are GenericFiles so we can use
+        # member_ids which requires fewer Fedora API calls than generic_file_ids.
+        # generic_file_ids requires loading all the members from Fedora but member_ids
+        # looks just at solr
+        solr_doc[Solrizer.solr_name('generic_file_ids', :symbol)] = object.member_ids
+        Solrizer.set_field(solr_doc, 'generic_type', 'Work', :facetable)
+      end
+    end
+
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/noid.rb

@@ -0,0 +1,23 @@
+require 'active_fedora/noid'
+
+module CurationConcerns
+  module Noid
+    extend ActiveSupport::Concern
+
+    ## This overrides the default behavior, which is to ask Fedora for an id
+    # @see ActiveFedora::Persistence.assign_id
+    def assign_id
+      service.mint if CurationConcerns.config.enable_noids
+    end
+
+    def to_param
+      id
+    end
+
+    private
+
+      def service
+        @service ||= ActiveFedora::Noid::Service.new
+      end
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/repository_audit_service.rb

@@ -0,0 +1,9 @@
+module CurationConcerns
+  class RepositoryAuditService
+    def self.audit_everything
+      ::GenericFile.find_each do |gf|
+        gf.audit
+      end
+    end
+  end
+end

data/curation_concerns-models/app/services/curation_concerns/versioning_service.rb

@@ -0,0 +1,27 @@
+module CurationConcerns
+  class VersioningService
+    # Make a version and record the version committer
+    # @param [ActiveFedora::File] content
+    # @param [User] user
+    def self.create(content, user=nil)
+      content.create_version
+      record_committer(content, user) if user
+    end
+
+    # @param [ActiveFedora::File] file
+    def self.latest_version_of(file)
+      file.versions.last
+    end
+
+    protected
+
+      # Record the version committer of the last version
+      # @param [ActiveFedora::File] content
+      # @param [User] user
+      def self.record_committer(content, user)
+        version = latest_version_of(content)
+        return if version.nil?
+        VersionCommitter.create(version_id: version.uri, committer_login: user.user_key)
+      end
+  end
+end

data/curation_concerns-models/config/locales/curation_concerns.en.yml

@@ -0,0 +1,6 @@
+en:
+  curation_concerns:
+    product_name:       "CurationConcerns"
+    product_twitter_handle: "@HydraSphere"
+    institution_name:   "Institution Name"
+    institution_name_full: "The Institution Name"

data/curation_concerns-models/curation_concerns-models.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+version = File.read(File.expand_path("../../VERSION", __FILE__)).strip
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "curation_concerns-models"
+  spec.version       = version
+  spec.authors       = ["Justin Coyne"]
+  spec.email         = ["justin@curationexperts.com"]
+  spec.summary       = %q{Simple institutional repository models for Hydra}
+  spec.description   = %q{An extensible repository data-model with works and and many attached files}
+  spec.homepage      = ""
+  spec.license       = "APACHE2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'active_attr'
+  spec.add_dependency 'nest', '~> 1.1'
+  spec.add_dependency 'hydra-collections', '~> 6.0.0.alpha'
+  spec.add_dependency 'hydra-head', '~> 9.2.2'
+  spec.add_dependency 'hydra-works', '~> 0.1'
+  spec.add_dependency 'active-fedora', '~> 9.3.0'
+  spec.add_dependency 'active_fedora-noid', '~> 1.0.2'
+  spec.add_dependency 'resque', '~> 1.23'
+  spec.add_dependency 'resque-pool', '~> 0.3'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+end

data/curation_concerns-models/lib/curation_concerns/messages.rb

@@ -0,0 +1,66 @@
+module CurationConcerns
+  module Messages
+    extend ActiveSupport::Concern
+
+    # Borrowed from AbstractController so we can render html content tags
+    attr_accessor :output_buffer
+    include ActionView::Helpers::TagHelper
+    include ActionView::Helpers::UrlHelper
+
+    def success_subject
+      I18n.t("curation_concerns.messages.success.subject")
+    end
+
+    def failure_subject
+      I18n.t("curation_concerns.messages.failure.subject")
+    end
+
+    def single_success id, file
+      content_tag :span, id: "ss-"+id do
+        [link_to_file(file), I18n.t("curation_concerns.messages.success.single")].join(" ").html_safe
+      end
+    end
+
+    def multiple_success id, files
+      content_tag :span, id: "ss-"+id do
+        [success_link(files), I18n.t("curation_concerns.messages.success.multiple.tag")].join(" ").html_safe
+      end
+    end
+
+    def single_failure id, file
+      content_tag :span, id: "ss-"+id do
+        [link_to_file(file), I18n.t("curation_concerns.messages.failure.single")].join(" ").html_safe
+      end
+    end
+
+    def multiple_failure id, files
+      content_tag :span, id: "ss-"+id do
+        [failure_link(files), I18n.t("curation_concerns.messages.failure.multiple.tag")].join(" ").html_safe
+      end
+    end
+
+    # Double-quotes are replaced with single ones so this list can be included in a data block. Ex:
+    #   <a href="#" data-content="<a href='#'>embedded link</a>" rel="popover">Click me</a>
+    def file_list files
+      files.map { |gf| link_to_file(gf) }.join(', ').gsub(/"/, "'")
+    end
+
+    def link_to_file file
+      link_to(file.to_s, Rails.application.class.routes.url_helpers.curation_concerns_generic_file_path(file))
+    end
+
+    private
+
+      def success_link files
+        link_to I18n.t("curation_concerns.messages.success.multiple.link"), "#",
+          rel: "popover",
+          data: { content: file_list(files).html_safe, title: I18n.t("curation_concerns.messages.success.title") }
+      end
+
+      def failure_link files
+        link_to I18n.t("curation_concerns.messages.failure.multiple.link"), "#",
+          rel: "popover",
+          data: { content: file_list(files).html_safe, title: I18n.t("curation_concerns.messages.failure.title") }
+      end
+  end
+end

data/curation_concerns-models/lib/curation_concerns/models/engine.rb

@@ -0,0 +1,61 @@
+module CurationConcerns
+  module Models
+
+    def self.config(&block)
+      @@config ||= Engine::Configuration.new
+      yield @@config if block
+      return @@config
+    end
+
+
+    class Engine < ::Rails::Engine
+
+      require 'curation_concerns/models/resque'
+
+      # Set some configuration defaults
+      config.persistent_hostpath = "http://localhost/files/"
+      config.enable_ffmpeg = false
+      config.ffmpeg_path = 'ffmpeg'
+      config.fits_message_length = 5
+      config.temp_file_base = nil
+      config.enable_noids = true
+      config.noid_template = '.reeddeeddk'
+      config.minter_statefile = '/tmp/minter-state'
+      config.redis_namespace = "curation_concerns"
+      config.fits_path = "fits.sh"
+      config.enable_local_ingest = nil
+      config.queue = CurationConcerns::Resque::Queue
+
+      # Defaulting analytic start date to whenever the file was uploaded by leaving it blank
+      config.analytic_start_date = nil
+
+      config.autoload_paths += %W(
+        #{config.root}/app/actors/concerns
+        #{config.root}/lib/curation_concerns
+        #{config.root}/app/models/datastreams
+      )
+
+      initializer 'requires' do
+        require 'active_fedora/noid'
+        require 'curation_concerns/noid'
+        require 'curation_concerns/permissions'
+      end
+
+      initializer 'configure' do
+        CurationConcerns.config.tap do |c|
+          Hydra::Derivatives.ffmpeg_path    = c.ffmpeg_path
+          Hydra::Derivatives.temp_file_base = c.temp_file_base
+          Hydra::Derivatives.fits_path      = c.fits_path
+          Hydra::Derivatives.enable_ffmpeg  = c.enable_ffmpeg
+
+          ActiveFedora::Base.translate_uri_to_id = ActiveFedora::Noid.config.translate_uri_to_id
+          ActiveFedora::Base.translate_id_to_uri = ActiveFedora::Noid.config.translate_id_to_uri
+          ActiveFedora::Noid.config.template = c.noid_template
+          ActiveFedora::Noid.config.statefile = c.minter_statefile
+        end
+      end
+
+    end
+
+  end
+end

data/curation_concerns-models/lib/curation_concerns/models/resque.rb

@@ -0,0 +1,36 @@
+# Borrowed from:
+# https://github.com/jeremy/resque-rails/blob/master/lib/resque/rails/queue.rb
+module CurationConcerns
+  module Resque
+    class Queue
+      attr_reader :default_queue_name
+
+      def initialize(default_queue_name)
+        @default_queue_name = default_queue_name
+      end
+
+      def push(job)
+        push_tries = 0
+        queue = job.respond_to?(:queue_name) ? job.queue_name : default_queue_name
+        begin
+          ::Resque.enqueue_to queue, MarshaledJob, Base64.encode64(Marshal.dump(job))
+        rescue Redis::CannotConnectError
+          ActiveFedora::Base.logger.error "Redis is down!"
+        rescue Redis::TimeoutError => error
+          ActiveFedora::Base.logger.warn "Redis Timed out.  Trying again! #{job.inspect}"
+          push_tries+=1
+          # fail for good if the tries is greater than 3
+          raise error if push_tries >=3
+          sleep 0.01
+          retry
+        end
+      end
+    end
+
+    class MarshaledJob
+      def self.perform(marshaled_job)
+        Marshal.load(Base64.decode64(marshaled_job)).run
+      end
+    end
+  end
+end

data/curation_concerns-models/lib/curation_concerns/models/utils.rb

@@ -0,0 +1,22 @@
+module CurationConcerns
+  module Utils
+    extend ActiveSupport::Concern
+
+    def retry_unless(number_of_tries, condition, &block)
+      self.class.retry_unless(number_of_tries, condition, &block)
+    end
+
+    module ClassMethods
+      def retry_unless(number_of_tries, condition, &block)
+        raise ArgumentError, "First argument must be an enumerator" unless number_of_tries.is_a? Enumerator
+        raise ArgumentError, "Second argument must be a lambda" unless condition.respond_to? :call
+        raise ArgumentError, "Must pass a block of code to retry" unless block_given?
+        number_of_tries.each do
+          result = block.call
+          return result unless condition.call
+        end
+        raise RuntimeError, "retry_unless could not complete successfully. Try upping the # of tries?"
+      end
+    end
+  end
+end

data/curation_concerns-models/lib/curation_concerns/models/version.rb

@@ -0,0 +1,5 @@
+module CurationConcerns
+  module Models
+    VERSION = "0.1.2"
+  end
+end

data/curation_concerns-models/lib/curation_concerns/models.rb

@@ -0,0 +1,32 @@
+require "curation_concerns/models/version"
+require "curation_concerns/models/engine"
+
+require 'hydra/head'
+require 'nest'
+# require "active_resource" # used by GenericFile to catch errors & by GeoNamesResource
+require 'resque/server'
+
+module CurationConcerns
+  extend ActiveSupport::Autoload
+
+  module Models
+  end
+
+  autoload :Utils, 'curation_concerns/models/utils'
+  autoload :Permissions
+  autoload :Messages
+
+  attr_writer :queue
+
+  def self.queue
+    @queue ||= config.queue.new('curation_concerns')
+  end
+
+  def self.config(&block)
+    @@config ||= CurationConcerns::Models::Engine::Configuration.new
+
+    yield @@config if block
+
+    return @@config
+  end
+end

data/curation_concerns-models/lib/generators/curation_concerns/models/abstract_migration_generator.rb

@@ -0,0 +1,30 @@
+# -*- encoding : utf-8 -*-
+require 'rails/generators'
+require 'rails/generators/migration'
+
+class CurationConcerns::Models::AbstractMigrationGenerator < Rails::Generators::Base
+  include Rails::Generators::Migration
+
+  # Implement the required interface for Rails::Generators::Migration.
+  # taken from http://github.com/rails/rails/blob/master/activerecord/lib/generators/active_record.rb
+  def self.next_migration_number(path)
+    if @prev_migration_nr
+      @prev_migration_nr += 1
+    else
+      if last_migration = Dir[File.join(path, '*.rb')].sort.last
+        @prev_migration_nr = last_migration.sub(File.join(path, '/'), '').to_i + 1
+      else
+        @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+      end
+    end
+    @prev_migration_nr.to_s
+  end
+
+  protected
+
+  def better_migration_template(file)
+    migration_template "migrations/#{file}", "db/migrate/#{file}"
+  rescue Rails::Generators::Error => e
+    say_status("warning", e.message, :yellow)
+  end
+end

data/curation_concerns-models/lib/generators/curation_concerns/models/clamav_generator.rb

@@ -0,0 +1,19 @@
+# -*- encoding : utf-8 -*-
+require 'rails/generators'
+
+class CurationConcerns::Models::ClamavGenerator < Rails::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+
+  desc """
+This generator makes the following changes to your application:
+ 1. Generates clamav initializer
+       """
+
+  def banner
+    say_status("info", "Generating clamav initializers", :blue)
+  end
+
+  def create_initializer_file
+    copy_file 'config/clamav.rb', 'config/initializers/clamav.rb'
+  end
+end

data/curation_concerns-models/lib/generators/curation_concerns/models/fulltext_generator.rb

@@ -0,0 +1,28 @@
+# -*- encoding : utf-8 -*-
+require 'rails/generators'
+
+class CurationConcerns::Models::FulltextGenerator < Rails::Generators::Base
+  source_root File.expand_path('../templates', __FILE__)
+
+  desc """
+This generator makes the following changes to your application:
+ 1. Copies solrconfig.xml into solr_conf/conf/
+ 2. Reconfigures jetty
+       """
+
+  def banner
+    say_status("warning", "GENERATING CURATION_CONCERNS FULL-TEXT", :yellow)
+  end
+
+  # Copy CurationConcerns's solrconfig into the dir from which the jetty:config task pulls
+  # CurationConcerns's solrconfig includes full-text extraction
+  def copy_solr_config
+    src = File.join(__FILE__, '..', '..', '..', '..', '..', '..', 'solr_conf', 'conf', 'solrconfig.xml')
+    copy_file File.expand_path(src), 'solr_conf/conf/solrconfig.xml', force: true
+  end
+
+  # Copy config, schema, and jars into jetty dir if it exists
+  def reconfigure_jetty
+    rake "curation_concerns:jetty:config" if File.directory?('jetty')
+  end
+end