ddr-models 2.0.1 → 2.1.0.rc1

data/lib/ddr/index/query_clause.rb

@@ -0,0 +1,52 @@
+module Ddr::Index
+  class QueryClause
+
+    PRESENT = "[* TO *]"
+    TERM = "{!term f=%s}%s"
+    BEFORE_DAYS = "[* TO NOW-%sDAYS]"
+
+    class << self
+      # Standard Solr query, no escaping applied
+      def build(field, value)
+        [field, value].join(":")
+      end
+
+      def unique_key(value)
+        term(UniqueKeyField.instance, value)
+      end
+      alias_method :id, :unique_key
+      alias_method :pid, :unique_key
+
+      def negative(field, value)
+        build "-#{field}", value
+      end
+
+      def present(field)
+        build field, PRESENT
+      end
+
+      def absent(field)
+        negative field, PRESENT
+      end
+
+      def or_values(field, *values)
+        build field, QueryValue.or_values(*values)
+      end
+
+      def before(field, date_time)
+        value = "[* TO %s]" % Ddr::Utils.solr_date(date_time)
+        build field, value
+      end
+
+      def before_days(field, days)
+        value = BEFORE_DAYS % days.to_i
+        build field, value
+      end
+
+      def term(field, value)
+        TERM % [field, value.gsub(/"/, '\"')]
+      end
+    end
+
+  end
+end

data/lib/ddr/index/query_result.rb

@@ -0,0 +1,70 @@
+module Ddr::Index
+  class QueryResult < AbstractQueryResult
+
+    PAGE_SIZE = 1000
+
+    delegate :csv, to: :query
+
+    def each(&block)
+      if params[:rows]
+        each_unpaginated(&block)
+      else
+        each_paginated(&block)
+      end
+    end
+
+    def each_unpaginated(&block)
+      conn.select(params).docs.each(&block)
+    end
+
+    def each_paginated(&block)
+      pages.each { |pg| pg.each(&block) }
+    end
+
+    def pids
+      Enumerator.new do |e|
+        each do |doc|
+          e << doc[Fields::PID]
+        end
+      end
+    end
+
+    def each_pid(&block)
+      pids.each(&block)
+    end
+
+    def docs
+      Enumerator.new do |e|
+        each do |doc|
+          e << DocumentBuilder.build(doc)
+        end
+      end
+    end
+
+    def all
+      to_a
+    end
+
+    def pages
+      num = 1
+      Enumerator.new do |e|
+        loop do
+          pg = page(num)
+          e << pg
+          unless pg.has_next?
+            break
+          end
+          num += 1
+        end
+      end
+    end
+
+    def page(num)
+      page_params = params.dup
+      page_size = page_params.delete(:rows) || PAGE_SIZE
+      response = conn.page num, page_size, "select", params: page_params
+      response.docs
+    end
+
+  end
+end

data/lib/ddr/index/query_value.rb

@@ -0,0 +1,16 @@
+module Ddr::Index
+  class QueryValue
+
+    class << self
+      def build(value)
+        RSolr.solr_escape(value)
+      end
+
+      def or_values(*values)
+        value = values.map { |v| build(v) }.join(" OR ")
+        "(#{value})"
+      end
+    end
+
+  end
+end

data/lib/ddr/index/response.rb

@@ -0,0 +1,13 @@
+module Ddr::Index
+  class Response < SimpleDelegator
+
+    def docs
+      self["response"]["docs"]
+    end
+
+    def num_found
+      self["response"]["numFound"].to_i
+    end
+
+  end
+end

data/lib/ddr/index/unique_key_field.rb

@@ -0,0 +1,12 @@
+require 'singleton'
+
+module Ddr::Index
+  class UniqueKeyField < Field
+    include Singleton
+
+    def initialize
+      super SOLR_DOCUMENT_ID
+    end
+
+  end
+end

data/lib/ddr/index_fields.rb

@@ -1,60 +1,14 @@
 module Ddr
   module IndexFields
 
-    def self.solr_name(*args)
-      ActiveFedora::SolrService.solr_name(*args)
-    end
+    warn "[DEPRECATION] `Ddr::IndexFields` is deprecated and will be removed in ddr-models 3.0." \
+         " Use `Ddr::Index::Fields` instead."
+
+    include Ddr::Index::Fields
 
-    ACCESS_ROLE                 = solr_name :access_role, :stored_sortable
-    ACTIVE_FEDORA_MODEL         = solr_name :active_fedora_model, :stored_sortable
-    ADMIN_SET                   = solr_name :admin_set, :stored_sortable
-    ADMIN_SET_FACET             = solr_name :admin_set_facet, :facetable
-    COLLECTION_FACET            = solr_name :collection_facet, :facetable
-    COLLECTION_URI              = solr_name :collection_uri, :symbol
-    CONTENT_CONTROL_GROUP       = solr_name :content_control_group, :searchable, type: :string
-    CONTENT_SIZE                = solr_name :content_size, :stored_sortable, type: :integer
-    CONTENT_SIZE_HUMAN          = solr_name :content_size_human, :symbol
-    CREATOR_FACET               = solr_name :creator_facet, :facetable
-    DATE_FACET                  = solr_name :date_facet, :facetable
-    DATE_SORT                   = solr_name :date_sort, :sortable
-    DEFAULT_LICENSE_DESCRIPTION = solr_name :default_license_description, type: :string
-    DEFAULT_LICENSE_TITLE       = solr_name :default_license_title, type: :string
-    DEFAULT_LICENSE_URL         = solr_name :default_license_url, type: :string
-    DISPLAY_FORMAT              = solr_name :display_format, :stored_sortable
-    EXTRACTED_TEXT              = solr_name :extracted_text, :searchable, type: :text
-    HAS_MODEL                   = solr_name :has_model, :symbol
-    IDENTIFIER_ALL              = solr_name :identifier_all, :symbol
-    INTERNAL_URI                = solr_name :internal_uri, :stored_sortable
-    IS_ATTACHED_TO              = solr_name :is_attached_to, :symbol
-    IS_EXTERNAL_TARGET_FOR      = solr_name :is_external_target_for, :symbol
-    IS_GOVERNED_BY              = solr_name :is_governed_by, :symbol
-    IS_MEMBER_OF                = solr_name :is_member_of, :symbol
-    IS_MEMBER_OF_COLLECTION     = solr_name :is_member_of_collection, :symbol
-    IS_PART_OF                  = solr_name :is_part_of, :symbol
-    LAST_FIXITY_CHECK_ON        = solr_name :last_fixity_check_on, :stored_sortable, type: :date
-    LAST_FIXITY_CHECK_OUTCOME   = solr_name :last_fixity_check_outcome, :symbol
-    LAST_VIRUS_CHECK_ON         = solr_name :last_virus_check_on, :stored_sortable, type: :date
-    LAST_VIRUS_CHECK_OUTCOME    = solr_name :last_virus_check_outcome, :symbol
-    LICENSE_DESCRIPTION         = solr_name :license_description, type: :string
-    LICENSE_TITLE               = solr_name :license_title, type: :string
-    LICENSE_URL                 = solr_name :license_url, type: :string
-    LOCAL_ID                    = solr_name :local_id, :stored_sortable
-    MEDIA_SUB_TYPE              = solr_name :content_media_sub_type, :facetable
-    MEDIA_MAJOR_TYPE            = solr_name :content_media_major_type, :facetable
-    MEDIA_TYPE                  = solr_name :content_media_type, :symbol
-    MULTIRES_IMAGE_FILE_PATH    = solr_name :multires_image_file_path, :stored_sortable
-    OBJECT_PROFILE              = solr_name :object_profile, :displayable
-    OBJECT_STATE                = solr_name :object_state, :stored_sortable
-    OBJECT_CREATE_DATE          = solr_name :system_create, :stored_sortable, type: :date
-    OBJECT_MODIFIED_DATE        = solr_name :system_modified, :stored_sortable, type: :date
-    PERMANENT_ID                = solr_name :permanent_id, :stored_sortable, type: :string
-    PERMANENT_URL               = solr_name :permanent_url, :stored_sortable, type: :string
-    POLICY_ROLE                 = solr_name :policy_role, :symbol
-    RESEARCH_HELP_CONTACT       = solr_name :research_help_contact, :stored_sortable
-    RESOURCE_ROLE               = solr_name :resource_role, :symbol
-    STRUCT_MAPS                 = solr_name :struct_maps, :stored_sortable
-    TITLE                       = solr_name :title, :stored_sortable
-    WORKFLOW_STATE              = solr_name :workflow_state, :stored_sortable
+    def self.get(name)
+      Ddr::Index::Fields.get(name)
+    end
 
   end
 end

data/lib/ddr/jobs.rb

@@ -3,7 +3,7 @@ module Ddr
     extend ActiveSupport::Autoload
 
     autoload :PermanentId
-    autoload :MigrateLegacyAuthorization
+    autoload :FitsFileCharacterization
 
     autoload_at 'ddr/jobs/permanent_id' do
       autoload :MakeUnavailable

data/lib/ddr/jobs/fits_file_characterization.rb

@@ -0,0 +1,51 @@
+require 'open3'
+
+module Ddr::Jobs
+  class FitsFileCharacterization
+
+    @queue = :file_characterization
+
+    EVENT_SUMMARY = 'FITS characterization of content file'.freeze
+
+    def self.perform(pid)
+      obj = ActiveFedora::Base.find(pid)
+      tmp_filename = Ddr::Utils::sanitize_filename(obj.original_filename) || obj.content.default_file_name
+      Dir.mktmpdir do |dir|
+        infile = create_temp_infile(dir, tmp_filename, obj.content.content)
+        fits_output, err, status = Open3.capture3(fits_command, '-i', infile)
+        if status.success? && fits_output.present?
+          obj.reload
+          obj.fits.content = fits_output
+          obj.save!
+        end
+        notify_event(pid, err, status)
+      end
+    end
+
+    def self.fits_command
+      File.join(Ddr::Models.fits_home, 'fits.sh')
+    end
+
+    def self.fits_version
+      `#{fits_command} -v`.strip
+    end
+
+    private
+
+    def self.create_temp_infile(dir, tmp_filename, content)
+      temp_infile = File.join(dir, tmp_filename)
+      File.open(temp_infile, 'wb') do |f|
+        f.write content
+      end
+      temp_infile
+    end
+
+    def self.notify_event(pid, err, status)
+      details = status.success? ? nil : err
+      event_args = { pid: pid, summary: EVENT_SUMMARY, software:  "fits #{fits_version}", detail: details }
+      event_args[:outcome] = status.success? ? Ddr::Events::Event::SUCCESS : Ddr::Events::Event::FAILURE
+      Ddr::Notifications.notify_event(:update, event_args)
+    end
+
+  end
+end

data/lib/ddr/managers.rb

@@ -5,6 +5,7 @@ module Ddr
     autoload :Manager
     autoload :DerivativesManager
     autoload :PermanentIdManager
+    autoload :TechnicalMetadataManager
     autoload :WorkflowManager
 
   end

data/lib/ddr/managers/technical_metadata_manager.rb

@@ -0,0 +1,104 @@
+module Ddr::Managers
+  class TechnicalMetadataManager < Manager
+
+    FITS_TIMESTAMP_FORMAT = "%D %l:%M %p" # Ex. 7/3/15 8:29 PM
+
+    delegate :content, :fits, to: :object
+
+    delegate :valid, :well_formed,
+             :media_type, :format_label, :format_version, :pronom_identifier,
+             :created, :modified, :creating_application, :extent,
+             :image_width, :image_height, :color_space,
+             to: :fits
+
+    alias_method :file_size, :extent
+    alias_method :last_modified, :modified
+
+    def fits?
+      !fits_version.nil?
+    end
+
+    def fits_version
+      fits.version.first
+    end
+
+    def fits_datetime
+      if fits_timestamp = fits.timestamp.first
+        no_zone = DateTime.strptime(fits_timestamp, FITS_TIMESTAMP_FORMAT)
+        no_zone.change(offset: no_zone.to_time.zone)
+      end
+    end
+
+    def file_size
+      extent.map(&:to_i)
+    end
+
+    def file_human_size
+      file_size.map do |fs|
+        "%s (%s bytes)" % [ ActiveSupport::NumberHelper.number_to_human_size(fs),
+                            ActiveSupport::NumberHelper.number_to_delimited(fs) ]
+      end
+    end
+
+    def checksum_digest
+      content.checksumType
+    end
+
+    def checksum_value
+      content.checksum
+    end
+
+    def invalid?
+      valid.any? { |v| v == "false" }
+    end
+
+    def valid?
+      !invalid?
+    end
+
+    def ill_formed?
+      well_formed.any? { |v| v == "false" }
+    end
+
+    def well_formed?
+      !ill_formed?
+    end
+
+    def creation_time
+      created.map { |datestr| coerce_to_time(datestr) }.compact
+    end
+
+    def modification_time
+      modified.map { |datestr| coerce_to_time(datestr) }.compact
+    end
+
+    def index_fields
+      { Ddr::Index::Fields::TECHMD_FITS_VERSION      => fits_version,
+        Ddr::Index::Fields::TECHMD_FITS_DATETIME     => Ddr::Utils.solr_date(fits_datetime),
+        Ddr::Index::Fields::TECHMD_CREATION_TIME     => Ddr::Utils.solr_dates(creation_time),
+        Ddr::Index::Fields::TECHMD_MODIFICATION_TIME => Ddr::Utils.solr_dates(modification_time),
+        Ddr::Index::Fields::TECHMD_COLOR_SPACE       => color_space,
+        Ddr::Index::Fields::TECHMD_CREATING_APPLICATION => creating_application,
+        Ddr::Index::Fields::TECHMD_FILE_SIZE         => file_size,
+        Ddr::Index::Fields::TECHMD_FORMAT_LABEL      => format_label,
+        Ddr::Index::Fields::TECHMD_FORMAT_VERSION    => format_version,
+        Ddr::Index::Fields::TECHMD_IMAGE_HEIGHT      => image_height,
+        Ddr::Index::Fields::TECHMD_IMAGE_WIDTH       => image_width,
+        Ddr::Index::Fields::TECHMD_MEDIA_TYPE        => media_type,
+        Ddr::Index::Fields::TECHMD_PRONOM_IDENTIFIER => pronom_identifier,
+        Ddr::Index::Fields::TECHMD_VALID             => valid,
+        Ddr::Index::Fields::TECHMD_WELL_FORMED       => well_formed,
+      }
+    end
+
+    private
+
+    def coerce_to_time(datestr)
+      datestr.sub! /\A(\d+):(\d+):(\d+)/, '\1-\2-\3'
+      DateTime.parse(datestr).to_time
+    rescue ArgumentError
+      nil
+    end
+
+  end
+end

data/lib/ddr/models.rb

@@ -9,32 +9,37 @@ require 'active_record'
 require 'hydra-core'
 require 'hydra/validations'
 
-require 'ddr/actions'
-require 'ddr/auth'
-require 'ddr/contacts'
-require 'ddr/datastreams'
-require 'ddr/derivatives'
-require 'ddr/events'
-require 'ddr/index_fields'
-require 'ddr/jobs'
-require 'ddr/managers'
-require 'ddr/metadata'
-require 'ddr/notifications'
-require 'ddr/utils'
-require 'ddr/vocab'
-
 module Ddr
+  extend ActiveSupport::Autoload
+
+  autoload :Actions
+  autoload :Auth
+  autoload :Contacts
+  autoload :Datastreams
+  autoload :Derivatives
+  autoload :Events
+  autoload :Index
+  autoload :IndexFields
+  autoload :Jobs
+  autoload :Managers
+  autoload :Metadata
+  autoload :Notifications
+  autoload :Utils
+  autoload :Vocab
+
   module Models
     extend ActiveSupport::Autoload
 
-    autoload :Base
     autoload :AccessControllable
-    autoload :Describable
-    autoload :EventLoggable
-    autoload :Error
+    autoload :Base
     autoload :ChecksumInvalid, 'ddr/models/error'
-    autoload :DerivativeGenerationFailure, 'ddr/models/error'
     autoload :ContentModelError, 'ddr/models/error'
+    autoload :DerivativeGenerationFailure, 'ddr/models/error'
+    autoload :Describable
+    autoload :EffectiveLicense
+    autoload :Error
+    autoload :EventLoggable
+    autoload :FileManagement
     autoload :FixityCheckable
     autoload :Governable
     autoload :HasAdminMetadata
@@ -42,15 +47,15 @@ module Ddr
     autoload :HasChildren
     autoload :HasContent
     autoload :HasMultiresImage
-    autoload :HasProperties
     autoload :HasStructMetadata
     autoload :HasThumbnail
     autoload :Indexing
-    autoload :FileManagement
-    autoload :Licensable
+    autoload :InheritedLicense
+    autoload :License
     autoload :SolrDocument
-    autoload :Structure
     autoload :StructDiv
+    autoload :Structure
+    autoload :YearFacet
 
     # Base directory of default external file store
     mattr_accessor :external_file_store
@@ -71,6 +76,17 @@ module Ddr
       "https://repository.lib.duke.edu/id/"
     end
 
+    # Home directory for FITS
+    mattr_accessor :fits_home
+
+    # Run file characterization or not?
+    mattr_accessor :characterize_files do
+      false
+    end
+    class << self
+      alias :characterize_files? :characterize_files
+    end
+
     # Yields an object with module configuration accessors
     def self.configure
       yield self