ddr-models 2.6.2 → 2.7.0.rc1

data/lib/ddr/events/update_event.rb

@@ -1,9 +1,16 @@
-module Ddr
-  module Events
-    class UpdateEvent < Event
-
-      self.description = "Object updated"
+module Ddr::Events
+  class UpdateEvent < Event
+    self.description = "Object updated"
 
+    def self.call(*args)
+      super do |event, notification|
+        attrs_changed = notification.payload[:attributes_changed]
+        ds_changed = notification.payload[:datastreams_changed]
+        event.detail = ["Attributes changed: #{attrs_changed}",
+                        "Datastreams changed: #{ds_changed.join(', ')}",
+                        event.detail,
+                       ].compact.join("\n\n")
+      end
     end
   end
 end

data/lib/ddr/index.rb

@@ -10,7 +10,6 @@ module Ddr
     autoload :FieldAttribute
     autoload :Fields
     autoload :Filter
-    autoload :LegacyLicenseFields
     autoload :Query
     autoload :QueryBuilder
     autoload :QueryClause

data/lib/ddr/index/csv_query_result.rb

@@ -26,6 +26,7 @@ module Ddr::Index
 
     def csv_opts
       { headers:        csv_headers,
+        converters:     [convert_semicolons, convert_escaped_newlines],
         return_headers: false,
         write_headers:  true,
       }
@@ -60,8 +61,15 @@ module Ddr::Index
     end
 
     def data
-      raw = Connection.get("select", params: solr_csv_params)
-      raw.gsub(/\\#{CSV_MV_SEPARATOR}/, CSV_MV_SEPARATOR)
+      Connection.get("select", params: solr_csv_params)
+    end
+
+    def convert_semicolons
+      lambda { |f| f.gsub(/\\#{CSV_MV_SEPARATOR}/, CSV_MV_SEPARATOR) rescue f }
+    end
+
+    def convert_escaped_newlines
+      lambda { |f| f.gsub(/\\r/, "\r").gsub(/\\n/, "\n") rescue f }
     end
 
   end

data/lib/ddr/index/fields.rb

@@ -29,6 +29,7 @@ module Ddr::Index
     CREATOR_FACET               = Field.new :creator_facet, :facetable
     DATE_FACET                  = Field.new :date_facet, :facetable
     DATE_SORT                   = Field.new :date_sort, :sortable
+    DC_IS_PART_OF               = Field.new :isPartOf, :symbol
     DEPOSITOR                   = Field.new :depositor, :stored_sortable
     DISPLAY_FORMAT              = Field.new :display_format, :stored_sortable
     DOI                         = Field.new :doi, :symbol
@@ -42,11 +43,14 @@ module Ddr::Index
     IDENTIFIER_ALL              = Field.new :identifier_all, :symbol
     ILLUSTRATED_FACET           = Field.new :illustrated_facet, :facetable
     ILLUSTRATOR_FACET           = Field.new :illustrator_facet, :facetable
+    INGESTED_BY                 = Field.new :ingested_by, :stored_sortable
+    INGESTION_DATE              = Field.new :ingestion_date, :stored_sortable, type: :date
     INSTRUMENTATION_FACET       = Field.new :instrumentation_facet, :facetable
     INTERNAL_URI                = Field.new :internal_uri, :stored_sortable
     INTERVIEWER_NAME_FACET      = Field.new :interviewer_name_facet, :facetable
     IS_ATTACHED_TO              = Field.new :is_attached_to, :symbol
     IS_EXTERNAL_TARGET_FOR      = Field.new :is_external_target_for, :symbol
+    IS_FORMAT_OF                = Field.new :isFormatOf, :symbol
     IS_GOVERNED_BY              = Field.new :is_governed_by, :symbol
     IS_LOCKED                   = Field.new :is_locked, :stored_sortable
     IS_MEMBER_OF                = Field.new :is_member_of, :symbol
@@ -80,11 +84,12 @@ module Ddr::Index
     PUBLISHER_FACET             = Field.new :publisher_facet, :facetable
     RESEARCH_HELP_CONTACT       = Field.new :research_help_contact, :stored_sortable
     RESOURCE_ROLE               = Field.new :resource_role, :symbol
+    RIGHTS_NOTE                 = Field.new :rights_note, :stored_searchable
     ROLL_NUMBER_FACET           = Field.new :roll_number_facet, :facetable
     SERIES_FACET                = Field.new :series_facet, :facetable
     SETTING_FACET               = Field.new :setting_facet, :facetable
     SPATIAL_FACET               = Field.new :spatial_facet, :facetable
-    STRUCT_MAPS                 = Field.new :struct_maps, :stored_sortable
+    STRUCTURE                   = Field.new :structure, :stored_sortable
     SUBJECT_FACET               = Field.new :subject_facet, :facetable
     SUBSERIES_FACET             = Field.new :subseries_facet, :facetable
     TECHMD_COLOR_SPACE          = Field.new :techmd_color_space, :symbol
@@ -99,6 +104,7 @@ module Ddr::Index
     TECHMD_ICC_PROFILE_VERSION  = Field.new :techmd_icc_profile_version, :symbol
     TECHMD_IMAGE_HEIGHT         = Field.new :techmd_image_height, :stored_searchable, type: :integer
     TECHMD_IMAGE_WIDTH          = Field.new :techmd_image_width, :stored_searchable, type: :integer
+    TECHMD_MD5                  = Field.new :techmd_md5, :stored_sortable
     TECHMD_MEDIA_TYPE           = Field.new :techmd_media_type, :symbol
     TECHMD_MODIFICATION_TIME    = Field.new :techmd_modification_time, :stored_searchable, type: :date
     TECHMD_PRONOM_IDENTIFIER    = Field.new :techmd_pronom_identifier, :symbol
@@ -131,15 +137,10 @@ module Ddr::Index
         Deprecation.warn(Ddr::Index::Fields,
                          "`Ddr::Index::Fields::#{name}` is deprecated." \
                          " Use `Ddr::Index::Fields::ID` instead.")
-        return ID
+        ID
+      else
+        super
       end
-      if const = LegacyLicenseFields.const_get(name)
-        # XXX Commented out b/c annoying, but maybe we want later ...
-        # Deprecation.warn(Ddr::Index::Fields,
-        #                  "`Ddr::Index::Fields::#{name}` is deprecated and will be removed in ddr-models 3.0.")
-        return const
-      end
-      super
     end
 
   end

data/lib/ddr/index/filter.rb

@@ -31,6 +31,7 @@ module Ddr::Index
         self.clauses += conditions.map do |field, v|
           Array(v).map { |value| QueryClause.negative(field, value) }
         end.flatten
+        self
       end
 
       def absent(field)

data/lib/ddr/managers/derivatives_manager.rb

@@ -1,124 +1,110 @@
 require "resque"
 
-module Ddr
-  module Managers
-    class DerivativesManager < Manager
+module Ddr::Managers
+  class DerivativesManager < Manager
+    extend Deprecation
 
-      SCHEDULE_LATER = :later
-      SCHEDULE_NOW = :now
-      SCHEDULES = [ SCHEDULE_LATER, SCHEDULE_NOW ]
+    SCHEDULE_LATER = :later
+    SCHEDULE_NOW = :now
+    SCHEDULES = [ SCHEDULE_LATER, SCHEDULE_NOW ]
 
-      ACTION_DELETE = "delete"
-      ACTION_GENERATE = "generate"
+    ACTION_DELETE = "delete"
+    ACTION_GENERATE = "generate"
 
-      def update_derivatives(schedule=SCHEDULE_LATER)
-        raise ArgumentError, "Must be one of #{SCHEDULES}" unless SCHEDULES.include?(schedule)
-        Ddr::Derivatives::DERIVATIVES.values.each do |derivative|
-          if Ddr::Derivatives.update_derivatives.include?(derivative.name)
-            # Need to update derivative if object has a datastream for this type of derivative and
-            # either (or both) of the following conditions are true:
-            # - object already has content in the derivative's datastream (need to delete or replace it)
-            # - the derivative can be generated for this object
-            if object.datastreams.include?(derivative.datastream) &&
-                  (object.datastreams[derivative.datastream].has_content? || generatable?(derivative))
-              schedule == SCHEDULE_NOW ? update_derivative(derivative) : Resque.enqueue(DerivativeJob, object.pid, derivative.name)
-            end
+    def update_derivatives(schedule=SCHEDULE_LATER)
+      raise ArgumentError, "Must be one of #{SCHEDULES}" unless SCHEDULES.include?(schedule)
+      Ddr::Derivatives::DERIVATIVES.values.each do |derivative|
+        if Ddr::Derivatives.update_derivatives.include?(derivative.name)
+          # Need to update derivative if object has a datastream for this type of derivative and
+          # either (or both) of the following conditions are true:
+          # - object already has content in the derivative's datastream (need to delete or replace it)
+          # - the derivative can be generated for this object
+          if object.datastreams.include?(derivative.datastream) &&
+             (object.datastreams[derivative.datastream].has_content? || generatable?(derivative))
+            schedule == SCHEDULE_NOW ? update_derivative(derivative) : Resque.enqueue(DerivativeJob, object.pid, derivative.name)
           end
         end
       end
+    end
 
-      def update_derivative(derivative)
-        raise ArgumentError, "This object does not have a datastream for #{derivative.name} derivatives" unless
-                                  object.datastreams.include?(derivative.datastream)
-        if generatable? derivative
-          generate_derivative derivative
-        else
-          # Delete existing derivative (if there is one) if that type of derivative is no longer
-          # applicable to the object
-          if object.datastreams[derivative.datastream].has_content?
-            delete_derivative derivative
-          end
+    def update_derivative(derivative)
+      raise ArgumentError, "This object does not have a datastream for #{derivative.name} derivatives" unless
+        object.datastreams.include?(derivative.datastream)
+      if generatable? derivative
+        generate_derivative derivative
+      else
+        # Delete existing derivative (if there is one) if that type of derivative is no longer
+        # applicable to the object
+        if object.datastreams[derivative.datastream].has_content?
+          delete_derivative derivative
         end
       end
+    end
 
-      def generate_derivative(derivative)
-        ActiveSupport::Notifications.instrument(Ddr::Notifications::UPDATE,
-                                                pid: object.pid,
-                                                summary: "Generate #{derivative.name.to_s} derivative"
-                                                ) do |payload|
-          generate_derivative! derivative
+    def generate_derivative!(derivative)
+      tempdir_path = File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname('',nil))
+      begin
+        tempdir = FileUtils.mkdir(tempdir_path).first
+        generator_source = create_source_file(tempdir)
+        generator_output = File.new(File.join(tempdir, "output.out"), 'wb')
+        exitstatus = derivative.generator.new(generator_source.path, generator_output.path, derivative.options).generate
+        generator_source.close unless generator_source.closed?
+        if exitstatus == 0
+          generator_output = File.open(generator_output, 'rb')
+          object.reload if object.persisted?
+          object.add_file generator_output, derivative.datastream, mime_type: derivative.generator.output_mime_type
+          object.save!
+        else
+          raise Ddr::Models::DerivativeGenerationFailure,
+                "Failure generating #{derivative.name} for #{object.pid}"
         end
+        generator_output.close unless generator_output.closed?
+      ensure
+        FileUtils.remove_dir(tempdir_path) if File.exist?(tempdir_path)
       end
+    end
 
-      def generate_derivative!(derivative)
-        tempdir_path = File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname('',nil))
-        begin
-          tempdir = FileUtils.mkdir(tempdir_path).first
-          generator_source = create_source_file(tempdir)
-          generator_output = File.new(File.join(tempdir, "output.out"), 'wb')
-          exitstatus = derivative.generator.new(generator_source.path, generator_output.path, derivative.options).generate
-          generator_source.close unless generator_source.closed?
-          if exitstatus == 0
-            generator_output = File.open(generator_output, 'rb')
-            object.reload if object.persisted?
-            object.add_file generator_output, derivative.datastream, mime_type: derivative.generator.output_mime_type
-            object.save!
-          else
-            raise Ddr::Models::DerivativeGenerationFailure,
-                    "Failure generating #{derivative.name} for #{object.pid}"
-          end
-          generator_output.close unless generator_output.closed?
-        ensure
-          FileUtils.remove_dir(tempdir_path) if File.exist?(tempdir_path)
-        end
-      end
+    alias_method :generate_derivative, :generate_derivative!
 
-      def delete_derivative(derivative)
-        ActiveSupport::Notifications.instrument(Ddr::Notifications::UPDATE,
-                                                pid: object.pid,
-                                                summary: "Delete derivative #{derivative.name.to_s}"
-                                                ) do |payload|
-          delete_derivative! derivative
-        end
-      end
+    def delete_derivative!(derivative)
+      File.unlink *object.datastreams[derivative.datastream].file_paths if
+        object.datastreams[derivative.datastream].external?
+      object.datastreams[derivative.datastream].delete
+      object.save!
+    end
 
-      def delete_derivative!(derivative)
-        File.unlink *object.datastreams[derivative.datastream].file_paths if
-                                                      object.datastreams[derivative.datastream].external?
-        object.datastreams[derivative.datastream].delete
-        object.save!
-      end
+    alias_method :delete_derivative, :delete_derivative!
 
-      class DerivativeJob
-        @queue = :derivatives
-        def self.perform(pid, derivative_name)
-          object = ActiveFedora::Base.find(pid)
-          derivative = Ddr::Derivatives::DERIVATIVES[derivative_name.to_sym]
-          object.derivatives.update_derivative(derivative)
-        end
+    class DerivativeJob
+      @queue = :derivatives
+      def self.perform(pid, derivative_name)
+        object = ActiveFedora::Base.find(pid)
+        derivative = Ddr::Derivatives::DERIVATIVES[derivative_name.to_sym]
+        object.derivatives.update_derivative(derivative)
       end
+    end
 
-      private
+    private
 
-      def create_source_file(dir)
-        generator_source = File.new(File.join(dir, "source"), "wb")
-        generator_source.write(object.content.content)
-        generator_source.close
-        generator_source
-      end
+    def create_source_file(dir)
+      generator_source = File.new(File.join(dir, "source"), "wb")
+      source_content = object.has_intermediate_file? ? object.intermediateFile.content : object.content.content
+      generator_source.write(source_content)
+      generator_source.close
+      generator_source
+    end
 
-      def generatable?(derivative)
-        return false unless object.has_content?
-        case derivative.name
-        when :multires_image
-          object.content_type == "image/tiff" || object.content_type == "image/jpeg"
-        when :thumbnail
-          object.image?
-        else
-          false
-        end
+    def generatable?(derivative)
+      return false unless object.has_content?
+      case derivative.name
+      when :multires_image
+        object.content_type == "image/tiff" || object.content_type == "image/jpeg"
+      when :thumbnail
+        object.image?
+      else
+        false
       end
-
     end
+
   end
 end

data/lib/ddr/managers/technical_metadata_manager.rb

@@ -5,11 +5,21 @@ module Ddr::Managers
 
     delegate :content, :fits, to: :object
 
-    delegate :valid, :well_formed,
-             :media_type, :format_label, :format_version, :pronom_identifier,
-             :created, :modified, :creating_application, :extent,
-             :image_width, :image_height, :color_space,
-             :icc_profile_name, :icc_profile_version,
+    delegate :color_space,
+             :created,
+             :creating_application,
+             :extent,
+             :format_label,
+             :format_version,
+             :icc_profile_name,
+             :icc_profile_version,
+             :image_height,
+             :image_width,
+             :media_type,
+             :modified,
+             :pronom_identifier,
+             :valid,
+             :well_formed,
              to: :fits
 
     alias_method :last_modified, :modified
@@ -40,6 +50,10 @@ module Ddr::Managers
       end
     end
 
+    def md5
+      fits.md5.first
+    end
+
     def checksum_digest
       content.checksumType
     end
@@ -86,6 +100,7 @@ module Ddr::Managers
         Ddr::Index::Fields::TECHMD_ICC_PROFILE_VERSION => icc_profile_version,
         Ddr::Index::Fields::TECHMD_IMAGE_HEIGHT      => image_height,
         Ddr::Index::Fields::TECHMD_IMAGE_WIDTH       => image_width,
+        Ddr::Index::Fields::TECHMD_MD5               => md5,
         Ddr::Index::Fields::TECHMD_MEDIA_TYPE        => media_type,
         Ddr::Index::Fields::TECHMD_PRONOM_IDENTIFIER => pronom_identifier,
         Ddr::Index::Fields::TECHMD_VALID             => valid,

data/lib/ddr/models.rb

@@ -1,11 +1,7 @@
 require 'ddr/models/engine'
 require 'ddr/models/version'
-
-# Awful hack to make Hydra::AccessControls::Permissions accessible
-$: << Gem.loaded_specs['hydra-access-controls'].full_gem_path + "/app/models/concerns"
-
+require 'action_view' # https://github.com/haml/haml/issues/695
 require 'active_record'
-
 require 'hydra-core'
 require 'hydra/validations'
 
@@ -19,7 +15,6 @@ module Ddr
   autoload :Derivatives
   autoload :Events
   autoload :Index
-  autoload :Jobs
   autoload :Managers
   autoload :Metadata
   autoload :Notifications
@@ -39,9 +34,9 @@ module Ddr
   module Models
     extend ActiveSupport::Autoload
 
-    autoload :AccessControllable
     autoload :AdminSet
     autoload :Base
+    autoload :Cache
     autoload :ChecksumInvalid, 'ddr/models/error'
     autoload :Contact
     autoload :ContentModelError, 'ddr/models/error'
@@ -58,6 +53,7 @@ module Ddr
     autoload :HasAttachments
     autoload :HasChildren
     autoload :HasContent
+    autoload :HasIntermediateFile
     autoload :HasMultiresImage
     autoload :HasStructMetadata
     autoload :HasThumbnail
@@ -65,7 +61,6 @@ module Ddr
     autoload :NotFoundError, 'ddr/models/error'
     autoload :PermanentId
     autoload :SolrDocument
-    autoload :StructDiv
     autoload :Structure
     autoload :WithContentFile
     autoload :YearFacet
@@ -78,14 +73,20 @@ module Ddr
       autoload :ParentLicense
     end
 
-    # Base directory of default external file store
-    mattr_accessor :external_file_store
-
-    # Base directory of external file store for multires image derivatives
-    mattr_accessor :multires_image_external_file_store
-
-    # Regexp for building external file subpath from hex digest
-    mattr_accessor :external_file_subpath_regexp
+    module Structures
+      extend ActiveSupport::Autoload
+
+      autoload :Agent
+      autoload :Div
+      autoload :File
+      autoload :FileGrp
+      autoload :FileSec
+      autoload :FLocat
+      autoload :Fptr
+      autoload :MetsHdr
+      autoload :Mptr
+      autoload :StructMap
+    end
 
     # Image server URL
     mattr_accessor :image_server_url
@@ -128,19 +129,18 @@ module Ddr
       false
     end
 
+    # File path to vips
+    mattr_accessor :vips_path
+
+    mattr_accessor :default_mime_type do
+      "application/octet-stream"
+    end
+
     # Yields an object with module configuration accessors
     def self.configure
       yield self
     end
 
-    def self.external_file_subpath_pattern= (pattern)
-      unless /^-{1,2}(\/-{1,2}){0,3}$/ =~ pattern
-        raise "Invalid external file subpath pattern: #{pattern}"
-      end
-      re = pattern.split("/").map { |x| "(\\h{#{x.length}})" }.join("")
-      self.external_file_subpath_regexp = Regexp.new("^#{re}")
-    end
-
   end
 end