ddr-models 2.11.0 → 3.0.0.alpha.1

data/lib/ddr/datastreams/fits_datastream.rb

@@ -22,15 +22,10 @@ module Ddr::Datastreams
         }
       }
       t.fileinfo {
-        t.created
+        t.size
         t.creatingApplicationName
-        t.creatingos
-        t.filename
-        t.filepath
-        t.fslastmodified
+        t.created
         t.lastmodified
-        t.md5checksum
-        t.size
       }
       t.filestatus {
         t.valid
@@ -41,8 +36,6 @@ module Ddr::Datastreams
           t.imageWidth
           t.imageHeight
           t.colorSpace
-          t.iccProfileName
-          t.iccProfileVersion
         }
         t.document {
           # TODO - configure to get from Tika?
@@ -54,29 +47,22 @@ module Ddr::Datastreams
       }
 
       ## proxy terms
-
       # identification / identity
+      t.media_type           proxy: [:identification, :identity, :mimetype]
       t.format_label         proxy: [:identification, :identity, :format_label]
       t.format_version       proxy: [:identification, :identity, :version]
-      t.media_type           proxy: [:identification, :identity, :mimetype]
       t.pronom_identifier    proxy: [:identification, :identity, :pronom_identifier]
-
       # filestatus
       t.valid                proxy: [:filestatus, :valid]
       t.well_formed          proxy: [:filestatus, :well_formed]
-
       # fileinfo
       t.created              proxy: [:fileinfo, :created]
       t.creating_application proxy: [:fileinfo, :creatingApplicationName]
       t.extent               proxy: [:fileinfo, :size]
-      t.md5                  proxy: [:fileinfo, :md5checksum]
-
       # image metadata
-      t.color_space          proxy: [:metadata, :image, :colorSpace]
-      t.icc_profile_name     proxy: [:metadata, :image, :iccProfileName]
-      t.icc_profile_version  proxy: [:metadata, :image, :iccProfileVersion]
-      t.image_height         proxy: [:metadata, :image, :imageHeight]
       t.image_width          proxy: [:metadata, :image, :imageWidth]
+      t.image_height         proxy: [:metadata, :image, :imageHeight]
+      t.color_space          proxy: [:metadata, :image, :colorSpace]
     end
 
     def self.xml_template
@@ -88,9 +74,10 @@ module Ddr::Datastreams
       builder.doc
     end
 
-    def prefix
-      "fits__"
-    end
+    # TODO fix or remove
+    # def prefix
+    #   "fits__"
+    # end
 
     def modified
       ng_xml

data/lib/ddr/datastreams/metadata_datastream.rb

@@ -2,26 +2,25 @@ module Ddr
   module Datastreams
     class MetadataDatastream < ActiveFedora::NtriplesRDFDatastream
 
-      # ASCII control chars, except newline
-      ILLEGAL_CHARS = Regexp.new('[\x00-\x09\x0B-\x1F]')
-
       def self.term_names
         properties.keys.map(&:to_sym).sort
       end
 
       # Returns ActiveTriplesTerm now that this is an RDF datastream
-      def values(term)
+      def values term
         term == :format ? self.format : self.send(term)
       end
 
       # Update term with values
       # Note that empty values (nil or "") are rejected from values array
-      def set_values(term, values)
-        vals = Array(values)
-               .map    { |v| sanitize_value(v) }
-               .reject { |v| reject_value?(v)  }
+      def set_values term, values
+        if values.respond_to?(:reject!)
+          values.reject! { |v| v.blank? }
+        else
+          values = nil if values.blank?
+        end
         begin
-          self.send("#{term}=", vals)
+          self.send("#{term}=", values)
         rescue NoMethodError
           raise ArgumentError, "No such term: #{term}"
         end
@@ -29,28 +28,15 @@ module Ddr
 
       # Add value to term
       # Note that empty value (nil or "") is not added
-      def add_value(term, value)
-        vals = values(term).to_a << value
-        set_values(term, vals)
-      end
-
-      def content_changed?
-        # Patches a bug in AF RDF datastreams where
-        # Content appears to change from nil to empty string
-        super && !empty?
-      end
-
-      private
-
-      def sanitize_value(value)
-        value
-          .to_s
-          .strip
-          .gsub(ILLEGAL_CHARS, "")
-      end
-
-      def reject_value?(value)
-        value.blank?
+      def add_value term, value
+        begin
+          unless value.blank?
+            values = values(term).to_a << value
+            set_values term, values
+          end
+        rescue NoMethodError
+          raise ArgumentError, "No such term: #{term}"
+        end
       end
 
     end

data/lib/ddr/derivatives.rb

@@ -2,12 +2,15 @@ module Ddr
   module Derivatives
     extend ActiveSupport::Autoload
 
-    autoload :Generator
-    autoload :PngGenerator
-    autoload :PtifGenerator
-    autoload :UpdateDerivatives
+    autoload :Derivative
+    autoload :MultiresImage
+    autoload :Thumbnail
 
-    Derivative = Struct.new(:name, :datastream, :generator, :options)
+    autoload_under 'generators' do
+      autoload :Generator
+      autoload :PngGenerator
+      autoload :PtifGenerator
+    end
 
     # Derivatives to generate.  Array of DERIVATIVE names
     mattr_accessor :update_derivatives
@@ -15,17 +18,11 @@ module Ddr
     # Eventually, we should inject the generator (probably) and the options (certainly) for each derivative
     # (e.g., from configuration)
     DERIVATIVES = {
-                    multires_image:
-                        Derivative.new( :multires_image,
-                                        Ddr::Datastreams::MULTIRES_IMAGE,
-                                        Ddr::Derivatives::PtifGenerator,
-                                        "jpeg:90,tile:256x256,pyramid"),
-                    thumbnail:
-                        Derivative.new( :thumbnail,
-                                        Ddr::Datastreams::THUMBNAIL,
-                                        Ddr::Derivatives::PngGenerator,
-                                        "-resize '100x100>'")
-                  }
+        multires_image: Ddr::Derivatives::MultiresImage.new(
+            Ddr::Derivatives::PtifGenerator.new("jpeg:90,tile:256x256,pyramid")),
+        thumbnail: Ddr::Derivatives::Thumbnail.new(
+            Ddr::Derivatives::PngGenerator.new("-resize '100x100>'"))
+    }
 
     # Yields an object with module configuration accessors
     def self.configure

data/lib/ddr/derivatives/derivative.rb

@@ -0,0 +1,68 @@
+module Ddr::Derivatives
+  class Derivative
+
+    attr_reader :generator
+    attr_accessor :workdir
+
+    def initialize(generator)
+      @generator = generator
+    end
+
+    def generate!(object)
+      self.workdir = FileUtils.mkdir(File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname('',nil))).first
+      source_path = create_source_file(object)
+      results = generator.generate(source_path, output_path(object))
+      if results.status.success?
+        store(object, results.output_path)
+      else
+        Rails.logger.error results.stderr
+        raise Ddr::Models::DerivativeGenerationFailure,
+                "Failure generating #{self.class.name} for #{object.id}: #{results.stderr}"
+      end
+      FileUtils.remove_dir(workdir)
+    end
+
+    # Whether a derivative can be generated for the object.
+    # Implemented in each subclass.
+    def self.generatable?(object)
+      raise NotImplementedError
+    end
+
+    # Whether the object has this derivative.
+    # Implemented in each subclass.
+    def self.has_derivative?(object)
+      raise NotImplementedError
+    end
+
+    # Delete the existing derivative
+    # Implemented in each subclass.
+    def delete!(object)
+      raise NotImplementedError
+    end
+
+    protected
+
+    # The path to which the generated output is written.
+    # Implemented in each subclass.
+    def output_path(object)
+      raise NotImplementedError
+    end
+
+    # The mechanism by which the generated derivative is registered with the ActiveFedora object.
+    # Implemented in each subclass.
+    def store(object, output_path)
+      raise NotImplementedError
+    end
+
+    private
+
+    def create_source_file(object)
+      file_path = File.join(workdir, "source")
+      file = File.new(file_path, "wb")
+      file.write(object.content.content)
+      file.close
+      file_path
+    end
+
+  end
+end

data/lib/ddr/derivatives/{generator.rb → generators/generator.rb}

@@ -3,13 +3,11 @@ module Ddr
     # @abstract
     class Generator
 
-      attr_reader :source, :output, :options
+      attr_reader :options
 
-      def initialize source, output, options=nil
-        raise ArgumentError, "Source must be a File or path to a file" unless Ddr::Utils.file_or_path?(source)
-        raise ArgumentError, "Output must be a File or path to a file" unless Ddr::Utils.file_or_path?(output)
-        @source = source
-        @output = output
+      GeneratorResult = Struct.new(:output_path, :stdout, :stderr, :status)
+
+      def initialize(options=nil)
         @options = options
       end
 
@@ -19,9 +17,15 @@ module Ddr
         raise NotImplementedError
       end
 
+      # The extension to use for the output generated.
+      # Implemented in each subclass
+      def self.output_extension
+        raise NotImplementedError
+      end
+
       # The actions required to generate the output from the source.
       # Implemented in each subclass.
-      def generate
+      def generate(source_path, output_path)
         raise NotImplementedError
       end
     end

data/lib/ddr/derivatives/generators/png_generator.rb

@@ -0,0 +1,23 @@
+require 'open3'
+
+module Ddr
+  module Derivatives
+    class PngGenerator < Generator
+
+      def self.output_mime_type
+        "image/png"
+      end
+
+      def self.output_extension
+        "png"
+      end
+
+      def generate(source_path, output_path)
+        command = "convert #{source_path} #{options} png:#{output_path}"
+        out, err, s = Open3.capture3(command)
+        GeneratorResult.new(output_path, out, err, s)
+      end
+
+    end
+  end
+end

data/lib/ddr/derivatives/generators/ptif_generator.rb

@@ -0,0 +1,56 @@
+require 'open3'
+
+module Ddr
+  module Derivatives
+    class PtifGenerator < Generator
+
+      def self.output_mime_type
+        "image/tiff"
+      end
+
+      def self.output_extension
+        "ptif"
+      end
+
+      def generate(source_path, output_path)
+        case source_depth(source_path).strip
+        when '16'
+          Dir.mktmpdir do |dir|
+            temp_8_bit_path = make_8_bit(dir, source_path)
+            run_generator(temp_8_bit_path, output_path)
+          end
+        when '8'
+          run_generator(source_path, output_path)
+        else
+          raise Ddr::Models::DerivativeGenerationFailure,
+                  "Unexpected depth -- #{source_depth} -- for source file #{Ddr::Utils.file_path(source_path)}"
+        end
+      end
+
+      private
+
+      def run_generator(source_to_use, output_path)
+        command = "vips im_vips2tiff #{source_to_use} #{output_path}:#{options}"
+        out, err, s = Open3.capture3(command)
+        GeneratorResult.new(output_path, out, err, s)
+      end
+
+      def make_8_bit(tempdir, source_path)
+        temp_8_bit_path = File.join(tempdir, "temp_8_bit.v")
+        command = "vips im_msb #{source_path} #{temp_8_bit_path}"
+        out, err, s = Open3.capture3(command)
+        if s.success?
+          return temp_8_bit_path
+        else
+          raise Ddr::Models::DerivativeGenerationFailure,
+                  "Error converting #{source_path} to 8-bit: #{err}"
+        end
+      end
+
+      def source_depth(source_path)
+        `identify -format '%[depth]' #{source_path}`
+      end
+
+    end
+  end
+end

data/lib/ddr/derivatives/multires_image.rb

@@ -0,0 +1,59 @@
+module Ddr::Derivatives
+  class MultiresImage < Derivative
+
+    def self.generatable?(object)
+      object.can_have_multires_image? && object.has_content? && object.content_type == "image/tiff"
+    end
+
+    def self.has_derivative?(object)
+      object.has_multires_image?
+    end
+
+    def delete!(object)
+      object.multires_image_file_path = nil
+      object.save
+    end
+
+    protected
+
+    def output_path(object)
+      File.join(create_external_file_path!, output_file_name(object))
+    end
+
+    def output_file_name(object)
+      basename = object.content.original_name.present? ? File.basename(object.content.original_name, '.*') : "multires_image"
+      "#{basename}.#{generator.class.output_extension}"
+    end
+
+    def store(object, output_path)
+      object.multires_image_file_path = output_path
+      object.save!
+    end
+
+    private
+
+    def create_external_file_path!
+      file_path = generate_external_file_path
+      FileUtils.mkdir_p(file_path)
+      file_path
+    end
+
+    #
+    # Generates a new external file storage location
+    #
+    # => {external_file_store}/1/e/69/1e691815-0631-4f9b-8e23-2dfb2eec9c70
+    #
+    def generate_external_file_path
+      File.join(Ddr::Models.multires_image_external_file_store, generate_external_directory_subpath)
+    end
+
+    def generate_external_directory_subpath
+      subdir = SecureRandom.uuid
+      m = Ddr::Models.external_file_subpath_regexp.match(subdir)
+      raise "File name does not match external file subpath pattern: #{file_name}" unless m
+      subpath_segments = m.to_a[1..-1]
+      File.join *subpath_segments, subdir
+    end
+
+  end
+end

data/lib/ddr/derivatives/thumbnail.rb

@@ -0,0 +1,36 @@
+module Ddr::Derivatives
+  class Thumbnail < Derivative
+
+    def self.generatable?(object)
+      object.can_have_thumbnail? && object.has_content? && object.image?
+    end
+
+    def self.has_derivative?(object)
+      object.has_thumbnail?
+    end
+
+    def delete!(object)
+      object.thumbnail.content = ''
+      object.save
+    end
+
+    protected
+
+    def output_path(object)
+      File.join(workdir, output_file_name(object))
+    end
+
+    def output_file_name(object)
+      basename = object.content.original_name.present? ? File.basename(object.content.original_name, '.*') : "thumbnail"
+      "#{basename}.#{generator.class.output_extension}"
+    end
+
+    def store(object, output_path)
+      output_file = File.open(output_path, 'rb')
+      object.reload if object.persisted?
+      object.add_file output_file, path: Ddr::Datastreams::THUMBNAIL, mime_type: generator.class.output_mime_type
+      object.save!
+    end
+
+  end
+end