hydra-derivatives 3.1.3 → 3.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e78f564d56a0e670ce4366f69e9a2b5ddbdca96e
-  data.tar.gz: 35205d1fcf5de953eee1d0a7e065dcf766f24dbf
+  metadata.gz: f9fe68ff09c49cf6061b2ef193ad428302dfac25
+  data.tar.gz: 1c0d21de93c9616a4d97cf4b8f6c46f68a66d96e
 SHA512:
-  metadata.gz: 7843da96087847b8215e3bcfc3840596cd94cc4d96cc1e7bd798d13a12ea21ae2c797c8a2e0a0e94127b4e5ea3abeb07dbbac51e7731b4768d85043a56599ecb
-  data.tar.gz: c2e424d665c7f5b3a3feabc82d1b4c8970280cc5f6a8e9dfcc2d5d42493aac176ea3ae3d39e4d340ff15bf2555e4d62b0f17bc6ac0897765adab4fa3c5f12f04
+  metadata.gz: 62bc8d5e8265a6f85214a530af6a6d36469473db17485b36a1cd622822ae591208ea5fa994ff2b23b4c1f5ab31d20131deb98061348941c47599be0ca403a2ee
+  data.tar.gz: d74669b4b6ec7c9b713ee33b72f35e7d43f4acfbc44a675ef162b6e8194c2df0363f17d9da958441209587118cdaa34b2d7c35688752a489acc6d563a1b76994

data/.rubocop.yml

@@ -0,0 +1,66 @@
+require: rubocop-rspec
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  DisplayCopNames: true
+  Include:
+    - '**/Rakefile'
+  Exclude:
+    - 'vendor/**/*'
+    - 'spec/internal/bin/*'
+    - 'spec/internal/db/schema.rb'
+
+Metrics/LineLength:
+  Enabled: false
+
+Style/CollectionMethods:
+  PreferredMethods:
+    collect: 'map'
+    collect!: 'map!'
+    inject: 'reduce'
+    detect: 'find'
+    find_all: 'select'
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/SignalException:
+  Enabled: false
+
+Style/IndentationConsistency:
+  EnforcedStyle: rails
+
+Style/PredicateName:
+  Exclude:
+    - spec/services/tempfile_service_spec.rb
+
+RSpec/ExampleWording:
+  CustomTransform:
+    be: is
+    have: has
+    not: does not
+    NOT: does NOT
+  IgnoredWords:
+    - only
+
+RSpec/FilePath:
+  Enabled: false
+
+RSpec/InstanceVariable:
+  Enabled: false
+
+RSpec/DescribeClass:
+  Exclude:
+    - spec/units/config_spec.rb
+    - spec/units/transcoding_spec.rb
+
+RSpec/AnyInstance:
+  Exclude:
+    - spec/processors/image_spec.rb
+    - spec/units/transcoding_spec.rb

data/.rubocop_todo.yml

@@ -0,0 +1,29 @@
+# List of files that we ought to fix based on what Rubocop is complaining about them
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - spec/units/transcoding_spec.rb
+
+Metrics/MethodLength:
+  Exclude:
+    - lib/hydra/derivatives/config.rb
+    - lib/hydra/derivatives/processors/document.rb
+    - lib/hydra/derivatives/processors/jpeg2k_image.rb
+    - lib/hydra/derivatives/processors/shell_based_processor.rb
+    - lib/hydra/derivatives/processors/video/processor.rb
+    - lib/hydra/derivatives/services/tempfile_service.rb
+    - spec/units/transcoding_spec.rb
+
+Metrics/ClassLength:
+  Exclude:
+    - lib/hydra/derivatives/processors/jpeg2k_image.rb
+
+Metrics/AbcSize:
+  Exclude:
+    - lib/hydra/derivatives/processors/document.rb
+    - lib/hydra/derivatives/processors/full_text.rb
+    - lib/hydra/derivatives/processors/jpeg2k_image.rb
+    - lib/hydra/derivatives/processors/shell_based_processor.rb
+    - lib/hydra/derivatives/services/persist_basic_contained_output_file_service.rb
+    - lib/hydra/derivatives/services/tempfile_service.rb
+    - spec/units/transcoding_spec.rb

data/Gemfile

@@ -5,4 +5,6 @@ gemspec
 
 group :development, :test do
   gem 'byebug' unless ENV['TRAVIS']
+  gem 'rubocop', '~> 0.37.2', require: false
+  gem 'rubocop-rspec', require: false
 end

data/Rakefile

@@ -10,11 +10,26 @@ RSpec::Core::RakeTask.new(:spec)
 require 'solr_wrapper/rake_task'
 require 'fcrepo_wrapper'
 require 'active_fedora/rake_support'
+require 'rubocop/rake_task'
 
-desc 'Start Fedora and Solr and run specs'
-task :ci do
-  with_test_server do
-    Rake::Task['spec'].invoke
+namespace :derivatives do
+  desc 'Run style checker'
+  RuboCop::RakeTask.new(:rubocop) do |task|
+    task.requires << 'rubocop-rspec'
+    task.fail_on_error = true
+  end
+
+  RSpec::Core::RakeTask.new(:rspec)
+
+  desc 'Start up Solr & Fedora and run tests'
+  task :spec do
+    with_test_server do
+      Rake::Task['derivatives:rspec'].invoke
+    end
   end
 end
+
+desc 'Run continuous integration build'
+task ci: ['derivatives:rubocop', 'derivatives:spec']
+
 task default: :ci

data/VERSION

@@ -1 +1 @@
-3.1.3
+3.1.4

data/lib/hydra/derivatives.rb

@@ -47,13 +47,13 @@ module Hydra
     end
 
     [:ffmpeg_path, :libreoffice_path, :temp_file_base, :fits_path, :kdu_compress_path,
-      :kdu_compress_recipes, :enable_ffmpeg, :source_file_service, :output_file_service].each do |method|
+     :kdu_compress_recipes, :enable_ffmpeg, :source_file_service, :output_file_service].each do |method|
       module_eval <<-RUBY
-        def self.#{method.to_s}
-          config.#{method.to_s}
+        def self.#{method}
+          config.#{method}
         end
-        def self.#{method.to_s}= val
-          config.#{method.to_s}= val
+        def self.#{method}= val
+          config.#{method}= val
         end
       RUBY
     end

data/lib/hydra/derivatives/config.rb

@@ -3,9 +3,9 @@ require 'tmpdir'
 module Hydra
   module Derivatives
     class Config
-      attr_writer :ffmpeg_path, :libreoffice_path, :temp_file_base, 
-        :source_file_service, :output_file_service, :fits_path,
-        :enable_ffmpeg, :kdu_compress_path, :kdu_compress_recipes
+      attr_writer :ffmpeg_path, :libreoffice_path, :temp_file_base,
+                  :source_file_service, :output_file_service, :fits_path,
+                  :enable_ffmpeg, :kdu_compress_path, :kdu_compress_recipes
 
       def ffmpeg_path
         @ffmpeg_path ||= 'ffmpeg'
@@ -41,7 +41,7 @@ module Hydra
 
       def kdu_compress_recipes
         @kdu_compress_recipes ||= {
-          default_color: %Q{-rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171
+          default_color: %(-rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171
             -jp2_space sRGB
             -double_buffering 10
             -num_threads 4
@@ -54,8 +54,8 @@ module Hydra
             Corder=RPCL
             ORGgen_plt=yes
             ORGtparts=R
-            "Stiles={1024,1024}" }.gsub(/\s+/, " ").strip,
-          default_grey: %Q{-rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171
+            "Stiles={1024,1024}" ).gsub(/\s+/, " ").strip,
+          default_grey: %(-rate 2.4,1.48331273,.91673033,.56657224,.35016049,.21641118,.13374944,.08266171
             -jp2_space sLUM
             -double_buffering 10
             -num_threads 4
@@ -68,10 +68,9 @@ module Hydra
             Corder=RPCL
             ORGgen_plt=yes
             ORGtparts=R
-            "Stiles={1024,1024}" }.gsub(/\s+/, " ").strip
+            "Stiles={1024,1024}" ).gsub(/\s+/, " ").strip
         }
       end
-
     end
   end
 end

data/lib/hydra/derivatives/logger.rb

@@ -1,15 +1,13 @@
 module Hydra::Derivatives
   class Logger
-
     class << self
-
-      def method_missing method_name, *arguments, &block
+      def method_missing(method_name, *arguments, &block)
         logger.send(method_name, *arguments, &block)
       rescue
         super
       end
 
-      def respond_to?(method_name, include_private = false)
+      def respond_to?(method_name, _include_private = false)
         logger.respond_to? method_name
       end
 
@@ -18,7 +16,6 @@ module Hydra::Derivatives
       def logger
         ActiveFedora::Base.logger || ::Logger.new(STDOUT)
       end
-
     end
   end
 end

data/lib/hydra/derivatives/processors/audio.rb

@@ -3,4 +3,3 @@ module Hydra::Derivatives::Processors
     include Ffmpeg
   end
 end
-

data/lib/hydra/derivatives/processors/document.rb

@@ -2,27 +2,42 @@ module Hydra::Derivatives::Processors
   class Document < Processor
     include ShellBasedProcessor
 
-    def self.encode(path, options, output_file)
-      format = File.extname(output_file).sub('.', '')
-      outdir = File.dirname(output_file)
+    def self.encode(path, format, outdir)
       execute "#{Hydra::Derivatives.libreoffice_path} --invisible --headless --convert-to #{format} --outdir #{outdir} #{path}"
     end
 
-    def encode_file(file_suffix, options = { })
-      new_output = ''
-      if file_suffix == 'jpg'
-        temp_file = File.join(Hydra::Derivatives.temp_file_base, [directives.fetch(:label).to_s, 'pdf'].join('.'))
-        new_output = File.join(Hydra::Derivatives.temp_file_base, [File.basename(temp_file).sub(File.extname(temp_file), ''), file_suffix].join('.'))
-        self.class.encode(source_path, options, temp_file)
-        self.class.encode(temp_file, options, output_file(file_suffix))
-        File.unlink(temp_file)
-      else
-        self.class.encode(source_path, options, output_file(file_suffix))
-        new_output = File.join(Hydra::Derivatives.temp_file_base, [directives.fetch(:label).to_s, file_suffix].join('.'))
-      end
-      out_file = File.open(new_output, "rb")
-      output_file_service.call(out_file, directives)
-      File.unlink(out_file)
+    # Converts the document to the format specified in the directives hash.
+    # TODO: file_suffix and options are passed from ShellBasedProcessor.process but are not needed.
+    #       A refactor could simplify this.
+    def encode_file(_file_suffix, _options = {})
+      convert_to_format
+    ensure
+      FileUtils.rm_f(converted_file)
     end
+
+    private
+
+      # For jpeg files, a pdf is created from the original source and then passed to the Image processor class
+      # so we can get a better conversion with resizing options. Otherwise, the ::encode method is used.
+      def convert_to_format
+        if directives.fetch(:format) == "jpg"
+          Hydra::Derivatives::Processors::Image.new(converted_file, directives).process
+        else
+          output_file_service.call(File.read(converted_file), directives)
+        end
+      end
+
+      def converted_file
+        @converted_file ||= if directives.fetch(:format) == "jpg"
+                              convert_to("pdf")
+                            else
+                              convert_to(directives.fetch(:format))
+                            end
+      end
+
+      def convert_to(format)
+        self.class.encode(source_path, format, Hydra::Derivatives.temp_file_base)
+        File.join(Hydra::Derivatives.temp_file_base, [File.basename(source_path, ".*"), format].join('.'))
+      end
   end
 end

data/lib/hydra/derivatives/processors/ffmpeg.rb

@@ -3,14 +3,13 @@ module Hydra::Derivatives::Processors
   module Ffmpeg
     extend ActiveSupport::Concern
 
-    INPUT_OPTIONS=:input_options
-    OUTPUT_OPTIONS=:output_options
+    INPUT_OPTIONS = :input_options
+    OUTPUT_OPTIONS = :output_options
 
     included do
       include ShellBasedProcessor
     end
 
-
     module ClassMethods
       def encode(path, options, output_file)
         inopts = options[INPUT_OPTIONS] ||= "-y"

data/lib/hydra/derivatives/processors/full_text.rb

@@ -36,7 +36,7 @@ module Hydra::Derivatives::Processors
 
       # @return [Hash] the request headers to send to the Solr extract service
       def request_headers
-        { Faraday::Request::UrlEncoded::CONTENT_TYPE => "#{mime_type}",
+        { Faraday::Request::UrlEncoded::CONTENT_TYPE => mime_type.to_s,
           Faraday::Adapter::CONTENT_LENGTH => original_size.to_s }
       end
 

data/lib/hydra/derivatives/processors/image.rb

@@ -5,54 +5,57 @@ module Hydra::Derivatives::Processors
     class_attribute :timeout
 
     def process
-      timeout ? process_with_timeout : process_without_timeout
+      timeout ? process_with_timeout : create_resized_image
     end
 
     def process_with_timeout
-      status = Timeout::timeout(timeout) do
-        process_without_timeout
-      end
-    rescue Timeout::Error => ex
+      Timeout.timeout(timeout) { create_resized_image }
+    rescue Timeout::Error
       raise Hydra::Derivatives::TimeoutError, "Unable to process image derivative\nThe command took longer than #{timeout} seconds to execute"
     end
 
-    def process_without_timeout
-      format = directives.fetch(:format)
-      name = directives.fetch(:label, format)
-      destination_name = output_filename_for(name)
-      size = directives.fetch(:size, nil)
-      quality = directives.fetch(:quality, nil)
-      create_resized_image(destination_name, size, format, quality)
-    end
-
     protected
 
-    def create_resized_image(destination_name, size, format, quality=nil)
-      create_image(destination_name, format, quality) do |xfrm|
-        xfrm.resize(size) if size.present?
+      # When resizing images, it is necessary to flatten any layers, otherwise the background
+      # may be completely black. This happens especially with PDFs. See #110
+      def create_resized_image
+        create_image do |xfrm|
+          if size
+            xfrm.flatten
+            xfrm.resize(size)
+          end
+        end
       end
-    end
 
-    def create_image(destination_name, format, quality=nil)
-      xfrm = load_image_transformer
-      yield(xfrm) if block_given?
-      xfrm.format(format)
-      xfrm.quality(quality.to_s) if quality
-      write_image(destination_name, format, xfrm)
-    end
+      def create_image
+        xfrm = load_image_transformer
+        yield(xfrm) if block_given?
+        xfrm.format(directives.fetch(:format))
+        xfrm.quality(quality.to_s) if quality
+        write_image(xfrm)
+      end
+
+      def write_image(xfrm)
+        output_io = StringIO.new
+        xfrm.write(output_io)
+        output_io.rewind
+        output_file_service.call(output_io, directives)
+      end
 
-    def write_image(destination_name, format, xfrm)
-      output_io = StringIO.new
-      xfrm.write(output_io)
-      output_io.rewind
+      # Override this method if you want a different transformer, or need to load the
+      # raw image from a different source (e.g. external file)
+      def load_image_transformer
+        MiniMagick::Image.open(source_path)
+      end
 
-      output_file_service.call(output_io, directives)
-    end
+    private
 
-    # Override this method if you want a different transformer, or need to load the
-    # raw image from a different source (e.g. external file)
-    def load_image_transformer
-      MiniMagick::Image.open(source_path)
-    end
+      def size
+        directives.fetch(:size, nil)
+      end
+
+      def quality
+        directives.fetch(:quality, nil)
+      end
   end
 end

data/lib/hydra/derivatives/processors/jpeg2k_image.rb

@@ -1,15 +1,97 @@
 require 'mini_magick'
 require 'nokogiri'
 
-
 module Hydra::Derivatives::Processors
   class Jpeg2kImage < Processor
     include ShellBasedProcessor
 
+    class << self
+      def srgb_profile_path
+        File.join [
+          File.expand_path('../../../../', __FILE__),
+          'color_profiles',
+          'sRGB_IEC61966-2-1_no_black_scaling.icc'
+        ]
+      end
+
+      def kdu_compress_recipe(args, quality, long_dim)
+        if args[:recipe].is_a? Symbol
+          recipe = [args[:recipe].to_s, quality].join('_').to_sym
+          if Hydra::Derivatives.kdu_compress_recipes.key? recipe
+            return Hydra::Derivatives.kdu_compress_recipes[recipe]
+          else
+            ActiveFedora::Base.logger.warn "No JP2 recipe for :#{args[:recipe]} ('#{recipe}') found in configuration. Using best guess."
+            return calculate_recipe(args, quality, long_dim)
+          end
+        elsif args[:recipe].is_a? String
+          return args[:recipe]
+        else
+          return calculate_recipe(args, quality, long_dim)
+        end
+      end
+
+      def calculate_recipe(args, quality, long_dim)
+        levels_arg = args.fetch(:levels, level_count_for_size(long_dim))
+        rates_arg = layer_rates(args.fetch(:layers, 8), args.fetch(:compression, 10))
+        tile_size = args.fetch(:tile_size, 1024)
+        tiles_arg = "#{tile_size},#{tile_size}"
+        jp2_space_arg = quality == 'gray' ? 'sLUM' : 'sRGB'
+
+        %(-rate #{rates_arg}
+            -jp2_space #{jp2_space_arg}
+            -double_buffering 10
+            -num_threads 4
+            -no_weights
+            Clevels=#{levels_arg}
+            "Stiles={#{tiles_arg}}"
+            "Cblk={64,64}"
+            Cuse_sop=yes
+            Cuse_eph=yes
+            Corder=RPCL
+            ORGgen_plt=yes
+            ORGtparts=R  ).gsub(/\s+/, " ").strip
+      end
+
+      def level_count_for_size(long_dim)
+        levels = 0
+        level_size = long_dim
+        while level_size >= 96
+          level_size /= 2
+          levels += 1
+        end
+        levels - 1
+      end
+
+      def layer_rates(layer_count, compression_numerator)
+        # e.g. if compression_numerator = 10 then compression is 10:1
+        rates = []
+        cmp = 24.0 / compression_numerator
+        layer_count.times do
+          rates << cmp
+          cmp = (cmp / 1.618).round(8)
+        end
+        rates.map(&:to_s).join(',')
+      end
+
+      protected
+
+        def encode(path, recipe, output_file)
+          kdu_compress = Hydra::Derivatives.kdu_compress_path
+          execute "#{kdu_compress} -quiet -i #{path} -o #{output_file} #{recipe}"
+        end
+
+        def tmp_file(ext)
+          Dir::Tmpname.create(['sufia', ext], Hydra::Derivatives.temp_file_base) {}
+        end
+
+        def long_dim(image)
+          [image[:width], image[:height]].max
+        end
+    end
+
     def process
       image = MiniMagick::Image.open(source_path)
       quality = image['%[channels]'] == 'gray' ? 'gray' : 'color'
-      name = directives.fetch(:label)
       long_dim = self.class.long_dim(image)
       file_path = self.class.tmp_file('.tif')
       to_srgb = directives.fetch(:to_srgb, true)
@@ -22,7 +104,7 @@ module Hydra::Derivatives::Processors
       File.unlink(file_path) unless file_path.nil?
     end
 
-    def encode_file(recipe, opts={})
+    def encode_file(recipe, opts = {})
       output_file = self.class.tmp_file('.jp2')
       if opts[:file_path]
         self.class.encode(opts[:file_path], recipe, output_file)
@@ -36,94 +118,14 @@ module Hydra::Derivatives::Processors
     end
 
     protected
-    def preprocess(image, opts={})
-      # resize: <geometry>, to_srgb: <bool>, src_quality: 'color'|'gray'
-      image.combine_options do |c|
-        c.resize(opts[:resize]) if opts[:resize]
-        c.profile self.class.srgb_profile_path if opts[:src_quality] == 'color' && opts[:to_srgb]
-      end
-      image
-    end
-
-    def self.encode(path, recipe, output_file)
-      kdu_compress = Hydra::Derivatives.kdu_compress_path
-      execute "#{kdu_compress} -quiet -i #{path} -o #{output_file} #{recipe}"
-    end
-
-    def self.srgb_profile_path
-      File.join [
-        File.expand_path('../../../../', __FILE__),
-        'color_profiles',
-        'sRGB_IEC61966-2-1_no_black_scaling.icc'
-      ]
-    end
 
-    def self.tmp_file(ext)
-      Dir::Tmpname.create(['sufia', ext], Hydra::Derivatives.temp_file_base){}
-    end
-
-    def self.long_dim(image)
-      [image[:width], image[:height]].max
-    end
-
-    def self.kdu_compress_recipe(args, quality, long_dim)
-      if args[:recipe].is_a? Symbol
-        recipe = [args[:recipe].to_s, quality].join('_').to_sym
-        if Hydra::Derivatives.kdu_compress_recipes.has_key? recipe
-          return Hydra::Derivatives.kdu_compress_recipes[recipe]
-        else
-          ActiveFedora::Base.logger.warn "No JP2 recipe for :#{args[:recipe].to_s} ('#{recipe}') found in configuration. Using best guess."
-          return calculate_recipe(args,quality,long_dim)
+      def preprocess(image, opts = {})
+        # resize: <geometry>, to_srgb: <bool>, src_quality: 'color'|'gray'
+        image.combine_options do |c|
+          c.resize(opts[:resize]) if opts[:resize]
+          c.profile self.class.srgb_profile_path if opts[:src_quality] == 'color' && opts[:to_srgb]
         end
-      elsif args[:recipe].is_a? String
-        return args[:recipe]
-      else
-        return calculate_recipe(args, quality, long_dim)
+        image
       end
-    end
-
-    def self.calculate_recipe(args, quality, long_dim)
-      levels_arg = args.fetch(:levels, level_count_for_size(long_dim))
-      rates_arg = layer_rates(args.fetch(:layers, 8), args.fetch(:compression, 10))
-      tile_size = args.fetch(:tile_size, 1024)
-      tiles_arg = "#{tile_size},#{tile_size}"
-      jp2_space_arg = quality == 'gray' ? 'sLUM' : 'sRGB'
-
-      %Q{-rate #{rates_arg}
-          -jp2_space #{jp2_space_arg}
-          -double_buffering 10
-          -num_threads 4
-          -no_weights
-          Clevels=#{levels_arg}
-          "Stiles={#{tiles_arg}}"
-          "Cblk={64,64}"
-          Cuse_sop=yes
-          Cuse_eph=yes
-          Corder=RPCL
-          ORGgen_plt=yes
-          ORGtparts=R  }.gsub(/\s+/, " ").strip
-    end
-
-    def self.level_count_for_size(long_dim)
-      levels = 0
-      level_size = long_dim
-      while level_size >= 96
-        level_size = level_size/2
-        levels+=1
-      end
-      levels-1
-    end
-
-    def self.layer_rates(layer_count,compression_numerator)
-      #e.g. if compression_numerator = 10 then compression is 10:1
-      rates = []
-      cmp = 24.0/compression_numerator
-      layer_count.times do
-        rates << cmp
-        cmp = (cmp/1.618).round(8)
-      end
-      rates.map(&:to_s ).join(',')
-    end
-
   end
 end