hydra-derivatives 2.0.0 → 3.0.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hydra-derivatives
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Justin Coyne
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-07 00:00:00.000000000 Z
+date: 2015-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,20 +80,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '9.0'
-- !ruby/object:Gem::Dependency
-  name: hydra-file_characterization
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: mini_magick
   requirement: !ruby/object:Gem::Requirement
@@ -179,26 +165,35 @@ files:
 - lib/color_profiles/license.txt
 - lib/color_profiles/sRGB_IEC61966-2-1_no_black_scaling.icc
 - lib/hydra/derivatives.rb
-- lib/hydra/derivatives/audio.rb
 - lib/hydra/derivatives/config.rb
-- lib/hydra/derivatives/document.rb
-- lib/hydra/derivatives/extract_metadata.rb
-- lib/hydra/derivatives/ffmpeg.rb
-- lib/hydra/derivatives/image.rb
 - lib/hydra/derivatives/io_decorator.rb
-- lib/hydra/derivatives/jpeg2k_image.rb
 - lib/hydra/derivatives/logger.rb
-- lib/hydra/derivatives/processor.rb
-- lib/hydra/derivatives/railtie.rb
-- lib/hydra/derivatives/raw_image.rb
+- lib/hydra/derivatives/processors.rb
+- lib/hydra/derivatives/processors/audio.rb
+- lib/hydra/derivatives/processors/document.rb
+- lib/hydra/derivatives/processors/ffmpeg.rb
+- lib/hydra/derivatives/processors/full_text.rb
+- lib/hydra/derivatives/processors/image.rb
+- lib/hydra/derivatives/processors/jpeg2k_image.rb
+- lib/hydra/derivatives/processors/processor.rb
+- lib/hydra/derivatives/processors/raw_image.rb
+- lib/hydra/derivatives/processors/shell_based_processor.rb
+- lib/hydra/derivatives/processors/video.rb
+- lib/hydra/derivatives/processors/video/config.rb
+- lib/hydra/derivatives/processors/video/processor.rb
+- lib/hydra/derivatives/runners/audio_derivatives.rb
+- lib/hydra/derivatives/runners/document_derivatives.rb
+- lib/hydra/derivatives/runners/full_text_extract.rb
+- lib/hydra/derivatives/runners/image_derivatives.rb
+- lib/hydra/derivatives/runners/jpeg2k_image_derivatives.rb
+- lib/hydra/derivatives/runners/pdf_derivatives.rb
+- lib/hydra/derivatives/runners/runner.rb
+- lib/hydra/derivatives/runners/video_derivatives.rb
+- lib/hydra/derivatives/services/mime_type_service.rb
 - lib/hydra/derivatives/services/persist_basic_contained_output_file_service.rb
 - lib/hydra/derivatives/services/persist_output_file_service.rb
 - lib/hydra/derivatives/services/retrieve_source_file_service.rb
 - lib/hydra/derivatives/services/tempfile_service.rb
-- lib/hydra/derivatives/shell_based_processor.rb
-- lib/hydra/derivatives/video.rb
-- lib/hydra/derivatives/video/config.rb
-- lib/hydra/derivatives/video/processor.rb
 - spec/fixtures/FlashPix.ppt
 - spec/fixtures/adobe1998.tif
 - spec/fixtures/countdown.avi
@@ -214,21 +209,22 @@ files:
 - spec/fixtures/test.xls
 - spec/fixtures/test.xlsx
 - spec/fixtures/world.png
-- spec/lib/hydra/derivatives/extract_metadata_spec.rb
+- spec/processors/full_text.rb
+- spec/processors/image_spec.rb
+- spec/processors/jpeg2k_spec.rb
+- spec/processors/processor_spec.rb
+- spec/processors/shell_based_processor_spec.rb
+- spec/processors/video_spec.rb
+- spec/services/audio_derivatives_spec.rb
 - spec/services/persist_basic_contained_output_file_service_spec.rb
 - spec/services/retrieve_source_file_service_spec.rb
 - spec/services/tempfile_service_spec.rb
 - spec/spec_helper.rb
 - spec/units/config_spec.rb
 - spec/units/derivatives_spec.rb
-- spec/units/extract_spec.rb
-- spec/units/image_spec.rb
-- spec/units/jpeg2k_spec.rb
+- spec/units/io_decorator_spec.rb
 - spec/units/logger_spec.rb
-- spec/units/processor_spec.rb
-- spec/units/shell_based_processor_spec.rb
 - spec/units/transcoding_spec.rb
-- spec/units/video_spec.rb
 homepage: https://github.com/projecthydra/hydra-derivatives
 licenses:
 - APACHE2
@@ -249,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Derivative generation plugin for hydra
@@ -269,18 +265,20 @@ test_files:
 - spec/fixtures/test.xls
 - spec/fixtures/test.xlsx
 - spec/fixtures/world.png
-- spec/lib/hydra/derivatives/extract_metadata_spec.rb
+- spec/processors/full_text.rb
+- spec/processors/image_spec.rb
+- spec/processors/jpeg2k_spec.rb
+- spec/processors/processor_spec.rb
+- spec/processors/shell_based_processor_spec.rb
+- spec/processors/video_spec.rb
+- spec/services/audio_derivatives_spec.rb
 - spec/services/persist_basic_contained_output_file_service_spec.rb
 - spec/services/retrieve_source_file_service_spec.rb
 - spec/services/tempfile_service_spec.rb
 - spec/spec_helper.rb
 - spec/units/config_spec.rb
 - spec/units/derivatives_spec.rb
-- spec/units/extract_spec.rb
-- spec/units/image_spec.rb
-- spec/units/jpeg2k_spec.rb
+- spec/units/io_decorator_spec.rb
 - spec/units/logger_spec.rb
-- spec/units/processor_spec.rb
-- spec/units/shell_based_processor_spec.rb
 - spec/units/transcoding_spec.rb
-- spec/units/video_spec.rb
+has_rdoc: 

data/lib/hydra/derivatives/audio.rb

@@ -1,19 +0,0 @@
-module Hydra
-  module Derivatives
-    class Audio < Processor
-      include Ffmpeg
-
-      protected
-
-      def new_mime_type(format)
-        case format
-        when 'mp3'
-          "audio/mpeg"
-        else
-          "audio/#{format}"
-        end
-      end
-    end
-  end
-end
-

data/lib/hydra/derivatives/document.rb

@@ -1,56 +0,0 @@
-module Hydra
-  module Derivatives
-    class Document < Processor
-      include ShellBasedProcessor
-
-      def self.encode(path, options, output_file)
-        format = File.extname(output_file).sub('.', '')
-        outdir = File.dirname(output_file)
-        execute "#{Hydra::Derivatives.libreoffice_path} --invisible --headless --convert-to #{format} --outdir #{outdir} #{path}"
-      end
-
-      def encode_file(destination_name, file_suffix, mime_type, options = { })
-        new_output = ''
-        Hydra::Derivatives::TempfileService.create(source_file) do |f|
-          if mime_type == 'image/jpeg'
-            temp_file = File.join(Hydra::Derivatives.temp_file_base, [File.basename(f.path).sub(File.extname(f.path), ''), 'pdf'].join('.'))
-            new_output = File.join(Hydra::Derivatives.temp_file_base, [File.basename(temp_file).sub(File.extname(temp_file), ''), file_suffix].join('.'))
-            self.class.encode(f.path, options, temp_file)
-            self.class.encode(temp_file, options, output_file(file_suffix))
-            File.unlink(temp_file)
-          else
-            self.class.encode(f.path, options, output_file(file_suffix))
-            new_output = File.join(Hydra::Derivatives.temp_file_base, [File.basename(f.path).sub(File.extname(f.path), ''), file_suffix].join('.'))
-          end
-        end
-        out_file = Hydra::Derivatives::IoDecorator.new(File.open(new_output, "rb"))
-        out_file.mime_type = mime_type
-        output_file_service.call(object, out_file, destination_name)
-        File.unlink(out_file)
-      end
-
-      def output_file(file_suffix)
-        Dir::Tmpname.create(["#{object.id.gsub('/', '_')}-content.", ".#{file_suffix}"], Hydra::Derivatives.temp_file_base){}
-      end
-
-      def new_mime_type(format)
-        case format
-        when 'pdf'
-          'application/pdf'
-        when 'odf'
-          'application/vnd.oasis.opendocument.text'
-        when 'docx'
-          'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
-        when 'xslx'
-          'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
-        when 'pptx'
-          'application/vnd.openxmlformats-officedocument.presentationml.presentation'
-        when 'jpg'
-          'image/jpeg'
-        else
-          raise "I don't know about the format '#{format}'"
-        end
-      end
-    end
-  end
-end

data/lib/hydra/derivatives/extract_metadata.rb

@@ -1,27 +0,0 @@
-require 'hydra-file_characterization'
-require 'mime/types'
-
-module Hydra
-  module Derivatives
-    module ExtractMetadata
-
-      def extract_metadata
-        return unless has_content?
-        Hydra::FileCharacterization.characterize(content, filename_for_characterization.join(""), :fits) do |config|
-          config[:fits] = Hydra::Derivatives.fits_path
-        end
-      end
-
-      protected
-
-      def filename_for_characterization
-        registered_mime_type = MIME::Types[mime_type].first
-        Logger.warn "Unable to find a registered mime type for #{mime_type.inspect} on #{uri}" unless registered_mime_type
-        extension = registered_mime_type ? ".#{registered_mime_type.extensions.first}" : ''
-        version_id = 1 # TODO fixme
-        m = /\/([^\/]*)$/.match(uri)
-        ["#{m[1]}-#{version_id}", "#{extension}"]
-      end
-    end
-  end
-end

data/lib/hydra/derivatives/ffmpeg.rb

@@ -1,31 +0,0 @@
-# An abstract class for asyncronous jobs that transcode files using FFMpeg
-
-require 'tmpdir'
-require 'open3'
-
-module Hydra
-  module Derivatives
-    module Ffmpeg
-      extend ActiveSupport::Concern
-
-      INPUT_OPTIONS=:input_options
-      OUTPUT_OPTIONS=:output_options
-
-      included do
-        include ShellBasedProcessor
-      end
-
-
-      module ClassMethods
-
-        def encode(path, options, output_file)
-          inopts = options[INPUT_OPTIONS] ||= "-y"
-          outopts = options[OUTPUT_OPTIONS] ||= ""
-          execute "#{Hydra::Derivatives.ffmpeg_path} #{inopts} -i \"#{path}\" #{outopts} #{output_file}"
-        end
-      end
-
-    end
-  end
-end
-

data/lib/hydra/derivatives/image.rb

@@ -1,73 +0,0 @@
-require 'mini_magick'
-
-module Hydra
-  module Derivatives
-    class Image < Processor
-      class_attribute :timeout
-
-      def process
-        timeout ? process_with_timeout : process_without_timeout
-      end
-
-      def process_with_timeout
-        status = Timeout::timeout(timeout) do
-          process_without_timeout
-        end
-      rescue Timeout::Error => ex
-        raise Hydra::Derivatives::TimeoutError, "Unable to process image derivative\nThe command took longer than #{timeout} seconds to execute"
-      end
-
-      def process_without_timeout
-        directives.each do |name, args|
-          opts = args.kind_of?(Hash) ? args : {size: args}
-          format = opts.fetch(:format, 'png')
-          destination_name = output_filename_for(name, opts)
-          create_resized_image(destination_name, opts[:size], format)
-        end
-      end
-
-      def output_filename_for(name, opts = {})
-        if opts.has_key? :datastream
-          Deprecation.warn Hydra::Derivatives::Image, 'The :datastream option is deprecated and will be removed in a future release.' 
-        end
-        opts.fetch(:datastream, output_file_id(name))
-      end
-
-      protected
-
-      def new_mime_type(format)
-        MIME::Types.type_for(format).first.to_s
-      end
-
-      def create_resized_image(destination_name, size, format, quality=nil)
-        create_image(destination_name, format, quality) do |xfrm|
-          xfrm.resize(size) if size.present?
-        end
-      end
-
-      def create_image(destination_name, format, quality=nil)
-        xfrm = load_image_transformer
-        yield(xfrm) if block_given?
-        xfrm.format(format)
-        xfrm.quality(quality.to_s) if quality
-        write_image(destination_name, format, xfrm)
-      end
-
-      def write_image(destination_name, format, xfrm)
-        output_io = Hydra::Derivatives::IoDecorator.new(StringIO.new) 
-        output_io.mime_type = new_mime_type(format)
-
-        xfrm.write(output_io)
-        output_io.rewind
-
-        output_file_service.call(object, output_io, destination_name)
-      end
-
-      # Override this method if you want a different transformer, or need to load the
-      # raw image from a different source (e.g.  external file)
-      def load_image_transformer
-        MiniMagick::Image.read(source_file.content)
-      end
-    end
-  end
-end

data/lib/hydra/derivatives/jpeg2k_image.rb

@@ -1,136 +0,0 @@
-require 'mini_magick'
-require 'nokogiri'
-
-
-module Hydra
-  module Derivatives
-    class Jpeg2kImage < Processor
-      include ShellBasedProcessor
-
-      def process
-        image = MiniMagick::Image.read(source_file.content)
-        quality = image['%[channels]'] == 'gray' ? 'gray' : 'color'
-        directives.each do |name, args|
-          long_dim = self.class.long_dim(image)
-          file_path = self.class.tmp_file('.tif')
-          to_srgb = args.fetch(:to_srgb, true)
-          if args[:resize] || to_srgb
-            preprocess(image, resize: args[:resize], to_srgb: to_srgb, src_quality: quality)
-          end
-          image.write file_path
-          recipe = self.class.kdu_compress_recipe(args, quality, long_dim)
-          output_file_name = args[:datastream] || output_file_id(name)
-          encode_file(output_file_name, recipe, file_path: file_path)
-          File.unlink(file_path) unless file_path.nil?
-        end
-      end
-
-      def encode_file(destination_name, recipe, opts={})
-        output_file = self.class.tmp_file('.jp2')
-        if opts[:file_path]
-          self.class.encode(opts[:file_path], recipe, output_file)
-        else
-          Hydra::Derivatives::TempfileService.create(source_file) do |f|
-            self.class.encode(f.path, recipe, output_file)
-          end
-        end
-        out_file = Hydra::Derivatives::IoDecorator.new(File.open(output_file, "rb"))
-        out_file.mime_type = "image/jp2"
-        output_file_service.call(object, out_file, destination_name)
-
-        File.unlink(output_file)
-      end
-
-      protected
-      def preprocess(image, opts={})
-        # resize: <geometry>, to_srgb: <bool>, src_quality: 'color'|'gray'
-        image.combine_options do |c|
-          c.resize(opts[:resize]) if opts[:resize]
-          c.profile self.class.srgb_profile_path if opts[:src_quality] == 'color' && opts[:to_srgb]
-        end
-        image
-      end
-
-      def self.encode(path, recipe, output_file)
-        kdu_compress = Hydra::Derivatives.kdu_compress_path
-        execute "#{kdu_compress} -i #{path} -o #{output_file} #{recipe}"
-      end
-
-      def self.srgb_profile_path
-        File.join [
-          File.expand_path('../../../', __FILE__),
-          'color_profiles',
-          'sRGB_IEC61966-2-1_no_black_scaling.icc'
-        ]
-      end
-
-      def self.tmp_file(ext)
-        Dir::Tmpname.create(['sufia', ext], Hydra::Derivatives.temp_file_base){}
-      end
-
-      def self.long_dim(image)
-        [image[:width], image[:height]].max
-      end
-
-      def self.kdu_compress_recipe(args, quality, long_dim)
-        if args[:recipe].is_a? Symbol
-          recipe = [args[:recipe].to_s, quality].join('_').to_sym
-          if Hydra::Derivatives.kdu_compress_recipes.has_key? recipe
-            return Hydra::Derivatives.kdu_compress_recipes[recipe]
-          else
-            Logger.warn "No JP2 recipe for :#{args[:recipe].to_s} ('#{recipe}') found in configuration. Using best guess."
-            return Hydra::Derivatives::Jpeg2kImage.calculate_recipe(args,quality,long_dim)
-          end
-        elsif args[:recipe].is_a? String
-          return args[:recipe]
-        else
-          return Hydra::Derivatives::Jpeg2kImage.calculate_recipe(args, quality, long_dim)
-        end
-      end
-
-      def self.calculate_recipe(args, quality, long_dim)
-        levels_arg = args.fetch(:levels, Hydra::Derivatives::Jpeg2kImage.level_count_for_size(long_dim))
-        rates_arg = Hydra::Derivatives::Jpeg2kImage.layer_rates(args.fetch(:layers, 8), args.fetch(:compression, 10))
-        tile_size = args.fetch(:tile_size, 1024)
-        tiles_arg = "#{tile_size},#{tile_size}"
-        jp2_space_arg = quality == 'gray' ? 'sLUM' : 'sRGB'
-
-        %Q{-rate #{rates_arg}
-            -jp2_space #{jp2_space_arg}
-            -double_buffering 10
-            -num_threads 4
-            -no_weights
-            Clevels=#{levels_arg}
-            "Stiles={#{tiles_arg}}"
-            "Cblk={64,64}"
-            Cuse_sop=yes
-            Cuse_eph=yes
-            Corder=RPCL
-            ORGgen_plt=yes
-            ORGtparts=R  }.gsub(/\s+/, " ").strip
-      end
-
-      def self.level_count_for_size(long_dim)
-        levels = 0
-        level_size = long_dim
-        while level_size >= 96
-          level_size = level_size/2
-          levels+=1
-        end
-        levels-1
-      end
-
-      def self.layer_rates(layer_count,compression_numerator)
-        #e.g. if compression_numerator = 10 then compression is 10:1
-        rates = []
-        cmp = 24.0/compression_numerator
-        layer_count.times do
-          rates << cmp
-          cmp = (cmp/1.618).round(8)
-        end
-        rates.map(&:to_s ).join(',')
-      end
-
-    end
-  end
-end