hydra-derivatives 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c023bf84dd9a0305bbecdeda90d7d8986153adc
-  data.tar.gz: 24543bbc4676d4a992249de13bae1750b188622c
+  metadata.gz: 2ca2b76caf4ff9ceb741524636c569044af87529
+  data.tar.gz: 8f166356f9cf966d055e0c51341b628afeb236e7
 SHA512:
-  metadata.gz: 7634f4729547506988e49107de96224037b3b036e9d8f7feada641e4f338bcf7e9c3f03eaa77ea5c2ea6301d3bd0a83ac6090f1b464fadc3f651c9e25d12c8c1
-  data.tar.gz: f0e16f0e1652fe06eb9728f76e51aef55b1ac23b7023b99de0587456ae4186ea68df7163bac7b75c2900023beef7dd0d311d24763504a31d8da3feec575e7e46
+  metadata.gz: c5fec89d3e2f7ec198baf9b6db62d39c8fd371abe3b38aeb336b9873b6d964d402f4bd1d8074969adaada0b0cca495b18af8cd88c22cb8dd9597a7f02ecd31de
+  data.tar.gz: fedc87879a089b05f20113cc207be5edfcbc33ecba3b35439a37d134c0cee360450001e6c914fd476d25802875a04526e0ff2ca9d5feecb8804913b88ba13f03

data/.travis.yml

@@ -1,4 +1,6 @@
 language: ruby
+cache: bundler
+sudo: false
 rvm:
   - 2.
 env:
@@ -8,7 +10,6 @@ notifications:
   email:
     recipients:
       - "ul-dlt-applications@lists.psu.edu"
-      - "michael@psu.edu"
     on_success: "change"
     on_failure: "always"
   irc:

data/History.md

@@ -1,3 +1,46 @@
+## 3.0.0 (2015-10-07)
+2015-10-07: Update to the containerized builds on travis [Justin Coyne]
+
+2015-10-07: Add full text extraction as a processor [Justin Coyne]
+
+2015-10-06: make quality be passed when creating an image [lutz]
+
+2015-10-02: Put the processors into their own namespace [Justin Coyne]
+
+2015-10-01: Make the IoDecorator initializer 1-3 args [Justin Coyne]
+
+2015-09-29: Refactor to allow saving at a uri [Justin Coyne]
+
+2015-09-28: Transcode local files [Justin Coyne]
+
+2015-09-30: Remove ExtractMetadata. Fixes #76 [Justin Coyne]
+
+2015-09-28: Rename the :datastream option to :output\_path [Justin Coyne]
+
+2015-09-25: Remove my PSU address from Travis config [Michael J. Giarlo]
+
+2015-09-21: Use IO.select so that all buffers get read (and the process can
+terminate). Ref #81 [Justin Coyne]
+
+2015-09-16: Log the exit code on failure [Justin Coyne]
+
+2015-09-09: Update README.md [Justin Coyne]
+
+2015-09-04: fits version specified [Nikitas Tampakis]
+
+2015-08-28: Update directions to install openoffice headless [ci skip] [Justin
+Coyne]
+
+2015-08-28: List supported version of libreoffice [Justin Coyne]
+
+2015-08-21: Update documentation to show IoDecorator [Justin Coyne]
+
+2015-08-21: Add a deprecation horizion to the deprecation message [Justin Coyne]
+
+2015-08-06: change call to transform\_file, in the deprecated method
+transform\_datastream, to not pass a default options values, since the method def
+for transform\_file already sets the default values. [Jose Blanco]
+
 ## 0.1.1 (2014-07-21)
  - Define a logger
 

data/README.md

@@ -11,47 +11,24 @@ If you have an ActiveFedora class like this:
         attr_accessor :mime_type
 
         # Use a block to declare which derivatives you want to generate
-        makes_derivatives do |obj|
-          case obj.mime_type
-          when 'application/pdf'
-            obj.transform_file :original_file, { :thumb => "100x100>" }
-          when 'audio/wav'
-            obj.transform_file :original_file, { :mp3 => {format: 'mp3'}, :ogg => {format: 'ogg'} }, processor: :audio
-          when 'video/avi'
-            obj.transform_file :original_file, { :mp4 => {format: 'mp4'}, :webm => {format: 'webm'} }, processor: :video
-          when 'image/png', 'image/jpg'
-            obj.transform_file :original_file, { :medium => "300x300>", :thumb => "100x100>" }
-          when 'image/tiff'
-            obj.transform_file :original_file, { :service => { resize: "3600x3600>" } }, processor: 'jpeg2k_image'
-          end
-        end
-    end
-```
-
-Or a class like this:
-
-```ruby
-    class GenericFile < ActiveFedora::Base
-        include Hydra::Derivatives
-
-        contains 'original_file'
-        attr_accessor :mime_type
-
-        # Use a callback method to declare which derivatives you want
-        makes_derivatives :generate_derivatives
 
-        def generate_derivatives
+        def create_derivatives(filename)
           case mime_type
           when 'application/pdf'
-            transform_file :original_file, { :thumb => "100x100>" }
+            PdfDerivatives.create(filename, outputs: [{ label: :thumb, size: "100x100>" }]
           when 'audio/wav'
-            transform_file :original_file, { :mp3 => {format: 'mp3'}, :ogg => {format: 'ogg'} }, processor: :audio
+            AudioDerivatives.create(self, source: :original_file, outputs: [{ label: :mp3, format: 'mp3', url: "#{uri}/mp3" }, { label: :ogg, format: 'ogg', url: "#{uri}/ogg" }])
           when 'video/avi'
-            transform_file :original_file, { :mp4 => {format: 'mp4'}, :webm => {format: 'webm'} }, processor: :video
+            VideoDerivatives.create(filename, outputs: [{ label: :mp4, format: 'mp4'}, { label: :webm, format: 'webm'}])
           when 'image/png', 'image/jpg'
-            transform_file :original_file, { :medium => "300x300>", :thumb => {size: "100x100>", datastream: 'thumbnail'} }
+            ImageDerivatives.create(self, source: :original_file,
+                                    outputs: [
+                                      { label: :medium, size: "300x300>", url: "#{uri}/medium" },
+                                      { label: :thumb, size: "100x100>", url: "#{uri}/thumb" }])
+          when 'application/vnd.ms-powerpoint'
+            DocumentDerivatives.create(filename, outputs[{ label: :preservation, format: 'pptx' }, { label: :access, format: 'pdf' }, { label: :thumnail, format: 'jpg' })
           when 'image/tiff'
-            transform_file :original_file, { :service => { recipe: :default } }, processor: 'jpeg2k_image'
+            Jpeg2kDerivatives.create(filename, outputs: [{ label: :service, resize: "3600x3600>" }])
           end
         end
     end
@@ -72,6 +49,14 @@ We recommend you run `obj.create_derivatives` in a background worker, because so
 
 ## Configuration
 
+### Retrieving from a basic container in Fedora
+
+Provide the object and `:source` option instead of a filename
+
+```ruby
+PdfDerivatives.create(active_fedora_object, source: :original_file, outputs: [{ label: :thumb, size: "100x100>" }]
+```
+
 ### Processing Timeouts
 
 hydra-derivatives can be configured to timeout derivatives processes.  Each process type has a separate timeout.
@@ -104,11 +89,11 @@ Just add `gem 'hydra-derivatives'` to your Gemfile.
 
 ## Dependencies
 
-* [FITS](http://fitstool.org/)
+* [FITS](http://fitstool.org/) - version 0.6.2
 * [FFMpeg](http://www.ffmpeg.org/)
-* [LibreOffice](https://www.libreoffice.org/)
+* [LibreOffice](https://www.libreoffice.org/) (openoffice.org-headless on Ubuntu/Debian to avoid "_X11 error: Can't open display:_")
 * [ImageMagick](http://www.imagemagick.org/)
-* Kakadu's [kdu_compress](http://www.kakadusoftware.com/)
+* Kakadu's [kdu_compress](http://www.kakadusoftware.com/) (optional)
 
 To enable LibreOffice, FFMpeg, ImageMagick, FITS support, and kdu_compress support, make sure they are on your path. Most people will put that in their .bash_profile or somewhere similar.
 
@@ -133,7 +118,7 @@ Unlike the other processors, the `Jpeg2kImage` processor does not generally acce
 
 Instead, each directive may contain these arguments:
 
-  * `:datastream` (String) : The name for the new datastream
+  * `:output_path` (String) : The name for the new file
   * `:to_srgb` (Boolean) : If `true` and the image is a color image it will map the source image color profile to sRGB. Default: `true`
   * `:resize` (String) : Geometry; the same syntax as the `Hydra::Derivatives::Image` processor
   * `:recipe` :

data/VERSION

@@ -1 +1 @@
-2.0.0
+3.0.0

data/hydra-derivatives.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.1'
 
   spec.add_dependency 'active-fedora', '~> 9.0'
-  spec.add_dependency 'hydra-file_characterization', '~> 0.3'
   spec.add_dependency 'mini_magick', '>= 3.2', '< 5'
   spec.add_dependency 'activesupport', '~> 4.0'
   spec.add_dependency 'mime-types', '< 3'

data/lib/hydra/derivatives.rb

@@ -1,5 +1,4 @@
 require 'active_fedora'
-require 'hydra/derivatives/railtie' if defined?(Rails)
 require 'deprecation'
 
 module Hydra
@@ -9,26 +8,32 @@ module Hydra
     extend Deprecation
     self.deprecation_horizon = "hydra-derivatives 1.0"
 
-    autoload :Processor
-    autoload :Image
-    autoload :Ffmpeg
-    autoload :Video
-    autoload :Audio
+    # Runners take a single input and produce one or more outputs
+    # The runner typically accomplishes this by using one or more processors
+    autoload_under 'runners' do
+      autoload :AudioDerivatives
+      autoload :DocumentDerivatives
+      autoload :FullTextExtract
+      autoload :ImageDerivatives
+      autoload :Jpeg2kImageDerivatives
+      autoload :PdfDerivatives
+      autoload :Runner
+      autoload :VideoDerivatives
+    end
+
+    autoload :Processors
     autoload :Config
-    autoload :Document
-    autoload :ExtractMetadata
-    autoload :ShellBasedProcessor
-    autoload :Jpeg2kImage
-    autoload :RawImage
     autoload :Logger
     autoload :TempfileService
     autoload :IoDecorator
 
-    # services
-    autoload :RetrieveSourceFileService,         'hydra/derivatives/services/retrieve_source_file_service'
-    autoload :PersistOutputFileService,          'hydra/derivatives/services/persist_output_file_service'
-    autoload :PersistBasicContainedOutputFileService, 'hydra/derivatives/services/persist_basic_contained_output_file_service'
-    autoload :TempfileService,                   'hydra/derivatives/services/tempfile_service'
+    autoload_under 'services' do
+      autoload :RetrieveSourceFileService
+      autoload :PersistOutputFileService
+      autoload :PersistBasicContainedOutputFileService
+      autoload :TempfileService
+      autoload :MimeTypeService
+    end
 
     # Raised if the timout elapses
     class TimeoutError < ::Timeout::Error; end
@@ -54,113 +59,8 @@ module Hydra
     end
 
     included do
-      class_attribute :transformation_schemes
-    end
-
-    # Runs all of the transformations immediately.
-    # You may want to run this job in the background as it may take a long time.
-    def create_derivatives
-      if transformation_schemes.present?
-        transformation_schemes.each do |transform_scheme|
-          if transform_scheme.instance_of?(Proc)
-            transform_scheme.call(self)
-          else
-            send(transform_scheme)
-          end
-        end
-      else
-        logger.warn "`create_derivatives' was called on an instance of #{self.class}, but no derivatives have been requested"
-      end
-    end
-
-    # Create derivatives from a file according to transformation directives
-    # @param file_name
-    # @param [Hash] transform_directives - each key corresponds to a desired derivative.  Associated values vary according to processor being used.
-    # @param [Hash] opts for specifying things like choice of :processor (processor defaults to :image)
-    # @option opts [Symbol] :processor (:image) Processor to use
-    # @option opts [Class] :source_file_service (Hydra::Derivatives::RetrieveSourceFileService) service to use when persisting generated derivatives.  The default for this can be set in your config file.
-    # @option opts [Class] :output_file_service (Hydra::Derivatives::PersistIndirectlyContainedOutputFile) service to use when retrieving the source.  The default for this can be set in your config file.
-    #
-    # @example This will create content_thumb
-    #   transform_file :content, { :thumb => "100x100>" }
-    #
-    # @example Specify the dsid for the output file
-    #   transform_file :content, { :thumb => {size: "200x300>", datastream: 'thumbnail'} }
-    #
-    # @example Create multiple derivatives with one set of directives.  This will create content_thumb and content_medium
-    #   transform_file :content, { :medium => "300x300>", :thumb => "100x100>" }
-    #
-    # @example Specify which processor you want to use (defaults to :image)
-    #   transform_file :content, { :mp3 => {format: 'mp3'}, :ogg => {format: 'ogg'} }, processor: :audio
-    #   transform_file :content, { :mp4 => {format: 'mp4'}, :webm => {format: 'webm'} }, processor: :video
-    #
-    # @example Specify an output file service to use when persisting generated derivatives
-    #   obj.transform_file :content, { mp4: { format: 'mp4' } }, processor: :video, output_file_service: My::System::PersistOutputFileToTapeStorage
-    #
-    # @example Specify a source file service to use when retrieving the source
-    #   obj.transform_file :content, { mp4: { format: 'mp4' } }, processor: :video, source_file_service: My::System::PersistOutputFileToTapeStorage
-
-    def transform_file(file_name, transform_directives, opts={})
-      initialize_processor(file_name, transform_directives, opts).process
-    end
-
-    def processor_class(processor)
-      case processor
-        when :video
-          Hydra::Derivatives::Video::Processor
-        else
-          constantize_processor(processor.to_s)
-        end
+      class_attribute :source_file_service
+      self.source_file_service = Hydra::Derivatives.source_file_service
     end
-
-    def constantize_processor(processor)
-      "Hydra::Derivatives::#{processor.classify}".constantize
-    rescue NameError
-      processor.classify.constantize
-    end
-
-    def transform_datastream(file_name, transform_directives, opts={})
-      transform_file(file_name, transform_directives, opts={})
-    end
-    deprecation_deprecate :transform_datastream
-
-
-    module ClassMethods
-      # Register transformation schemes for generating derivatives.
-      # You can do this using a block or by defining a callback method.
-      #
-      # @example Define transformation scheme using a block
-      #    makes_derivatives do |obj|
-      #      case obj.mime_type
-      #      when 'application/pdf'
-      #        obj.transform_file :content, { :thumb => "100x100>" }
-      #      when 'audio/wav'
-      #        obj.transform_file :content, { :mp3 => {format: 'mp3'}, :ogg => {format: 'ogg'} }, processor: :audio
-      #
-      # @example Define transformation scheme using a callback method
-      #      makes_derivatives :generate_image_derivatives
-      #
-      #      def generate_image_derivatives
-      #        case mime_type
-      #        when 'image/png', 'image/jpg'
-      #          transform_file :content, { :medium => "300x300>", :thumb => "100x100>" }
-      #        end
-      #      end
-      def makes_derivatives(*callback_method_names, &block)
-        self.transformation_schemes ||= []
-        if block_given?
-          self.transformation_schemes << block
-        end
-        callback_method_names.each do |callback_name|
-          self.transformation_schemes << callback_name
-        end
-      end
-    end
-
-    private
-    def initialize_processor(file_name, transform_directives, opts={})
-      processor_class(opts[:processor] || :image).new(self, file_name, transform_directives, opts)
-    end
-
   end
 end

data/lib/hydra/derivatives/io_decorator.rb

@@ -1,5 +1,5 @@
 # Nieve implementation of IO wrapper class that adds mime_type and original_name attributes.
-# This is done so the attributes do not have to be passed as additional arguments, 
+# This is done so the attributes do not have to be passed as additional arguments,
 #  and are attached properly to the object they describe.
 #
 #
@@ -10,6 +10,12 @@ module Hydra
   module Derivatives
     class IoDecorator < SimpleDelegator
       attr_accessor :mime_type, :original_name
+
+      def initialize(file, mime_type = nil, original_name = nil)
+        super(file)
+        self.mime_type = mime_type
+        self.original_name = original_name
+      end
     end
   end
 end

data/lib/hydra/derivatives/processors.rb

@@ -0,0 +1,19 @@
+module Hydra::Derivatives
+  module Processors
+    extend ActiveSupport::Autoload
+
+    eager_autoload do
+      autoload :Processor
+    end
+
+    autoload :Audio
+    autoload :Document
+    autoload :Ffmpeg
+    autoload :FullText
+    autoload :Image
+    autoload :Jpeg2kImage
+    autoload :RawImage
+    autoload :ShellBasedProcessor
+    autoload :Video
+  end
+end

data/lib/hydra/derivatives/processors/audio.rb

@@ -0,0 +1,6 @@
+module Hydra::Derivatives::Processors
+  class Audio < Processor
+    include Ffmpeg
+  end
+end
+

data/lib/hydra/derivatives/processors/document.rb

@@ -0,0 +1,28 @@
+module Hydra::Derivatives::Processors
+  class Document < Processor
+    include ShellBasedProcessor
+
+    def self.encode(path, options, output_file)
+      format = File.extname(output_file).sub('.', '')
+      outdir = File.dirname(output_file)
+      execute "#{Hydra::Derivatives.libreoffice_path} --invisible --headless --convert-to #{format} --outdir #{outdir} #{path}"
+    end
+
+    def encode_file(file_suffix, options = { })
+      new_output = ''
+      if file_suffix == 'jpg'
+        temp_file = File.join(Hydra::Derivatives.temp_file_base, [directives.fetch(:label).to_s, 'pdf'].join('.'))
+        new_output = File.join(Hydra::Derivatives.temp_file_base, [File.basename(temp_file).sub(File.extname(temp_file), ''), file_suffix].join('.'))
+        self.class.encode(source_path, options, temp_file)
+        self.class.encode(temp_file, options, output_file(file_suffix))
+        File.unlink(temp_file)
+      else
+        self.class.encode(source_path, options, output_file(file_suffix))
+        new_output = File.join(Hydra::Derivatives.temp_file_base, [directives.fetch(:label).to_s, file_suffix].join('.'))
+      end
+      out_file = File.open(new_output, "rb")
+      output_file_service.call(out_file, directives)
+      File.unlink(out_file)
+    end
+  end
+end

data/lib/hydra/derivatives/processors/ffmpeg.rb

@@ -0,0 +1,22 @@
+# An abstract class for asyncronous jobs that transcode files using FFMpeg
+module Hydra::Derivatives::Processors
+  module Ffmpeg
+    extend ActiveSupport::Concern
+
+    INPUT_OPTIONS=:input_options
+    OUTPUT_OPTIONS=:output_options
+
+    included do
+      include ShellBasedProcessor
+    end
+
+
+    module ClassMethods
+      def encode(path, options, output_file)
+        inopts = options[INPUT_OPTIONS] ||= "-y"
+        outopts = options[OUTPUT_OPTIONS] ||= ""
+        execute "#{Hydra::Derivatives.ffmpeg_path} #{inopts} -i \"#{path}\" #{outopts} #{output_file}"
+      end
+    end
+  end
+end