hydra-derivatives 0.0.5 → 0.0.6

data/CONTRIBUTING.md

@@ -78,6 +78,8 @@ You should also add yourself to the `CONTRIBUTORS.md` file in the root of the pr
 
 ### Submitting Changes
 
+[Detailed Walkthrough of One Pull Request per Commit](http://ndlib.github.io/practices/one-commit-per-pull-request/)
+
 * Read the article ["Using Pull Requests"](https://help.github.com/articles/using-pull-requests) on GitHub.
 * Make sure your branch is up to date with its parent branch (i.e. master)
   * `git checkout master`

data/README.md

@@ -1,5 +1,6 @@
 hydra-derivatives
 =================
+[![Gem Version](https://badge.fury.io/rb/hydra-derivatives.png)](http://badge.fury.io/rb/hydra-derivatives)
 
 Derivative generation for hydra
 
@@ -47,8 +48,8 @@ Or a class like this:
             transform_datastream :content, { :mp3 => {format: 'mp3'}, :ogg => {format: 'ogg'} }, processor: :audio
           when 'video/avi'
             transform_datastream :content, { :mp4 => {format: 'mp4'}, :webm => {format: 'webm'} }, processor: :video
-          when 'image/png', 'image/jpg'
-            transform_datastream :content, { :medium => "300x300>", :thumb => "100x100>" }
+          when 'image/png', 'image/jpg', 'image/tiff'
+            transform_datastream :content, { :medium => "300x300>", :thumb => {size: "100x100>", datastream: 'thumbnail'} }
           end
         end
     end
@@ -63,20 +64,31 @@ And you add some content to it:
    obj.save
 ```
 
-Then when you call `obj.create_derivatives` a new datastream, called 'thumbnail' will have been created, with a downsized image in it.
+Then when you call `obj.create_derivatives` two new datastreams, 'thumbnail' and 'content_medium', will have been created with downsized images in them.
 
 We recommend you run `obj.create_derivatives` in a background worker, because some derivative creation (especially videos) can take a long time.
 
 # Installation 
 
+Just add `gem 'hydra-derivatives'` to your Gemfile.
+
 ## Dependencies
 
 * [Fits](https://code.google.com/p/fits/)
+* [FFMpeg](http://www.ffmpeg.org/)
 * [LibreOffice](https://www.libreoffice.org/)
 
-To enable LibreOffice and Fits.sh support, you must make sure they are on your path.  Most people will put that in their .bash_profile or somewhere similar.  For example:
+To enable LibreOffice, FFMpeg and Fits.sh support, you make sure they are on your path.  Most people will put that in their .bash_profile or somewhere similar.  For example:
 
 ```bash
 # in .bash_profile
 export PATH=${PATH}:/Users/justin/workspace/fits-0.6.2:/Applications/LibreOffice.app/Contents/MacOS
 ```
+
+Alternatively, you can configure their paths:
+```ruby
+Hydra::Derivatives.ffmpeg_path = '/opt/local/ffmpeg/bin/ffmpeg'
+Hydra::Derivatives.fits_path = '/opt/local/fits/bin/fits.sh'
+Hydra::Derivatives.libreoffice_path = '/opt/local/libreoffice_path/bin/soffice'
+
+```

data/VERSION

@@ -1 +1 @@
-0.0.5
+0.0.6

data/hydra-derivatives.gemspec

@@ -23,7 +23,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec"
 
   spec.add_dependency 'active-fedora'
-  spec.add_dependency 'rmagick'
+  spec.add_dependency 'hydra-file_characterization'
+  spec.add_dependency 'mini_magick'
   spec.add_dependency 'activesupport', '>= 3.2.13', '< 5.0'
 end
 

data/lib/hydra/derivatives.rb

@@ -1,5 +1,4 @@
 require 'active_fedora'
-require 'RMagick'
 module Hydra
   module Derivatives
     extend ActiveSupport::Concern

data/lib/hydra/derivatives/document.rb

@@ -27,6 +27,12 @@ module Hydra
           'application/pdf'
         when 'odf'
           'application/vnd.oasis.opendocument.text'
+        when 'docx'
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+        when 'xslx'
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+        when 'pptx'
+            'application/vnd.openxmlformats-officedocument.presentationml.presentation'
         else
           raise "I don't know about the format '#{format}'"
         end

data/lib/hydra/derivatives/extract_metadata.rb

@@ -1,58 +1,24 @@
-require 'open3'
+require 'hydra-file_characterization'
 module Hydra
   module Derivatives
     module ExtractMetadata
-      include Open3
 
       def extract_metadata
-        out = nil
-        to_tempfile do |f|
-          out = run_fits!(f.path)
-        end
-        out
-      end
-
-      def to_tempfile &block
         return unless has_content?
-        type = MIME::Types[mimeType].first
-        logger.warn "Unable to find a registered mime type for #{mimeType.inspect} on #{pid}" unless type
-        extension = type ? ".#{type.extensions.first}" : ''
-
-        f = Tempfile.new(["#{pid}-#{dsVersionID}", extension])
-        f.binmode
-        if content.respond_to? :read
-          f.write(content.read)
-        else
-          f.write(content)
+        Hydra::FileCharacterization.characterize(content, filename_for_characterization, :fits) do |config|
+          config[:fits] = Hydra::Derivatives.fits_path
         end
-        f.close
-        content.rewind if content.respond_to? :rewind
-        yield(f)
-        f.unlink
       end
 
-      private 
-
-
-        def run_fits!(file_path)
-            command = "#{fits_path} -i \"#{file_path}\""
-            stdin, stdout, stderr, wait_thr = popen3(command)
-            stdin.close
-            out = stdout.read
-            stdout.close
-            err = stderr.read
-            stderr.close
-            exit_status = wait_thr.value
-            raise "Unable to execute command \"#{command}\"\n#{err}" unless exit_status.success?
-            out
-        end
-
-
-        def fits_path
-          Hydra::Derivatives.fits_path
-        end
+      protected
 
+      def filename_for_characterization
+        mime_type = MIME::Types[mimeType].first
+        logger.warn "Unable to find a registered mime type for #{mimeType.inspect} on #{pid}" unless mime_type
+        extension = mime_type ? ".#{mime_type.extensions.first}" : ''
+        "#{pid}-#{dsVersionID}#{extension}"
       end
+
     end
+  end
 end
-

data/lib/hydra/derivatives/image.rb

@@ -1,3 +1,4 @@
+require 'mini_magick'
 module Hydra
   module Derivatives
     class Image < Processor
@@ -16,33 +17,32 @@ module Hydra
         MIME::Types.type_for(format).first.to_s
       end
 
-      def create_resized_image(output_ds, size, format, quality=nil)
-        create_image(output_ds, format, quality) do |xfrm|
-          if size
-            xfrm.change_geometry!(size) do |cols, rows, img|
-             img.resize!(cols, rows)
-            end
-          end
+      def create_resized_image(output_datastream, size, format, quality=nil)
+        create_image(output_datastream, format, quality) do |xfrm|
+          xfrm.resize(size) if size.present?
         end
-        output_ds.mimeType = new_mime_type(format)
+        output_datastream.mimeType = new_mime_type(format)
       end
 
       def create_image(output_datastream, format, quality=nil)
         xfrm = load_image_transformer
         yield(xfrm) if block_given?
-        output_datastream.content = if quality
-          xfrm.to_blob { self.quality = quality; self.format = format.upcase }
-        else
-          xfrm.to_blob { self.format = format.upcase }
-        end
+        xfrm.format(format)
+        xfrm.quality(quality.to_s) if quality
+        write_image(output_datastream, xfrm)
+      end
+
+      def write_image(output_datastream, xfrm)
+        stream = StringIO.new
+        xfrm.write(stream)
+        stream.rewind
+        output_datastream.content = stream
       end
 
       # Override this method if you want a different transformer, or need to load the 
       # raw image from a different source (e.g.  external datastream)
       def load_image_transformer
-        Magick::ImageList.new.tap do |xformer|
-          xformer.from_blob(source_datastream.content)
-        end
+        MiniMagick::Image.read(source_datastream.content)
       end
     end
   end

data/lib/hydra/derivatives/video.rb

@@ -28,7 +28,7 @@ module Hydra
       def codecs(format)
         case format
         when 'mp4'
-          "-vcodec libx264 -acodec libfaac"
+          "-vcodec libx264 -acodec libfdk_aac"
         when 'webm'
           "-acodec libvorbis"
         else

data/spec/lib/hydra/derivatives/extract_metadata_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+module Hydra::Derivatives
+  describe ExtractMetadata do
+    let(:class_with_metadata_extraction) {
+      Class.new do
+        attr_reader :content, :mimeType, :pid, :dsVersionID
+        def initialize(options = {})
+          @content = options.fetch(:content, '')
+          @mimeType = options.fetch(:mime_type, nil)
+          @pid = options.fetch(:pid, 'pid-123')
+          @dsVersionID = options.fetch(:dsVersionID, 'version-id-1')
+        end
+        include Hydra::Derivatives::ExtractMetadata
+        def has_content?; content.present?; end
+      end
+    }
+    let(:initialization_options) { {content: 'abc', mime_type: 'text/plain'} }
+    subject { class_with_metadata_extraction.new(initialization_options) }
+
+    context '#extract_metadata' do
+      context 'without content' do
+        let(:initialization_options) { {content: '', mime_type: 'text/plain'} }
+        its(:extract_metadata) { should be_nil}
+      end
+
+      context 'with content' do
+        let(:mime_type) { 'image/jpeg' }
+        its(:extract_metadata) { should match("<identity format=\"Plain text\" mimetype=\"text/plain\"")}
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -7,3 +7,4 @@ require 'hydra/derivatives'
 RSpec.configure do |config|
 end
 
+$in_travis = !ENV['TRAVIS'].nil? && ENV['TRAVIS'] == 'true'

data/spec/units/extract_spec.rb

@@ -0,0 +1,22 @@
+require "spec_helper"
+
+class ExtractThing < ActiveFedora::Datastream
+  include Hydra::Derivatives::ExtractMetadata
+  attr_accessor :pid
+end
+
+describe Hydra::Derivatives::ExtractMetadata, :unless => $in_travis do
+  let(:subject) { ExtractThing.new }
+  let(:attachment) { File.open(File.expand_path('../../fixtures/world.png', __FILE__))}
+
+  describe "Image Content" do
+    it "should get a mime type" do
+      subject.content = attachment
+      subject.pid = "abc"
+      xml = subject.extract_metadata
+      doc = Nokogiri::HTML(xml)
+      identity = doc.xpath('//identity').first
+      identity.attr('mimetype').should == 'image/png'
+    end
+  end
+end

data/spec/units/transcoding_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe "Transcoder" do
-  before do
+  before(:all) do
     class ContentDatastream < ActiveFedora::Datastream
       include Hydra::Derivatives::ExtractMetadata
     end
@@ -14,7 +14,7 @@ describe "Transcoder" do
       delegate :mime_type, :to => :characterization, :unique => true
       has_file_datastream 'content', type: ContentDatastream
 
-      makes_derivatives do |obj| 
+      makes_derivatives do |obj|
         case obj.mime_type
         when 'application/pdf'
           obj.transform_datastream :content, { :thumb => "100x100>" }
@@ -25,10 +25,15 @@ describe "Transcoder" do
         when 'image/png', 'image/jpg'
           obj.transform_datastream :content, { :medium => "300x300>", :thumb => "100x100>", :access => {format: 'jpg', datastream: 'access'} }
         when 'application/vnd.ms-powerpoint'
-          obj.transform_datastream :content, { :access => { :format=>'pdf' } }, processor: 'document'
+          obj.transform_datastream :content, { :preservation=> {:format => 'pptx'}, :access => { :format=>'pdf' } }, processor: 'document'
         when 'text/rtf'
           obj.transform_datastream :content, { :preservation=> {:format => 'odf' }, :access => { :format=>'pdf' } }, processor: 'document'
+        when 'application/msword'
+          obj.transform_datastream :content, { :access => { :format=>'pdf' }, :preservation=> {:format => 'docx' } }, processor: 'document'
+        when 'application/vnd.ms-excel'
+          obj.transform_datastream :content, { :access => { :format=>'pdf' }, :preservation=> {:format => 'xslx' } }, processor: 'document'
         end
+
       end
       
       makes_derivatives :generate_special_derivatives
@@ -41,6 +46,12 @@ describe "Transcoder" do
 
     end
   end
+
+  after(:all) do
+    GenericFile.all.each(&:destroy)
+    Object.send(:remove_const, :GenericFile);
+    Object.send(:remove_const, :ContentDatastream);
+  end
   
   describe "with an attached image" do
     let(:attachment) { File.open(File.expand_path('../../fixtures/world.png', __FILE__))}
@@ -89,7 +100,7 @@ describe "Transcoder" do
     let(:file) { GenericFile.new(mime_type: 'audio/wav').tap { |t| t.content.content = attachment; t.content.mimeType = 'audio/vnd.wav'; t.save } }
 
     it "should transcode" do
-      expect(logger).to receive(:warn).with("Unable to find a registered mime type for \"audio/vnd.wav\" on #{file.pid}").at_least(1).times
+      expect(logger).to receive(:warn).with("Unable to find a registered mime type for \"audio/vnd.wav\" on #{file.pid}").twice
       file.create_derivatives
       file.datastreams['content_mp3'].should have_content
       file.datastreams['content_mp3'].mimeType.should == 'audio/mpeg'
@@ -131,6 +142,8 @@ describe "Transcoder" do
       file.create_derivatives
       file.datastreams['content_access'].should have_content
       file.datastreams['content_access'].mimeType.should == 'application/pdf'
+      file.datastreams['content_preservation'].should have_content
+      file.datastreams['content_preservation'].mimeType.should == 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
     end
   end
 
@@ -146,4 +159,32 @@ describe "Transcoder" do
       file.datastreams['content_preservation'].mimeType.should == 'application/vnd.oasis.opendocument.text'
     end
   end
+
+  describe "with an attached word doc format", unless: ENV['TRAVIS'] == 'true' do
+    let(:attachment) { File.open(File.expand_path('../../fixtures/test.doc', __FILE__))}
+    let(:file) { GenericFile.new(mime_type: 'application/msword').tap { |t| t.content.content = attachment; t.save } }
+
+    it "should transcode" do
+      file.create_derivatives
+      file.datastreams['content_access'].should have_content
+      file.datastreams['content_access'].mimeType.should == 'application/pdf'
+      file.datastreams['content_preservation'].should have_content
+      file.datastreams['content_preservation'].mimeType.should == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+    end
+  end
+
+  describe "with an attached excel format", unless: ENV['TRAVIS'] == 'true' do
+    let(:attachment) { File.open(File.expand_path('../../fixtures/test.xls', __FILE__))}
+    let(:file) { GenericFile.new(mime_type: 'application/vnd.ms-excel').tap { |t| t.content.content = attachment; t.save } }
+
+    it "should transcode" do
+      file.create_derivatives
+      puts file.datastreams.inspect
+      file.datastreams['content_access'].should have_content
+      file.datastreams['content_access'].mimeType.should == 'application/pdf'
+      file.datastreams['content_preservation'].should have_content
+      file.datastreams['content_preservation'].mimeType.should == 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    end
+  end
+
 end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: hydra-derivatives
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
+  prerelease: 
 platform: ruby
 authors:
 - Justin Coyne
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-25 00:00:00.000000000 Z
+date: 2013-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,78 +30,105 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: jettywrapper
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: active-fedora
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rmagick
+  name: hydra-file_characterization
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mini_magick
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.2.13
     - - <
@@ -107,8 +137,9 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.2.13
     - - <
@@ -148,46 +179,55 @@ files:
 - spec/fixtures/countdown.avi
 - spec/fixtures/piano_note.wav
 - spec/fixtures/sample.rtf
+- spec/fixtures/test.doc
 - spec/fixtures/test.pdf
+- spec/fixtures/test.xls
 - spec/fixtures/world.png
+- spec/lib/hydra/derivatives/extract_metadata_spec.rb
 - spec/spec_helper.rb
 - spec/units/config_spec.rb
+- spec/units/extract_spec.rb
 - spec/units/image_spec.rb
 - spec/units/transcoding_spec.rb
 - spec/units/video_spec.rb
 homepage: https://github.com/projecthydra/hydra-derivatives
 licenses:
 - APACHE2
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.5
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Derivative generation plugin for hydra
 test_files:
 - spec/fixtures/FlashPix.ppt
 - spec/fixtures/countdown.avi
 - spec/fixtures/piano_note.wav
 - spec/fixtures/sample.rtf
+- spec/fixtures/test.doc
 - spec/fixtures/test.pdf
+- spec/fixtures/test.xls
 - spec/fixtures/world.png
+- spec/lib/hydra/derivatives/extract_metadata_spec.rb
 - spec/spec_helper.rb
 - spec/units/config_spec.rb
+- spec/units/extract_spec.rb
 - spec/units/image_spec.rb
 - spec/units/transcoding_spec.rb
 - spec/units/video_spec.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 168e645949c93b6ed1981c164f86969d71024cea
-  data.tar.gz: cecd38bd8f89d9de5d6e709cd3a409ee1792f4c7
-SHA512:
-  metadata.gz: 5ccf06b680a87e68f1375d662439edaa451560bd3895c0262032fd8357b7d4f235b2ebf142ac5d2eb78d2aeeac3da2f337cf04473b329ce0e3063a329c4f46fd
-  data.tar.gz: dc693c6c09dde061948bccae85c67576cea915e668d5b7d7bb5e9ca9544950ba9678e087df0b68360e4be5d90e1b9ed60c96c1c7276df37e73dd80c70ea9900a