sufia 0.1.0 → 1.0.0

data/lib/sufia/generic_file.rb

@@ -25,7 +25,7 @@ module Sufia
       has_metadata :name => "descMetadata", :type => GenericFileRdfDatastream
       has_metadata :name => "properties", :type => PropertiesDatastream
       has_file_datastream :name => "content", :type => FileContentDatastream
-      has_file_datastream :name => "thumbnail", :type => FileContentDatastream
+      has_file_datastream :name => "thumbnail"
 
       belongs_to :batch, :property => :is_part_of
 
@@ -36,6 +36,8 @@ module Sufia
                                   :publisher, :date_created, :subject,
                                   :resource_type, :identifier, :language]
       around_save :characterize_if_changed, :retry_warming
+      before_save :remove_blank_assertions
+
     end
 
     def delete
@@ -43,6 +45,12 @@ module Sufia
        super
     end
 
+    def remove_blank_assertions
+      terms_for_editing.each do |key|
+        self[key] = nil if self[key] == ['']
+      end
+    end
+
 
     def record_version_committer(user)
       version = content.latest_version
@@ -55,15 +63,21 @@ module Sufia
     end
 
     def pdf?
-      ["application/pdf"].include? self.mime_type
+      ['application/pdf'].include? self.mime_type
     end
 
     def image?
-      ["image/png","image/jpeg", 'image/jpg', 'image/bmp', "image/gif"].include? self.mime_type
+      ['image/png','image/jpeg', 'image/jpg', 'image/jp2', 'image/bmp', 'image/gif'].include? self.mime_type
     end
 
     def video?
-      ["video/mpeg", "video/mp4", "video/x-msvideo", "video/avi", "video/quicktime"].include? self.mime_type
+      ['video/mpeg', 'video/mp4', 'video/webm', 'video/x-msvideo', 'video/avi', 'video/quicktime', 'application/mxf'].include? self.mime_type
+    end
+
+    def audio?
+      # audio/x-wave is the mime type that fits 0.6.0 returns for a wav file.
+      # audio/mpeg is the mime type that fits 0.6.0 returns for an mp3 file.
+      ['audio/mp3', 'audio/mpeg', 'audio/x-wave', 'audio/x-wav', 'audio/ogg'].include? self.mime_type
     end
 
     def persistent_url
@@ -111,21 +125,25 @@ module Sufia
       relateds = begin
                    self.batch.generic_files
                  rescue NoMethodError => e
-                   #batch is nil
+                   #batch is nil - When would this ever happen?
                    batch_id = self.object_relations["isPartOf"].first || self.object_relations[:is_part_of].first
                    return [] if batch_id.nil?
-                   self.class.find(:is_part_of_s => batch_id)
+                   self.class.find(Solrizer.solr_name('is_part_of', :symbol) => batch_id)
                  end
       relateds.reject { |gf| gf.pid == self.pid }
     end
 
+    # Unstemmed, searchable, stored
+    def self.noid_indexer
+      @noid_indexer ||= Solrizer::Descriptor.new(:text, :indexed, :stored)
+    end 
 
     def to_solr(solr_doc={}, opts={})
       super(solr_doc, opts)
-      solr_doc["label_t"] = self.label
-      solr_doc["noid_s"] = noid
-      solr_doc["file_format_t"] = file_format
-      solr_doc["file_format_facet"] = solr_doc["file_format_t"]
+      solr_doc[Solrizer.solr_name('label')] = self.label
+      solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile.noid_indexer)] = noid
+      solr_doc[Solrizer.solr_name('file_format')] = file_format
+      solr_doc[Solrizer.solr_name('file_format', :facetable)] = file_format
       return solr_doc
     end
 
@@ -161,7 +179,7 @@ module Sufia
 
     def terms_for_editing
       terms_for_display -
-       [:part_of, :date_modified, :date_uploaded, :format, :resource_type]
+       [:part_of, :date_modified, :date_uploaded, :format] #, :resource_type]
     end
 
     def terms_for_display

data/lib/sufia/generic_file/audit.rb

@@ -89,7 +89,7 @@ module Sufia
         end
 
         def audit_everything(force = false)
-          ::GenericFile.find(:all, :rows => ::GenericFile.count).each do |gf|
+          ::GenericFile.find_each do |gf|
             gf.per_version do |ver|
               ::GenericFile.audit(ver, force)
             end

data/lib/sufia/generic_file/thumbnail.rb

@@ -4,29 +4,46 @@ module Sufia
       # Create thumbnail requires that the characterization has already been run (so mime_type, width and height is available)
       # and that the object is already has a pid set
       def create_thumbnail
-        return if self.content.content.nil?
+        return unless self.content.has_content?
         if pdf?
           create_pdf_thumbnail
         elsif image?
           create_image_thumbnail
-        # elsif video?
-        #   create_video_thumbnail
+        elsif video?
+          create_video_thumbnail
         end
       end
 
+      protected
+      def create_video_thumbnail
+        return unless Sufia::Engine.config.enable_ffmpeg
+      
+        output_file = Dir::Tmpname.create(['sufia', ".png"], Sufia::Engine.config.temp_file_base){}
+        content.to_tempfile do |f|
+          # we could use something like this in order to find a frame in the middle.
+          #ffprobe -show_files video.avi 2> /dev/null | grep duration | cut -d= -f2 53.399999  
+          command = "#{Sufia::Engine.config.ffmpeg_path} -i \"#{f.path}\" -loglevel quiet -vf \"scale=338:-1\"  -r  1  -t  1 #{output_file}"
+          system(command)
+          raise "Unable to execute command \"#{command}\"" unless $?.success?
+        end
+
+        self.thumbnail.content = File.open(output_file, 'rb').read
+        self.thumbnail.mimeType = 'image/png'
+        self.save
+      end
+
       def create_pdf_thumbnail
         retryCnt = 0
         stat = false;
         for retryCnt in 1..3
           begin
-            pdf = Magick::ImageList.new
-            pdf.from_blob(content.content)
+            pdf = load_image_transformer
             first = pdf.to_a[0]
             first.format = "PNG"
             thumb = first.scale(338, 493)
             self.thumbnail.content = thumb.to_blob { self.format = "PNG" }
-            #logger.debug "Has the content changed before saving? #{self.content.changed?}"
-            stat = self.save
+            self.thumbnail.mimeType = 'image/png'
+            self.save
             break
           rescue => e
             logger.warn "Rescued an error #{e.inspect} retry count = #{retryCnt}"
@@ -37,30 +54,38 @@ module Sufia
       end
 
       def create_image_thumbnail
-        img = Magick::ImageList.new
-        img.from_blob(content.content)
-        # horizontal img
+        self.thumbnail.content = scale_image.to_blob { self.format = "PNG" }
+        self.thumbnail.mimeType = 'image/png'
+        #logger.debug "Has the content before saving? #{self.content.changed?}"
+        self.save
+      end
+
+      def scale_image
+        img = load_image_transformer
         height = Float(self.height.first.to_i)
         width = Float(self.width.first.to_i)
-        scale = height / width
-        if width > height
-          if width > 150 and height > 105
-            thumb = img.scale(150, height/scale)
-          else
-            thumb = img.scale(width, height)
-          end
-        # vertical img
+        if width > height && width > 150 && height > 105
+          # horizontal img
+          scale  = 150 / width
+          img.scale(150, height * scale)
+        elsif height >= width && width > 150 && height > 200
+          # vertical or square
+          scale  = 200 / height
+          img.scale(width*scale, 200)
         else
-          if width > 150 and height > 200
-            thumb = img.scale(150*scale, 200)
-          else
-            thumb = img.scale(width, height)
-          end
+          # Too small to worry about resizing
+          img
         end
-        self.thumbnail.content = thumb.to_blob
-        #logger.debug "Has the content before saving? #{self.content.changed?}"
-        self.save
       end
+
+      # Override this method if you want a different transformer, or need to load the 
+      # raw image from a different source (e.g.  external datastream)
+      def load_image_transformer
+        xformer = Magick::ImageList.new
+        xformer.from_blob(content.content)
+        xformer
+      end
+
     end
   end
 end

data/lib/sufia/id_service.rb

@@ -15,26 +15,43 @@
 require 'noid'
 
 module Sufia
-  class IdService
-    @@minter = ::Noid::Minter.new(:template => '.reeddeeddk')
-    @@namespace = Sufia::Engine.config.id_namespace
+  module IdService
+    @minter = ::Noid::Minter.new(:template => '.reeddeeddk')
+    @pid = $$
+    @namespace = Sufia::Engine.config.id_namespace
+    @semaphore = Mutex.new
     def self.valid?(identifier)
       # remove the fedora namespace since it's not part of the noid
       noid = identifier.split(":").last
-      return @@minter.valid? noid
+      return @minter.valid? noid
     end
     def self.mint
-      while true
-        pid = self.next_id
-        break unless ActiveFedora::Base.exists?(pid)
+      @semaphore.synchronize do
+        while true
+          pid = self.next_id
+          return pid unless ActiveFedora::Base.exists?(pid)
+        end
       end
-      return pid
     end
+
     protected
+
     def self.next_id
-      # seed with process id so that if two processes are running they do not come up with the same id.
-      @@minter.seed($$)
-      return  "#{@@namespace}:#{@@minter.mint}"
+      pid = ''
+      File.open("tmp/minter-state", File::RDWR|File::CREAT, 0644) {|f|
+        f.flock(File::LOCK_EX)
+        yaml = YAML::load(f.read)
+        yaml = {:template => '.reeddeeddk'} unless yaml
+        minter = ::Noid::Minter.new(yaml)
+        pid =  "#{@namespace}:#{minter.mint}"
+        f.rewind
+        yaml = YAML::dump(minter.dump)
+        f.write yaml
+        f.flush
+        f.truncate(f.pos)
+      }
+      return pid
     end
+    
   end
 end

data/lib/sufia/jobs/batch_update_job.rb

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 class BatchUpdateJob
-  include Hydra::AccessControlsEnforcement
+  include Hydra::PermissionsQuery
   include GenericFileHelper
   include Rails.application.routes.url_helpers 
 
@@ -55,7 +55,7 @@ class BatchUpdateJob
 
   def update_file(gf, user)
     unless user.can? :edit, gf
-      logger.error "User #{user.user_key} DEEEENIED access to #{gf.pid}!"
+      logger.error "User #{user.user_key} DENIED access to #{gf.pid}!"
       @denied << gf
       return
     end

data/lib/sufia/jobs/characterize_job.rb

@@ -18,18 +18,26 @@ class CharacterizeJob
     :characterize
   end
 
-  attr_accessor :generic_file_id
+  attr_accessor :generic_file_id, :generic_file
 
   def initialize(generic_file_id)
     self.generic_file_id = generic_file_id
   end
 
   def run
-    generic_file = GenericFile.find(generic_file_id)
+    self.generic_file = GenericFile.find(generic_file_id)
     generic_file.characterize
-    generic_file.create_thumbnail
+    after_characterize
+  end
+
+  def after_characterize
+    if generic_file.pdf? || generic_file.image? || generic_file.video?
+      generic_file.create_thumbnail
+    end
     if generic_file.video?
       Sufia.queue.push(TranscodeVideoJob.new(generic_file_id, 'content'))
+    elsif generic_file.audio?
+      Sufia.queue.push(TranscodeAudioJob.new(generic_file_id, 'content'))
     end
   end
 end

data/lib/sufia/jobs/ffmpeg_transcode_job.rb

@@ -0,0 +1,61 @@
+# Created by: Justin Coyne
+# 7 Feb 2013
+# An abstract class for asyncronous jobs that transcode files using FFMpeg
+
+require 'tmpdir'
+
+class FfmpegTranscodeJob
+  extend Open3
+
+  attr_accessor :generic_file_id, :datastream_in, :datastream, :generic_file
+
+  def initialize(generic_file_id, datastream_in)
+    self.generic_file_id = generic_file_id
+    self.datastream_in = datastream_in
+  end
+
+  def process
+    raise "You attempted to call process() on an abstract class.  Implement process() on the concrete class"
+  end
+
+  def run
+    return unless Sufia::Engine.config.enable_ffmpeg
+    self.generic_file = GenericFile.find(generic_file_id)
+    self.datastream = generic_file.datastreams[datastream_in]
+    if datastream
+      process 
+      generic_file.save!
+    else
+      logger.warn "No datastream for transcoding!!!!! pid: #{generic_file_id} dsid: #{datastream_in}"
+    end
+  end
+
+  def encode_datastream(dest_dsid, mime_type, options)
+    file_suffix = dest_dsid
+    out_file = nil
+    output_file = Dir::Tmpname.create(['sufia', ".#{file_suffix}"], Sufia::Engine.config.temp_file_base){}
+    datastream.to_tempfile do |f|
+      self.class.encode(f.path, options, output_file)
+    end
+    out_file = File.open(output_file, "rb")
+    generic_file.add_file_datastream(out_file.read, :dsid=>dest_dsid, :mimeType=>mime_type)
+    File.unlink(output_file)
+  end
+
+  def self.encode(path, options, output_file)
+    command = "#{ffmpeg_path} -y -i \"#{path}\" #{options} #{output_file}"
+    stdin, stdout, stderr, wait_thr = popen3(command)
+    stdin.close
+    out = stdout.read
+    stdout.close
+    err = stderr.read
+    stderr.close
+    raise "Unable to execute command \"#{command}\"\n#{err}" unless wait_thr.value.success?
+  end
+
+  def self.ffmpeg_path
+    Sufia::Engine.config.ffmpeg_path
+  end
+end
+
+

data/lib/sufia/jobs/resolrize_job.rb

@@ -18,6 +18,6 @@ class ResolrizeJob
   end
 
   def run
-    Solrizer::Fedora::Solrizer.new.solrize_objects(:suppress_errors => false)
+    ActiveFedora::Base.reindex_everything
   end
 end

data/lib/sufia/jobs/transcode_audio_job.rb

@@ -0,0 +1,40 @@
+# Created by: Justin Coyne
+# 7 Feb 2013
+# An asyncronous job for transcoding audio files using FFMpeg
+
+class TranscodeAudioJob < FfmpegTranscodeJob
+  def queue_name
+    :audio
+  end
+
+  def process
+    encode_mp3()
+    encode_ogg()
+  end
+
+  private 
+    def encode_ogg
+      opts = ""
+      if generic_file.mime_type == 'audio/ogg'
+        # Don't re-encode, just copy
+        generic_file.add_file_datastream(generic_file.content.read, :dsid=>'ogg', :mimeType=>'audio/ogg')
+        #generic_file.content.rewind
+      else
+        encode_datastream('ogg', 'audio/ogg', opts)
+      end
+    end
+
+    def encode_mp3
+      opts = ""
+      if generic_file.mime_type == 'audio/mpeg'
+        # Don't re-encode, just copy
+        generic_file.add_file_datastream(generic_file.content.read, :dsid=>'mp3', :mimeType=>'audio/mp3')
+        #generic_file.content.rewind
+      else
+        encode_datastream('mp3', 'audio/mp3', opts)
+      end
+    end
+
+
+end
+

data/lib/sufia/jobs/transcode_video_job.rb

@@ -2,35 +2,23 @@
 # 13 Dec 2012
 # An asyncronous job for transcoding video files using FFMpeg
 
-require 'tmpdir'
-
-class TranscodeVideoJob
-  extend Open3
+class TranscodeVideoJob < FfmpegTranscodeJob
   def queue_name
     :video
   end
 
-  attr_accessor :generic_file_id, :datastream_in
-
-  def initialize(generic_file_id, datastream_in)
-    self.generic_file_id = generic_file_id
-    self.datastream_in = datastream_in
+  def process
+    encode_mp4()
+    encode_webm()
   end
 
-  def run
-    @generic_file = GenericFile.find(generic_file_id)
-    @datastream = @generic_file.datastreams[datastream_in]
-    if @datastream
-      encode_mp4()
-      encode_webm()
-      @generic_file.save!
-    else
-      logger.warn "No datastream for transcoding!!!!! pid: #{generic_file_id} dsid: #{datastream_in}"
-    end
-  end
+  private 
 
   def encode_webm
-    opts = "#{size_attributes}  -b:v 345k -acodec libvorbis #{audio_attributes}"
+    # -g 30 enforces keyframe generation every second (30fps)
+    # -b:v is the video bitrate
+    # -acodec is the audio codec
+    opts = "#{size_attributes}  -g 30 -b:v 345k -acodec libvorbis #{audio_attributes}"
     encode_datastream('webm', 'video/webm', opts)
   end
 
@@ -47,33 +35,5 @@ class TranscodeVideoJob
   def audio_attributes 
     "-ac 2 -ab 96k -ar 44100"
   end
-
-  def encode_datastream(dest_dsid, mime_type, options)
-    file_suffix = dest_dsid
-    out_file = nil
-    output_file = Dir::Tmpname.create('sufia'){} + ".#{file_suffix}"
-    @datastream.to_tempfile do |f|
-      self.class.encode(f.path, options, output_file)
-    end
-    out_file = File.open(output_file, "rb")
-    @generic_file.add_file_datastream(out_file.read, :dsid=>dest_dsid, :mimeType=>mime_type)
-    File.unlink(output_file)
-  end
-
-  # TODO tmp file for output
-  def self.encode(path, options, output_file)
-    command = "#{ffmpeg_path} -y -i #{path} #{options} #{output_file}"
-    stdin, stdout, stderr, wait_thr = popen3(command)
-    stdin.close
-    out = stdout.read
-    stdout.close
-    err = stderr.read
-    stderr.close
-    raise "Unable to execute command \"#{command}\"\n#{err}" unless wait_thr.value.success?
-  end
-
-  def self.ffmpeg_path
-    Sufia::Engine.config.ffmpeg_path
-  end
 end