sufia-models 0.0.1

data/lib/sufia/models.rb

@@ -0,0 +1,11 @@
+require "sufia/models/version"
+require "sufia/models/engine"
+
+module Sufia
+  module Models
+    extend ActiveSupport::Autoload
+
+    autoload :Resque, 'models/queue/resque'
+    autoload :User, 'models/user'
+  end
+end

data/lib/sufia/models/active_fedora/redis.rb

@@ -0,0 +1,49 @@
+module ActiveFedora
+  class UnsavedDigitalObject
+    def assign_pid
+      @pid ||= Sufia::IdService.mint
+    end
+  end
+
+  class Base
+    def stream
+      Nest.new(self.class.name, $redis)[to_param]
+    rescue
+      nil
+    end
+
+    def self.stream
+      Nest.new(name, $redis)
+    rescue
+      nil
+    end
+
+    def events(size=-1)
+      stream[:event].lrange(0, size).map do |event_id|
+        {
+          action: $redis.hget("events:#{event_id}", "action"),
+          timestamp: $redis.hget("events:#{event_id}", "timestamp")
+        }
+      end
+    rescue
+      []
+    end
+
+    def create_event(action, timestamp)
+      event_id = $redis.incr("events:latest_id")
+      $redis.hmset("events:#{event_id}", "action", action, "timestamp", timestamp)
+      event_id
+    rescue
+      nil
+    end
+
+    def log_event(event_id)
+      stream[:event].lpush(event_id)
+    rescue
+      nil
+    end
+  end
+end
+
+
+

data/lib/sufia/models/active_record/redis.rb

@@ -0,0 +1,56 @@
+ActiveRecord::Base.class_eval do
+  def stream
+    Nest.new(self.class.name, $redis)[to_param]
+  rescue
+    nil
+  end
+
+  def self.stream
+    Nest.new(name, $redis)
+  rescue
+    nil
+  end
+
+  def events(size=-1)
+    stream[:event].lrange(0, size).map do |event_id|
+      {
+        action: $redis.hget("events:#{event_id}", "action"),
+        timestamp: $redis.hget("events:#{event_id}", "timestamp")
+      }
+    end
+  rescue
+    []
+  end
+
+  def profile_events(size=-1)
+    stream[:event][:profile].lrange(0, size).map do |event_id|
+      {
+        action: $redis.hget("events:#{event_id}", "action"),
+        timestamp: $redis.hget("events:#{event_id}", "timestamp")
+      }
+    end
+  rescue
+    []
+  end
+
+  def create_event(action, timestamp)
+    event_id = $redis.incr("events:latest_id")
+    $redis.hmset("events:#{event_id}", "action", action, "timestamp", timestamp)
+    event_id
+  rescue
+    nil
+  end
+
+  def log_event(event_id)
+    stream[:event].lpush(event_id)
+  rescue
+    nil
+  end
+
+  def log_profile_event(event_id)
+    stream[:event][:profile].lpush(event_id)
+  rescue
+    nil
+  end
+end
+

data/lib/sufia/models/engine.rb

@@ -0,0 +1,34 @@
+module Curate
+  module Models
+    class Engine < ::Rails::Engine
+      config.autoload_paths += %W(
+        #{config.root}/lib/sufia/models/jobs
+      )
+      initializer "Patch active_fedora" do
+        require 'sufia/models/active_fedora/redis'
+      end
+
+      initializer "Patch active_record" do
+        require 'sufia/models/active_record/redis'
+      end
+
+      initializer 'requires' do
+        require 'sufia/models/model_methods'
+        require 'sufia/models/noid'
+        require 'sufia/models/file_content'
+        require 'sufia/models/file_content/extract_metadata'
+        require 'sufia/models/file_content/versions'
+        require 'sufia/models/generic_file/actions'
+        require 'sufia/models/generic_file/audit'
+        require 'sufia/models/generic_file/characterization'
+        require 'sufia/models/generic_file/export'
+        require 'sufia/models/generic_file/permissions'
+        require 'sufia/models/generic_file/thumbnail'
+        require 'sufia/models/generic_file'
+        require 'sufia/models/user'
+        require 'sufia/models/id_service'
+        require 'sufia/models/solr_document_behavior'
+      end
+    end
+  end
+end

data/lib/sufia/models/file_content.rb

@@ -0,0 +1,9 @@
+module Sufia
+  module FileContent
+    extend ActiveSupport::Autoload
+
+    autoload :ExtractMetadata, 'sufia/models/file_content/extract_metadata'
+    autoload :Versions, 'sufia/models/file_content/versions'
+
+  end
+end

data/lib/sufia/models/file_content/extract_metadata.rb

@@ -0,0 +1,60 @@
+require 'open3'
+module Sufia
+  module FileContent
+    module ExtractMetadata
+      include Open3
+
+      def extract_metadata
+        out = nil
+        to_tempfile do |f|
+          out = run_fits!(f.path)
+        end
+        out
+      end
+
+      def to_tempfile &block
+        return unless has_content?
+        tmp_base = Sufia::Engine.config.temp_file_base
+        f = Tempfile.new("#{pid}-#{dsVersionID}")
+        f.binmode
+        if content.respond_to? :read
+          f.write(content.read)
+        else
+          f.write(content)
+        end
+        f.close
+        content.rewind if content.respond_to? :rewind
+        yield(f)
+        f.unlink
+      end
+
+      # Return true if the content is present
+      # You can override this method if your content is an external datastream
+      def has_content?
+        !content.nil?
+      end
+
+      private 
+
+
+        def run_fits!(file_path)
+            command = "#{fits_path} -i \"#{file_path}\""
+            stdin, stdout, stderr, wait_thr = popen3(command)
+            stdin.close
+            out = stdout.read
+            stdout.close
+            err = stderr.read
+            stderr.close
+            exit_status = wait_thr.value
+            raise "Unable to execute command \"#{command}\"\n#{err}" unless exit_status.success?
+            out
+        end
+
+
+        def fits_path
+          Sufia::Engine.config.fits_path
+        end
+
+      end
+    end
+end

data/lib/sufia/models/file_content/versions.rb

@@ -0,0 +1,23 @@
+module Sufia
+  module FileContent
+    module Versions
+      def get_version(version_id)
+        self.versions.select { |v| v.versionID == version_id}.first
+      end
+
+      def latest_version
+        self.versions.first
+      end
+
+      def version_committer(version)
+        vc = VersionCommitter.where(:obj_id => version.pid,
+                                    :datastream_id => version.dsid,
+                                    :version_id => version.versionID)
+        return vc.empty? ? nil : vc.first.committer_login
+      end
+
+
+    end
+  end
+end
+

data/lib/sufia/models/generic_file.rb

@@ -0,0 +1,183 @@
+require 'datastreams/generic_file_rdf_datastream'
+require 'datastreams/properties_datastream'
+require 'datastreams/file_content_datastream'
+
+module Sufia
+  module GenericFile
+    extend ActiveSupport::Concern
+    extend ActiveSupport::Autoload
+    include Sufia::ModelMethods
+    include Sufia::Noid
+    include Sufia::GenericFile::Thumbnail
+    include Sufia::GenericFile::Export
+    include Sufia::GenericFile::Characterization
+    include Sufia::GenericFile::Audit
+    include Sufia::GenericFile::Permissions
+
+    included do
+      has_metadata :name => "descMetadata", :type => GenericFileRdfDatastream
+      has_metadata :name => "properties", :type => PropertiesDatastream
+      has_file_datastream :name => "content", :type => FileContentDatastream
+      has_file_datastream :name => "thumbnail"
+
+      belongs_to :batch, :property => :is_part_of
+
+      delegate_to :properties, [:relative_path, :depositor], :unique => true
+      delegate_to :descMetadata, [:date_uploaded, :date_modified], :unique => true
+      delegate_to :descMetadata, [:related_url, :based_near, :part_of, :creator,
+                                  :contributor, :title, :tag, :description, :rights,
+                                  :publisher, :date_created, :subject,
+                                  :resource_type, :identifier, :language]
+
+      around_save :characterize_if_changed, :retry_warming
+      before_save :remove_blank_assertions
+      before_destroy :cleanup_trophies
+    end
+
+    def remove_blank_assertions
+      terms_for_editing.each do |key|
+        self[key] = nil if self[key] == ['']
+      end
+    end
+
+
+    def record_version_committer(user)
+      version = content.latest_version
+      # content datastream not (yet?) present
+      return if version.nil?
+      VersionCommitter.create(:obj_id => version.pid,
+                              :datastream_id => version.dsid,
+                              :version_id => version.versionID,
+                              :committer_login => user.user_key)
+    end
+
+    def pdf?
+      ['application/pdf'].include? self.mime_type
+    end
+
+    def image?
+      ['image/png','image/jpeg', 'image/jpg', 'image/jp2', 'image/bmp', 'image/gif'].include? self.mime_type
+    end
+
+    def video?
+      ['video/mpeg', 'video/mp4', 'video/webm', 'video/x-msvideo', 'video/avi', 'video/quicktime', 'application/mxf'].include? self.mime_type
+    end
+
+    def audio?
+      # audio/x-wave is the mime type that fits 0.6.0 returns for a wav file.
+      # audio/mpeg is the mime type that fits 0.6.0 returns for an mp3 file.
+      ['audio/mp3', 'audio/mpeg', 'audio/x-wave', 'audio/x-wav', 'audio/ogg'].include? self.mime_type
+    end
+
+    def persistent_url
+      "#{Sufia::Engine.config.persistent_hostpath}#{noid}"
+    end
+
+    def retry_warming
+        save_tries = 0
+        conflict_tries = 0
+        begin
+          yield
+        rescue RSolr::Error::Http => error
+          save_tries += 1
+          logger.warn "Retry Solr caught RSOLR error on #{self.pid}: #{error.inspect}"
+          # fail for good if the tries is greater than 3
+          raise if save_tries >=3
+          sleep 0.01
+          retry
+        rescue  ActiveResource::ResourceConflict => error
+          conflict_tries += 1
+          logger.warn "Retry caught Active Resource Conflict #{self.pid}: #{error.inspect}"
+          raise if conflict_tries >=10
+          sleep 0.01
+          retry
+        rescue =>error
+          if (error.to_s.downcase.include? "conflict")
+            conflict_tries += 1
+            logger.warn "Retry caught Active Resource Conflict #{self.pid}: #{error.inspect}"
+            raise if conflict_tries >=10
+            sleep 0.01
+            retry
+          else
+            raise
+          end
+        end
+    end
+
+    def cleanup_trophies
+      Trophy.destroy_all(generic_file_id: self.noid)
+    end
+
+    def related_files
+      relateds = begin
+                   self.batch.generic_files
+                 rescue NoMethodError => e
+                   #batch is nil - When would this ever happen?
+                   batch_id = self.object_relations["isPartOf"].first || self.object_relations[:is_part_of].first
+                   return [] if batch_id.nil?
+                   self.class.find(Solrizer.solr_name('is_part_of', :symbol) => batch_id)
+                 end
+      relateds.reject { |gf| gf.pid == self.pid }
+    end
+
+    # Unstemmed, searchable, stored
+    def self.noid_indexer
+      @noid_indexer ||= Solrizer::Descriptor.new(:text, :indexed, :stored)
+    end
+
+    def to_solr(solr_doc={}, opts={})
+      super(solr_doc, opts)
+      solr_doc[Solrizer.solr_name('label')] = self.label
+      solr_doc[Solrizer.solr_name('noid', Sufia::GenericFile.noid_indexer)] = noid
+      solr_doc[Solrizer.solr_name('file_format')] = file_format
+      solr_doc[Solrizer.solr_name('file_format', :facetable)] = file_format
+      return solr_doc
+    end
+
+    def file_format
+      return nil if self.mime_type.blank? and self.format_label.blank?
+      return self.mime_type.split('/')[1]+ " ("+self.format_label.join(", ")+")" unless self.mime_type.blank? or self.format_label.blank?
+      return self.mime_type.split('/')[1] unless self.mime_type.blank?
+      return self.format_label
+    end
+
+    # Redefine this for more intuitive keys in Redis
+    def to_param
+      noid
+    end
+
+    def label=(new_label)
+      @inner_object.label = new_label
+      if self.title.empty?
+        self.title = new_label
+      end
+    end
+
+    def to_jq_upload
+      return {
+        "name" => self.title,
+        "size" => self.file_size,
+        "url" => "/files/#{noid}",
+        "thumbnail_url" => self.pid,
+        "delete_url" => "deleteme", # generic_file_path(:id => id),
+        "delete_type" => "DELETE"
+      }
+    end
+
+    def terms_for_editing
+      terms_for_display -
+       [:part_of, :date_modified, :date_uploaded, :format] #, :resource_type]
+    end
+
+    def terms_for_display
+      self.descMetadata.class.config.keys
+    end
+
+    # Is this file in the middle of being processed by a batch?
+    def processing?
+       return false if self.batch.blank?
+       return false if !self.batch.methods.include? :status
+       return (!self.batch.status.empty?) && (self.batch.status.count == 1) && (self.batch.status[0] == "processing")
+    end
+  end
+end

data/lib/sufia/models/generic_file/actions.rb

@@ -0,0 +1,39 @@
+module Sufia::GenericFile
+  # Actions are decoupled from controller logic so that they may be called from a controller or a background job.
+  module Actions
+    def self.create_metadata(generic_file, user, batch_id)
+
+      generic_file.apply_depositor_metadata(user.user_key)
+      generic_file.date_uploaded = Date.today
+      generic_file.date_modified = Date.today
+      generic_file.creator = user.name
+
+      if batch_id
+        generic_file.add_relationship("isPartOf", "info:fedora/#{Sufia::Noid.namespaceize(batch_id)}")
+      else
+        logger.warn "unable to find batch to attach to"
+      end
+      generic_file.save!
+    end
+    
+    def self.create_content(generic_file, file, file_name, dsid, user)
+      generic_file.add_file(file, dsid, file_name)
+
+      save_tries = 0
+      begin
+        generic_file.save!
+      rescue RSolr::Error::Http => error
+        logger.warn "GenericFilesController::create_and_save_generic_file Caught RSOLR error #{error.inspect}"
+        save_tries+=1
+        # fail for good if the tries is greater than 3
+        raise error if save_tries >=3
+        sleep 0.01
+        retry
+      end
+
+      generic_file.record_version_committer(user)
+      Sufia.queue.push(UnzipJob.new(generic_file.pid)) if generic_file.content.mimeType == 'application/zip'
+      Sufia.queue.push(ContentDepositEventJob.new(generic_file.pid, user.user_key))
+    end
+  end
+end