talia_core 0.4.0

data/lib/talia_core/background_jobs/progress_job.rb

@@ -0,0 +1,68 @@
+module TaliaCore
+  module BackgroundJobs
+    
+    # Helper table to track the current status of a long-running task
+    class ProgressJob < ActiveRecord::Base
+      
+      # Minimum interval for database updates. 
+      DB_UPDATE_INTERVAL = 2
+      
+      validates_numericality_of :job_id, :only_integer => true
+      validates_numericality_of :processed_count, :only_integer => true, :greater_than_or_equal => 0
+      validates_numericality_of :item_count, :only_integer => true, :greater_than => 0
+      
+      # Create a new progress job
+      def self.create_progress!(job_id, message = '', item_count = 1)
+        job_prog = new(:job_id => job_id, :progress_message => message, :item_count => item_count, :processed_count => 0)
+        job_prog.save!
+        job_prog
+      end
+      
+      # Clears the progress for processes no longer active
+      def self.clear
+        find(:all).each do |job_prog|
+          delete(job_prog.id) if(!Bg.table.job.exists?(job_id) || Bg.tablejob.find(job_id).finished?)
+        end
+      end
+      
+      # Increments the number of processed items. To avoid flooding the db, the same object will
+      # only save this value at most all DB_UPDATE_INTERVAL seconds. Will return true if the
+      # element was saved, false otherwise. 
+      def inc(inc_value = 1)
+        pc_old = self.processed_count
+        self.processed_count = pc_old + inc_value
+        if(!@last_update || ((Time.now - @last_update) > DB_UPDATE_INTERVAL))
+          save!
+          @last_update = Time.now
+          true
+        else
+          false
+        end
+      end
+      
+      def finish
+        self.processed_count = self.item_count
+        save!
+      end
+      
+      # The percentage completed
+      def percentage
+        [((self.processed_count * 100) / self.item_count), 100].min
+      end
+      
+      # Elapsed time in seconds
+      def elapsed
+        return unless(self.started_at)
+        Time.now - started_at
+      end
+      
+      # Returns the estimated time remaining on the current job
+      def eta
+        ((elapsed * 100) / percentage) - elapsed
+      end
+      
+      
+    end
+    
+  end
+end

data/lib/talia_core/collection.rb

@@ -0,0 +1,13 @@
+module TaliaCore
+
+  class Collection < DcResource
+
+    has_rdf_type N::DCNS.Collection
+    has_rdf_type N::SKOS.Collection
+    has_rdf_type N::DCMIT.Collection
+
+    simple_property :items, N::DCNS.hasPart
+
+  end
+
+end

data/lib/talia_core/data_types/data_loader.rb

@@ -0,0 +1,92 @@
+module TaliaCore
+  module DataTypes
+
+    # Used for attaching data items by laoding them from files and/or URLs. This will also attempt to
+    # create the correct data type for any given file.
+    module DataLoader
+
+      module ClassMethods
+
+        # Load the data from the given URL. If the mime_type option is given, the handler will always
+        # use the parameter for the MIME type (which can be a Mime::Type object or a string like
+        # 'text/html', or a mime type symbol).
+        #
+        # *Attention:* This method will return an *Array* of data objects. This is for those cases,
+        # where a single data file will be processed into multiple objects (e.g. IIP data).
+        #
+        # If the mime type is not given, the method will attempt to automatically determine the
+        # type, using the file extension or the response code.
+        #
+        # The :http_credentials option may be used to pass login information for http like this:
+        #   http_credentials = { :http_basic_authentication => [login, password] }
+        # See the openuri documentation for more.
+        #
+        # You may pass the :location parameter to identify the "location" value for the new
+        # data record. In general, this is not neccessary. If the location is given, the system
+        # will *always* attempt to determine the mime type through the location parameter, unless
+        # an explicit mime type is given.
+        def create_from_url(uri, options = {})
+          mime_type = options[:mime_type] || options['mime_type']
+          location = options[:location] || options['location']
+          # If a Mime type is given, use that.
+          if(mime_type)
+            mime_type = Mime::Type.lookup(mime_type) if(mime_type.is_a?(String))
+          elsif(location)
+            mime_type = Mime::Type.lookup_by_extension(File.extname(location)[1..-1])
+          end
+
+          data_records = []
+
+          # Remove file:// from URIs to allow standard file URIs
+          uri = file_url(uri)
+          
+          # We have diffent code paths for local and remote files. This is mainly because
+          # the system will try to not open local files at all and just copy them around -
+          # which will greatly speed up the operation.
+          is_file = File.exist?(uri)
+
+          location ||= File.basename(uri) if(is_file)
+          # If we have a "standard" uri, we cut off at the last slash (the
+          # File.basename would use the system file separator)
+          location ||= uri.rindex('/') ? uri[(uri.rindex('/') + 1)..-1] : uri
+
+          if(is_file)
+            mime_type ||= Mime::Type.lookup_by_extension(File.extname(location)[1..-1])
+            open_and_create(mime_type, location, uri, true)
+          else
+            open_from_url(uri, options[:http_credentials]) do |io|
+              mime_type ||= Mime::Type.lookup(io.content_type)
+              open_and_create(mime_type, location, io, false)
+            end
+          end
+
+        end
+
+        private
+
+        # The main loader. This will handle the lookup from the mapping and the creating of the
+        # data objects. Depending on the setting of is_file, the source parameter will be interpreted
+        # in a different way. If it is a file, the file name will be passed in here. If it is
+        # a URL, the method will receive the io object of the open connection as the source
+        def open_and_create(mime_type, location, source, is_file)
+          data_type = loader_type_from(mime_type)
+          if(data_type.is_a?(Symbol))
+            raise(ArgumentError, "No handler found for loading: #{data_type}") unless(self.respond_to?(data_type))
+            self.send(data_type, mime_type, location, source, is_file)
+          else
+            raise(ArgumentError, "Registered handler for loading must be a method symbol or class. (#{data_type})") unless(data_type.is_a?(Class))
+            data_record = data_type.new
+            is_file ? data_record.create_from_file(location, source) : data_record.create_from_data(location, source)
+            data_record.mime = mime_type.to_s
+            data_record.location = location
+            [ data_record ]
+          end
+        end
+
+
+        
+      end # Class methods end
+      
+    end # Closing modules and such
+  end
+end

data/lib/talia_core/data_types/data_record.rb

@@ -0,0 +1,105 @@
+module TaliaCore
+  
+  # Contains all data types that are handled by the Talia system. All data elements
+  # should be subclasses of DataRecord
+  module DataTypes
+   
+    # ActiveRecord interface to the data record in the database
+    class DataRecord < ActiveRecord::Base
+      # Attention: These need to come before the extends, otherwise it'll blow the
+      # tests
+      belongs_to :source, :class_name => 'TaliaCore::ActiveSource'
+      before_create :set_mime_type # Mime type must be saved before the record is written
+      
+      extend MimeMapping
+
+      # Declaration of main abstract methods ======================
+      # Some notes: every subclasses of DataRecord must implement
+      #             at least the following methods
+      # See also:   single-table inheritance    
+
+      # returns all bytes in the object as an array of unsigned integers
+      def all_bytes
+      end
+    
+      # Returns all_bytes as an binary string
+      def content_string
+        all_bytes.pack('C*') if(all_bytes)
+      end
+
+      # returns the next byte from the object, or nil at EOS  
+      def get_byte(close_after_single_read=false)
+      end
+    
+      # returns the current position of the read cursor
+      def position
+      end
+    
+      # adjust the position of the read cursor
+      def seek(new_position)
+      end
+
+      # returns the size of the object in bytes
+      def size
+      end
+    
+      # reset the cursor to the initial state
+      def reset
+      end
+      
+      def extract_mime_type(location)
+        # Lookup the mime type for the extension (removing the dot
+        # in front of the file extension) Works only for the file
+        # types supported by Rails' Mime class.
+        Mime::Type.lookup_by_extension((File.extname(location).downcase)[1..-1]).to_s
+      end
+      
+      def mime_type
+        self.mime
+      end
+    
+      attr_accessor :temp_path
+      
+      # class methods ============================================
+      class << self
+
+        # Find all data records about a specified source    
+        def find_data_records(id)
+          find(:all, :conditions => { :source_id => id })
+        end
+
+        def find_by_type_and_location!(source_data_type, location)
+          # TODO: Should it directly instantiate the STI sub-class?
+          # In this case we should use the following line instead.
+          #
+          # source_data = source_data_type.classify.constantize.find_by_location(location, :limit => 1)
+          #
+          source_data = self.find(:first, :conditions => ["type = ? AND location = ?", source_data_type.camelize, location])
+          raise ActiveRecord::RecordNotFound if source_data.nil?
+          source_data
+        end
+
+      end
+
+      private
+      
+    
+      # Returns demodulized class name.
+      def class_name
+        self.class.name.demodulize
+      end
+    
+      
+      # set mime type if it hasn't been assigned already
+      def set_mime_type
+        assit(!self.location.blank?, "Location for #{self} should not be blank")
+        if(!self.location.blank? && self.mime.blank?)
+          # Set mime type for the record
+          self.mime = extract_mime_type(self.location)
+          assit_not_nil(self.mime, "Mime should not be nil (location was #{self.location})!")
+        end
+      end
+      
+    end
+  end
+end

data/lib/talia_core/data_types/delayed_copier.rb

@@ -0,0 +1,76 @@
+require 'fileutils'
+
+module TaliaCore
+  module DataTypes
+    
+    # This is used for "delayed" copy operations. Basically this will created
+    # a file called "delayed_copy.sh" in the RAILS_ROOT, which can later be
+    # run as a bash script. This will allow the user to run the
+    # copy operation and be potentially faster than using the builtin copy
+    # operations (especially using JRuby)
+    class DelayedCopier
+      
+      # Returns (and creates, if necessary) the file to write the delayed 
+      # copy operations to
+      def self.delayed_copy_file
+        @delayed_copy_file ||= begin
+          backup_file if(File.exists?(delay_file_name))
+          file = File.open(delay_file_name, 'w')
+          file.puts('#!/bin/bash')
+          file
+        end
+      end
+      
+      def self.cp(source, target)
+        unless(dir_seen?(File.expand_path(target)))
+          mkdir_string = 'mkdir -vp "'
+          mkdir_string << File.dirname(File.expand_path(target))
+          mkdir_string << '"'
+          delayed_copy_file.puts(mkdir_string)
+        end
+        cp_string = 'cp -v "'
+        cp_string << File.expand_path(source)
+        cp_string << '" "'
+        cp_string << File.expand_path(target)
+        cp_string << '"'
+        delayed_copy_file.puts(cp_string)
+        delayed_copy_file.flush
+      end
+      
+      # Close the delayed copy file
+      def self.close
+        if(@delayed_copy_file)
+          @delayed_copy_file.close
+          @delayed_copy_file = nil
+        end
+      end
+      
+      private
+      
+      
+      def self.dir_seen?(directory)
+        @seen_dirs = {}
+        return true if(@seen_dirs[directory])
+        @seen_dirs[directory] = true
+        false
+      end
+      
+      # The file name for the delayed copy
+      def self.delay_file_name
+        File.join(RAILS_ROOT, 'delayed_copy.sh')
+      end
+      
+      # Backs up an existing file if necessary
+      def self.backup_file
+        round = 1
+        file_name = 'nil'
+        while(File.exists?(file_name = File.join(RAILS_ROOT, "delayed_copy_old_#{round}.sh")))
+          round += 1
+        end
+        FileUtils.mv(delay_file_name, file_name) 
+      end
+      
+    end
+    
+  end
+end

data/lib/talia_core/data_types/file_record.rb

@@ -0,0 +1,59 @@
+module TaliaCore
+  module DataTypes
+    
+    # Base class for all data records that use a plain file for data storage
+    class FileRecord < DataRecord
+      include FileStore
+      extend FileStore::ClassMethods
+      
+      include PathHelpers
+      extend PathHelpers::ClassMethods
+      
+      include TempFileHandling
+      extend TempFileHandling::ClassMethods
+      
+      include DataLoader
+      extend DataLoader::ClassMethods
+      extend IipLoader
+      extend TaliaUtil::IoHelper # Data IO for class methods
+      
+      after_save :save_attachment, :write_file_after_save
+      
+      before_destroy :destroy_attachment
+      
+      # Returns and, if necessary, creates the file for "delayed" copy operations
+      
+      # returns all bytes in the object as an array
+      def all_bytes
+        read_all_bytes
+      end
+      
+      # returns the next byte from the object, or nil at EOS
+      def get_byte(close_after_single_read=false)
+        next_byte(close_after_single_read)
+      end
+
+      # returns the current position of the read cursor (binary access)
+      def position
+        return (@position != nil) ? @position : 0
+      end
+   
+      # reset the cursor to the initial state
+      def reset
+        set_position(0)
+      end
+    
+      # set the new position of the reding cursors
+      def seek(new_position)
+        set_position(new_position)
+      end
+    
+      # returns the size of the object in bytes
+      def size
+        data_size
+      end
+      
+    end
+    
+  end
+end