talia_core 0.4.0

data/lib/talia_core/data_types/file_store.rb

@@ -0,0 +1,306 @@
+require 'fileutils'
+
+module TaliaCore
+  module DataTypes
+    module FileStore
+  
+      # the handle for the file
+      @file_handle = nil
+      # position of the reading cursors
+      @position    = 0      
+   
+      # Class for data paths
+      class DataPath < String ; end
+      
+      module ClassMethods
+      
+        # Find or create a record for the given location and source_id, then it saves the given file.
+        def find_or_create_and_assign_file(params)
+          data_record = self.find_or_create_by_location_and_source_id(extract_filename(params[:file]), params[:source_id])
+          data_record.file = params[:file]
+          data_record.save # force attachment save and it also saves type attribute.
+        end
+      
+      end
+    
+      # This will create the data object from a given file. This will simply move the
+      # given file to the correct location upon save. This will avoid multiple
+      # read/write operations during import.
+      #
+      # The original file must not be touched by external processes until the
+      # record is saved.
+      #
+      # If the delete_original flag is set, the original file will be removed
+      # on save
+      def create_from_file(location, file_path, delete_original = false)
+        close_file
+        self.location = location
+        @file_data_to_write = DataPath.new(file_path)
+        @delete_original_file = delete_original
+      end
+  
+      # Add data as string into file
+      def create_from_data(file_location, data, options = {})
+        # close file if opened
+        close_file
+    
+        # Set the location for the record
+        self.location = file_location
+    
+        if(data.respond_to?(:read))
+          @file_data_to_write = data.read
+        else
+          @file_data_to_write = data
+        end
+    
+      end
+      
+      # returns the complete text
+      def all_text
+        if(!is_file_open?)
+          open_file
+        end
+        @file_handle.read(self.size)
+      end
+  
+      # This is a placeholder in case file is used in a form.
+      def file() nil; end
+    
+      # Assign the file data (<tt>StringIO</tt> or <tt>File</tt>).
+      def file=(file_data)
+        return nil if file_data.nil? || file_data.size == 0 
+        self.assign_type file_data.content_type
+        self.location = file_data.original_filename
+        if file_data.is_a?(StringIO)
+          file_data.rewind
+          self.temp_data = file_data.read
+        else
+          self.temp_path = file_data.path
+        end
+        @save_attachment = true
+      end
+    
+      def write_file_after_save 
+        # check if there are data to write
+        return unless(@file_data_to_write)
+    
+        # check if file already exists
+#        raise(RuntimeError, "File already exists: #{file_path}") if(File.exists?(file_path))
+
+        begin
+          self.class.benchmark("\033[36m\033[1m\033[4mFileStore\033[0m Saving file for #{self.id}") do
+            # create data directory path
+            FileUtils.mkdir_p(data_directory)
+    
+            if(@file_data_to_write.is_a?(DataPath))
+              copy_data_file
+            else
+              save_cached_data
+            end
+          
+            @file_data_to_write = nil
+          end
+        rescue Exception => e
+          assit_fail("Exception on writing file #{self.location}: #{e}")
+        end
+
+      end
+   
+      # Return true if the specified data file is open, false otherwise
+      def is_file_open?
+        (@file_handle != nil)
+      end
+       
+      
+      # Assign the STI subclass, perfoming a mime-type lookup.
+      def assign_type(content_type)
+        self.type = self.class.class_type_from(content_type).name
+      end
+      
+      
+      # private methods ==================================================================
+      private
+      
+      # This saves the cached data from the file creation
+      def save_cached_data
+        # open file for writing
+        @file_handle = File.open(file_path, 'w')
+      
+        # write data string into file
+        @file_handle << (@file_data_to_write.respond_to?(:read) ? @file_data_to_write.read : @file_data_to_write)
+    
+        # close file
+        close_file
+    
+      end
+      
+      # This copies the data file with which this object was created to the
+      # actual storage lcoation
+      def copy_data_file
+        copy_or_move(@file_data_to_write, file_path)
+      end
+      
+      # Open a specified file name and return a file handle.
+      # If the file is already opened, return the file handle
+      def open_file(file_name = file_path, options = 'rb')
+        # chek if the file name really exists, otherwise raise an exception
+        if !File.exists?(file_name)
+          raise(IOError, "File #{file_name} could not be opened.", caller)
+        end
+    
+        # try to open the specified file if is not already open
+        if @file_handle == nil
+          @file_handle = File.open(file_name, options)
+      
+          # check and set the initial position of the reading cursors.
+          # It's necessary to do this thing because we don't know if the user
+          # has specified the initial reading cursors befort starting working on file
+          @position ||= @file_handle.pos
+      
+          @file_handle.pos = @position
+        end
+      end
+
+      # Close an already opened file
+      def close_file
+        if is_file_open?
+          @file_handle.close
+      
+          # reset 'flags' variables and position
+          @file_handle = nil
+          @position    = 0
+        end
+      end
+
+      # Read all bytes from a file  
+      def read_all_bytes
+        # 1. Open file with option "r" (reading) and "b" (binary, useful for window system)
+        open_file
+      
+        # 2. Read all bytes
+        begin
+          bytes = @file_handle.read(self.size).unpack("C*")
+          return bytes
+        rescue
+          # re-raise system the excepiton
+          raise
+          return nil
+        ensure
+          # 3. Close the file
+          close_file
+        end
+      end
+
+      # return the next_byte
+      def next_byte(close)
+        if !is_file_open?
+          open_file
+        end
+    
+        begin
+          current_byte = @file_handle.getc
+      
+          if current_byte == nil or close
+            close_file
+          else
+            @position += 1
+          end
+
+          return current_byte
+        rescue
+          # re-raise system the excepiton
+          raise
+          close_file
+          return nil
+        end
+    
+      end
+
+      # Copy or move the source file to the target. Working around all the
+      # things that suck in JRuby. This will honour two environment settings:
+      # 
+      # * delay_file_copies - will not copy the files, but create a batch file
+      #     so that the copy can be done later. Uses the DelayedCopier class.
+      # * fast_copies - will use the "normal" copy method from FileUtils that
+      #     is faster. Since it crashed the system for us, the default is to
+      #     use a "safe" workaround. The workaround is probably necessary for
+      #     JRuby only.
+      def copy_or_move(original, target)
+        if(@delete_original_file)
+          FileUtils.move(original, target)
+        else
+          # Delay can be enabled through enviroment
+          if(delay_copies)
+            DelayedCopier.cp(original, target)
+          elsif(fast_copies)
+            FileUtils.copy(original, target)
+          else
+            # Call the copy as an external command. This is to work around the
+            # crashes that occurred using the builtin copy
+            from_file = File.expand_path(original)
+            to_file = File.expand_path(target)
+            system_success = system("cp '#{from_file}' '#{to_file}'")
+            raise(IOError, "copy error '#{from_file}' '#{to_file}'") unless system_success
+          end
+        end
+      end
+      
+      
+      # Returns true if the 'delayed write' is enabled in the environment
+      def delay_copies
+        ENV['delay_file_copies'] == 'true' || ENV['delay_file_copies'] == 'yes'
+      end
+        
+      # Returns true if the 'fast copy' is enabled in the environment.
+      # Otherwise the class will use a workaround that is less likely to 
+      # crash the whole system using JRuby.
+      def fast_copies
+        ENV['fast_copies'] == 'true' || ENV['fast_copies'] == 'yes'
+      end
+      
+      # Return the data size
+      def data_size
+        File.size(file_path)
+      end
+
+      # set the position of the reading cursor
+      def set_position(position)
+        if (position != nil and position =~ /\A\d+\Z/)
+          if (position < size)
+            set_position(position)
+          else
+            raise(IOError, 'Position out of range', caller)
+          end
+        else
+          raise(IOError, 'Position not valid. It must be an integer')
+        end
+      end
+    
+      # Check if the attachment should be saved.
+      def save_attachment?
+        @save_attachment
+      end
+    
+      # Save the attachment, copying the file from the temp_path to the data_path.
+      def save_attachment
+        return unless save_attachment?
+        save_file
+        @save_attachment = false
+        true
+      end
+
+      # Destroy the attachment
+      def destroy_attachment
+        FileUtils.rm(full_filename) if File.exists?(full_filename)
+      end
+
+      # Save the attachment on the data_path directory.
+      def save_file
+        FileUtils.mkdir_p(File.dirname(full_filename))
+        FileUtils.cp(temp_path, full_filename)
+        FileUtils.chmod(0644, full_filename)
+      end
+
+    end
+  end
+end

data/lib/talia_core/data_types/iip_data.rb

@@ -0,0 +1,153 @@
+module TaliaCore
+  module DataTypes
+    
+    # Class to manage IIP Image data type
+    class IipData < FileRecord
+      
+      # Returns the IIP server configured for the application
+      def self.iip_server_uri
+        TaliaCore::CONFIG['iip_server_uri'] ||= 'http://localhost/fcgi-bin/iipsrv.fcgi'
+      end
+      
+      # This is the mime type for the thumbnail - always tiff
+      def set_mime_type
+        self.mime = 'image/gif'
+      end
+      
+      alias :get_thumbnail :all_bytes
+     
+      # Create from existing thumb and pyramid images
+      def create_from_existing(thumb, pyramid, delete_originals = false)
+        @file_data_to_write = [thumb, pyramid]
+        @delete_original_file = delete_originals
+        self.location = ''
+      end
+      
+      # return IIP Server Path
+      def iip_server_path
+        self.location
+      end
+      
+      def write_file_after_save
+        return unless(@file_data_to_write)
+        
+        # Check if we have the images already given, in this case we prepare
+        # them and call the super method
+        return super if(direct_write!)
+        
+        # create name for orginal temp file and destination temp file
+        original_file_path, orig_is_temp = prepare_original_file
+        will_delete_source = orig_is_temp || @delete_original_file
+        destination_thumbnail_file_path = File.join(Dir.tmpdir, "thumbnail_#{random_tempfile_filename}.gif")
+        
+        begin # Begin the file creation operation
+          self.class.benchmark("\033[36mIipData\033[0m Making thumb and pyramid for #{self.id}", Logger::INFO) do
+          
+            TaliaUtil::ImageConversions::create_thumb(original_file_path, destination_thumbnail_file_path)
+            create_pyramid(original_file_path)
+        
+            # Run the super implementation for the thumbnail
+            # We will simply tell the system that we have to move the newly create
+            # thumb file
+            @file_data_to_write = DataPath.new(destination_thumbnail_file_path)
+            @delete_original_file = true
+          
+          end # end benchmarking
+          super
+          
+        ensure
+          # delete teƒmp files
+          File.delete original_file_path if(File.exists?(original_file_path) && will_delete_source)
+        end
+      end
+      
+      
+      # Checks if we have file paths given to directly copy thum and image file.
+      # Will always return true if such paths were given.
+      def direct_write!
+        return false unless(@file_data_to_write.kind_of?(Array))
+        
+        thumb, pyramid = @file_data_to_write
+        self.class.benchmark("\033[36mIipData\033[0m Direct write for #{self.id}", Logger::INFO) do
+          prepare_for_pyramid
+        
+          copy_or_move(pyramid, get_iip_root_file_path)
+        
+        end # end benchmark
+          
+        @file_data_to_write = DataPath.new(thumb)
+        
+        true
+      end
+
+      # This prepares the original file that needs to be converted. This will
+      # see if the data to be written is binary data or a file path. If this
+      # is binary data, it will create a temporary file on the disk.
+      #
+      # This returns an array with two elements: The name of the file to 
+      # be used (a file system path) and a flag indicating if the file is
+      # a temporary file or not.
+      def prepare_original_file
+        if(@file_data_to_write.is_a?(DataPath))
+          [@file_data_to_write, false]
+        else
+          temp_file = File.join(Dir.tmpdir, "original_#{random_tempfile_filename}")
+          # write the original file
+          File.open(temp_file, 'w') do |original_file|
+            if(@file_data_to_write.respond_to?(:read))
+              original_file << @file_data_to_write.read
+            else
+              original_file << @file_data_to_write
+            end
+          end
+          
+          [temp_file, true]
+        end
+      end
+      
+      # Prepare for copying or creating the pyramid image
+      def prepare_for_pyramid
+        # set location
+        self.location = get_iip_root_file_path(true)
+        
+        # create data directory path
+        FileUtils.mkdir_p(iip_root_directory)
+      end
+      
+      # Creates the pyramid image for IIP by running the configured system
+      # command. This automatically creates the file in the correct location 
+      # (IIP root)
+      def create_pyramid(source)
+        # check if file already exists
+        raise(IOError, "File already exists: #{get_iip_root_file_path}") if(File.exists?(get_iip_root_file_path))
+       
+        prepare_for_pyramid
+
+        TaliaUtil::ImageConversions::create_pyramid(source, get_iip_root_file_path)
+      end
+      
+      # Return the iip root directory for a specific iip image file
+      def iip_root_directory(relative = false)
+        if relative == false
+          File.join(TaliaCore::CONFIG["iip_root_directory_location"], ("00" + self.id.to_s)[-3..-1])
+        else
+          File.join(("00" + self.id.to_s)[-3..-1])
+        end
+      end
+      
+      # Return the full file path related to the data directory
+      def get_iip_root_file_path(relative = false)
+        File.join(iip_root_directory(relative), self.id.to_s + '.tif')
+      end
+      
+      private
+  
+      # Generates a unique filename for a Tempfile. 
+      def random_tempfile_filename
+        "#{rand 10E16}"
+      end
+      
+    end
+    
+  end
+end

data/lib/talia_core/data_types/iip_loader.rb

@@ -0,0 +1,127 @@
+module TaliaCore
+  module DataTypes
+
+    # Special module that contains the DataLoader methods to create IIP image data
+    module IipLoader
+
+
+      # Loads an image for the given file. This is a tad more complex than loading
+      # the data into a data record: It will create both an IIP data object and
+      # an Image data object. 
+      def create_iip(mime_type, location, source, is_file)
+        iip_record = TaliaCore::DataTypes::IipData.new
+        image_record = TaliaCore::DataTypes::ImageData.new
+        records = [iip_record, image_record]
+        return records if(is_file && prepare_image_from_existing!(iip_record, image_record, source, location))
+
+        if(convert_original?(mime_type))
+          # We have an image that needs to be converted
+          open_original_image(source, is_file, mime_type) do |io|
+            create_from_stream(location, io, records)
+            image_record.location = orig_location(location)
+          end
+        else
+          if(is_file)
+            create_from_files(location, source, records) 
+          else 
+            create_from_stream(location, source, records)
+          end
+        end
+        # IipRecord is always a (multi-leve) tiff
+        iip_record.mime = 'image/tiff'
+
+        records
+      end
+
+      # Rewrite the file location for original image files (to .png)
+      def orig_location(location)
+        File.basename(location, File.extname(location)) + '.png'
+      end
+
+      # Indicates if the given mime type requires a conversion for the
+      # original image
+      def convert_original?(mime_type)
+        !([:jpeg, :png].include?(mime_type.to_sym))
+      end
+
+      # Create the elements from a file
+      def create_from_files(location, file, records)
+        records.each { |rec| rec.create_from_file(location, file) }
+      end
+
+      # Create the elements from a stream
+      def create_from_stream(location, io, records)
+        data = io.read
+        records.each { |rec| rec.create_from_data(location, data)}
+      end
+
+      # Attempts to create an IipData object with pre-prepared images if possible
+      # Returns true if (and only if) the object has been created with existing
+      # data. Always fals if the data_record is not an IipData object or the
+      # :prepared_images option is not set.
+      def prepare_image_from_existing!(iip_record, image_record, url, location)
+        return false unless(iip_record.is_a?(TaliaCore::DataTypes::IipData) && image_record.is_a?(TaliaCore::DataTypes::ImageData))
+        return false unless((prep = ENV['prepared_images']) && prep.to_s.downcase != 'no' && prep.to_s.downcase != 'false')
+
+        file_ext = File.extname(url)
+        file_base = File.basename(url, file_ext)
+
+        thumb_file = File.join( ENV['prepared_images'], 'thumbs', "#{file_base}.gif")
+        pyramid_file = File.join( ENV['prepared_images'], 'pyramids', "#{file_base}.tif")
+        orig_file_pattern = File.join(ENV['prepared_images'], 'originals', "#{file_base}.*")
+        # We need to fix the pattern, also the Dir[] doesn't like unescaped brackets
+        orig_file_pattern.gsub!(/\[/, '\\[')
+          orig_file_pattern.gsub!(/\]/, '\\]')
+          orig_file_l = Dir[orig_file_pattern]
+          raise(ArgumentError, 'Original find not found for ' + url) unless(orig_file_l.size > 0)
+          orig_file = orig_file_l.first
+          assit_block { %w(.jpg .jpeg .png).include?(File.extname(orig_file).downcase) }
+
+          iip_record.create_from_existing(thumb_file, pyramid_file)
+          image_record.create_from_file(location, orig_file)
+
+          true
+        end
+
+        # Little helper to decide how to open the original image
+        def open_original_image(thing, is_file, current_type, &block)
+          if(is_file)
+            open_original_image_file(thing, &block) 
+          else
+            open_original_image_stream(thing, current_type, &block)
+          end
+        end
+
+        # Same as open_original_image_file, but getting the data from a stream.
+        def open_original_image_stream(io, type, &block)
+          # First load this from the web to a temp file
+          tempfile = File.join(Dir.tmpdir, "talia_down_#{rand 10E16}.#{type.to_sym}")
+          begin
+            File.open(tempfile, 'w') do |fio|
+              fio << io.read # Download the file
+            end
+            assit(File.exist?(tempfile))
+            open_original_image_file(tempfile, &block)
+          ensure
+            FileUtils.rm(tempfile) if(File.exist?(tempfile))
+          end
+        end
+
+        # Opens the "original" image for the given file. This will convert the
+        # image to PNG image and the yield the io object for the PNG.
+        def open_original_image_file(filename)
+          converted_file = File.join(Dir.tmpdir, "talia_convert_#{rand 10E16}.png")
+          begin
+            TaliaUtil::ImageConversions.to_png(filename, converted_file)
+            File.open(converted_file) do |io|
+              yield(io)
+            end
+          ensure
+            FileUtils.rm(converted_file) if(File.exist?(converted_file))
+          end
+        end
+
+
+      end # Ending modules and such
+    end
+  end

data/lib/talia_core/data_types/image_data.rb

@@ -0,0 +1,32 @@
+module TaliaCore
+  module DataTypes
+    
+    # Class to manage image data type
+    class ImageData < FileRecord
+      
+      # return the mime_type for a file
+      def extract_mime_type(location)
+        case File.extname(location).downcase
+        when '.bmp'
+          'image/bmp'
+        when '.cgm'
+          'image/cgm'
+        when '.fit', '.fits'
+          'image/fits'
+        when '.g3'
+          'image/g3fax'
+        when '.gif'
+          'image/gif'
+        when '.jpg', '.jpeg', '.jpe'
+          'image/jpeg'
+        when '.png'
+          'image/png'
+        when '.tif', '.tiff'
+          'image/tiff'
+        end
+      end
+       
+    end
+    
+  end
+end

data/lib/talia_core/data_types/media_link.rb

@@ -0,0 +1,19 @@
+module TaliaCore
+  module DataTypes
+    
+    # Data class that contains just a link to a remote data element (and no
+    # locally stored data at all). The uri for the link is stored in the location
+    # field
+    class MediaLink < DataRecord
+      
+      def all_bytes
+        raise(RuntimeError, "Media Links have no data")
+      end
+      
+      def get_byte(close_after_single_read=false)
+        raise(RuntimeRror, "Media Links have no data")
+      end
+      
+    end
+  end
+end

data/lib/talia_core/data_types/mime_mapping.rb

@@ -0,0 +1,45 @@
+module TaliaCore
+  module DataTypes
+    
+    # Mapping from Mime types to data classes and importing methods. Currently uses a fixed 
+    # default mapping
+    module MimeMapping
+      
+      def mapping_for(mime_type)
+        mime_type = Mime::Type.lookup(mime_type) if(mime_type.is_a?(String))
+        mapping = mapping_hash[mime_type.to_sym]
+        raise(ArgumentError, "No data class registered for type #{mime_type.inspect}") unless(mapping)
+        mapping
+      end
+      
+      # Gets the data class for the given mime type
+      def class_type_from(mime_type)
+        mapping_for(mime_type)[:type]
+      end
+      
+      def loader_type_from(mime_type)
+        map = mapping_for(mime_type)
+        map[:loader] || map[:type]
+      end
+      
+      # Currently this is only the default mapping
+      def mapping_hash
+        @mapping ||= {
+          :xml => { :type => XmlData },
+          :html =>{ :type => XmlData },
+          :tei => { :type => XmlData },
+          :hnml => { :type => XmlData },
+          :jpeg => { :type => ImageData, :loader => :create_iip },
+          :tiff => { :type => ImageData, :loader => :create_iip },
+          :png => { :type => ImageData, :loader => :create_iip },
+          :gif => { :type => ImageData, :loader => :create_iip },
+          :pdf => { :type => PdfData },
+          :text => { :type => SimpleText }
+        }
+      end
+      
+      
+    end
+    
+  end
+end