talia_core 0.5.4 → 0.7.0

data/lib/talia_core/data_types/delayed_copier.rb

@@ -21,7 +21,15 @@ module TaliaCore
         end
       end
       
+      # This writes the "cp" command to the output script. It will also
+      # add a "mkdir" command to create the directory for the target file,
+      # if necessary.
+      #
+      # At the moment,
+      # this will always use UNIX-style "cp" and "mkdir" commands.
       def self.cp(source, target)
+        # We use the << in-place string concenation, 'cause if there
+        # are a lot of files, it really makes a speed difference
         unless(dir_seen?(File.expand_path(target)))
           mkdir_string = 'mkdir -vp "'
           mkdir_string << File.dirname(File.expand_path(target))
@@ -47,7 +55,8 @@ module TaliaCore
       
       private
       
-      
+      # Returns true if the directory has already been seen by
+      # the copier before.
       def self.dir_seen?(directory)
         @seen_dirs = {}
         return true if(@seen_dirs[directory])
@@ -55,12 +64,13 @@ module TaliaCore
         false
       end
       
-      # The file name for the delayed copy
+      # The file name for the delayed copy (the file where the
+      # commands are written out)
       def self.delay_file_name
         File.join(RAILS_ROOT, 'delayed_copy.sh')
       end
       
-      # Backs up an existing file if necessary
+      # Backs up an existing file with delayed commands, if necessary
       def self.backup_file
         round = 1
         file_name = 'nil'

data/lib/talia_core/data_types/file_record.rb

@@ -1,54 +1,65 @@
 module TaliaCore
   module DataTypes
     
-    # Base class for all data records that use a plain file for data storage
+    # Base class for all data records that use a plain file for data storage. This 
+    # implements the DataRecord API so that all byte methods work on a file in the 
+    # File system. 
+    #
+    # Most of the operations are defined in the FileStore module, see there on how
+    # to create and work with file records.
+    #
+    # See the DataLoader and MimeMapping modules to see new file records are 
+    # created automatically, depending on the MIME type.
+    #
+    # The data paths are set automatically by the class, see PathHelpers
+    #
+    # There is also an IipLoader module, that contains the loader mechanism for
+    # creating Iip images - you can also use that as an example to create
+    # new loaders for other file types.
     class FileRecord < DataRecord
       include FileStore
-      extend FileStore::ClassMethods
       
       include PathHelpers
       extend PathHelpers::ClassMethods
       
-      include TempFileHandling
-      extend TempFileHandling::ClassMethods
-      
       include DataLoader
       extend DataLoader::ClassMethods
       extend IipLoader
       extend TaliaUtil::IoHelper # Data IO for class methods
       
-      after_save :save_attachment, :write_file_after_save
+      after_save :write_file_after_save
       
-      before_destroy :destroy_attachment
+      before_destroy :destroy_file
       
       # Returns and, if necessary, creates the file for "delayed" copy operations
       
-      # returns all bytes in the object as an array
+      # Return all bytes from the file as a byte array.
       def all_bytes
         read_all_bytes
       end
       
-      # returns the next byte from the object, or nil at EOS
+      # Returns the next byte from the file (at the position of the
+      # read cursor), or EOS if the end of the file has been reached.
       def get_byte(close_after_single_read=false)
         next_byte(close_after_single_read)
       end
 
-      # returns the current position of the read cursor (binary access)
+      # Returns the current position of the read cursor
       def position
         return (@position != nil) ? @position : 0
       end
    
-      # reset the cursor to the initial state
+      # Reset the cursor to the beginning of the file
       def reset
         set_position(0)
       end
     
-      # set the new position of the reding cursors
+      # Set a new position for the read cursor
       def seek(new_position)
         set_position(new_position)
       end
     
-      # returns the size of the object in bytes
+      # Returns the file size in bytes
       def size
         data_size
       end

data/lib/talia_core/data_types/file_store.rb

@@ -2,36 +2,72 @@ require 'fileutils'
 
 module TaliaCore
   module DataTypes
+    
+    # The "hevy lifting" for FileRecords, handling the actual creation and
+    # manipulation of the files. 
+    #
+    # A record can be created with create_from_file (passing a filename) or 
+    # with create_from_data, passing a byte array.
+    #
+    # While a "location" can be passed to the record (which will be save 
+    # in the database), the actual file name will be determined by the system.
+    # Also see the PathHelpers for info
+    #
+    # The base directory for file storage can be configured in the 
+    # talia_core.yml file as "data_directory_location". Otherwise,
+    # RAILS_ROOT/data. Inside, each record will be stored at 
+    # "<data class of the record>/<last three numbers of the id>/<record id>"
+    # 
+    # So, for example, a "XmlData" record with the id "123456" would be stored
+    # as "DATA_ROOT/XmlData/456/123456"
+    #
+    # = Creating new records
+    #
+    # When creating a new record using create_from_data, the data will be cached
+    # inside the object and only be saved when the record itself is saved. In 
+    # this case, the file data is simply written to file during the save operation.
+    #
+    # When the record is created using create_from_file, the behaviour depens on
+    # the parameters passed and the system settings.
+    #
+    # * In case the delete_original flag is set, the system will try to move the
+    #   file to the new location. If both are on the same file system, this will
+    #   be quicker than a copy operation.
+    # * Currently, the move operation uses the "mv" command. This is does not work
+    #   on windows, but is a workaround to stability problems with the file handling
+    #   in JRuby
+    # * If the delete_original flag is not set, the system will attempt to copy the
+    #   files:
+    #   * If the "delay_file_copies" options is set in the _environment_, no copy
+    #     operation will be done. Instead, the system will create a "delayed_copies.sh"
+    #     script that can be executed from a UNIX shell to do the actual copying.
+    #     This is extremely fast and stable, as no actual copying is done.
+    #   * Otherwise Talia will attempt to copy the file by itself. If the "fast_copies"
+    #     flag is set in _environment_, it will use the internal copy routine
+    #     which will work on any system. Otherwise, it will call the system's "cp"
+    #     command, which can sometimes be more stable with jruby.
+    #
+    # Also see the DataLoader module to see how the creation of records automatically
+    # selects the record type and loader, depending on the MIME type of the data.
+    #
+    # *Note*: The above behaviour means that for files that are treated through "copy"
+    # or "move", the original file must not be touched by external processes until
+    # the record is saved
     module FileStore
   
-      # the handle for the file
+      # The file handle of the current record
       @file_handle = nil
-      # position of the reading cursors
+      # Read cursor within the current record
       @position    = 0      
    
-      # Class for data paths
+      # Class used to represent data paths in file records. This type is used for
+      # strings that contain a path to a file, to distinguish them from "normal"
+      # string, which may contain plain data.
       class DataPath < String ; end
-      
-      module ClassMethods
-      
-        # Find or create a record for the given location and source_id, then it saves the given file.
-        def find_or_create_and_assign_file(params)
-          data_record = self.find_or_create_by_location_and_source_id(extract_filename(params[:file]), params[:source_id])
-          data_record.file = params[:file]
-          data_record.save # force attachment save and it also saves type attribute.
-        end
-      
-      end
     
-      # This will create the data object from a given file. This will simply move the
-      # given file to the correct location upon save. This will avoid multiple
-      # read/write operations during import.
-      #
-      # The original file must not be touched by external processes until the
-      # record is saved.
-      #
-      # If the delete_original flag is set, the original file will be removed
-      # on save
+      # Creates a new record from an existing file on the file system.
+      # The file will be copied or moved when the new record is saved - see
+      # the module documentation to see how this works in detail.
       def create_from_file(location, file_path, delete_original = false)
         close_file
         self.location = location
@@ -39,13 +75,15 @@ module TaliaCore
         @delete_original_file = delete_original
       end
   
-      # Add data as string into file
-      def create_from_data(file_location, data, options = {})
+      # Creates a new record from the given data (binary array). See
+      # the module documentation for the details; the data will be cached
+      # and written to disk once the new record is saved.
+      def create_from_data(location, data, options = {})
         # close file if opened
         close_file
     
         # Set the location for the record
-        self.location = file_location
+        self.location = location
     
         if(data.respond_to?(:read))
           @file_data_to_write = data.read
@@ -55,38 +93,21 @@ module TaliaCore
     
       end
       
-      # returns the complete text
+      # Returns the contents of the file as a text string.
       def all_text
         if(!is_file_open?)
           open_file
         end
         @file_handle.read(self.size)
       end
-  
-      # This is a placeholder in case file is used in a form.
-      def file() nil; end
-    
-      # Assign the file data (<tt>StringIO</tt> or <tt>File</tt>).
-      def file=(file_data)
-        return nil if file_data.nil? || file_data.size == 0 
-        self.assign_type file_data.content_type
-        self.location = file_data.original_filename
-        if file_data.is_a?(StringIO)
-          file_data.rewind
-          self.temp_data = file_data.read
-        else
-          self.temp_path = file_data.path
-        end
-        @save_attachment = true
-      end
     
+      # Callback for writing the data from create_from_data or create_from_file. If there is
+      # a problem saving this file, only an internal assertion is thrown so that it won't crash
+      # production environments.
       def write_file_after_save 
         # check if there are data to write
         return unless(@file_data_to_write)
     
-        # check if file already exists
-#        raise(RuntimeError, "File already exists: #{file_path}") if(File.exists?(file_path))
-
         begin
           self.class.benchmark("\033[36m\033[1m\033[4mFileStore\033[0m Saving file for #{self.id}") do
             # create data directory path
@@ -106,7 +127,7 @@ module TaliaCore
 
       end
    
-      # Return true if the specified data file is open, false otherwise
+      # Return true if the data file is open
       def is_file_open?
         (@file_handle != nil)
       end
@@ -117,11 +138,10 @@ module TaliaCore
         self.type = MimeMapping.class_type_from(content_type).name
       end
       
-      
-      # private methods ==================================================================
       private
       
-      # This saves the cached data from the file creation
+      # This saves the cached data from create_from_data. Simply writes the
+      # data to disk.
       def save_cached_data
         # open file for writing
         @file_handle = File.open(file_path, 'w')
@@ -135,7 +155,7 @@ module TaliaCore
       end
       
       # This copies the data file with which this object was created to the
-      # actual storage lcoation
+      # actual storage lcoation. This is for records created with create_from_file
       def copy_data_file
         copy_or_move(@file_data_to_write, file_path)
       end
@@ -172,7 +192,7 @@ module TaliaCore
         end
       end
 
-      # Read all bytes from a file  
+      # Read all bytes from the file  
       def read_all_bytes
         # 1. Open file with option "r" (reading) and "b" (binary, useful for window system)
         open_file
@@ -191,7 +211,7 @@ module TaliaCore
         end
       end
 
-      # return the next_byte
+      # Gets the next byte of the file
       def next_byte(close)
         if !is_file_open?
           open_file
@@ -216,7 +236,9 @@ module TaliaCore
     
       end
 
-      # Copy or move the source file to the target. Working around all the
+      # Copy or move the source file to the target depending on the 
+      # <tt>delete_original</tt> setting in #create_from_file. 
+      # Working around all the
       # things that suck in JRuby. This will honour two environment settings:
       # 
       # * delay_file_copies - will not copy the files, but create a batch file
@@ -227,35 +249,50 @@ module TaliaCore
       #     JRuby only.
       def copy_or_move(original, target)
         if(@delete_original_file)
-          FileUtils.move(original, target)
+          begin
+            FileUtils.move(original, target)
+          rescue Errno::EACCES
+            # Workaround for File.rename bug with JRuby (jira.codehaus.org/browse/JRUBY-3381),
+            # based on the code from Lenny Marks 03/Jun/10.
+            safe_copy original, target
+            FileUtils.rm original
+          end
         else
           # Delay can be enabled through enviroment
           if(delay_copies)
             DelayedCopier.cp(original, target)
-          elsif(fast_copies)
-            FileUtils.copy(original, target)
           else
-            # Call the copy as an external command. This is to work around the
-            # crashes that occurred using the builtin copy
-            from_file = File.expand_path(original)
-            to_file = File.expand_path(target)
-            system_success = system("cp '#{from_file}' '#{to_file}'")
-            raise(IOError, "copy error '#{from_file}' '#{to_file}'") unless system_success
+            safe_copy original, target
           end
         end
       end
+
+      # Copies the file using some workarounds for jruby if necessary.
+      # See also #copy_or_move.
+      def safe_copy(original, target)
+        if(fast_copies)
+          FileUtils.copy(original, target)
+        else
+          # Call the copy as an external command. This is to work around the
+          # crashes that occurred using the builtin copy
+          from_file = File.expand_path(original)
+          to_file = File.expand_path(target)
+          system_success = system("cp '#{from_file}' '#{to_file}'")
+          raise(IOError, "copy error '#{from_file}' '#{to_file}'") unless system_success
+        end
+      end
+
       
-      
-      # Returns true if the 'delayed write' is enabled in the environment
+      # Returns true if the 'delay_file_copies' option is set in the environment
       def delay_copies
-        ENV['delay_file_copies'] == 'true' || ENV['delay_file_copies'] == 'yes'
+        ENV['delay_file_copies'].yes?
       end
         
-      # Returns true if the 'fast copy' is enabled in the environment.
+      # Returns true if the 'fast_copies' option is enabled in the environment.
       # Otherwise the class will use a workaround that is less likely to 
       # crash the whole system using JRuby.
       def fast_copies
-        ENV['fast_copies'] == 'true' || ENV['fast_copies'] == 'yes'
+        ENV['fast_copies'].yes?
       end
       
       # Return the data size
@@ -263,7 +300,7 @@ module TaliaCore
         File.size(file_path)
       end
 
-      # set the position of the reading cursor
+      # Sets the position of the reading cursor
       def set_position(position)
         if (position != nil and position =~ /\A\d+\Z/)
           if (position < size)
@@ -275,32 +312,12 @@ module TaliaCore
           raise(IOError, 'Position not valid. It must be an integer')
         end
       end
-    
-      # Check if the attachment should be saved.
-      def save_attachment?
-        @save_attachment
-      end
-    
-      # Save the attachment, copying the file from the temp_path to the data_path.
-      def save_attachment
-        return unless save_attachment?
-        save_file
-        @save_attachment = false
-        true
-      end
-
-      # Destroy the attachment
-      def destroy_attachment
+      
+      # Delete the file connected to this record
+      def destroy_file
         FileUtils.rm(full_filename) if File.exists?(full_filename)
       end
 
-      # Save the attachment on the data_path directory.
-      def save_file
-        FileUtils.mkdir_p(File.dirname(full_filename))
-        FileUtils.cp(temp_path, full_filename)
-        FileUtils.chmod(0644, full_filename)
-      end
-
     end
   end
-end
+end

data/lib/talia_core/data_types/iip_data.rb

@@ -1,87 +1,162 @@
 module TaliaCore
   module DataTypes
     
-    # Class to manage IIP Image data type. FIXME: Check if this correctly destroys existing files.
+    # Class to manage IIP Image data type. This differs from the "normal" 
+    # file record in various ways:
+    #
+    # * The record itself only contains a thumbnail version of the image as 
+    #   data (#get_thumbnail will return the file data for the thumbnail)
+    # * The location of the record contains the path to the image on the
+    #   IIP server. This is also returned by #iip_server_path
+    # * The records should be created using the IipLoader - see DataLoader
+    #   to find out how to configure Talia to do that
+    #
+    # = What is IIP?
+    #
+    # IIP is a protocol to use "pyramidal images" - which are images which
+    # contain a tiled version of the original image in various resolutions.
+    # It is used to serve high-resolution images in a way that a client can 
+    # request only the portion of the image that is currently needed.
+    #
+    # = How does IIP work with Talia?
+    #
+    # To use IIP, an IIP server is needed.
+    # The IIP client will connect to the server and request an image from using
+    # HTTP requests. 
+    #
+    # The IIP server is a separate piece of sofware, which is not part of
+    # Talia. The one commonly used is http://iipimage.sourceforge.net/ which
+    # runs as a fastcgi module in Apache (or another web server).
+    #
+    # For talia this means:
+    #
+    # * The pyramidal images have to be put in a place where the IIP server can
+    #   access them - they do *not* go in the normal data directory.
+    # * The location of the directory for the pyramidal images can be configured
+    #   as <tt>iip_root_directory_location</tt> in the <tt>talia_core.yml</tt>
+    #   file.
+    # * The <tt>location</tt> field of each IipData record contains the _relative_
+    #   path for accessing the file on the IIP server.
+    # * The URI of the server itself can be set in <tt>talia_core.yml</tt> as
+    #   <tt>iip_server_uri</tt>
+    #
+    # = How can the IIP images be shown in Talia?
+    #
+    # There are several clients or "viewers" for IIP images available, and one
+    # is also included in the "muruca_widgets" gem that can be installed in
+    # addition to talia.
+    #
+    # Generally you'll have to include the client in your view templates, and
+    # use the #iip_server_uri and #iip_server_path to point it to the image
+    # on the server.
+    #
+    # Obviously you'll also need a running IIP server.
+    #
+    # = What are the requirements for IIP?
+    #
+    # Apart from the IIP server itself, you'll need some software to actually
+    # create the pyramidal images - see the TaliaUtil::ImageConversions class for details.
     class IipData < FileRecord
       
-      # Returns the IIP server configured for the application
+      # Returns the IIP server configured for the application. This is usually the
+      # value configured as the <tt>iip_server_uri</tt> in the <tt>talia_core.yml</tt>
       def self.iip_server_uri
         TaliaCore::CONFIG['iip_server_uri'] ||= 'http://localhost/fcgi-bin/iipsrv.fcgi'
       end
       
-      # This is the mime type for the thumbnail - always tiff
+      # This is the mime type for the thumbnail. Always gif.
       def set_mime_type
         self.mime = 'image/gif'
       end
       
       alias :get_thumbnail :all_bytes
      
-      # Create from existing thumb and pyramid images
+      # Create a new record from an existing thumb and pyramidal image. 
+      # This is used to make a record from pre-existing "prepared" images
+      # and does not do any image conversion.
       def create_from_existing(thumb, pyramid, delete_originals = false)
         @file_data_to_write = [thumb, pyramid]
         @delete_original_file = delete_originals
         self.location = ''
       end
       
-      # return IIP Server Path
+      # The relative path to the image on the IIP server
       def iip_server_path
         self.location
       end
       
+      # Callback to write the data when the record is saved, which is more involved than
+      # the same method on the superclass.
+      #
+      # * If the thumb and pyramid file are already available, they are used directly 
+      #   with #direct_write!
+      # * Otherwise it will take the original file and pass it through the TaliaUtil::ImageConversions
+      #   to create the thumbnail and pyramidal image.
+      # * Any temporary files created in the process are cleaned up afterwards
       def write_file_after_save
         return unless(@file_data_to_write)
         
-        # Check if we have the images already given, in this case we prepare
-        # them and call the super method
+        # Check if we have the converted images already. In this case we write
+        # them to the appropriate directories directly and call the super method
         return super if(direct_write!)
         
-        # create name for orginal temp file and destination temp file
+        # "Prepare" the original file for conversion, indicate if a temp file is being used
         original_file_path, orig_is_temp = prepare_original_file
+        # Check if we need to delete the original/temp file
         will_delete_source = orig_is_temp || @delete_original_file
+        # Path to the temporary thumbnail file
         destination_thumbnail_file_path = File.join(Dir.tmpdir, "thumbnail_#{random_tempfile_filename}.gif")
         
-        begin # Begin the file creation operation
+        begin
           self.class.benchmark("\033[36mIipData\033[0m Making thumb and pyramid for #{self.id}", Logger::INFO) do
           
+            # Create the thumbnail at the temporary location
             TaliaUtil::ImageConversions::create_thumb(original_file_path, destination_thumbnail_file_path)
+            # Create the pyramidal image from the original
             create_pyramid(original_file_path)
         
-            # Run the super implementation for the thumbnail
-            # We will simply tell the system that we have to move the newly create
-            # thumb file
+            # Run the super implementation for the thumbnail, by using the temporary thumb file as the "data"
             @file_data_to_write = DataPath.new(destination_thumbnail_file_path)
+            # The temp thumb file needs to be deleted by the superclass
             @delete_original_file = true
           
           end # end benchmarking
           super
           
         ensure
-          # delete teƒmp files
+          # Delete the temporary "original" file, if necessary
           File.delete original_file_path if(File.exists?(original_file_path) && will_delete_source)
         end
       end
       
       
-      # Checks if we have file paths given to directly copy thum and image file.
-      # Will always return true if such paths were given.
+      # Checks if there if the thumb and pyramidal file already exist and can simply be moved
+      # to the correct location.
+      #
+      # If yes, the method will move or copy the files to the correct locations, and return true
+      # Otherwise, the method will do nothing and return false.
       def direct_write!
+        # If we have an array of data files, we can assume that these are 
+        # pre-prepared thumb and pyramid images
         return false unless(@file_data_to_write.kind_of?(Array))
         
         thumb, pyramid = @file_data_to_write
         self.class.benchmark("\033[36mIipData\033[0m Direct write for #{self.id}", Logger::INFO) do
-          prepare_for_pyramid
-        
+          prepare_for_pyramid # Setup the record for the new image
+          # Copy or move the pyramid file to the correct location
           copy_or_move(pyramid, get_iip_root_file_path)
         
         end # end benchmark
-          
+        
+        # Set the thumb file as the data file for the current FileRecord (which
+        # is automatically handled by the superclass)
         @file_data_to_write = DataPath.new(thumb)
         
         true
       end
 
       # This prepares the original file that needs to be converted. This will
-      # see if the data to be written is binary data or a file path. If this
+      # see if the data to be written is binary data or a file path. If it
       # is binary data, it will create a temporary file on the disk.
       #
       # This returns an array with two elements: The name of the file to 
@@ -105,7 +180,8 @@ module TaliaCore
         end
       end
       
-      # Prepare for copying or creating the pyramid image
+      # Prepare for copying or creating the pyramid image. Sets the <tt>location</tt>
+      # of the record and creates the directory to store the IIP images
       def prepare_for_pyramid
         # set location
         self.location = get_iip_root_file_path(true)
@@ -116,7 +192,7 @@ module TaliaCore
       
       # Creates the pyramid image for IIP by running the configured system
       # command. This automatically creates the file in the correct location 
-      # (IIP root)
+      # (IIP root). The conversion is done using the TaliaUtil::ImageConversions
       def create_pyramid(source)
         # check if file already exists
         raise(IOError, "File already exists: #{get_iip_root_file_path}") if(File.exists?(get_iip_root_file_path))
@@ -126,8 +202,13 @@ module TaliaCore
         TaliaUtil::ImageConversions::create_pyramid(source, get_iip_root_file_path)
       end
       
-      # Return the iip root directory for a specific iip image file
+      # Return the iip root directory for a specific iip image file. This is the
+      # directory where the pyramid file will be stored on disk.
+      #
+      # If the <tt>relative</tt> flag is set, it will return a relative path.
       def iip_root_directory(relative = false)
+        # TODO: The relative paths are (also?) used for web access, is it o.k. to
+        # use File.join ?
         if relative == false
           File.join(TaliaCore::CONFIG["iip_root_directory_location"], ("00" + self.id.to_s)[-3..-1])
         else
@@ -135,7 +216,7 @@ module TaliaCore
         end
       end
       
-      # Return the full file path related to the data directory
+      # Return the full file path for the IIP image. See #iip_root_directory
       def get_iip_root_file_path(relative = false)
         File.join(iip_root_directory(relative), self.id.to_s + '.tif')
       end