talia_core 0.4.0

data/lib/talia_util/image_conversions.rb

@@ -0,0 +1,82 @@
+module TaliaUtil
+
+  # Helper class that provides an interface to convert images, create
+  # thumbnails and pyramid images. This provides a central point from which to
+  # call external conversion tools.
+  #
+  # Since it calls the command line, it should be compatible both with JRuby
+  # and plain Ruby.
+  class ImageConversions
+
+    class << self
+
+      # Returns the command that is used for converting images
+      def vips_command
+        @vips_command ||= if(defined?(TaliaCore))
+          TaliaCore::CONFIG['vips_command'] || '/opt/local/bin/vips'
+        elsif(defined?(VIPS_COMMAND))
+          VIPS_COMMAND
+        else
+          raise(ArgumentError('Unconfigured vips command. If not in Talia, set VIPS_COMMAND'))
+        end
+      end
+
+      # Returns the command that is used for converting thumbnails
+      def convert_command
+        @convert_command ||= if(defined?(TaliaCore))
+          TaliaCore::CONFIG['convert_command'] || '/opt/local/bin/convert'
+        elsif(defined?(CONVERT_COMMAND))
+          CONVERT_COMMAND
+        else
+          raise(ArgumentError('Unconfigured convert command. If not in Talia, set CONVERT_COMMAND'))
+        end
+      end
+
+      # Returns the options for the thumbnail
+      def thumb_options
+        @thumb_options ||= if(defined?(TaliaCore))
+          TaliaCore::CONFIG['thumb_options'] || { 'width' => '80', 'height' => '120' }
+        elsif(defined?(THUMB_OPTIONS))
+          THUMB_OPTIONS
+        else
+          raise(ArgumentError('Unconfigured thumbnail options. If not in Talia, set THUMB_OPTIONS'))
+        end
+      end
+
+      # Create the thumbnail by running the configured creation command.
+      def create_thumb(source, destination)
+        thumbnail_size = "#{thumb_options['width']}x#{thumb_options['height']}"
+        thumbnail_command = "#{convert_command} \"#{source}\" -quality 85 -thumbnail \"#{thumbnail_size}>\" -background transparent -gravity center -extent #{thumbnail_size} \"#{destination}\""
+        execute_command(thumbnail_command, destination)
+      end
+
+      # Creates the pyramid image for IIP by running the configured system
+      # command. This automatically creates the file in the correct location
+      # (IIP root)
+      def create_pyramid(source, destination)
+        pyramid_command = "#{vips_command} im_vips2tiff \"#{source}\" \"#{destination}\":jpeg:85,tile:256x256,pyramid"
+        execute_command(pyramid_command, destination)
+      end
+
+      # Transforms the given image into a PNG image. Note that the .png suffix
+      # will automatically added to the destination name
+      def to_png(source, destination)
+        destination = "#{destination}.png" unless(File.extname(destination) == '.png')
+        convert_line = "#{convert_command} \"#{source}\" \"#{destination}\""
+        execute_command(convert_line, destination)
+      end
+
+      private
+
+      # Executes the given command and raises an error if not successful.
+      # The error will also be raised if a file is given and does not exist.
+      def execute_command(command, file_to_exist = nil)
+        system_result = system(command)
+        # check if successful
+        raise(IOError, "Command #{command} failed (#{$?}).") unless ((file_to_exist && File.exists?(file_to_exist)) || !system_result)
+      end
+
+    end
+  end
+end
+

data/lib/talia_util/import_job_helper.rb

@@ -0,0 +1,132 @@
+require 'hpricot'
+
+module TaliaUtil
+
+  # Helper methods that will be used during import job runs.
+  # The import jobs may use the following environment parameters:
+  #
+  #  [*base_url*] - The base URL or directory. This will be prefixed to all urls, or if it is 
+  #                 a local directory, it will be made the current directory during the import
+  #  [*index*] - If given, the importer will try to read this document. While this will still
+  #              support the old-style "hyper" format with sigla, it should usually contain a
+  #              main element called "index" followed by "url" entries. 
+  #  [*xml*] - URL of an XML file to import. This is incompatible with the "index" option. 
+  #            If neither "xml" nor "index" are given, the class will try to read the XML data from
+  #            STDIN
+  #  [*importer*] - Name of the importer class to be used for the data. Uses the default class if not given
+  # [*reset_store*] - If this is set, the data store will be cleared before the import
+  # [*user*] - Username for HTTP authentication, if required
+  # [*pass*] - Password for HTTP authentication, if required
+  # [*callback*] - Name of a class. If given, the import will call the #before_import and #after_import
+  #                 methods on an object of that class. The call will receive a block which may be
+  #                 yielded to for each progress step and which can receive the overall number of
+  #                 steps
+  # [*extension*] - Only used with index files; file extension to use
+  class ImportJobHelper
+
+    include IoHelper
+
+    attr_reader :importer, :credentials, :index_data, :xml_data, :reset, :callback, :base_url, :message_stream, :progressor
+
+    # The message_stream will be used for printing progress messages
+    def initialize(message_stream = STDOUT, progressor = TaliaCore::BackgroundJobs::Job)
+      @progressor = progressor
+      @message_stream = message_stream
+      @importer = ENV['importer'] || 'TaliaCore::ActiveSourceParts::Xml::SourceReader'
+      @credentials = { :http_basic_authentication => [ENV['user'], ENV['pass']] } unless(ENV['user'].blank?)
+      assit(!(ENV['xml'] && ENV['index']), 'Not both xml and index parameters allowed')
+      @reset = ['yes', 'true'].include?(ENV['reset_store'].downcase) if(ENV['reset_store'])
+
+      @base_url = ENV['base_url'].blank? ? '' : ENV['base_url']
+      if(base_url && File.directory?(base_url))
+        message_stream.puts "Setting directory to #{base_url}"
+        FileUtils.cd(base_url)
+      end
+
+      init_data
+      
+      @callback = ENV['callback'].classify.constantize.new unless(ENV['callback'].blank?)
+      
+      message_stream.puts "Registered callback (#{callback.class.name}) - (#{callback.respond_to?(:before_import)}|#{callback.respond_to?(:after_import)})" if(callback)
+      
+      callback.progressor = progressor if(callback && callback.respond_to?(:'progressor='))
+    end
+
+    def init_data
+      if(ENV['index'].blank?)
+        @xml_data = if(ENV['xml'].blank?)
+          STDIN.read
+        else
+          xml_url = ENV['xml']
+          xml_url = base_url + xml_url unless(File.exists?(xml_url))
+          open_generic(xml_url, credentials) { |io| io.read }
+        end
+      else
+        index = make_url_from(ENV['index'])
+        @index_data = open_generic(index, credentials) { |io| io.read }
+      end
+    end
+
+    def do_import
+      if(reset)
+        TaliaUtil::Util.full_reset
+        puts "Data Store has been completely reset"
+      end
+      errors = []
+      run_callback(:before_import)
+      if(index_data)
+        import_from_index(errors)
+      else
+        puts "Importing from single data file."
+        TaliaCore::ActiveSource.create_from_xml(xml_data, :progressor => progressor, :reader => importer, :errors => errors)
+      end
+      if(errors.size > 0)
+        puts "WARNING: #{errors.size} errors during import:"
+        errors.each { |e| puts e }
+      end
+      run_callback(:after_import)
+    end
+
+    def import_from_index(errors)
+      doc = Hpricot.XML(index_data)
+      hyper_format = (doc.root.name == 'sigla')
+      elements = hyper_format ? (doc/:siglum) : (doc/:url)
+      puts "Import from Index file, #{elements.size} elements"
+      # Read the Attributes from the urls
+      source_attributes = []
+      my_importer = importer.classify.constantize
+      progressor.run_with_progress('Reading w/ index', elements.size) do |prog|
+        elements.each do |element|
+          url = make_url_from("#{element.inner_text}#{ENV['extension']}")
+          begin
+            open_generic(url, credentials) do |io|
+              this_attribs = my_importer.sources_from(io)
+              source_attributes = source_attributes + this_attribs
+            end
+          rescue Exception => e
+            message_stream.puts "Problem importing #{url} (#{e.message})"
+            message_stream.puts e.backtrace
+          end
+          prog.inc
+        end
+      end
+      # Write the data
+      TaliaCore::ActiveSource.progressor = progressor
+      TaliaCore::ActiveSource.create_multi_from(source_attributes, :errors => errors)
+    end
+
+    def make_url_from(url)
+      return url if(File.exist?(url))
+      "#{base_url}#{url}"
+    end
+
+    def run_callback(name)
+      if(callback && callback.respond_to?(name))
+        puts "Running callback #{name}"
+        callback.send(name)
+      end
+    end
+
+
+  end # End class
+end

data/lib/talia_util/io_helper.rb

@@ -0,0 +1,54 @@
+require 'open-uri'
+
+module TaliaUtil
+  
+  # Import data files into the Talia store. This can be used to bootstrap
+  # simple installations
+  module IoHelper
+
+    # Generic "open" method for files and urls. This won't choke on file:// URLs and
+    # will do some extra escaping on the URL. 
+    #
+    # See open_from_url for an explanation of the options
+    def open_generic(url, options = {})
+      url = file_url(url)
+      # Even though open-uri would also open local files, we avoid to mangle
+      # the URL in open_from_url
+      if(File.exist?(url))
+        File.open(url) { |io| yield(io) }
+      else
+        open_from_url(url, options) { |io| yield(io) }
+      end
+    end
+
+    # Opens the given (web) URL, using URL encoding and necessary substitutions.
+    # The user must pass a block which will receive the io object from
+    # the url.
+    #
+    # The options may contain the http authentication information and such. See
+    # the documentation for open-uri for more information. Example for options:
+    # 
+    #   :http_basic_authentication => [login, password]
+    def open_from_url(url, options = {})
+      url = URI.encode(url)
+      url.gsub!(/\[/, '%5B') # URI class doesn't like unescaped brackets
+      url.gsub!(/\]/, '%5D')
+      open_args = [ url ]
+      open_args << options if(options)
+
+      begin
+        open(*open_args) do |io|
+          yield(io)
+        end
+      rescue Exception => e
+        raise(IOError, "Error loading #{url} (when file: #{url}, open_args: [#{open_args.join(', ')}]) #{e}")
+      end
+    end
+    
+    # Get the "file url" for the given url, stripping a possible file:// from the front
+    def file_url(uri)
+      uri.gsub(/\Afile:\/\//, '')
+    end
+    
+  end # End modules
+end

data/lib/talia_util/progressable.rb

@@ -0,0 +1,38 @@
+module TaliaUtil
+  
+  # Mix-in for a class that wishes to use decoupled progress meters.
+  module Progressable
+    
+    # This is the object that will receive the progress messages
+    def progressor
+      @progressor
+    end
+    
+    # Set the progressor class. The progress class should simply respond to
+    # a #run_with_progress(message, size, &block) class 
+    def progressor=(progr)
+      raise(ArgumentError, "Illegal progressor") unless((progr == nil) || progr.respond_to?(:run_with_progress))
+      @progressor = progr
+    end
+    
+    # Runs some block with a progress meter. The containing block will be
+    # passed an object on which #inc can be called to increase the meter.
+    # 
+    # If no progressor object is passed in manually, the one configured
+    # in the class is used
+    def run_with_progress(message, size, progr = nil, &block)
+      if(progr_object = (progr || progressor))
+        progr_object.run_with_progress(message, size, &block)
+      else
+        dummy_prog = Object.new
+        class << dummy_prog
+          def inc
+          end
+        end
+        block.call(dummy_prog)
+      end
+    end
+    
+  end
+  
+end

data/lib/talia_util/progressbar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar # :nodoc:
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "o"
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols >= 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/talia_util/rdf_update.rb

@@ -0,0 +1,80 @@
+module TaliaUtil
+  
+  # Some methods to update the RDF store. There is no real inferencing, just
+  # some hardcoded rules that help to provide basic functionality with simple
+  # RDF stores.
+  class RdfUpdate
+    
+    class << self
+      
+      # This is the wrapper for rdfs_from_owl for the rake task
+      def owl_to_rdfs
+        puts "Checking for OWL classes."
+        # Register the namespaces for ActiveRDF
+        Namespace.register(:rdfs, N::RDFS.to_s)
+        Namespace.register(:owl, N::OWL.to_s)
+        progress = nil
+        size, modified, blanks = rdfs_from_owl do |size|
+          progress ||= begin
+            puts "#{size} OWL classes found, adding rdfs:Class type for each."
+            ProgressBar.new("Updating RDF database", size)
+          end
+          progress.inc
+        end
+        
+        progress.finish if(progress)
+        puts "Finished updating. Updated #{modified} of #{size} classes. #{blanks} blank nodes were ignored."
+        
+      end
+      
+      # This checks for owl:Classes and adds and rdfs:Class triple to them. This
+      # doesn't check if the triple already exists, and thus may cause duplicates.
+      # You can pass a block that will be called with the overall size
+      # as a parameter.
+      #
+      # It returns the overall size, the number of modified elements and the
+      # number of blank nodes
+      def rdfs_from_owl
+        # Remove previous auto rdfs triples
+        FederationManager.clear(N::TALIA.auto_rdfs.context)
+        
+        # Get all OWL classes
+        qry = Query.new(N::URI).distinct.select(:klass, :type)
+        qry.where(:klass, N::RDF::type, N::OWL + 'Class')
+        qry.where(:klass, N::RDF::type, :type)
+        classes_with_types = qry.execute
+        modified = 0
+        blanks = 0
+        
+        class_hash = {}
+        
+        # We have a list of all combinations of klass elements with their types
+        # The next step is to find all things where a RDFS.Class type already
+        # exists
+        classes_with_types.each do |class_with_type|
+          if(class_with_type.first.is_a?(RDFS::BNode))
+            blanks = blanks + 1
+            next
+          end
+          
+          klass, type = class_with_type
+          
+          class_hash[klass.to_s] ||= :no_rdfs_class
+          class_hash[klass.to_s] = :has_rdfs_class if(type == N::RDFS.Class)
+        end
+        
+        # Now go through all klasses and add the missing triples
+        class_hash.each do |klass, status|
+          if(status == :has_rdfs_class)
+            modified = modified + 1
+            FederationManager.add(N::URI.new(klass), N::RDF.type, N::RDFS.Class, N::TALIA.auto_rdfs_context)
+          end
+          yield(class_hash.size) if(block_given?)
+        end
+        
+        return [class_hash.size, modified, blanks]
+      end
+      
+    end
+  end
+end