talia_core 0.5.4 → 0.7.0

data/lib/talia_util/import_job_helper.rb

@@ -44,14 +44,25 @@ module TaliaUtil
   #
   # [*trace*] Enable tracing output for errors. (By default, this takes the rake task's setting
   #           if possible)
+  #
+  # The import itself consists in calling #initialize and the do_import
   class ImportJobHelper
 
     include IoHelper
 
     attr_reader :importer, :credentials, :index_data, :xml_data, :reset, :callback, :base_url, :message_stream, :progressor, :duplicates, :trace
 
-    # The message_stream will be used for printing progress messages
-    def initialize(message_stream = STDOUT, progressor = TaliaCore::BackgroundJobs::Job)
+    # The message_stream will be used for printing progress messages.
+    #
+    # The procedure of the import is the following:
+    #
+    # * Set up all the attributes of this class from the respective environment variables (from the
+    #   rake task)
+    # * Initialize the data: If an index file is given, read the index file. Otherwise read the
+    #   file given by the 'xml' environment variable, or from STDIN if 'xml' isn't set. See init_data
+    # * Create the callback class, if given
+    # * Set up the progressor for the import, if any
+    def initialize(message_stream = STDOUT, progressor = TaliaUtil::BarProgressor)
       @trace = (defined?(Rake) ? Rake.application.options.trace : false) || ENV['trace']
       @progressor = progressor
       @message_stream = message_stream
@@ -59,7 +70,7 @@ module TaliaUtil
       @importer = ENV['importer'] || 'TaliaCore::ActiveSourceParts::Xml::SourceReader'
       @credentials = { :http_basic_authentication => [ENV['user'], ENV['pass']] } unless(ENV['user'].blank?)
       assit(!(ENV['xml'] && ENV['index']), 'Not both xml and index parameters allowed')
-      @reset = ['yes', 'true'].include?(ENV['reset_store'].downcase) if(ENV['reset_store'])
+      @reset = ENV['reset_store'].yes?
 
       @base_url = ENV['base_url'].blank? ? '' : ENV['base_url']
       if(base_url && File.directory?(base_url))
@@ -76,6 +87,10 @@ module TaliaUtil
       callback.progressor = progressor if(callback && callback.respond_to?(:'progressor='))
     end
 
+    # Reads the data for the coming import. If the 'index' parameter is found in the
+    # environment, this will be used as the file name for the index file, which will be
+    # read into the object. Otherwise, if the 'xml' environment variable is set, this will
+    # will be read and used as the XML data for the import
     def init_data
       if(ENV['index'].blank?)
         @xml_data = if(ENV['xml'].blank?)
@@ -92,6 +107,14 @@ module TaliaUtil
       end
     end
 
+    # Does the actual importing: 
+    #
+    # * If required, reset the data store
+    # * Run the "before_import" callback
+    # * In case there is plain xml data, TaliaCore::ActiveSource.create_from_xml
+    #   will handle all the import
+    # * If an index is given, the import will be done by import_from_index
+    # * Run the "after_import" callback
     def do_import
       if(reset)
         TaliaUtil::Util.full_reset
@@ -112,11 +135,25 @@ module TaliaUtil
       run_callback(:after_import)
     end
 
+    # Prints the message and, if the "trace" option is set,
+    # also the stack trace of the Exception e
     def print_error(e)
       puts e.message
       puts e.backtrace if(trace)
     end
 
+    # This is *only* used if an index file is given for the import. All "plain"
+    # imports go directly to #create_from_xml in the ActiveSource class
+    #
+    # * The index file is parsed as XML
+    # * If the root element is "sigla", the old hyper format is used
+    # * In case the hyper format is used, sigla (local names for URIs)
+    #   are expected as "siglum" elements. Otherwise, the import URIs are expected
+    #   in "url" tags.
+    # * For each import url, #sources_from_url is called on the selected importer,
+    #   and the attributes added to the import data
+    # * The result is passed to TaliaCore::ActiveSource.create_multi from
+    #   to create the sources
     def import_from_index(errors)
       doc = Hpricot.XML(index_data)
       hyper_format = (doc.root.name == 'sigla')

data/lib/talia_util/io_helper.rb

@@ -2,13 +2,18 @@ require 'open-uri'
 
 module TaliaUtil
   
-  # Import data files into the Talia store. This can be used to bootstrap
-  # simple installations
+  # Collection of IO methods than can be mixed into other classes
   module IoHelper
 
     # Generic "open" method for files and urls. This won't choke on file:// URLs and
     # will do some extra escaping on the URL. 
     #
+    # *Usage*:
+    #
+    #  open_generic('someurl.foo') do |io|
+    #    ...
+    #  end
+    #
     # See open_from_url for an explanation of the options
     def open_generic(url, options = {})
       url = file_url(url)
@@ -21,7 +26,7 @@ module TaliaUtil
       end
     end
 
-    # Will try to figure out the "base" (that is the parent directory or path)
+    # Will try to figure out the "base", which is the parent directory or path.
     # If the base is a directory, this will return the directory name, but if
     # it is an URL, this will return an URI object.
     def base_for(url)
@@ -46,6 +51,12 @@ module TaliaUtil
     # the documentation for open-uri for more information. Example for options:
     # 
     #   :http_basic_authentication => [login, password]
+    #
+    # *Example*:
+    #
+    #  open_from_url('http://foobar.com/', :http_basic_authentication => ['user', 'secret']) do |io|
+    #    ...
+    #  end
     def open_from_url(url, options = {})
       # Encode the URI (the inner decode will save already-encoded URI and should
       # do nothing to non-encoded URIs)
@@ -65,7 +76,7 @@ module TaliaUtil
       end
     end
     
-    # Get the "file url" for the given url, stripping a possible file:// from the front
+    # Get the "file url" for the given uri, stripping a possible file:// from the front
     def file_url(uri)
       uri.gsub(/\Afile:\/\//, '')
     end

data/lib/talia_util/progressable.rb

@@ -1,6 +1,55 @@
 module TaliaUtil
   
-  # Mix-in for a class that wishes to use decoupled progress meters.
+  # Mix-in for a class that wishes to use decoupled progress meters. 
+  # This creates an read/write accessor for a progressor, which is used
+  # to report progress to.
+  #
+  # Inside the object #run_with_progress may be used to run a 
+  # progress-reporting operation. If no progressor has been assigned,
+  # run_with_progress will still work as expected, providing a progressor
+  # object that will do nothing.
+  #
+  # = Example
+  #
+  #  class Foo
+  #    
+  #    include Progressable
+  #
+  #    def long_thing
+  #      run_with_progress("doing long thing", 100) { |prog| (1..100).each { prog.inc } }
+  #    end
+  #  end
+  #
+  #  Foo.new.long_thing # Will do nothing
+  #  foo_real = Foo.new
+  #  foo_real.progressor = some_progressor
+  #  foo_real.long_thing # reports progress to some_progressor
+  #
+  # = How a progressor must look like
+  #
+  # A progressor is simply required to provide a method called "run_with_progress(message, size)",
+  # with the first parameter being the progress message and the second the overall 
+  # count of operations. The method *must* take a block, and call that block with an object
+  # which responds to a method calle "inc", which increases the current count/progress.
+  #
+  # = Example progressor
+  #
+  #  class DummyProgress
+  #    def inc
+  #      print '.'
+  #    end
+  #  end
+  #
+  #  class SimpleProgressor
+  #    def self.run_with_progress(message, size)
+  #      puts message
+  #      yield(DummyProgress.new)
+  #      puts "Done"
+  #    end
+  #  end
+  #
+  # This will print a dot for each thing that is processed. See also the BarProgressor for
+  # another example
   module Progressable
     
     # This is the object that will receive the progress messages

data/lib/talia_util/rake_tasks.rb

@@ -53,17 +53,10 @@ namespace :talia_core do
     t.libs << 'lib'
     # This will always take the files from the talia_core directory
     t.test_files = FileList["#{File.dirname(__FILE__)}/../../test/**/*_test.rb"]
-    t.verbose = true
+    t.verbose = false
   end
 
-  # Queue an import in the background
-  desc "Xml Background import. Options: [index=<indexfile>] [xml=<datafile>] [importer=<importclass>] [reset_store=true] [...]"
-  task :xml_background_import do 
-    background_job('xml_import', :tag => 'import')
-    puts "Queued XML background import."
-  end
-
-  desc "Xml Import. Options: see xml_background_import"
+  desc "Xml import. Options: [index=<indexfile>] [xml=<datafile>] [importer=<importclass>] [reset_store=true] [...]"
   task :xml_import => :init do
     importer = TaliaUtil::ImportJobHelper.new(STDOUT, TaliaUtil::BarProgressor)
     importer.do_import
@@ -185,15 +178,4 @@ namespace :talia_core do
 
   # Helper methods
 
-  # Queue the long-running task in the background processing queue.
-  # This will simply queue the job, and doesn't start the runner
-  # by itself
-  def background_job(job, options)
-    # Use the current environment for the job
-    options[:env] = ENV.merge(options[:env] || {})
-    # Avoid "tickling" to start the background job from here
-    options[:no_tickle] = true
-    TaliaCore::BackgroundJobs::Job.submit_with_progress(job, options)
-  end
-
-end
+end

data/lib/talia_util/test_helpers.rb

@@ -21,8 +21,9 @@ module TaliaUtil
     def assert_property(property, *values)
       assert_kind_of(TaliaCore::SemanticCollectionWrapper, property) # Just to be sure
       assert_equal(values.size, property.size, "Expected #{values.size} values instead of #{property.size}.")
+      value_str = values.inject("Expected:\n") { |str, value| str << "#{value.inspect}  (#{value.class.name})\n"  }
       property.each do |value|
-        assert(values.detect { |val| val.respond_to?(:uri) ? (val.uri.to_s == value.uri.to_s) : (value == val) }, "Found unexpected value #{value}. Value is a #{value.class}\nExpected:\n#{values.join("\n")}")
+        assert(values.detect { |val| val.respond_to?(:uri) ? (value.respond_to?(:uri) && (val.uri.to_s == value.uri.to_s)) : (value == val) }, "Found unexpected value #{value.inspect} (#{value.class.name})\n#{value_str}")
       end
     end
     
@@ -129,7 +130,11 @@ class Test::Unit::TestCase
   
   # Lets the class suppress the fixtures for the tests
   def self.suppress_fixtures
+    # Activates the suppress mechanism for older versions of rails
     @suppress_fixtures = true
+    # This overwrites the setup/teardown callbacks for Rails 2.3.*
+    define_method(:setup_fixtures) { }
+    define_method(:teardown_fixtures) { }
   end
   
   def self.suppress_fixtures?

data/lib/talia_util/util.rb

@@ -5,7 +5,12 @@ module TaliaUtil
   class Util
     class << self
 
-      # Get the list of files from the "files" option
+      # Rake task helper. Get the list of files the <tt>files</tt>
+      # environment variable (passed as <tt>file=[files]</tt>). Exits the
+      # runtime and prints an error message if the variable is not set.
+      #
+      # This returns a FileList object with all the files that match the
+      # pattern.
       def get_files
         puts "Files given: #{ENV['files']}"
         unless(ENV['files'])
@@ -17,12 +22,19 @@ module TaliaUtil
         FileList.new(ENV['files'])
       end
 
-      # Get the configured folder for the ontologies
+      # Gets the <tt>ontology_folder</tt> environment variable, as passed
+      # to the rake task with <tt>ontology_folder=[folder]</tt>
+      #
+      # Defaults to <tt>RAILS_ROOT/ontologies</tt> if the variable is not
+      # set.
       def ontology_folder
         ENV['ontology_folder'] || File.join(RAILS_ROOT, 'ontologies')
       end
 
-      # Set up the ontologies from the given folder
+      # Set up the ontologies from the ontology_folder. This clears
+      # the RDF context for the ontologies, if possible. Then it will
+      # load all ontologies in the ontology_folder into the store,
+      # and run the RdfUpdate::owl_to_rdfs update.
       def setup_ontologies
         # Clear the ontologies from RDF, if possible
         adapter = ActiveRDF::ConnectionPool.write_adapter
@@ -41,7 +53,13 @@ module TaliaUtil
         RdfUpdate::owl_to_rdfs
       end
 
-      # Init the talia core system
+      # Init the talia core system. Use to initialize the system for the 
+      # rake tasks. This will just exit if the system is already initialized,
+      # e.g. through Rails.
+      #
+      # See the rake:talia_core help task for options supported. Options
+      # include <tt>talia_root</tt>, <tt>environment</tt>, <tt>config</tt>,
+      # <tt>reset_db</tt> and <tt>reset_rdf</tt>
       def init_talia
         return if(TaliaCore::Initializer.initialized)
 
@@ -82,7 +100,8 @@ module TaliaUtil
         end
       end
 
-      # Get info from the Talia configuraion
+      # Rake helper. Prints the talia configuration to the 
+      # console.
       def talia_config
         puts "Talia configuration"
         puts ""
@@ -93,25 +112,32 @@ module TaliaUtil
         puts "Local Domain: #{N::LOCAL}"
       end
 
-      # Put the title for Talia
+      # Rake/Startup helper. Prints out the talia version/header to the the console.
       def title
         puts "\nTalia Digital Library system. Version: #{TaliaCore::Version::STRING}" 
         puts "http://www.muruca.org/\n\n"
       end
 
-      # Flush the database. This only flushes the main data tables!
+      # Flush the SQL database. This deletes all entries *only* from the main Talia
+      # tables in the db. Additional tables for user-defined models (e.g. translations)
+      # will _not_ be touched.
       def flush_db
         [ 'active_sources', 'data_records', 'semantic_properties', 'semantic_relations', 'workflows'].reverse.each { |f| ActiveRecord::Base.connection.execute "DELETE FROM #{f}" }
         # Also remove the "unsaved cache" for the wrappers (may be important during testing)
         TaliaCore::SemanticCollectionWrapper.instance_variable_set(:'@unsaved_source_cache', {})
       end
 
-      # Flush the RDF store
+      # Flush the RDF store. This clears the whole store, including triples that
+      # were added through other means than the Talia API.
       def flush_rdf
         ActiveRDF::ConnectionPool.write_adapter.clear
       end
 
-      # Remove the data directories
+      # Remove the data directories. Removes the data directory (configured
+      # as <tt>data_directory_location</tt> in talia_core.yml) and the iip directory
+      # (configured as <tt>iip_root_location</tt> in talia_core.yml). 
+      #
+      # This ignores non-existing directories without an error message.
       def clear_data
         data_dir = TaliaCore::CONFIG['data_directory_location']
         iip_dir = TaliaCore::CONFIG['iip_root_directory_location']
@@ -120,7 +146,9 @@ module TaliaUtil
       end
 
 
-      # Do a full reset of the data store
+      # Do a full reset of the data store. Equivalent to clearing
+      # the Talia SQL tables, the RDF store and the data directories.
+      # This will re-initialize the ontologies afterwards.
       def full_reset
         flush_db
         flush_rdf
@@ -128,11 +156,29 @@ module TaliaUtil
         setup_ontologies
       end
 
-      # Rewrite the RDF for the whole database. Will yield without parameters
-      # for progress-reporting blocks.
-      # FIXME: At the moment, this
-      # looses all RDF data that is not in the database.
-      def rewrite_rdf
+      # Rewrite the RDF for the whole database. Erases the RDF  store 
+      # completely and re-builds the graph from the data in the SQL
+      # tables. 
+      #
+      # *Warning:* This will *loose* all information contained in the
+      # RDF that is not duplicated. This includes all SWICKY notebooks!
+      #
+      # Unless the turn_off_safety flag is set, or the environment variable
+      # <tt>i_know_what_i_am_doing</tt> is set to "yes", this method will
+      # print an error message and raise an exception.
+      #
+      # For each triple written, this will yield to the block (if one is given)
+      # without parameters. For progress reporting, the overall number of
+      # triples that will be rewritten can be acquired with #rewrite_count
+      def rewrite_rdf(turn_off_safety=false)
+        unless((ENV['i_know_what_i_am_doing'].yes?) || turn_off_safety)
+          puts "WARNING: Rewriting the RDF will ERASE all data that does not come from the Talia API"
+          puts "This includes ALL SWICKY notebooks"
+          puts 
+          puts "To proceed run this task again, and give the following option:"
+          puts "i_know_what_i_am_doing=yes"
+          raise ArgumentError, "Can't proceed without confirmation."
+        end
         flush_rdf
         # We'll get all data from single query.
         fat_rels = TaliaCore::SemanticRelation.find(:all, :joins => fat_record_joins,
@@ -160,12 +206,14 @@ module TaliaUtil
         end
       end
 
-      # This gives the number of triples that would be rewritten on #rewrite_rdf
+      # The number of triples that would be rewritten with #rewrite_rdf
       def rewrite_count
         TaliaCore::SemanticRelation.count + TaliaCore::ActiveSource.count
       end
 
-      # Load the fixtures
+      # Load the database fixtures. This "manually" loads the database fixtures for
+      # the Talia tables for the core unit tests. The fixtures are those contained
+      # in the talia_core folder, _not_ the ones from the application's tests.
       def load_fixtures
         # fixtures = ENV['FIXTURES'] ? ENV['FIXTURES'].split(/,/) : Dir.glob(File.join(File.dirname(__FILE__), 'test', 'fixtures', '*.{yml,csv}'))  
         fixtures = [ 'active_sources', 'semantic_relations', 'semantic_properties' 'data_records']
@@ -175,21 +223,23 @@ module TaliaUtil
         end  
       end
 
-      # Do migrations
+      # Runs the migrations for the main Talia tables. This will use the migrations from 
+      # the "talia" generator, _not_ the ones from the Rails application.
       def do_migrations
         migration_path = File.join("generators", "talia", "templates", "migrations")
         ActiveRecord::Migrator.migrate(migration_path, ENV["VERSION"] ? ENV["VERSION"].to_i : nil )
       end
 
-      # Check if the given flag is set on the command line
+      # Check if the given flag is set on the command line. This will assert that the flag
+      # is set, otherwise it's equivalent to String#yes? (from the core extensions) on 
+      # the variable
       def flag?(the_flag)
         assit_not_nil(the_flag)
-        ENV[the_flag] && (ENV[the_flag] == "yes" || ENV[the_flag] == "true")
+        ENV[the_flag].yes?
       end
 
-      # For selecting "fat" records on the semantic properties, including the
-      # objects. Used for rewriting the rdf. TODO: Review after merging
-      # with optimized branch
+      # SQL select portion selecting "fat" records from the semantic_relatations table. 
+      # This will select all data needed to create all triples. Used for rewriting the rdf. 
       def fat_record_select
         select = 'semantic_relations.id AS id, semantic_relations.created_at AS created_at, '
         select << 'semantic_relations.updated_at AS updated_at, '
@@ -204,8 +254,7 @@ module TaliaUtil
         select
       end
 
-      # Select semantic properties joined with all connected tables.
-      # TODO: Review after merging with optimized branch
+      # SQL join snippet for selecting "fat" records. See fat_record_select
       def fat_record_joins
         joins =  " LEFT JOIN active_sources AS obj_sources ON semantic_relations.object_id = obj_sources.id AND semantic_relations.object_type = 'TaliaCore::ActiveSource'"
         joins << " LEFT JOIN semantic_properties AS obj_props ON semantic_relations.object_id = obj_props.id AND semantic_relations.object_type = 'TaliaCore::SemanticProperty'"
@@ -213,7 +262,7 @@ module TaliaUtil
         joins
       end
 
-      # print the common options for the tasks
+      # Print the options for the rake tasks to the console.
       def print_options
         puts "\nGeneral options (not all options are valid for all tasks):"
         puts "files=<pattern>     - Files for the task (a pattern to match the files)"
@@ -228,6 +277,38 @@ module TaliaUtil
         puts ""
       end
 
+      # Force-loads all Talia related models. This will attempt to load all classes in
+      # RAILS_ROOT/app/models and in TALIA_CODE_ROOT/lib/talia_core/source_types.
+      #
+      # Use this to make sure the whole hierarchy of ActiveSource subclasses is in memory..
+      def load_all_models
+        return if @models_loaded
+        load_models_from File.join(RAILS_ROOT, 'app', 'models', '**', '*.rb') if(defined? RAILS_ROOT)
+        load_models_from File.join(TALIA_CODE_ROOT, 'lib', 'talia_core', 'source_types',  '**',  '*.rb'), 'TaliaCore::SourceTypes::'
+        TaliaCore::Source
+        TaliaCore::Collection
+
+        @models_loaded = true
+      end
+
+      # Helper to load all classes from a given directory. This will attempt to instanciate all 
+      # classes found in the dir. The name of the class to be instantiated will be taken from the
+      # file name, prepending the prefix (e.g. 'ModuleName::') if set. 
+      def load_models_from(dir, prefix='')
+        # Appends a file system directory separator to the directory if needed.
+        dir = File.join(dir, '')
+        Dir[File.join(dir, '**', '*.rb')].each do |f| 
+          # For every rb file we try to gues and instantiate the contained class.
+          model_name = f.gsub(/#{dir}|\.rb/, '')
+          begin
+            (prefix + model_name).camelize.constantize
+          rescue Exception => e
+            # Errors at this point could be ignored, as there may be files that do not contain classes.
+            TaliaCore.logger.warn "Could not load class #{(prefix + model_name).camelize}: #{e.message}"
+          end
+        end
+      end
+
     end
   end
-end
+end