talia_core 0.5.4 → 0.7.0

data/lib/talia_core/active_source_parts/finders.rb

@@ -1,20 +1,31 @@
 module TaliaCore
   module ActiveSourceParts
 
-    # All class methods that are used for finding sources.
+    # Class method for ActiveSource that deal with #find and friends, and other forms of querying the
+    # data store. 
     module Finders
 
 
-      # Finder also accepts uris as "ids". There are also some additional options
-      # that are accepted:
+      # Extends the functionality of the ActiveRecord #find. This version also accepts URIs as
+      # "ids" and has a few additional options:
       # 
-      # [*:find_through*] accepts and array with an predicate name and an object
-      #                   value/uri, to search for predicates that match the given predicate/value 
-      #                   combination
-      # [*:type*] specifically looks for sources with the given type.
-      # [*:find_through_inv*] like :find_through, but for the "inverse" lookup
-      # [*:prefetch_relations*] if set to "true", this will pre-load all semantic
-      #                         relations for the sources (experimental, not fully implemented yet)
+      # [*find_through*] accepts and array with an predicate name and an object
+      #                  value/uri, to search for predicates that match the given predicate/value 
+      #                  combination
+      # [*type*] specifically looks for sources with the given type.
+      # [*find_through_inv*] like :find_through, but for the "inverse" lookup
+      # [*prefetch_relations*] if set to "true", this will pre-load all semantic
+      #                        relations for the sources (experimental, not fully implemented yet)
+      #
+      # == Examples:
+      #  # With a URI as id
+      #  ActiveSource.find(N::LOCAL.mySource) 
+      #  
+      #  # A URI as a string, same as above
+      #  ActiveSource.find('http://www.foobar.org')
+      #  
+      #  # Find through a given attribute, and prefetch all attributes for the found sources
+      #  ActiveSource.find(:all, :find_through => [N::DCT.creator, N::LOCAL.schopenhauer], :prefetch_relations => true)
       def find(*args)
         prefetching = false
         if(args.last.is_a?(Hash))
@@ -45,8 +56,7 @@ module TaliaCore
         result
       end
       
-      # Modify the count helper so that it can use the advanced options of the 
-      # ActiveSource #find routine
+      # The count for ActiveSource will accept the same options as the find method
       def count(*args)
         if((options = args.last).is_a?(Hash))
           options.to_options!
@@ -57,23 +67,28 @@ module TaliaCore
       end
 
       # Find a list of sources which contains the given token inside the local name.
-      # This means that the namespace it will be excluded.
+      # This means that the namespace it will be excluded from the toke search
+      #
+      # == Example
       #
-      #   Sources in system:
-      #     * http://talia.org/one
-      #     * http://talia.org/two
+      # Sources in system:
+      # * http://talia.org/one
+      # * http://talia.org/two
+      #
+      # With these sources, you will get:
       #
       #   Source.find_by_uri_token('a') # => [ ]
       #   Source.find_by_uri_token('o') # => [ 'http://talia.org/one', 'http://talia.org/two' ]
       #
-      # NOTE: It internally use a MySQL function, as sql condition, to find the local name of the uri.
+      # NOTE: It internally uses a MySQL function, as sql condition, to find the local name of the uri.
       def find_by_uri_token(token, options = {})
         find(:all, { 
           :conditions => [ "LOWER(SUBSTRING_INDEX(uri, '/', -1)) LIKE ?", '%' + token.downcase + '%' ], 
           :order => "uri ASC" }.merge!(options))
       end
 
-      # Find the Sources within the given namespace by a partial local name
+      # Find the Sources within the given namespace by a partial local name. Works like
+      # #find_by_uri_token, except that only sources from the given namspace are returned
       def find_by_partial_local(namespace, local_part, options = {})
         namesp = N::URI[namespace]
         return [] unless(namesp)
@@ -82,12 +97,8 @@ module TaliaCore
           :order => "uri ASC"}.merge!(options))
       end
 
-      # Find the fist Source that matches the given URI.
-      # It's useful for admin pane, because users visit:
-      #   /admin/sources/<source_id>/edit
-      # but that information is not enough, since we store
-      # into the database the whole reference as URI:
-      #   http://localnode.org/av_media_sources/source_id
+      # Find the fist Source that matches the given URI. This works like #find_by_uri_token,
+      # except that the whole URI is matched, not only the local name.
       def find_by_partial_uri(id, options = {})
         find(:all, { :conditions => ["uri LIKE ?", '%' + id + '%'] }.merge!(options))
       end
@@ -111,18 +122,30 @@ module TaliaCore
           raise(ArgumentError, "Cannot pass custom join conditions with :find_through") if(options.has_key?(:joins))
           predicate = f_through[0]
           obj_val = f_through[1]
-          search_prop = (f_through.size > 2) ? f_through[2] : !(obj_val.to_s =~ /:/)
+          search_prop = check_if_search_value(f_through)
           options[:joins] = default_joins(!search_prop, search_prop)
           options[:conditions] ||= {}
           options[:conditions]['semantic_relations.predicate_uri'] = predicate.to_s
           if(search_prop)
             options[:conditions]['obj_props.value'] = obj_val.to_s
           else
-            options[:conditions]['obj_sources.uri'] = obj_val.to_s
+            options[:conditions]['obj_sources.uri'] = (obj_val.respond_to?(:uri) ? obj_val.uri.to_s : obj_val.to_s)
           end
         end
       end
 
+      # Checks if the given find_through options should search for a value or
+      # an object. See #check_for_find_through
+      def check_if_search_value(finder_array)
+        if(finder_array.size > 2)
+          finder_array[2]
+        elsif(finder_array[1].respond_to?(:uri))
+          false
+        else
+          !(finder_array[1].to_s =~ /:/)
+        end
+      end
+
       # Check for the :find_through_inv option. This expects the 2 basic values
       # in the same way as :find_through.
       #

data/lib/talia_core/active_source_parts/predicate_handler.rb

@@ -1,9 +1,20 @@
 module TaliaCore
   module ActiveSourceParts
+    
+    # Methods for ActiveSource objects for accessing and handling predicates, which are the
+    # properties connected to the source. When accessing a predicate/property of an ActiveSource,
+    # the system will return a SemanticCollectionWrapper.
+    #
+    # Once a predicate is loaded, the ActiveSource will cache the SemanticCollectionWrapper
+    # internally, and will re-use it on subsequent accesses. 
+    #
+    # If the relations are prefetched (usually by providing the :prefetch_relations option to
+    # the find Method, see the Finders module), all wrappers for the source are loaded at once;
+    # the actual prefetching is done by the prefetch_relations_for method, which can also be
+    # used directly on a collection of sources.
     module PredicateHandler
-      # This file contains the handling of the "predicate wrapper" lists
-      # that represent the properties/objects a class has for a given predicate
 
+      # Predicate-related class methods. See the PredicateHandler module for more
       module ClassMethods
 
         # Attempts to fetch all relations on the given sources at once, so that
@@ -12,18 +23,21 @@ module TaliaCore
         # For safety reasons, there is a limit on the number of sources that is
         # accepted. (For a web application, if you go over the default, you're
         # probably doing it wrong).
+        #
+        # When prefetching, all the relations/properties for the given sources are
+        # loaded in a single request, and the data is injected in the internal cache
+        # of the sources. 
+        #
+        # A source with prefetched relations will not cause database queries if you
+        # access its properties.
         def prefetch_relations_for(sources, limit = 1024)
           sources = [ sources ] if(sources.is_a?(ActiveSource))
           raise(RangeError, "Too many sources for prefetching.") if(sources.size > limit)
           src_hash = {}
           sources.each { |src| src_hash[src.id] = src }
-          conditions = { :subject_id => src_hash.keys }
-          joins = ActiveSource.sources_join
-          joins << ActiveSource.props_join
-          relations = SemanticRelation.find(:all, :conditions => conditions,
-          :joins => joins,
-          :select => SemanticRelation.fat_record_select
-          )
+          
+          relations = SemanticRelation.find(:all, :conditions => { :subject_id => src_hash.keys }, :include => [:subject, :object])
+          
           relations.each do |rel|
             src_hash[rel.subject_id].inject_predicate(rel)
           end
@@ -37,20 +51,28 @@ module TaliaCore
 
       end # End class methods
 
-      # Gets the types
+      # Gets the RDF types for the source. This is equivalent to accessing the
+      # rdf:type predicate.
       def types
         get_objects_on(N::RDF.type.to_s)
       end
       
-      # Returns if the Facsimile is of the given type
+      # Checks if the source has the given RDF type
       def has_type?(type)
         (self.types.include?(type))
       end
 
-      # Returns the objects on the given predicate. This will be cached internally
-      # so that the object will always be the same as long as the parent source
-      # lives.
-      def get_objects_on(predicate)
+      # Returns the SemanticCollectionWrapper for the given predicate. The collection
+      # wrapper will be cached internally, so that subsequent calls will receive the
+      # same collection wrapper again.
+      #
+      # This also means that any modifications to the wrapper are preserved in the 
+      # cache - if a wrapper is modified in memory, and accessed again _on the same
+      # source_, a subsequent access will return the modified wrapper.
+      #
+      # Modified wrappers are saved when the ActiveSource itself is saved (through
+      # save_wrappers, which is automatically called)
+      def get_wrapper_on(predicate)
         @type_cache ||= {}
         active_wrapper = @type_cache[predicate.to_s]
 
@@ -66,8 +88,30 @@ module TaliaCore
         
         active_wrapper
       end
+      
+      # Returns the values for the given predicate. This will work like #get_wrapper_on
+      # _except_ if the predicate is declared as a :singular_property with ActiveSouce.property_options
+      # (or singular_property, or multi_property). In that case, it will 
+      # return only a single value. However, if the predicate is declare singular and
+      # there already is more than one value, it will return the wrapper (and fail an assit).
+      # In general, a property that was defined singular and contains more than one value 
+      # indicates a problem with the application.
+      def get_objects_on(predicate)
+        singular = property_options_for(predicate)[:singular_property].true?
+        wrapper = get_wrapper_on(predicate) 
+
+        if(singular && wrapper.size <= 1) 
+          wrapper.first
+        else
+          assit(!singular, 'Was expecting a single value (property is defined as singular). Got more than one.')
+          wrapper
+        end
+      end
 
-      # Go through the existing relation wrappers and save the (new) items
+      # Goes through the existing SemanticCollectionWrappers in the cache, and
+      # saves any modifications that may exist.
+      #
+      # This is automatically called when the source is saved.
       def save_wrappers
         each_cached_wrapper do |wrap|
           # Load unloaded if we're not rdf_autosaving. Quick hack since otherwise
@@ -78,22 +122,26 @@ module TaliaCore
         end
       end
 
-      # Loops through the cache and passes each existing wrapper to the block
+      # Loops through the wrapper cache and passes each of the SemanticCollectionWrappers
+      # in the cache to the block given to this method
       def each_cached_wrapper
         return unless(@type_cache)
         @type_cache.each_value {  |wrap| yield(wrap) }
       end
 
-      # Clear the source (this will force reloading of elements and discard
-      # unsaved changes
+      # Resets the internal cache of wrappers/properties. Any unsaved changes on
+      # the wrappers are lost, and get_object_on will have to reload all data when it is
+      # called again
       def reset!
         @type_cache = nil
       end
 
-      # Injects a 'fat' predicate relation on a source
-      def inject_predicate(fat_relation)
-        wrapper = get_objects_on(fat_relation.predicate_uri)
-        wrapper.inject_fat_item(fat_relation)
+      # Injects a 'fat relation' into the cache/wrappter. A "fat" relation is a 
+      # SemanticRelation which contains additional fields (e.g. the subject uri, all
+      # object information, etc.) - See also the SemanticCollectionWrapper documentation.
+      def inject_predicate(relation)
+        wrapper = get_wrapper_on(relation.predicate_uri)
+        wrapper.insert_item(relation)
       end
     end
   end

data/lib/talia_core/active_source_parts/rdf/ntriples_reader.rb

@@ -0,0 +1,13 @@
+module TaliaCore
+  module ActiveSourceParts
+    module Rdf
+      class NtriplesReader < RdfReader
+        require 'rdf/ntriples'
+        def format
+          :ntriples
+        end
+      end
+    end
+  end
+end
+

data/lib/talia_core/active_source_parts/rdf/rdf_reader.rb

@@ -0,0 +1,99 @@
+require 'rdf'
+
+module TaliaCore
+  module ActiveSourceParts
+    module Rdf
+
+      # Import class for rdf ntriples files using rdf.rb (http://rdf.rubyforge.org/).
+      # See GenericReader for more information on Talia import classes in general.
+      class RdfReader
+
+        extend TaliaUtil::IoHelper
+        include TaliaUtil::IoHelper
+        include TaliaUtil::Progressable
+
+        class << self
+          # See TaliaCore::ActiveSourceParts::Xml::GenericReader#sources_from_url
+          def sources_from_url(url, options=nil, progressor=nil)
+            open_generic(url, options) {|io| sources_from(io, progressor, url)}
+          end
+
+          # See TaliaCore::ActiveSourceParts::Xml::GenericReader#sources_from
+          def sources_from(source, progressor=nil, base_url=nil)
+            reader = self.new(source)
+            reader.progressor = progressor
+            reader.sources
+          end
+
+        end # End class methods
+
+        # On inititalization: 
+        # * We are going to use Class.subclasses_of method to determine the talia type of the source.
+        #   Due to the autoload functionality of rails we need to be sure any possible source type class file 
+        #   is loaded when we actually use that method. This is what TaliaUtil::Util::load_all_models does.
+        # * Works only with format=:ntriples for now.
+        def initialize(source)
+          TaliaUtil::Util.load_all_models
+          source = StringIO.new(source) if(source.is_a? String)
+          @reader = RDF::Reader.for(format).new(source)
+        end
+        
+        # See TaliaCore::ActiveSourceParts::Xml::GenericReader#sources
+        def sources
+          return @sources if(@sources)
+          @sources = {}
+          run_with_progress('RdfRead', 0) do |progress|
+            @reader.each_statement do |statement|
+              source = (@sources[statement.subject.to_s] ||= {})
+              source['uri'] ||= statement.subject.to_s
+              update_source_type(source, statement)
+              source[statement.predicate.to_s] ||= []
+              object = if(statement.object.literal?)
+                         parsed_string = PropertyString.parse(statement.object.value)
+                         parsed_string.lang = statement.object.language.to_s if(statement.object.language)
+                         parsed_string
+                       else
+                         "<#{statement.object.to_s}>"
+                       end
+              source[statement.predicate.to_s] << object
+              progress.inc
+            end
+          end
+          # Set all empty source types to ActiveSource, to prevent DummySource type objects to
+          # be created. (Reason: When we import RDF, we assume that all sources are "valid", and should
+          # never be marked as DummySource)
+          @sources = @sources.values
+          @sources.each { |s| s['type'] ||= 'TaliaCore::ActiveSource' }
+          @sources
+        end
+
+        # Update the Talia source type. The type can be contained explicitly as object of a N::TALIA.type
+        # predicate or can be inferred from the rdf type if a N::RDF.type predicate is present.
+        #
+        # The method works in the way that a N::TALIA type attribute always overwrites the type, while an
+        # N::RDF.type will only be used if no type has been set on the source
+        def update_source_type(source, statement)
+          return if(statement.object.literal?)
+          case(statement.predicate.to_s)
+          when N::TALIA.type.to_s
+            source['type'] = statement.object.to_s
+          when N::RDF.type.to_s
+            source['type'] ||= rdf_to_talia_type statement.object.to_s
+          end
+        end
+
+        # Tries to gues at the talia type of a source given its rdf type.
+        def rdf_to_talia_type(rdf_type)
+          Class.subclasses_of(TaliaCore::ActiveSource).detect do |c|
+            c.additional_rdf_types.include? rdf_type
+          end.try :name
+        end
+
+        def format
+          raise NotImplementedError
+        end
+
+      end
+    end
+  end
+end

data/lib/talia_core/active_source_parts/rdf/rdfxml_reader.rb

@@ -0,0 +1,12 @@
+module TaliaCore
+  module ActiveSourceParts
+    module Rdf
+      class RdfxmlReader < RdfReader
+        require 'rdf/raptor'
+        def format
+          :rdfxml
+        end
+      end
+    end
+  end
+end

data/lib/talia_core/active_source_parts/{rdf.rb → rdf_handler.rb}

@@ -1,24 +1,35 @@
 module TaliaCore
   module ActiveSourceParts
 
-    module Rdf
-      # This file contains the RDF handling elements of the ActiveSource class
+    # Methods for the ActiveSource class to automatically create the RDF triples
+    # for a source and to access the RDF data of the source.
+    #
+    # The RDF for a source will be automatically created through the auto_create_rdf
+    # callback when the source is set (and autosave_rdf is set)
+    module RdfHandler
 
-      # This can be used to turn of automatic rdf creation. *Attention:* Improperly
-      # used this may compromise the integrity of the RDF data. However, it may
-      # be used in order to speed up "create" operations that save a record
-      # several times and don't need the RDF data in the meantime.
+      # Returns the value of the autosave_rdf flag, as set by autosave_rdf=
       def autosave_rdf?
         @autosave_rdf = true unless(defined?(@autosave_rdf))
         @autosave_rdf
       end
 
-      # Set the autosave property. See autosave_rdf?
+      # This can be used to turn of automatic rdf creation. If set to true, 
+      # create_rdf will *not* be called automatically after saving the source.
+      # 
+      # *Attention*: Improper use will compromise the integrity of the RDF data. 
+      # However, it may
+      # be used in order to speed up operations that save a record
+      # several times and don't need the RDF data in the meantime.
       def autosave_rdf=(value)
         @autosave_rdf = value
       end
 
-      # Returns the RDF object to use for this ActiveSource
+      # Returns the RDF object to use for this ActiveSource. This
+      # will return a RdfResource, which has a similiar (but more
+      # limited API) than the ActiveSource itself. All operations and
+      # queries on that resource will go to the RDF store instead of the
+      # database
       def my_rdf
         @rdf_resource ||= begin
           src = RdfResource.new(uri)
@@ -36,10 +47,20 @@ module TaliaCore
       # will not changed, but if database objects were not added through the
       # standard API they'll be missed
       #
-      # The force option may have three values: :false for normal operation,
-      # :force for forcing a complete rewrite and :create - the latter will
-      # avoid the cleaning of the elements in the RDF store in case the
-      # source is completely new and no triples exist.
+      # The force option may have three values: 
+      #
+      # [*false*] Normal operation. This retrieves the data for each of the 
+      #           cached SemanticCollectionWrappers that may have been modified and rewrites
+      #           the respective attribute in the RDF store. (see the PredicateHandler 
+      #           module for an explanation of the wrapper cache). It will ignore wrappers that 
+      #           are obviously "clean", but will do a "retrieve and write" for each wrapper 
+      #           separately.
+      # [*force*] Force a complete rewrite of the RDF data for this source. This will erase the
+      #           RDF and write all triples for this source in one go. It will also remove
+      #           any triples for this source that may have been added externally.
+      # [*create*] Do not check for any existing data, just write out the data that is in the
+      #            cache. This is fast, but must *only* be used for new sources where it is
+      #            certain that no data for the source exists in the RDF store.
       def create_rdf(force = :false)
         self.class.benchmark("\033[32m\033[4m\033[1mActiveSource::RD\033[0m Creating RDF for source", Logger::DEBUG, false) do
           assit(!new_record?, "Record must exist here: #{self.uri}")
@@ -58,6 +79,7 @@ module TaliaCore
             # the value. If not the RDF handler will detect the #uri method and
             # will add it as Resource.
             obj = sem_ref.object
+            assit(obj, "Must have an object here. #{sem_ref.inspect}")
             value = obj.is_a?(SemanticProperty) ? obj.value : obj
             my_rdf.direct_write_predicate(N::URI.new(sem_ref.predicate_uri), value)
           end
@@ -66,7 +88,8 @@ module TaliaCore
         end
       end
       
-      # Creates an RDF/XML resprentation of the source
+      # Creates an RDF/XML resprentation of the source. See the Xml::RdfBuilder and the
+      # Xml::SourceReader for more information.
       def to_rdf
         rdf = String.new
 
@@ -80,7 +103,9 @@ module TaliaCore
       private
 
       # Get the "standard" predicates to write (which is just the ones changed
-      # through the standard API. This will erase the
+      # through the standard API. This will go through each of the cached
+      # SemanticCollectionWrapper(s) and it will erase the triples for each
+      # of the wrappers separately.
       def prepare_predicates_to_write
         preds_to_write = []
         each_cached_wrapper do |wrap|
@@ -89,17 +114,17 @@ module TaliaCore
           # Remove the existing data. TODO: Not using contexts
           my_rdf.remove(N::URI.new(wrap.instance_variable_get(:@assoc_predicate)))
           items = wrap.send(:items) # Get the items
-          items.each { |it| preds_to_write << it.relation }
+          items.each { |it| preds_to_write << it }
         end
         preds_to_write
       end
 
       # This will get all existing predicates from the database. This will also
-      # erase the rdf for this source completely
-      # TODO: Could load with a single sql
+      # erase the rdf for this source completely in one go
       def prepare_all_predicates_to_write
+        # TODO: Could load with a single sql
         my_rdf.clear_rdf # TODO: Not using contexts here
-        SemanticRelation.find(:all, :conditions => { :subject_id => self.id })
+        SemanticRelation.find(:all, :conditions => { :subject_id => self.id }, :include => [ :object ])
       end
       
       # ATTENTION: This is a speed hack that avoids the usual checks based
@@ -111,7 +136,7 @@ module TaliaCore
           next if(wrap.clean?)
           # Evil, we get the items directly to avoid a useless load
           items = wrap.instance_variable_get(:@items)
-          items.each { |it| preds_to_create << it.relation }
+          items.each { |it| preds_to_create << it }
         end
         preds_to_create
       end
@@ -125,6 +150,14 @@ module TaliaCore
       def auto_create_rdf
         create_rdf(:create) if(autosave_rdf?)
       end
+      
+      # Cleans out the RDF data before the source is destroyed
+      def clear_rdf
+        if autosave_rdf?
+          ActiveRDF::FederationManager.delete(self, :predicate, :object)
+          ActiveRDF::FederationManager.delete(:subject, :predicate, self)
+        end
+      end
 
     end
   end