talia_core 0.4.0

data/lib/talia_core/active_source_parts/class_methods.rb

@@ -0,0 +1,378 @@
+module TaliaCore
+  module ActiveSourceParts
+    module ClassMethods
+
+      # Accessor for addtional rdf types that will automatically be added to each
+      # object of that Source class
+      def additional_rdf_types 
+        @additional_rdf_types ||= []
+      end
+
+      # New method for ActiveSources. If a URL of an existing Source is given as the only parameter, 
+      # that source will be returned. This makes the class work smoothly with our ActiveRDF version
+      # query interface.
+      #
+      # Note that any semantic properties that were passed in to the constructor will be assigned
+      # *after* the ActiveRecord "create" callbacks have been called.
+      #
+      # The option hash may contain a "files" option, which can be used to add data files directly
+      # on creation. This will call the attach_files method on the object.
+      def new(*args)
+        the_source = if((args.size == 1) && (args.first.is_a?(Hash)))
+          # We have an option hash to init the source
+          files = args.first.delete(:files) || args.first.delete('files')
+          attributes = split_attribute_hash(args.first)
+          the_source = super(attributes[:db_attributes])
+          the_source.add_semantic_attributes(false, attributes[:semantic_attributes])
+          the_source.attach_files(files) if(files)
+          the_source
+        elsif(args.size == 1 && ( uri_s = uri_string_for(args[0]))) # One string argument should be the uri
+          # Either the current object from the db, or a new one if it doesn't exist in the db
+          find(:first, :conditions => { :uri => uri_s } ) || super(:uri => uri_s)
+        else
+          # In this case, it's a generic "new" call
+          super
+        end
+        the_source.add_additional_rdf_types if(the_source.new_record?)
+        the_source
+      end
+
+      # Retrieves a new source with the given type. This gets a propety hash
+      # like #new, but it will correctly initialize a source of the type given
+      # in the hash. If no type is given, this will create a plain ActiveSource.
+      def create_source(args)
+        type = args.delete(:type) || args.delete('type') || 'ActiveSource'
+        klass = type.constantize
+        klass.new(args)
+      end
+
+      # Create sources from XML. The result is either a single source or an Array
+      # of sources, depending on wether the XML contains multiple sources.
+      #
+      # The imported sources will be saved during import, to ensure that relations
+      # between them are resolved correctly. If one of the imported elements
+      # does already exist, the existing source will be rewritten using ActiveSource#rewrite_attributes
+      #
+      # The options may contain:
+      #
+      # [*reader*] The reader class that the import should use
+      # [*progressor*] The progress reporting object, which must respond to run_with_progress(message, size, &block)
+      # [*errors*] If given, all erors will be looged to this array instead of raising
+      #            an exception
+      def create_from_xml(xml, options = {})
+        reader = options[:reader] ? options[:reader].to_s.classify.constantize : TaliaCore::ActiveSourceParts::Xml::SourceReader
+        source_properties = reader.sources_from(xml, options[:progressor])
+        self.progressor = options[:progressor]
+        sources = create_multi_from(source_properties)
+        (sources.size > 1) ? sources : sources.first
+      end
+
+      # Creates multiple sources from the given array of attribute hashes. The
+      # sources are saved during import, ensuring that the relations are resolved
+      # correctly.
+      #
+      # Options:
+      # [*errors*] If given, all erors will be looged to this array instead of raising
+      #            an exception
+      def create_multi_from(sources, options = {})
+        source_objects = []
+        run_with_progress('Writing imported', sources.size) do |progress|
+          source_objects = sources.collect do |props|
+            src = nil
+            begin
+              if(src = ActiveSource.find(:first, :conditions => { :uri => (props[:uri] || props['uri']) }))
+                # Deal with already existing sources
+                src.rewrite_attributes(props)
+                # Rewrite the type, if neccessary
+                type = props[:type] || props['type']
+                switch_type = type && (src.type != type)
+                # Warn to the log if we have a problematic type change
+                TaliaCore.logger.warn("WARNING: Type change from #{src.type} to #{type}") if(switch_type && !src.is_a?(DummySource))
+                src.type = type if(switch_type)
+                src
+              else
+                src = ActiveSource.create_source(props)
+              end
+              src.save!
+            rescue Exception => e
+              if(options[:errors]) 
+                options[:errors] << "ERROR during import of #{props['uri'] || props[:uri]}: #{e.message}" 
+                TALIA_CORE.logger.warn("Problems importing #{props['uri'] || props[:uri]} (logged): #{e.message}")
+              else
+                raise
+              end
+            end
+            progress.inc
+            src
+          end
+        end
+        source_objects
+      end
+
+      # This method is slightly expanded to allow passing uris and uri objects
+      # as an "id"
+      def exists?(value)
+        if(uri_s = uri_string_for(value))
+          super(:uri => uri_s)
+        else
+          super
+        end
+      end
+
+
+      # Finder also accepts uris as "ids". There are also some additional options
+      # that are accepted:
+      # 
+      # [*:find_through*] accepts and array with an predicate name and an object
+      #                   value/uri, to search for predicates that match the given predicate/value 
+      #                   combination
+      # [*:type] specifically looks for sources with the given type.
+      # [*:find_through_inv*] like :find_through, but for the "inverse" lookup
+      # [*:prefetch_relations*] if set to "true", this will pre-load all semantic
+      #                         relations for the sources (experimental, not fully implemented yet)
+      def find(*args)
+        prefetching = false
+        if(args.last.is_a?(Hash))
+          options = args.last
+          prefetching =  options.delete(:prefetch_relations)
+          if(options.empty?) # If empty we remove the args hash, so that the 1-param uri search works
+            args.pop
+          else
+            prepare_options!(args.last)
+          end
+        end
+        result = if(args.size == 1 && (uri_s = uri_string_for(args[0])))
+          src = super(:first, :conditions => { :uri => uri_s })
+          raise(ActiveRecord::RecordNotFound, "Not found: #{uri_s}") unless(src)
+          src
+        else
+          super
+        end
+
+        prefetch_relations_for(result) if(prefetching)
+
+        result
+      end
+
+      # Semantic version of ActiveRecord::Base#update - the id may be a record id or an URL,
+      # and the attributes may contain semantic attributes. See the update_attributes method
+      # for details on how the semantic attributes behave.
+      def update(id, attributes)
+        record = find(id)
+        raise(ActiveRecord::RecordNotFound) unless(record)
+        record.update_attributes(attributes)
+      end
+
+      # Like update, only that it will overwrite the given attributes instead
+      # of adding to them
+      def rewrite(id, attributes)
+        record = find(id)
+        raise(ActiveRecord::RecordNotFound) unless(record)
+        record.rewrite_attributes(attributes)
+      end
+
+      # The pagination will also use the prepare_options! to have access to the
+      # advanced finder options
+      def paginate(*args)
+        prepare_options!(args.last) if(args.last.is_a?(Hash))
+        super
+      end
+
+      # If will return itself unless the value is a SemanticProperty, in which
+      # case it will return the property's value.
+      def value_for(thing)
+        thing.is_a?(SemanticProperty) ? thing.value : thing
+      end
+
+      # Returns true if the given attribute is one that is stored in the database
+      def db_attr?(attribute)
+        db_attributes.include?(attribute.to_s)
+      end
+
+      # Tries to expand a generic URI value that is either given as a full URL
+      # or a namespace:name value.
+      #
+      # This will assume a full URL if it finds a ":/" string inside the URI. 
+      # Otherwise it will construct a namespace - name URI
+      def expand_uri(uri) # TODO: Merge with uri_for ?
+        assit_block do |errors| 
+          unless(uri.respond_to?(:uri) || uri.kind_of?(String)) || uri.kind_of?(Symbol)
+            errors << "Found strange object of type #{uri.class}"
+          end
+          true
+        end
+        uri = uri.respond_to?(:uri) ? uri.uri.to_s : uri.to_s
+        return uri if(uri.include?(':/'))
+        N::URI.make_uri(uri).to_s
+      end
+
+      # Splits the attribute hash that is given for new, update and the like. This
+      # will return another hash, where result[:db_attributes] will contain the
+      # hash of the database attributes while result[:semantic_attributes] will
+      # contain the other attributes. 
+      #
+      # The semantic attributes will be expanded to full URIs whereever possible.
+      #
+      # This method will *not* check for attributes that correspond to singular
+      # property names.
+      def split_attribute_hash(attributes)
+        assit_kind_of(Hash, attributes)
+        db_attributes = {}
+        semantic_attributes = {}
+        attributes.each do |field, value|
+          if(db_attr?(field))
+            db_attributes[field] = value
+          else
+            semantic_attributes[expand_uri(field)] = value
+          end
+        end
+        { :semantic_attributes => semantic_attributes, :db_attributes => db_attributes }
+      end
+
+      private
+
+      # The attributes stored in the database
+      def db_attributes
+        @db_attributes ||= ActiveSource.new.attribute_names
+      end
+
+      # Helper to define a "additional type" in subclasses which will 
+      # automatically be added on Object creation
+      def has_rdf_type(*types)
+        @additional_rdf_types ||= []
+        types.each { |t| @additional_rdf_types << t.to_s }
+      end
+
+      # Helper to define a "singular accessor" for something (e.g. siglum, catalog)
+      # This accessor will provide an "accessor" method that returns the
+      # single property value directly and an assignment method that replaces
+      # the property with the value.
+      #
+      # The Source will cache newly set singular properties internally, so that
+      # the new value is immediately reflected on the object. However, the
+      # change will only be made permanent on #save! - and saving will also clear
+      # the cache
+      def singular_property(prop_name, property)
+        prop_name = prop_name.to_s
+        @singular_props ||= []
+        return if(@singular_props.include?(prop_name))
+        raise(ArgumentError, "Cannot overwrite method #{prop_name}") if(self.instance_methods.include?(prop_name) || self.instance_methods.include?("#{prop_name}="))
+        # define the accessor
+        define_method(prop_name) do
+          prop = self[property]
+          assit_block { |err| (prop.size > 1) ? err << "Must have at most 1 value for singular property #{prop_name} on #{self.uri}. Values #{self[property]}" : true }
+          prop.size > 0 ? prop[0] : nil
+        end
+
+        # define the writer
+        define_method("#{prop_name}=") do |value|
+          prop = self[property]
+          prop.remove
+          prop << value
+        end
+
+        # define the finder
+        (class << self ; self; end).module_eval do
+          define_method("find_by_#{prop_name}") do |value, *optional|
+            raise(ArgumentError, "Too many options") if(optional.size > 1)
+            options = optional.last || {}
+            finder = options.merge( :find_through => [property, value] )
+            find(:all, finder)
+          end
+        end
+
+        @singular_props << prop_name
+        true
+      end
+
+      # Helper to creat an accessor for the given predicate. This will shortcut
+      # the prop_name method to self[property]
+      def simple_property(prop_name, property)
+        define_method(prop_name) do
+          self[property]
+        end
+      end
+
+      # This gets the URI string from the given value. This will just return
+      # the value if it's a string. It will return the result of value.uri, if
+      # that method exists; otherwise it'll return nil
+      def uri_string_for(value)
+        result = if value.is_a? String
+          # if this is a local name, prepend the local namespace
+          (value =~ /:/) ? value : (N::LOCAL + value).uri
+        elsif(value.respond_to?(:uri))
+          value.uri
+        else
+          nil
+        end
+        result = result.to_s if result
+        result
+      end
+
+
+      # Takes the "advanced" options that can be passed to the find method and
+      # converts them into "standard" find options.
+      def prepare_options!(options)
+        check_for_find_through!(options)
+        check_for_type_find!(options)
+        check_for_find_through_inv!(options)
+      end
+
+      # Checks if the :find_through option is set. If so, this expects the
+      # option to have 2 values: The first representing the URL of the predicate
+      # and the second the URL or value that should be matched.
+      #
+      # An optional third parameter can be used to force an object search on the
+      # semantic_properties table (instead of active_sources) - if not present
+      # this will be auto-guessed from the "object value", checking if it appears
+      # to be an URL or not.
+      #
+      #   ...find(:find_through => [N::RDF::something, 'value', true]
+      def check_for_find_through!(options)
+        if(f_through = options.delete(:find_through))
+          assit_kind_of(Array, f_through)
+          raise(ArgumentError, "Passed non-hash conditions with :find_through") if(options.has_key?(:conditions) && !options[:conditions].is_a?(Hash))
+          raise(ArgumentError, "Cannot pass custom join conditions with :find_through") if(options.has_key?(:joins))
+          predicate = f_through[0]
+          obj_val = f_through[1]
+          search_prop = (f_through.size > 2) ? f_through[2] : !(obj_val.to_s =~ /:/)
+          options[:joins] = default_joins(!search_prop, search_prop)
+          options[:conditions] ||= {}
+          options[:conditions]['semantic_relations.predicate_uri'] = predicate.to_s
+          if(search_prop)
+            options[:conditions]['obj_props.value'] = obj_val.to_s
+          else
+            options[:conditions]['obj_sources.uri'] = obj_val.to_s
+          end
+        end
+      end
+
+      # Check for the :find_through_inv option. This expects the 2 basic values
+      # in the same way as :find_through.
+      #
+      # find(:find_through_inv => [N::RDF::to_me, my_uri]
+      def check_for_find_through_inv!(options)
+        if(f_through = options.delete(:find_through_inv))
+          assit_kind_of(Array, f_through)
+          raise(ArgumentError, "Passed non-hash conditions with :find_through") if(options.has_key?(:conditions) && !options[:conditions].is_a?(Hash))
+          raise(ArgumentError, "Cannot pass custom join conditions with :find_through") if(options.has_key?(:joins))
+          options[:joins] = default_inv_joins
+          options[:conditions] ||= {}
+          options[:conditions]['semantic_relations.predicate_uri'] = f_through[0].to_s
+          options[:conditions]['sub_sources.uri'] = f_through[1].to_s
+        end
+      end
+
+
+      # Checks for the :type option in the find options. This is the same as
+      # doing a :find_through on the rdf type
+      def check_for_type_find!(options)
+        if(f_type = options.delete(:type))
+          options[:find_through] = [N::RDF::type, f_type.to_s, false]
+          check_for_find_through!(options)
+        end
+      end
+
+    end
+  end
+end

data/lib/talia_core/active_source_parts/predicate_handler.rb

@@ -0,0 +1,89 @@
+module TaliaCore
+  module ActiveSourceParts
+    module PredicateHandler
+      # This file contains the handling of the "predicate wrapper" lists
+      # that represent the properties/objects a class has for a given predicate
+
+      module ClassMethods
+
+        # Attempts to fetch all relations on the given sources at once, so that
+        # there is potentially only one.
+        #
+        # For safety reasons, there is a limit on the number of sources that is
+        # accepted. (For a web application, if you go over the default, you're
+        # probably doing it wrong).
+        def prefetch_relations_for(sources, limit = 1024)
+          sources = [ sources ] if(sources.is_a?(ActiveSource))
+          raise(RangeError, "Too many sources for prefetching.") if(sources.size > limit)
+          src_hash = {}
+          sources.each { |src| src_hash[src.id] = src }
+          conditions = ['subject_id in (?)', src_hash.keys.join(', ')]
+          joins = ActiveSource.sources_join
+          joins << ActiveSource.props_join
+          relations = SemanticRelation.find(:all, :conditions => conditions,
+          :joins => joins,
+          :select => SemanticRelation.fat_record_select
+          )
+          relations.each do |rel|
+            src_hash[rel.subject_id].inject_predicate(rel)
+          end
+
+          # Set all as loaded
+          sources.each do |src|
+            src.each_cached_wrapper { |wrap| wrap.instance_variable_set(:'@loaded', true) }
+          end
+        end
+
+      end # End class methods
+
+      # Gets the types
+      def types
+        get_objects_on(N::RDF.type.to_s)
+      end
+
+      # Returns the objects on the given predicate. This will be cached internally
+      # so that the object will always be the same as long as the parent source
+      # lives.
+      def get_objects_on(predicate)
+        @type_cache ||= {}
+        active_wrapper = @type_cache[predicate.to_s]
+
+        if(active_wrapper.nil?)
+          active_wrapper = SemanticCollectionWrapper.new(self, predicate)
+          @type_cache[predicate.to_s] = active_wrapper
+        end
+
+        active_wrapper
+      end
+
+      # Go through the existing relation wrappers and save the (new) items
+      def save_wrappers
+        each_cached_wrapper do |wrap|
+          # Load unloaded if we're not rdf_autosaving. Quick hack since otherwise
+          # since the blanking of unloaded properties could cause problems with
+          # the rdf writing otherwise
+          wrap.send(:load!) unless(wrap.loaded? || autosave_rdf?)
+          wrap.save_items!
+        end
+      end
+
+      # Loops through the cache and passes each existing wrapper to the block
+      def each_cached_wrapper
+        return unless(@type_cache)
+        @type_cache.each_value {  |wrap| yield(wrap) }
+      end
+
+      # Clear the source (this will force reloading of elements and discard
+      # unsaved changes
+      def reset!
+        @type_cache = nil
+      end
+
+      # Injects a 'fat' predicate relation on a source
+      def inject_predicate(fat_relation)
+        wrapper = get_objects_on(fat_relation.predicate_uri)
+        wrapper.inject_fat_item(fat_relation)
+      end
+    end
+  end
+end

data/lib/talia_core/active_source_parts/rdf.rb

@@ -0,0 +1,131 @@
+module TaliaCore
+  module ActiveSourceParts
+
+    module Rdf
+      # This file contains the RDF handling elements of the ActiveSource class
+
+      # This can be used to turn of automatic rdf creation. *Attention:* Improperly
+      # used this may compromise the integrity of the RDF data. However, it may
+      # be used in order to speed up "create" operations that save a record
+      # several times and don't need the RDF data in the meantime.
+      def autosave_rdf?
+        @autosave_rdf = true unless(defined?(@autosave_rdf))
+        @autosave_rdf
+      end
+
+      # Set the autosave property. See autosave_rdf?
+      def autosave_rdf=(value)
+        @autosave_rdf = value
+      end
+
+      # Returns the RDF object to use for this ActiveSource
+      def my_rdf
+        @rdf_resource ||= begin
+          src = RdfResource.new(uri)
+          src.object_class = TaliaCore::ActiveSource
+          src
+        end
+      end
+
+      # This creates the RDF subgraph for this Source and saves it to disk. This
+      # may be an expensive operation since it removes the existing elements.
+      # (Could be optimised ;-)
+      #
+      # Unless the force option is specified, this will ignore predicates that
+      # remain unchanged. This means that writing will be faster if a predicate
+      # will not changed, but if database objects were not added through the
+      # standard API they'll be missed
+      #
+      # The force option may have three values: :false for normal operation,
+      # :force for forcing a complete rewrite and :create - the latter will
+      # avoid the cleaning of the elements in the RDF store in case the
+      # source is completely new and no triples exist.
+      def create_rdf(force = :false)
+        self.class.benchmark("\033[32m\033[4m\033[1mActiveSource::RD\033[0m Creating RDF for source", Logger::DEBUG, false) do
+          assit(!new_record?, "Record must exist here: #{self.uri}")
+          # Get the stuff to write. This will also erase the old data
+          
+          s_rels = case force 
+            when :force 
+              prepare_all_predicates_to_write 
+            when :create
+              prepare_predicates_to_create
+            else
+              prepare_predicates_to_write
+            end
+          s_rels.each do |sem_ref|
+            # We pass the object on. If it's a SemanticProperty, we need to add
+            # the value. If not the RDF handler will detect the #uri method and
+            # will add it as Resource.
+            obj = sem_ref.object
+            value = obj.is_a?(SemanticProperty) ? obj.value : obj
+            my_rdf.direct_write_predicate(N::URI.new(sem_ref.predicate_uri), value)
+          end
+          my_rdf.direct_write_predicate(N::RDF.type, (N::TALIA + self.class.name.demodulize))
+          my_rdf.save
+        end
+      end
+      
+      # Creates an RDF/XML resprentation of the source
+      def to_rdf
+        rdf = String.new
+
+        ActiveSourceParts::Xml::RdfBuilder.open(:target => rdf, :indent => 2) do |builder|
+          builder.write_source(self)
+        end
+
+        rdf
+      end
+
+      private
+
+      # Get the "standard" predicates to write (which is just the ones changed
+      # through the standard API. This will erase the
+      def prepare_predicates_to_write
+        preds_to_write = []
+        each_cached_wrapper do |wrap|
+          # If it wasn't loaded, it hasn't been written to
+          next if(wrap.clean?)
+          # Remove the existing data. TODO: Not using contexts
+          my_rdf.remove(N::URI.new(wrap.instance_variable_get(:@assoc_predicate)))
+          items = wrap.send(:items) # Get the items
+          items.each { |it| preds_to_write << it.relation }
+        end
+        preds_to_write
+      end
+
+      # This will get all existing predicates from the database. This will also
+      # erase the rdf for this source completely
+      # TODO: Could load with a single sql
+      def prepare_all_predicates_to_write
+        my_rdf.clear_rdf # TODO: Not using contexts here
+        SemanticRelation.find(:all, :conditions => { :subject_id => self.id })
+      end
+      
+      # ATTENTION: This is a speed hack that avoids the usual checks based
+      # on the assumption that this source was created from scratch, 
+      # no attributes are in the store and all attributes are in-memory.
+      def prepare_predicates_to_create
+        preds_to_create = []
+        each_cached_wrapper do |wrap|
+          next if(wrap.clean?)
+          # Evil, we get the items directly to avoid a useless load
+          items = wrap.instance_variable_get(:@items)
+          items.each { |it| preds_to_create << it.relation }
+        end
+        preds_to_create
+      end
+      
+      def auto_update_rdf
+        create_rdf if(autosave_rdf?)
+      end
+
+      # On creation we force the full write, there's no use 
+      # doing all checks if we know that nothing exists
+      def auto_create_rdf
+        create_rdf(:create) if(autosave_rdf?)
+      end
+
+    end
+  end
+end

data/lib/talia_core/active_source_parts/sql_helper.rb

@@ -0,0 +1,36 @@
+module TaliaCore
+  module ActiveSourceParts
+    
+    # This contains sql joins and segments that can be used by the ActiveSource classes.
+    module SqlHelper
+      
+      # Returns the "default" join (meaning that it joins all the "triple tables"
+      # together. The flags signal whether the relations and properties should 
+      # be joined.
+      def default_joins(include_rels = true, include_props = true)
+        join = "LEFT JOIN semantic_relations ON semantic_relations.subject_id = active_sources.id "
+        join << sources_join if(include_rels)
+        join << props_join  if(include_props)
+        join
+      end
+
+      # Joins sources on semantic relations
+      def sources_join
+        " LEFT JOIN active_sources AS obj_sources ON semantic_relations.object_id = obj_sources.id AND semantic_relations.object_type = 'TaliaCore::ActiveSource'"
+      end
+
+      # Joins properties on semantic relations
+      def props_join
+        " LEFT JOIN semantic_properties AS obj_props ON semantic_relations.object_id = obj_props.id AND semantic_relations.object_type = 'TaliaCore::SemanticProperty'"
+      end
+
+      # Returns the "default" join for reverse lookups
+      def default_inv_joins
+        join = "LEFT JOIN semantic_relations ON semantic_relations.object_id = active_sources.id AND semantic_relations.object_type = 'TaliaCore::ActiveSource' "
+        join << " LEFT JOIN active_sources AS sub_sources ON semantic_relations.subject_id = sub_sources.id"
+        join
+      end
+      
+    end
+  end
+end

data/lib/talia_core/active_source_parts/xml/base_builder.rb

@@ -0,0 +1,47 @@
+module TaliaCore
+  module ActiveSourceParts
+    module Xml
+      
+      # Base class for builders that create source-related XML. This uses a Builder::XmlMarkup object
+      # in the background which does the actual XML writing.
+      # 
+      # All builders will be used through the #open method, which can be passed either a Builder::XmlMarkup
+      # object, or the options to create one.
+      class BaseBuilder
+        
+        # Creates a new builder. The options are equivalent for the options of the
+        # underlying Xml builder. The builder itself will be passed to the block that
+        # is called by this method.
+        # If you pass a :builder option instead, it will use the given builder instead
+        # of creating a new one
+        def self.open(options)
+          my_builder = self.new(options)
+          my_builder.send(:build_structure) do
+            yield(my_builder)
+          end
+        end
+        
+        # Quick helper: Returns the xml for one source as a string
+        def self.build_source(source)
+          xml = ''
+          
+          open(:target => xml, :indent => 2) do |builder|
+            builder.write_source(source)
+          end
+          
+          xml
+        end
+        
+        private
+        
+        # Create a new builder
+        def initialize(options)
+          @builder = options[:builder]
+          @builder ||= Builder::XmlMarkup.new(options)
+        end
+        
+      end
+      
+    end
+  end
+end