talia_core 0.4.3 → 0.4.4

data/lib/talia_core/source.rb

@@ -6,7 +6,7 @@ require 'dummy_handler'
 require 'rdf_resource'
 
 module TaliaCore
-  
+
   # This represents a Source in the Talia core system.
   #
   # Since data for the Source exists both in the database and in the RDF store, the 
@@ -19,41 +19,32 @@ module TaliaCore
   # * To ensure that the data is written, the save method should be called as 
   #   necessary.
   class Source < ActiveSource
-    
+    # FIXME: Remove methods for old admin panel
+
     has_one :workflow, :class_name => 'TaliaCore::Workflow::Base', :dependent => :destroy
-    
+
     # The uri will be wrapped into an object
     def uri
       N::URI.new(self[:uri])
     end
-    
+
     # Indicates if this source belongs to the local store
     def local
       uri.local?
     end
 
-    alias_method :ar_update_attributes, :update_attributes    
-    # Wrapping for <tt>ActiveRecord</tt> <tt>update_attributes</tt>.
-    def update_attributes(attributes)
-      attributes, rdf_attributes = extract_attributes!(attributes)
-      rdf_attributes.each do |k,v|
-        send(k + "=", v)
-        send('save_' + k)
-      end
-    end    
-    
     # Shortcut for assigning the primary_source status
     def primary_source=(value)
       value = value ? 'true' : 'false'
       predicate_set(:talia, :primary_source, value)
     end
-    
+
     # Indicates if the current source is considered "primary" in the local 
     # library
     def primary_source
       predicate(:talia, :primary_source) == true
     end
-    
+
     # Searches for sources where <tt>property</tt> has one of the values given
     # to this method. The result is a hash that contains one result list for
     # each of the values, with the value as a key.
@@ -72,22 +63,22 @@ module TaliaCore
       joins = 'LEFT JOIN semantic_relations ON semantic_relations.subject_id = active_sources.id '
       joins << "LEFT JOIN active_sources AS t_sources ON semantic_relations.object_id = t_sources.id AND semantic_relations.object_type = 'TaliaCore::ActiveSource' "
       joins << "LEFT JOIN semantic_properties ON semantic_relations.object_id = semantic_properties.id AND semantic_relations.object_type = 'TaliaCore::SemanticProperty' "
-      
+
       property = uri_string_for(property)
       results = {}
       for val in values
         find(:all )
         val_str = uri_string_for(val)
         find_parms = params.merge(
-          :conditions => ['semantic_properties.value = ? OR t_sources.uri = ?', val_str, val_str],
-          :joins => joins
+        :conditions => ['semantic_properties.value = ? OR t_sources.uri = ?', val_str, val_str],
+        :joins => joins
         )
         results[val] = find(:all, find_parms)
       end
-      
+
       results
     end
-    
+
     # Try to find a source for the given uri, if not exists it instantiate
     # a new one, combining the N::LOCAL namespace and the given local name
     #
@@ -101,35 +92,6 @@ module TaliaCore
       result = find_by_uri(uri)
       result ||= self.new(N::LOCAL.to_s + local_name.to_permalink)
     end
-    
-    # Find a list of sources which contains the given token inside the local name.
-    # This means that the namespace it will be excluded.
-    #
-    #   Sources in system:
-    #     * http://talia.org/one
-    #     * http://talia.org/two
-    #
-    #   Source.find_by_uri_token('a') # => [ ]
-    #   Source.find_by_uri_token('o') # => [ 'http://talia.org/one', 'http://talia.org/two' ]
-    #
-    # NOTE: It internally use a MySQL function, as sql condition, to find the local name of the uri.
-    def self.find_by_uri_token(token, options = {})
-      find(:all, { 
-          :conditions => [ "LOWER(SUBSTRING_INDEX(uri, '/', -1)) LIKE ?", '%' + token.downcase + '%' ], 
-          :select => :uri,
-          :order => "uri ASC",
-          :limit => 10 }.merge!(options))
-    end
-   
-    # Find the fist Source that matches the given URI.
-    # It's useful for admin pane, because users visit:
-    #   /admin/sources/<source_id>/edit
-    # but that information is not enough, since we store
-    # into the database the whole reference as URI:
-    #   http://localnode.org/av_media_sources/source_id
-    def self.find_by_partial_uri(id)
-      find(:first, :conditions => ["uri LIKE ?", '%' + id + '%'])
-    end
 
     # Return an hash of direct predicates, grouped by namespace.
     def grouped_direct_predicates
@@ -178,146 +140,136 @@ module TaliaCore
       end
     end
 
-    # Save, associate/disassociate given predicates attributes.
-    def save_predicates_attributes
-      each_predicate do |namespace, name, objects|
-        objects.each { |object| object.save if object.is_a?(Source) && object.new_record? }
-        self.predicate_replace(namespace, name, objects.to_s) if predicate_changed?(namespace, name, objects)
+      # Save, associate/disassociate given predicates attributes.
+      def save_predicates_attributes
+        each_predicate do |namespace, name, objects|
+          objects.each { |object| object.save if object.is_a?(Source) && object.new_record? }
+          self.predicate_replace(namespace, name, objects.to_s) if predicate_changed?(namespace, name, objects)
+        end
       end
-    end
 
-    
-    # Returns an array of labels for this source. You may give the name of the
-    # property that is used as a label, by default it uses rdf:label(s). If
-    # the given property is not set, it will return the local part of this
-    # Source's URI.
-    #
-    # In any case, the result will always be an Array with at least one elment.
-    def labels(type = N::RDFS::label)
-      labels = get_attribute(type)
-      unless(labels && labels.size > 0)
-        labels = [uri.local_name]
+
+      # Returns an array of labels for this source. You may give the name of the
+      # property that is used as a label, by default it uses rdf:label(s). If
+      # the given property is not set, it will return the local part of this
+      # Source's URI.
+      #
+      # In any case, the result will always be an Array with at least one elment.
+      def labels(type = N::RDFS::label)
+        labels = get_attribute(type)
+        unless(labels && labels.size > 0)
+          labels = [uri.local_name]
+        end
+
+        labels
       end
 
-      labels
-    end
-    
-    # This returns a single label of the given type. (If multiple labels
-    # exist in the RDF, just the first is returned.)
-    def label(type = N::RDFS::label)
-      labels(type)[0]
-    end
-    
-    # Return the titleized uri local name.
-    #
-    #   http://localnode.org/source # => Source
-    def titleized
-      self.uri.local_name.titleize
-    end
-    
-    # Equality test. Two sources are equal if they have the same URI
-    def ==(value)
-      value.is_a?(Source) && (value.uri == uri)
-    end
-    
-    def normalize_uri(uri, label = '')
-      self.class.normalize_uri(uri, label)
-    end
-    
-    protected
-    
-    # Separates given attributes distinguishing between ActiveSource and RDF.
-    def extract_attributes!(attributes)
-      active_source_attributes = attributes.inject({}) do |active_source_attributes, column_values|
-        active_source_attributes[column_values.first] = attributes.delete(column_values.first) if self.class.column_names.include? column_values.first
-        active_source_attributes
+      # This returns a single label of the given type. (If multiple labels
+      # exist in the RDF, just the first is returned.)
+      def label(type = N::RDFS::label)
+        labels(type)[0]
       end
 
-      [ active_source_attributes, attributes ]
-    end
-    
-    # Look at the given attributes and choose to instantiate
-    # a Source or a RDF object (triple endpoint).
-    #
-    # Cases:
-    #   Homer Simpson
-    #     # => Should instantiate a source with
-    #     http://localnode.org/Homer_Simpson using N::LOCAL constant.
-    #
-    #   "Homer Simpson"
-    #     # => Should return the string itself, without the double quoting
-    #     in order to add it directly to the RDF triple.
-    #
-    #   http://springfield.org/Homer_Simpson
-    #     # => Should instantiate a source with the given uri
-    def instantiate_source_or_rdf_object(attributes)
-      name_or_uri = attributes['titleized']
-      if /^\"[\w\s\d]+\"$/.match name_or_uri
-        name_or_uri[1..-2]
-      elsif attributes['uri'].blank? and attributes['source'].blank?
-        name_or_uri
-      elsif /^http:\/\//.match name_or_uri
-        Source.new(name_or_uri)
-      else
-        Source.find_or_instantiate_by_uri(normalize_uri(attributes['uri']), name_or_uri)
+      # Return the titleized uri local name.
+      #
+      #   http://localnode.org/source # => Source
+      def titleized
+        self.uri.local_name.titleize
       end
-    end
 
-    # Iterate through grouped_predicates_attributes, yielding the given code.
-    def each_predicate(&block)
-      grouped_predicates_attributes.each do |namespace, predicates|
-        predicates.each do |predicate, objects|
-          block.call(namespace, predicate, objects.flatten)
-        end
+      # Equality test. Two sources are equal if they have the same URI
+      def ==(value)
+        value.is_a?(Source) && (value.uri == uri)
       end
-    end
 
-    # Class methods
-    class << self
-      
-      # Normalize the given uri.
-      #
-      # Example:
-      #   normalize_uri('Lucca') # => http://www.talia.discovery-project.org/sources/Lucca
-      #   normalize_uri('http://xmlns.com/foaf/0.1/Group') # => http://xmlns.com/foaf/0.1/Group
-      #   normalize_uri('http://www.talia.discovery-project.org/sources/Lucca')
-      #     # => http://www.talia.discovery-project.org/sources/Lucca
       def normalize_uri(uri, label = '')
-        uri = N::LOCAL if uri.blank?
-        uri = N::LOCAL+label.gsub(' ', '_') if uri == N::LOCAL.to_s
-        uri.to_s
+        self.class.normalize_uri(uri, label)
       end
-      
-    end
-    
-    # End of class methods
-    
-    
-    # Missing methods: This just check if the given method corresponds to a
-    # registered namespace. If yes, this will return a "dummy" handler that
-    # allows access to properties.
-    # 
-    # This will allow invocations as namespace::name
-    def method_missing(method_name, *args)
-      # TODO: Add permission checking for all updates to the model
-      # TODO: Add permission checking for read access?
-      
-      update = method_name.to_s[-1..-1] == '='
-      
-      shortcut = if update 
-        method_name.to_s[0..-2]
-      else
-        method_name.to_s
+
+      protected
+
+      # Look at the given attributes and choose to instantiate
+      # a Source or a RDF object (triple endpoint).
+      #
+      # Cases:
+      #   Homer Simpson
+      #     # => Should instantiate a source with
+      #     http://localnode.org/Homer_Simpson using N::LOCAL constant.
+      #
+      #   "Homer Simpson"
+      #     # => Should return the string itself, without the double quoting
+      #     in order to add it directly to the RDF triple.
+      #
+      #   http://springfield.org/Homer_Simpson
+      #     # => Should instantiate a source with the given uri
+      def instantiate_source_or_rdf_object(attributes)
+        name_or_uri = attributes['titleized']
+        if /^\"[\w\s\d]+\"$/.match name_or_uri
+          name_or_uri[1..-2]
+        elsif attributes['uri'].blank? and attributes['source'].blank?
+          name_or_uri
+        elsif /^http:\/\//.match name_or_uri
+          Source.new(name_or_uri)
+        else
+          Source.find_or_instantiate_by_uri(normalize_uri(attributes['uri']), name_or_uri)
+        end
+      end
+
+      # Iterate through grouped_predicates_attributes, yielding the given code.
+      def each_predicate(&block)
+        grouped_predicates_attributes.each do |namespace, predicates|
+          predicates.each do |predicate, objects|
+            block.call(namespace, predicate, objects.flatten)
+          end
+        end
+      end
+
+      # Class methods
+      class << self
+
+        # Normalize the given uri.
+        #
+        # Example:
+        #   normalize_uri('Lucca') # => http://www.talia.discovery-project.org/sources/Lucca
+        #   normalize_uri('http://xmlns.com/foaf/0.1/Group') # => http://xmlns.com/foaf/0.1/Group
+        #   normalize_uri('http://www.talia.discovery-project.org/sources/Lucca')
+        #     # => http://www.talia.discovery-project.org/sources/Lucca
+        def normalize_uri(uri, label = '')
+          uri = N::LOCAL if uri.blank?
+          uri = N::LOCAL+label.gsub(' ', '_') if uri == N::LOCAL.to_s
+          uri.to_s
+        end
+
+      end
+
+      # End of class methods
+
+
+      # Missing methods: This just check if the given method corresponds to a
+      # registered namespace. If yes, this will return a "dummy" handler that
+      # allows access to properties.
+      # 
+      # This will allow invocations as namespace::name
+      def method_missing(method_name, *args)
+        # TODO: Add permission checking for all updates to the model
+        # TODO: Add permission checking for read access?
+
+        update = method_name.to_s[-1..-1] == '='
+
+        shortcut = if update 
+          method_name.to_s[0..-2]
+        else
+          method_name.to_s
+        end
+
+        # Otherwise, check for the RDF predicate
+        registered = N::URI[shortcut.to_s]
+
+        return super(method_name, *args) unless(registered) # normal handler if not a registered uri
+        raise(ArgumentError, "Must give a namspace as argument") unless(registered.is_a?(N::Namespace))
+
+        DummyHandler.new(registered, self)
       end
 
-      # Otherwise, check for the RDF predicate
-      registered = N::URI[shortcut.to_s]
-      
-      return super(method_name, *args) unless(registered) # normal handler if not a registered uri
-      raise(ArgumentError, "Must give a namspace as argument") unless(registered.is_a?(N::Namespace))
-      
-      DummyHandler.new(registered, self)
     end
-    
-  end
-end
+  end

data/lib/talia_core/source_types/collection.rb

@@ -0,0 +1,15 @@
+module TaliaCore
+  module SourceTypes
+
+    class Collection < DcResource
+
+      has_rdf_type N::DCNS.Collection
+      has_rdf_type N::SKOS.Collection
+      has_rdf_type N::DCMIT.Collection
+
+      simple_property :items, N::DCNS.hasPart
+
+    end
+
+  end
+end

data/lib/talia_core/source_types/dc_resource.rb

@@ -0,0 +1,22 @@
+module TaliaCore
+  module SourceTypes
+
+    # A generic resource that should contain the most important 
+    # Dublin Core metadata fields
+    class DcResource < Source
+
+      # General metadata
+      singular_property :identifier, N::DCNS.identifier
+      simple_property :creators, N::DCNS.creator
+      singular_property :date, N::DCNS.date
+      singular_property :description, N::DCNS.description
+      simple_property :publishers, N::DCNS.publisher
+      singular_property :language, N::DCNS.language
+      simple_property :dc_subjects, N::DCNS.subject
+      singular_property :rights, N::DCNS.rights
+      singular_property :title, N::DCNS.title
+
+    end
+
+  end
+end

data/lib/talia_core/source_types/dummy_source.rb

@@ -0,0 +1,22 @@
+module TaliaCore
+  module SourceTypes
+
+    # Dummy source class. This will be created by some mechanisms that need to create a relation to a 
+    # not-yet-existing source. The DummySource should only exist temporarily, if some are found inside
+    # the data store it may be a sign of an inconsistent or not completely initialized store.
+    class DummySource < Source
+
+      # Converts the current source into one with a "real" klass. Returns the new, converted sourc
+      def self.make_real(klass)
+        assit_kind_of(Class, klass)
+        self['type'] = klass.name
+        save!
+        new_src = ActiveSource.find(uri)
+        assit_kind_of(klass, new_src)
+        new_src
+      end
+
+    end
+
+  end
+end

data/lib/talia_core.rb

@@ -3,5 +3,4 @@ require File.dirname(__FILE__) + '/talia_dependencies'
 
 # Load local things
 require 'talia_core/errors'
-require 'talia_core/rails_ext'
 require 'talia_core/initializer'

data/lib/talia_util/import_job_helper.rb

@@ -5,33 +5,57 @@ module TaliaUtil
   # Helper methods that will be used during import job runs.
   # The import jobs may use the following environment parameters:
   #
-  #  [*base_url*] - The base URL or directory. This will be prefixed to all urls, or if it is 
+  #  [*base_url*] The base URL or directory. This will be prefixed to all urls, or if it is 
   #                 a local directory, it will be made the current directory during the import
-  #  [*index*] - If given, the importer will try to read this document. While this will still
+  #  [*index*] If given, the importer will try to read this document. While this will still
   #              support the old-style "hyper" format with sigla, it should usually contain a
   #              main element called "index" followed by "url" entries. 
-  #  [*xml*] - URL of an XML file to import. This is incompatible with the "index" option. 
+  #  [*xml*] URL of an XML file to import. This is incompatible with the "index" option. 
   #            If neither "xml" nor "index" are given, the class will try to read the XML data from
   #            STDIN
-  #  [*importer*] - Name of the importer class to be used for the data. Uses the default class if not given
+  #  [*importer*] Name of the importer class to be used for the data. Uses the default class if not given
   # [*reset_store*] - If this is set, the data store will be cleared before the import
-  # [*user*] - Username for HTTP authentication, if required
-  # [*pass*] - Password for HTTP authentication, if required
-  # [*callback*] - Name of a class. If given, the import will call the #before_import and #after_import
+  # [*user*] Username for HTTP authentication, if required
+  # [*pass*] Password for HTTP authentication, if required
+  # [*callback*] Name of a class. If given, the import will call the #before_import and #after_import
   #                 methods on an object of that class. The call will receive a block which may be
   #                 yielded to for each progress step and which can receive the overall number of
   #                 steps
-  # [*extension*] - Only used with index files; file extension to use
+  # [*extension*] Only used with index files; file extension to use
+  # [*duplicates*] How to deal with elements that already exist in the datastore. This may be
+  #                  set to one of the following options (default: :skip):
+  #                  * :add - Database fields will be updated and the system will add semantic
+  #                    properties as additional values, without removing any of the existing
+  #                    semantic relations. Example: If the data store already
+  #                    contains a title for an element, and the import file contains another
+  #                    for that element, the element will have two titles after the import. 
+  #                    The system will not check for duplicates. Files will always be imported
+  #                    in addition to the existing ones.
+  #                  * :update - Database fields will be updated, and semantic properties will
+  #                    be overwritten with the new value(s). Semantic properties that are not
+  #                    included in the import data will be left untouched. In the example above,
+  #                    the element would only contain the new title. If the element also contained
+  #                    author information, and no author information was in the import file, the
+  #                    existing author information will be untouched. Existing files are replaced
+  #                    if the import contains new files
+  #                  * :overwrite - Database fields will be updated. All semantic data will be
+  #                    deleted before the import. Files are always removed.
+  #                  * :skip - If an element already exists, the import will be skipped.
+  #
+  # [*trace*] Enable tracing output for errors. (By default, this takes the rake task's setting
+  #           if possible)
   class ImportJobHelper
 
     include IoHelper
 
-    attr_reader :importer, :credentials, :index_data, :xml_data, :reset, :callback, :base_url, :message_stream, :progressor
+    attr_reader :importer, :credentials, :index_data, :xml_data, :reset, :callback, :base_url, :message_stream, :progressor, :duplicates, :trace
 
     # The message_stream will be used for printing progress messages
     def initialize(message_stream = STDOUT, progressor = TaliaCore::BackgroundJobs::Job)
+      @trace = (defined?(Rake) ? Rake.application.options.trace : false) || ENV['trace']
       @progressor = progressor
       @message_stream = message_stream
+      @duplicates = ENV['duplicates'].to_sym if(ENV['duplicates'])
       @importer = ENV['importer'] || 'TaliaCore::ActiveSourceParts::Xml::SourceReader'
       @credentials = { :http_basic_authentication => [ENV['user'], ENV['pass']] } unless(ENV['user'].blank?)
       assit(!(ENV['xml'] && ENV['index']), 'Not both xml and index parameters allowed')
@@ -59,6 +83,7 @@ module TaliaUtil
         else
           xml_url = ENV['xml']
           xml_url = base_url + xml_url unless(File.exists?(xml_url))
+          @true_root = base_for(xml_url)
           open_generic(xml_url, credentials) { |io| io.read }
         end
       else
@@ -78,15 +103,20 @@ module TaliaUtil
         import_from_index(errors)
       else
         puts "Importing from single data file."
-        TaliaCore::ActiveSource.create_from_xml(xml_data, :progressor => progressor, :reader => importer, :errors => errors)
+        TaliaCore::ActiveSource.create_from_xml(xml_data, :progressor => progressor, :reader => importer, :base_file_uri => @true_root, :errors => errors, :duplicates => duplicates)
       end
       if(errors.size > 0)
         puts "WARNING: #{errors.size} errors during import:"
-        errors.each { |e| puts e }
+        errors.each { |e| print_error e }
       end
       run_callback(:after_import)
     end
 
+    def print_error(e)
+      puts e.message
+      puts e.backtrace if(trace)
+    end
+
     def import_from_index(errors)
       doc = Hpricot.XML(index_data)
       hyper_format = (doc.root.name == 'sigla')
@@ -99,10 +129,8 @@ module TaliaUtil
         elements.each do |element|
           url = make_url_from("#{element.inner_text}#{ENV['extension']}")
           begin
-            open_generic(url, credentials) do |io|
-              this_attribs = my_importer.sources_from(io)
-              source_attributes = source_attributes + this_attribs
-            end
+            this_attribs = my_importer.sources_from_url(url, credentials)
+            source_attributes = source_attributes + this_attribs
           rescue Exception => e
             message_stream.puts "Problem importing #{url} (#{e.message})"
             message_stream.puts e.backtrace
@@ -112,7 +140,7 @@ module TaliaUtil
       end
       # Write the data
       TaliaCore::ActiveSource.progressor = progressor
-      TaliaCore::ActiveSource.create_multi_from(source_attributes, :errors => errors)
+      TaliaCore::ActiveSource.create_multi_from(source_attributes, :errors => errors, :duplicates => duplicates)
     end
 
     def make_url_from(url)

data/lib/talia_util/io_helper.rb

@@ -21,6 +21,23 @@ module TaliaUtil
       end
     end
 
+    # Will try to figure out the "base" (that is the parent directory or path)
+    # If the base is a directory, this will return the directory name, but if
+    # it is an URL, this will return an URI object.
+    def base_for(url)
+      url = file_url(url)
+      if(File.exist?(url))
+        file = File.expand_path(url)
+        File.directory?(file) ? file : File.dirname(file)
+      else
+        uri = URI.parse(url)
+        # Remove everything after the last '/'
+        uri.path.gsub!(/\/[^\/]+\Z/, '/')
+        uri.fragment = nil
+        uri
+      end
+    end
+
     # Opens the given (web) URL, using URL encoding and necessary substitutions.
     # The user must pass a block which will receive the io object from
     # the url.
@@ -30,7 +47,10 @@ module TaliaUtil
     # 
     #   :http_basic_authentication => [login, password]
     def open_from_url(url, options = {})
-      url = URI.encode(url)
+      # Encode the URI (the inner decode will save already-encoded URI and should
+      # do nothing to non-encoded URIs)
+      url = URI.encode(URI.decode(url))
+      
       url.gsub!(/\[/, '%5B') # URI class doesn't like unescaped brackets
       url.gsub!(/\]/, '%5D')
       open_args = [ url ]