activerdf_net7 1.6.16 → 1.7.0

data/activerdf-rdflite/lib/activerdf_rdflite/rdflite.rb

@@ -10,28 +10,19 @@ require 'queryengine/ntriples_parser'
 require 'open-uri'
 require 'mime/types'
 
-ActiveRdfLogger::log_info "Loading RDFLite adapter", self
-
-
-
-# RDFLite is a lightweight RDF database on top of sqlite3. It can act as adapter 
-# in ActiveRDF. It supports on-disk and in-memory usage, and allows keyword 
+# RDFLite is a lightweight RDF database on top of sqlite3. It can act as adapter
+# in ActiveRDF. It supports on-disk and in-memory usage, and allows keyword
 # search if ferret is installed.
-class RDFLite < ActiveRdfAdapter
+module ActiveRDF
+  class RDFLite < ActiveRdfAdapter
+    ActiveRdfLogger::log_info "Loading RDFLite adapter", self
 
-  class << self
-    begin 
+    begin
       require 'ferret'
-      @have_ferret = true
+      @@have_ferret = true
     rescue LoadError
-      ActiveRdfLogger::log_info "Keyword search is disabled since we could not load Ferret. To 
-      enable, please do \"gem install ferret\"", self
-        @have_ferret = false
-      end
-
-      def has_ferret?
-        @have_ferret
-      end
+      ActiveRdfLogger::log_info "Keyword search is disabled since we could not load Ferret. To enable, please do \"gem install ferret\"", self
+      @@have_ferret = false
     end
 
     ConnectionPool.register_adapter(:rdflite,self)
@@ -46,19 +37,27 @@ class RDFLite < ActiveRdfAdapter
       super()
       ActiveRdfLogger::log_info(self) { "Initialised rdflite with params #{params.to_s}" }
 
-      @reads = true
-      @writes = true
+      @reasoning = truefalse(params[:reasoning], false)
+      @subprops = {} if @reasoning
 
       # if no file-location given, we use in-memory store
       file = params[:location] || ':memory:'
-      @db = SQLite3::Database.new(file) 
+      @db = SQLite3::Database.new(file)
 
       # disable keyword search by default, enable only if ferret is found
-      @keyword_search = params[:keyword].nil? ? false : params[:keyword]
-      @keyword_search &= self.class.has_ferret?
+      @keyword_search = truefalse(params[:keyword], false) && @@have_ferret
 
-      @reasoning = params[:reasoning] || false
-      @subprops = {} if @reasoning
+      # turn off filesystem synchronisation for speed
+      @db.synchronous = 'off'
+
+      # drop the table if a new datastore is requested
+      @db.execute('drop table if exists triple') if truefalse(params[:new],false)
+
+      # create triples table. ignores duplicated triples
+      @db.execute('create table if not exists triple(s,p,o,c, unique(s,p,o,c) on conflict ignore)')
+
+      create_indices(params)
+      @db
 
       if keyword_search?
         # we initialise the ferret index, either as a file or in memory
@@ -74,15 +73,6 @@ class RDFLite < ActiveRdfAdapter
           Ferret::I.new(:field_infos => infos)
         end
       end
-
-      # turn off filesystem synchronisation for speed
-      @db.synchronous = 'off'
-
-      # create triples table. ignores duplicated triples
-      @db.execute('create table if not exists triple(s,p,o,c, unique(s,p,o,c) on conflict ignore)')
-
-      create_indices(params)
-      @db
     end
 
     # returns the number of triples in the datastore (incl. possible duplicates)
@@ -110,19 +100,16 @@ class RDFLite < ActiveRdfAdapter
 
     # deletes triple(s,p,o,c) from datastore
     # symbol parameters match anything: delete(:s,:p,:o) will delete all triples
-    # you can specify a context to limit deletion to that context: 
+    # you can specify a context to limit deletion to that context:
     # delete(:s,:p,:o, 'http://context') will delete all triples with that context
-    def delete(s, p, o, c=nil)
-      # convert non-nil input to internal format
-      quad = [s,p,o,c].collect {|r| r.nil? ? nil : internalise(r) }
-
-      # construct where clause for deletion (for all non-nil input)
+    def delete(s, p, o=nil, c=nil)
       where_clauses = []
-      conditions = []		
-      quad.each_with_index do |r,i|
-        unless r.nil?
-          conditions << r
+      conditions = []
+
+      [s,p,o,c].each_with_index do |r,i|
+        unless r.nil? or check_type(r, Symbol)
           where_clauses << "#{SPOC[i]} = ?"
+          conditions << r.to_literal_s
         end
       end
 
@@ -130,7 +117,7 @@ class RDFLite < ActiveRdfAdapter
       ds = 'delete from triple'
       ds << " where #{where_clauses.join(' and ')}" unless where_clauses.empty?
 
-      # execute delete string with possible deletion conditions (for each 
+      # execute delete string with possible deletion conditions (for each
       # non-empty where clause)
       ActiveRdfLogger::log_debug(self) { "Deleting #{[s,p,o,c].join(' ')}" }
       @db.execute(ds, *conditions)
@@ -150,14 +137,10 @@ class RDFLite < ActiveRdfAdapter
       raise(ActiveRdfError, "adding non-resource #{s} while adding (#{s},#{p},#{o},#{c})") unless s.respond_to?(:uri)
       raise(ActiveRdfError, "adding non-resource #{p} while adding (#{s},#{p},#{o},#{c})") unless p.respond_to?(:uri)
 
-      triple = [s, p, o].collect{|r| serialise(r) }
-      add_ntriples(triple.join(' ') + " .\n", serialise(c) )
-
-      ## get internal representation (array)
-      #quad = [s,p,o,c].collect {|r| internalise(r) }
+      c = c.to_literal_s unless c.nil?
 
-      ## insert the triple into the datastore
-      #@db.execute('insert into triple values (?,?,?,?)', *quad)
+      # insert triple into database
+      @db.execute('insert into triple values (?,?,?,?);',s.to_literal_s,p.to_literal_s,o.to_literal_s,c)
 
       ## if keyword-search available, insert the object into keyword search
       #@ferret << {:subject => s, :object => o} if keyword_search?
@@ -165,28 +148,31 @@ class RDFLite < ActiveRdfAdapter
 
     # flushes openstanding changes to underlying sqlite3
     def flush
-      # since we always write changes into sqlite3 immediately, we don't do 
+      # since we always write changes into sqlite3 immediately, we don't do
       # anything here
       true
     end
 
     # loads triples from file in ntriples format
-    def load(location)
-      context = if URI.parse(location).host
-        location
+    def load(location, syntax = nil)
+      context = if @contexts
+        if URI.parse(location).host
+          RDFS::Resource.new(location)
+        else
+          RDFS::Resource.new("file:#{location}")
+        end
       else
-        internalise(RDFS::Resource.new("file:#{location}"))
+        nil
       end
 
-      case MIME::Types.of(location)
-      when MIME::Types['application/rdf+xml']
+      if MIME::Types.of(location) == MIME::Types['application/rdf+xml'] or syntax == 'rdfxml'
         # check if rapper available
-        begin 
+        begin
           # can only parse rdf/xml with redland
           # die otherwise
           require 'rdf/redland'
           model = Redland::Model.new
-          Redland::Parser.new.parse_into_model(model, location)
+          Redland::Parser.new(syntax).parse_into_model(model, location)
           add_ntriples(model.to_string('ntriples'), location)
         rescue LoadError
           raise ActiveRdfError, "cannot parse remote rdf/xml file without Redland: please install Redland (librdf.org) and its Ruby bindings"
@@ -198,21 +184,16 @@ class RDFLite < ActiveRdfAdapter
     end
 
     # adds ntriples from given context into datastore
-    def add_ntriples(ntriples, context)
+    def add_ntriples(ntriples, context = nil)
       # add each triple to db
       @db.transaction
-      insert = @db.prepare('insert into triple values (?,?,?,?);')
 
       ntriples = NTriplesParser.parse(ntriples)
       ntriples.each do |s,p,o|
-        # convert triples into internal db format
-        subject, predicate, object = [s,p,o].collect {|r| internalise(r) }
-
-        # insert triple into database
-        insert.execute(subject, predicate, object, context)
+        add(s,p,o,context)
 
         # if keyword-search available, insert the object into keyword search
-        @ferret << {:subject => subject, :object => object} if keyword_search?
+        @ferret << {:subject => s.to_s, :object => o.to_s} if keyword_search?
       end
 
       @db.commit
@@ -220,11 +201,11 @@ class RDFLite < ActiveRdfAdapter
     end
 
     # executes ActiveRDF query on datastore
-    def query(query)
+    def execute(query)
       # construct query clauses
       sql, conditions = translate(query)
 
-      # executing query, passing all where-clause values as parameters (so that 
+      # executing query, passing all where-clause values as parameters (so that
       # sqlite will encode quotes correctly)
       results = @db.execute(sql, *conditions)
 
@@ -288,91 +269,68 @@ class RDFLite < ActiveRdfAdapter
       if not query.sort_clauses.empty?
         sort = query.sort_clauses.collect { |term| variable_name(query, term) }
         " order by (#{sort.join(',')})"
-      elsif not query.reverse_sort_clauses.empty?
-        sort = query.reverse_sort_clauses.collect { |term| variable_name(query, term) }
-        " order by (#{sort.join(',')}) DESC"
       else
         ""
       end
     end
 
     # construct join clause
-    # TODO: joins don't work this way, they have to be linear (in one direction 
-    # only, and we should only alias tables we didnt alias yet)
-    # we should only look for one join clause in each where-clause: when we find 
-    # one, we skip the rest of the variables in this clause.
+    # begins with first element of construct traverses tables referenced by
     def construct_join(query)
-      join_stmt = ''
-
-      # no join necessary if only one where clause given
-      return ' from triple as t0 ' if query.where_clauses.size == 1
-
-      where_clauses = query.where_clauses.flatten
-      considering = where_clauses.uniq.select{|w| w.is_a?(Symbol)}
+      join_stmt = []
+      seen_aliases = ['t0']  # first table already seen
+      seen_vars = []
+
+      # get a hash with indices for for all terms
+      # if the term doesnt have a join-buddy, we can skip it
+      occurances = term_occurances(query).reject{|var,terms| terms.size < 2}
+
+      # start with variables in first where clause
+      var_queue = query.where_clauses[0].find_all{|obj| check_type(obj, Symbol) and occurances.include?(obj)}
+
+      while !var_queue.empty? do
+        var = var_queue.shift
+        seen_vars << var
+        terms = occurances[var]
+
+        # look for a term in a table that has already been seen
+        table,field = terms.find{|table,field| seen_aliases.include?(table)}
+        raise ActiveRdfError, "query is confused. haven't seen #{table} for :#{var} yet." unless table
+        terms -= [[table,field]]   # ignore this previously seen buddy term table
+        first_term = "#{table}.#{field}"
+
+        terms.each do |table,field|
+          join = ''
+          unless seen_aliases.include?(table)
+            seen_aliases << table
+            join << "join triple as #{table} on "
+          else
+            join << "and "
+          end
+          join << "#{first_term}=#{table}.#{field} "
+          join_stmt << join
 
-      # constructing hash with indices for all terms
-      # e.g. {?s => [1,3,5], ?p => [2], ... }
-      term_occurrences = Hash.new()
-      where_clauses.each_with_index do |term, index|
-        ary = (term_occurrences[term] ||= [])
-        ary << index 
+          # add any terms from this table that haven't been seen yet to the queue
+          var_queue.concat query.where_clauses[table[1..-1].to_i].find_all{|obj| check_type(obj, Symbol) and !seen_vars.include?(obj) and occurances.include?(obj)}
+        end
       end
 
-      aliases = {}
-
-      where_clauses.each_with_index do |term, index|
-        # if the term has been joined with his buddy already, we can skip it
-        next unless considering.include?(term)
-
-        # we find all (other) occurrences of this term
-        indices = term_occurrences[term]
-
-        # if the term doesnt have a join-buddy, we can skip it
-        next if indices.size == 1
-
-        # construct t0,t1,... as aliases for term
-        # and construct join condition, e.g. t0.s
-        termalias = "t#{index / 4}"
-        termjoin = "#{termalias}.#{SPOC[index % 4]}"
-
-        join = if join_stmt.include?(termalias)
-          ""
-        else
-          "triple as #{termalias}"
-        end
+      join = ' from triple as t0 '
+      join_stmt.empty? ? join : join + "#{join_stmt.join(' ')} "
+    end
 
-        indices.each do |i|
-          # skip the current term itself
-          next if i==index
-
-          # construct t0,t1, etc. as aliases for buddy,
-          # and construct join condition, e.g. t0.s = t1.p
-          buddyalias = "t#{i/4}"
-          buddyjoin = "#{buddyalias}.#{SPOC[i%4]}"
-
-          # TODO: fix reuse of same table names as aliases, e.g.
-          # "from triple as t1 join triple as t2 on ... join t1 on ..."
-          # is not allowed as such by sqlite
-          # but on the other hand, restating the aliases gives ambiguity:
-          # "from triple as t1 join triple as t2 on ... join triple as t1 ..."
-          # is ambiguous
-          if join_stmt.include?(buddyalias)
-            join << "and #{termjoin} = #{buddyjoin}"
-          else
-            join << " join triple as #{buddyalias} on #{termjoin} = #{buddyjoin} "
+    # Returns a hash of arrays, keyed by the term symbol. Each element of the array is nested array of a pair of values: [table alias, field('s'|'p'|'o'|'c')]
+    # {:s => [[table,field],['t0','s'],...], :p => [['t0','p'],['t1','p']]}
+    def term_occurances(query)
+      term_occurances = {}
+      query.where_clauses.each_with_index do |clause, table_index|
+        clause.zip(SPOC).each do |obj,field|
+          if check_type(obj, Symbol)
+            (term_occurances[obj] ||= []) << ["t#{table_index}",field]
           end
         end
-        join_stmt << join
-
-        # remove term from 'todo' list of still-considered terms
-        considering.delete(term)
-      end
-
-      if join_stmt == ''
-        return " from triple as t0 "
-      else
-        return " from #{join_stmt} "
       end
+      term_occurances
     end
 
     # construct where clause
@@ -380,25 +338,45 @@ class RDFLite < ActiveRdfAdapter
       # collecting where clauses, these will be added to the sql string later
       where = []
 
-      # collecting all the right-hand sides of where clauses (e.g. where name = 
-      # 'abc'), to add to query string later using ?-notation, because then 
+      # collecting all the right-hand sides of where clauses (e.g. where name =
+      # 'abc'), to add to query string later using ?-notation, because then
       # sqlite will automatically encode quoted literals correctly
       right_hand_sides = []
 
-      # convert each where clause to SQL:
-      # add where clause for each subclause, except if it's a variable
-      query.where_clauses.each_with_index do |clause,level|
-        raise ActiveRdfError, "where clause #{clause} is not a triple" unless clause.is_a?(Array)
-        clause.each_with_index do |subclause, i|
-          # dont add where clause for variables
-          unless subclause.is_a?(Symbol) || subclause.nil?
-            conditions = compute_where_condition(i, subclause, query.reasoning? && reasoning?)
-            if conditions.size == 1
-              where << "t#{level}.#{SPOC[i]} = ?"
-              right_hand_sides << conditions.first
+      query.where_clauses.each_with_index do |clause, table_index|
+        clause.zip(SPOC).each do |clause_elem,field|
+          if !(check_type(clause_elem, Symbol) or clause_elem.nil?)
+            # include querying on subproperty fields
+            if field == 'p' and query.reasoning? and $activerdf_internal_reasoning
+              predicate = clause_elem
+              predicates = [predicate] + predicate.sub_predicates
+              if predicates.size > 1
+                where << "t#{table_index}.#{field} in ('#{predicates.collect{|res| res.to_literal_s}.join("','")}')"
+              else
+                where << "t#{table_index}.#{field} = ?"
+                right_hand_sides << predicates[0].to_literal_s
+              end
             else
-              conditions = conditions.collect {|c| "'#{c}'"}
-              where << "t#{level}.#{SPOC[i]} in (#{conditions.join(',')})"
+              # match plain strings to all strings
+              if query.all_types? and !clause_elem.respond_to?(:uri)  # dont wildcard resources
+                where << "t#{table_index}.#{field} like ?"
+                right_hand_sides << "\"#{clause_elem.to_s}\"%"
+              else
+                where << "t#{table_index}.#{field} = ?"
+                right_hand_sides << clause_elem.to_literal_s
+              end
+            end
+            # process filters
+          elsif field == 'o' and check_type(clause_elem, Symbol) and (filter = query.filter_clauses[clause_elem])
+            operator, operand = filter
+            case operator
+            when :datatype
+              where << "t#{table_index}.o like ?"
+              right_hand_sides << "%\"^^<#{operand}>"
+            when :lang
+              lang, exact = operand
+              where << "t#{table_index}.o like ?"
+              right_hand_sides << (exact ? "%\"@#{lang}" : "%\"@%#{lang}%")
             end
           end
         end
@@ -426,100 +404,34 @@ class RDFLite < ActiveRdfAdapter
       end
     end
 
-    def compute_where_condition(index, subclause, reasoning)
-      conditions = [subclause]
-
-      # expand conditions with rdfs rules if reasoning enabled
-      if reasoning
-        case index
-        when 0: ;
-          # no rule for subjects
-        when 1:
-          # expand properties to include all subproperties
-          conditions = subproperties(subclause) if subclause.respond_to?(:uri)
-        when 2:
-          # no rule for objects
-        when 3:
-          # no rule for contexts
-        end
-      end
-
-      # convert conditions into internal format
-      #conditions.collect { |c| c.respond_to?(:uri) ? "<#{c.uri}>" : c.to_s }
-      conditions.collect { |c| internalise(c) }
-    end
-
-    def subproperties(resource)
-      # compute and store subproperties of this resource 
-      # or use earlier computed value if available
-      unless @subprops[resource]
-        subproperty = Namespace.lookup(:rdfs,:subPropertyOf)
-        children_query = Query.new.distinct(:sub).where(:sub, subproperty, resource)
-        children_query.reasoning = false
-        children = children_query.execute
-
-        if children.empty?
-          @subprops[resource] = [resource]
-        else
-          @subprops[resource] = [resource] + children.collect{|c| subproperties(c)}.flatten.compact
-        end
+    # returns first sql variable name found for a queryterm
+    def variable_name(query,term)
+      if (indices = term_occurances(query)[term])
+        table, field = indices[0]
+        "#{table}.#{field}"
       end
-      @subprops[resource]
     end
 
-    # returns sql variable name for a queryterm
-    def variable_name(query,term)
-      # look up the first occurence of this term in the where clauses, and compute 
-      # the level and s/p/o position of it
-      index = query.where_clauses.flatten.index(term)
-
-      if index.nil? 
-        # term does not appear in where clause
-        # but maybe it appears in a keyword clause
-
-        # index would not be nil if we had:
-        # select(:o).where(knud, knows, :o).where(:o, :keyword, 'eyal')
-        #
-        # the only possibility that index is nil is if we have:
-        # select(:o).where(:o, :keyword, :eyal) (selecting subject)
-        # or if we use a select clause that does not appear in any where clause
-
-        # so we check if we find the term in the keyword clauses, otherwise we throw 
-        # an error
-        if query.keywords.keys.include?(term)
-          return "t0.s"
-        else
-          raise ActiveRdfError, "unbound variable :#{term.to_s} in select of #{query.to_sp}"
-        end
+    # "safe" type checking that will not trigger additional queries
+    # on RDFS::Resource. This assumes that the klass is not a Resource
+    # type itself
+    def check_type(item, klass)
+      if(item.respond_to?(:uri))
+        item.class == klass
+      else
+        item.is_a?(klass)
       end
-
-      termtable = "t#{index / 4}"
-      termspo = SPOC[index % 4]
-      return "#{termtable}.#{termspo}"
     end
 
-    # wrap resources into ActiveRDF resources, literals into Strings. result_type
-    # is the type that should be used for ActiveRDF resources
+    # wrap resources into ActiveRDF resources, literals into Strings
     def wrap(results, result_type)
       results.collect do |row|
         row.collect { |result| parse(result, result_type) }
       end
     end
 
-    # Return the result as a correct type. result_type is the type that should 
-    # be used for "resouce" elements
-    def parse(result, result_type = RDFS::Resource)
-      NTriplesParser.parse_node(result, result_type) || result
-      # case result
-      # when Literal
-      #   # replace special characters to allow string interpolation for e.g. 'test\nbreak'
-      #   $1.double_quote
-      # when Resource
-      #   result_type.new($1)
-      # else
-      #   # when we do a count(*) query we get a number, not a resource/literal
-      #   result
-      # end
+    def parse(result, result_type)
+      NTriplesParser.parse_node(result, result_type)
     end
 
     def create_indices(params)
@@ -532,7 +444,7 @@ class RDFLite < ActiveRdfAdapter
       opidx = params[:opidx] || false
 
       # creating lookup indices
-      @db.transaction do 
+      @db.transaction do
         @db.execute('create index if not exists sidx on triple(s)') if sidx
         @db.execute('create index if not exists pidx on triple(p)') if pidx
         @db.execute('create index if not exists oidx on triple(o)') if oidx
@@ -542,41 +454,5 @@ class RDFLite < ActiveRdfAdapter
         @db.execute('create index if not exists opidx on triple(o,p)') if opidx
       end
     end
-
-    # transform triple into internal format <uri> and "literal"
-    def internalise(r)
-      if r.nil? or r.is_a? Symbol
-        nil
-      else
-        r.to_ntriple
-      end
-    end
-
-    # transform resource/literal into ntriples format
-    def serialise(r)
-      # Fixme: r is most likely NOT a resource
-      if r.nil?
-        nil
-      else
-        r.to_ntriple
-      end
-    end
-
-    Resource = /<([^>]*)>/
-    Literal = /"((?:\\"|[^"])*)"/
-
-    public :subproperties
-  end
-
-  class String
-    def double_quote
-      Thread.new do
-        $SAFE = 12
-        begin
-          eval('"%s"' % self)
-        rescue Exception => e
-          self
-        end
-      end.value
-    end
   end
+end

data/activerdf-rdflite/lib/activerdf_rdflite/suggesting.rb

@@ -26,7 +26,7 @@ class SuggestingAdapter < FetchingAdapter
 
   # suggests additional predicates that might be applicable for the given resource
   def suggest(resource)
-    ActiveRdfLogger::log_debug(self) { "Starting suggestions for #{size} triples" }
+		ActiveRdfLogger::log_debug(self) { "Starting suggestions for #{size} triples" }
     time = Time.now
 
     predicates = []
@@ -53,7 +53,7 @@ class SuggestingAdapter < FetchingAdapter
       end
       [candidate, score]
     end
-    ActiveRdfLogger::log_debug(self) { "Suggestions for #{resource} took #{Time.now-time}s" }
+		ActiveRdfLogger::log_debug(self) { "Suggestions for #{resource} took #{Time.now-time}s" }
     suggestions
   end
 

data/activerdf-rdflite/test/test_fetching.rb

@@ -2,32 +2,17 @@
 # Copyright:: (c) 2005-2006
 # License:: LGPL
 
-require 'rubygems'
 require 'test/unit'
 require 'active_rdf'
+common_test_dir = File.dirname(File.expand_path(__FILE__)) + '/../../test'
+require "#{common_test_dir}/common"
+require "#{common_test_dir}/adapters/test_network_aware_adapter"
 
 class TestFetchingAdapter < Test::Unit::TestCase
-  def setup
-    ConnectionPool.clear
-    @adapter = ConnectionPool.add(:type => :fetching)
-  end
-
-  def teardown
-  end
+  include SetupAdapter
+  include TestNetworkAwareAdapter
 
-  def test_parse_foaf    
-    @adapter.fetch("http://eyaloren.org/foaf.rdf#me")
-    assert @adapter.size > 0
-  end
-  
-  def test_sioc_schema
-    @adapter.fetch("http://rdfs.org/sioc/ns#")
-    assert_equal 560, @adapter.size 
-  end
-    
-  def test_foaf_schema
-    @adapter.fetch("http://xmlns.com/foaf/0.1/")
-    # foaf contains 563 triples but with two duplicates
-    assert_equal 561, @adapter.size    
+  def setup
+    super(:type => :fetching)
   end
 end