activerdf_net7 1.6.11

data/activerdf-rdflite/README

@@ -0,0 +1,16 @@
+== Welcome to rdflite
+
+rdflite is a small RDF datastore, based on sqlite3. It can be used as a 
+datastore for ActiveRDF: it acts as a gemplugin for ActiveRDF. rdflite does not 
+scale well for large datasets, but is very fast for small ones. It supports the 
+full ActiveRDF query grammar (basically, select/distinct/count with arbitrary 
+joins in the where clauses) and offers full-text search over the literals if 
+ferret is installed. 
+
+See http://www.activerdf.org/ for more information.
+
+== License
+rdflite is distributed under the LGPL license.
+
+== Authors
+* Eyal Oren

data/activerdf-rdflite/Rakefile

@@ -0,0 +1,73 @@
+require 'meta_project'
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/xforge'
+require 'rubygems'
+require 'fileutils'
+require '../tools/rakehelp'
+
+$version  = IO.read('VERSION').strip
+$name     = 'activerdf_rdflite'
+$project  = MetaProject::Project::XForge::RubyForge.new('activerdf')
+
+# setup tests
+setup_tests
+setup_clean ["pkg", "lib/*.bundle", "*.gem", ".config"]
+
+# setup rdoc 
+Rake::RDocTask.new do |rdoc|
+	rdoc.rdoc_dir = 'doc/rdoc'
+	rdoc.options << '--line-numbers'
+	rdoc.rdoc_files.add ['README', 'LICENSE', 'lib/**/*.rb', 'doc/**/*.rdoc']
+end
+
+desc "Does a full compile, test run"
+task :default => [:test, :package]
+
+setup_gem($name, $version) do |spec|
+  spec.summary = "an RDF database for usage in ActiveRDF (based on sqlite3)" 
+  spec.description = spec.summary
+  spec.author="Eyal Oren <eyal.oren@deri.org"
+  spec.add_dependency('gem_plugin', '>= 0.2.1')
+  spec.add_dependency('uuidtools')
+  spec.add_dependency('activerdf', '>= 1.6.4')
+  spec.add_dependency('mime-types')
+  spec.add_dependency('sqlite3-ruby', '>= 1.2.1')
+end
+
+task :verify_rubyforge do
+  raise "RUBYFORGE_USER environment variable not set!" unless ENV['RUBYFORGE_USER']
+  raise "RUBYFORGE_PASSWORD environment variable not set!" unless ENV['RUBYFORGE_PASSWORD']
+end
+
+desc "Release files on RubyForge."
+task :release => [:clean, :verify_rubyforge, :package ] do
+  release_files = FileList["pkg/#$name-#$version.gem"]
+
+  Rake::XForge::Release.new($project) do |release|
+    release.user_name     = ENV['RUBYFORGE_USER']
+    release.password      = ENV['RUBYFORGE_PASSWORD']
+    release.files         = release_files.to_a
+    release.release_name  = "#$name #$version"
+    release.package_name  = "activerdf-rdflite"
+		release.release_notes = ""
+
+    changes = []
+    File.open("CHANGELOG") do |file|
+      current = true
+
+      file.each do |line|
+        line.chomp!
+				if current and line =~ /^==/
+					current = false; next 
+				end
+        break if line.empty? and not current
+        changes << line
+      end
+    end
+    release.release_changes = changes.join("\n")
+  end
+end

data/activerdf-rdflite/VERSION

@@ -0,0 +1 @@
+1.4.1

data/activerdf-rdflite/lib/activerdf_rdflite/fetching.rb

@@ -0,0 +1,34 @@
+# Author:: Eyal Oren
+# Copyright:: (c) 2005-2006 Eyal Oren
+# License:: LGPL
+
+# FetchingAdapter is an extension to rdflite for fetching RDF from online sources.
+class FetchingAdapter < RDFLite
+  ConnectionPool.register_adapter(:fetching, self)
+
+  # TODO: check that rapper is installed
+
+  # fetches RDF/XML data from given url and adds it to the datastore, using the 
+  # source url as context identifier.
+  def fetch url
+    # check if url starts with http://
+    return unless url.match(/http:\/\/(.*)/)
+
+    ActiveRdfLogger::log_debug(self) { "Fetching from #{url}" }
+
+    #model = Redland::Model.new
+    #parser = Redland::Parser.new('rdfxml')
+    #scan = Redland::Uri.new('http://feature.librdf.org/raptor-scanForRDF')
+    #enable = Redland::Literal.new('1')
+    #Redland::librdf_parser_set_feature(parser, scan.uri, enable.node)
+    #parser.parse_into_model(model, url)
+    #triples = Redland::Serializer.ntriples.model_to_string(nil, model)
+
+    triples = `rapper --scan --quiet "#{url}"`
+    lines = triples.split($/)
+    ActiveRdfLogger::log_debug(self) { "Found #{lines.size} triples" }
+
+    context = Query.resource_class.new(url)
+    add_ntriples(triples, context)
+  end
+end

data/activerdf-rdflite/lib/activerdf_rdflite/init.rb

@@ -0,0 +1,13 @@
+
+# add the directory in which this file is located to the ruby loadpath
+file =
+if File.symlink?(__FILE__)
+  File.readlink(__FILE__)
+else
+  __FILE__
+end
+$: << File.dirname(File.expand_path(file))
+
+require 'rdflite'
+require 'fetching'
+require 'suggesting'

data/activerdf-rdflite/lib/activerdf_rdflite/rdflite.rb

@@ -0,0 +1,582 @@
+# Author:: Eyal Oren
+# Copyright:: (c) 2005-2006 Eyal Oren
+# License:: LGPL
+
+require 'sqlite3'
+require 'active_rdf'
+require 'federation/connection_pool'
+require 'uuidtools'
+require 'queryengine/ntriples_parser'
+require 'open-uri'
+require 'mime/types'
+
+ActiveRdfLogger::log_info "Loading RDFLite adapter", self
+
+
+
+# RDFLite is a lightweight RDF database on top of sqlite3. It can act as adapter 
+# in ActiveRDF. It supports on-disk and in-memory usage, and allows keyword 
+# search if ferret is installed.
+class RDFLite < ActiveRdfAdapter
+
+  class << self
+    begin 
+      require 'ferret'
+      @have_ferret = true
+    rescue LoadError
+      ActiveRdfLogger::log_info "Keyword search is disabled since we could not load Ferret. To 
+      enable, please do \"gem install ferret\"", self
+        @have_ferret = false
+      end
+
+      def has_ferret?
+        @have_ferret
+      end
+    end
+
+    ConnectionPool.register_adapter(:rdflite,self)
+    bool_accessor :keyword_search, :reasoning
+
+    # instantiates RDFLite database
+    # available parameters:
+    # * :location => filepath (defaults to memory)
+    # * :keyword => true/false (defaults to false)
+    # * :pidx, :oidx, etc. => true/false (enable/disable these indices)
+    def initialize(params = {})
+      super()
+      ActiveRdfLogger::log_info(self) { "Initialised rdflite with params #{params.to_s}" }
+
+      @reads = true
+      @writes = true
+
+      # if no file-location given, we use in-memory store
+      file = params[:location] || ':memory:'
+      @db = SQLite3::Database.new(file) 
+
+      # disable keyword search by default, enable only if ferret is found
+      @keyword_search = params[:keyword].nil? ? false : params[:keyword]
+      @keyword_search &= self.class.has_ferret?
+
+      @reasoning = params[:reasoning] || false
+      @subprops = {} if @reasoning
+
+      if keyword_search?
+        # we initialise the ferret index, either as a file or in memory
+        infos = Ferret::Index::FieldInfos.new
+
+        # we setup the fields not to store object's contents
+        infos.add_field(:subject, :store => :yes, :index => :no, :term_vector => :no)
+        infos.add_field(:object, :store => :no) #, :index => :omit_norms)
+
+        @ferret = if params[:location]
+          Ferret::I.new(:path => params[:location] + '.ferret', :field_infos => infos)
+        else
+          Ferret::I.new(:field_infos => infos)
+        end
+      end
+
+      # turn off filesystem synchronisation for speed
+      @db.synchronous = 'off'
+
+      # create triples table. ignores duplicated triples
+      @db.execute('create table if not exists triple(s,p,o,c, unique(s,p,o,c) on conflict ignore)')
+
+      create_indices(params)
+      @db
+    end
+
+    # returns the number of triples in the datastore (incl. possible duplicates)
+    def size
+      @db.execute('select count(*) from triple')[0][0].to_i
+    end
+
+    # returns all triples in the datastore
+    def dump
+      @db.execute('select s,p,o,c from triple').collect do |s,p,o,c|
+        [s,p,o,c].join(' ')
+      end
+    end
+
+    # deletes all triples from datastore
+    def clear
+      @db.execute('delete from triple')
+    end
+
+    # close adapter and remove it from the ConnectionPool
+    def close
+      ConnectionPool.remove_data_source(self)
+      @db.close
+    end
+
+    # deletes triple(s,p,o,c) from datastore
+    # symbol parameters match anything: delete(:s,:p,:o) will delete all triples
+    # you can specify a context to limit deletion to that context: 
+    # delete(:s,:p,:o, 'http://context') will delete all triples with that context
+    def delete(s, p, o, c=nil)
+      # convert non-nil input to internal format
+      quad = [s,p,o,c].collect {|r| r.nil? ? nil : internalise(r) }
+
+      # construct where clause for deletion (for all non-nil input)
+      where_clauses = []
+      conditions = []		
+      quad.each_with_index do |r,i|
+        unless r.nil?
+          conditions << r
+          where_clauses << "#{SPOC[i]} = ?"
+        end
+      end
+
+      # construct delete string
+      ds = 'delete from triple'
+      ds << " where #{where_clauses.join(' and ')}" unless where_clauses.empty?
+
+      # execute delete string with possible deletion conditions (for each 
+      # non-empty where clause)
+      ActiveRdfLogger::log_debug(self) { "Deleting #{[s,p,o,c].join(' ')}" }
+      @db.execute(ds, *conditions)
+
+      # delete literal from ferret index
+      @ferret.search_each("subject:\"#{s}\", object:\"#{o}\"") do |idx, score|
+        @ferret.delete(idx)
+      end if keyword_search?
+
+      @db
+    end
+
+    # adds triple(s,p,o) to datastore
+    # s,p must be resources, o can be primitive data or resource
+    def add(s,p,o,c=nil)
+      # check illegal input
+      raise(ActiveRdfError, "adding non-resource #{s} while adding (#{s},#{p},#{o},#{c})") unless s.respond_to?(:uri)
+      raise(ActiveRdfError, "adding non-resource #{p} while adding (#{s},#{p},#{o},#{c})") unless p.respond_to?(:uri)
+
+      triple = [s, p, o].collect{|r| serialise(r) }
+      add_ntriples(triple.join(' ') + " .\n", serialise(c) )
+
+      ## get internal representation (array)
+      #quad = [s,p,o,c].collect {|r| internalise(r) }
+
+      ## insert the triple into the datastore
+      #@db.execute('insert into triple values (?,?,?,?)', *quad)
+
+      ## if keyword-search available, insert the object into keyword search
+      #@ferret << {:subject => s, :object => o} if keyword_search?
+    end
+
+    # flushes openstanding changes to underlying sqlite3
+    def flush
+      # since we always write changes into sqlite3 immediately, we don't do 
+      # anything here
+      true
+    end
+
+    # loads triples from file in ntriples format
+    def load(location)
+      context = if URI.parse(location).host
+        location
+      else
+        internalise(RDFS::Resource.new("file:#{location}"))
+      end
+
+      case MIME::Types.of(location)
+      when MIME::Types['application/rdf+xml']
+        # check if rapper available
+        begin 
+          # can only parse rdf/xml with redland
+          # die otherwise
+          require 'rdf/redland'
+          model = Redland::Model.new
+          Redland::Parser.new.parse_into_model(model, location)
+          add_ntriples(model.to_string('ntriples'), location)
+        rescue LoadError
+          raise ActiveRdfError, "cannot parse remote rdf/xml file without Redland: please install Redland (librdf.org) and its Ruby bindings"
+        end
+      else
+        data = open(location).read
+        add_ntriples(data, context)
+      end
+    end
+
+    # adds ntriples from given context into datastore
+    def add_ntriples(ntriples, context)
+      # add each triple to db
+      @db.transaction
+      insert = @db.prepare('insert into triple values (?,?,?,?);')
+
+      ntriples = NTriplesParser.parse(ntriples)
+      ntriples.each do |s,p,o|
+        # convert triples into internal db format
+        subject, predicate, object = [s,p,o].collect {|r| internalise(r) }
+
+        # insert triple into database
+        insert.execute(subject, predicate, object, context)
+
+        # if keyword-search available, insert the object into keyword search
+        @ferret << {:subject => subject, :object => object} if keyword_search?
+      end
+
+      @db.commit
+      @db
+    end
+
+    # executes ActiveRDF query on datastore
+    def query(query)
+      # construct query clauses
+      sql, conditions = translate(query)
+
+      # executing query, passing all where-clause values as parameters (so that 
+      # sqlite will encode quotes correctly)
+      results = @db.execute(sql, *conditions)
+
+      # if ASK query, we check whether we received a positive result count
+      if query.ask?
+        return [[results[0][0].to_i > 0]]
+      elsif query.count?
+        return [[results[0][0].to_i]]
+      else
+        # otherwise we convert results to ActiveRDF nodes and return them
+        return wrap(results, query.resource_class)
+      end
+    end
+
+    # translates ActiveRDF query into internal sqlite query string
+    def translate(query)
+      where, conditions = construct_where(query)
+      [construct_select(query) + construct_join(query) + where + construct_sort(query) + construct_limit(query), conditions]
+    end
+
+    private
+    # constants for extracting resources/literals from sql results
+    SPOC = ['s','p','o','c']
+
+    # construct select clause
+    def construct_select(query)
+      # ASK queries counts the results, and return true if results > 0
+      return "select count(*)" if query.ask?
+
+      # add select terms for each selectclause in the query
+      # the term names depend on the join conditions, e.g. t0.s or t1.p
+      select = query.select_clauses.collect do |term|
+        variable_name(query, term)
+      end
+
+      # add possible distinct and count functions to select clause
+      select_clause = ''
+      select_clause << 'distinct ' if query.distinct?
+      select_clause << select.join(', ')
+      select_clause = "count(#{select_clause})" if query.count?
+
+      "select " + select_clause
+    end
+
+    # construct (optional) limit and offset clauses
+    def construct_limit(query)
+      clause = ""
+
+      # if no limit given, use limit -1 (no limit)
+      limit = query.limits.nil? ? -1 : query.limits
+
+      # if no offset given, use offset 0
+      offset = query.offsets.nil? ? 0 : query.offsets
+
+      clause << " limit #{limit} offset #{offset}"
+      clause
+    end
+
+    # sort query results on variable clause (optionally)
+    def construct_sort(query)
+      if not query.sort_clauses.empty?
+        sort = query.sort_clauses.collect { |term| variable_name(query, term) }
+        " order by (#{sort.join(',')})"
+      elsif not query.reverse_sort_clauses.empty?
+        sort = query.reverse_sort_clauses.collect { |term| variable_name(query, term) }
+        " order by (#{sort.join(',')}) DESC"
+      else
+        ""
+      end
+    end
+
+    # construct join clause
+    # TODO: joins don't work this way, they have to be linear (in one direction 
+    # only, and we should only alias tables we didnt alias yet)
+    # we should only look for one join clause in each where-clause: when we find 
+    # one, we skip the rest of the variables in this clause.
+    def construct_join(query)
+      join_stmt = ''
+
+      # no join necessary if only one where clause given
+      return ' from triple as t0 ' if query.where_clauses.size == 1
+
+      where_clauses = query.where_clauses.flatten
+      considering = where_clauses.uniq.select{|w| w.is_a?(Symbol)}
+
+      # constructing hash with indices for all terms
+      # e.g. {?s => [1,3,5], ?p => [2], ... }
+      term_occurrences = Hash.new()
+      where_clauses.each_with_index do |term, index|
+        ary = (term_occurrences[term] ||= [])
+        ary << index 
+      end
+
+      aliases = {}
+
+      where_clauses.each_with_index do |term, index|
+        # if the term has been joined with his buddy already, we can skip it
+        next unless considering.include?(term)
+
+        # we find all (other) occurrences of this term
+        indices = term_occurrences[term]
+
+        # if the term doesnt have a join-buddy, we can skip it
+        next if indices.size == 1
+
+        # construct t0,t1,... as aliases for term
+        # and construct join condition, e.g. t0.s
+        termalias = "t#{index / 4}"
+        termjoin = "#{termalias}.#{SPOC[index % 4]}"
+
+        join = if join_stmt.include?(termalias)
+          ""
+        else
+          "triple as #{termalias}"
+        end
+
+        indices.each do |i|
+          # skip the current term itself
+          next if i==index
+
+          # construct t0,t1, etc. as aliases for buddy,
+          # and construct join condition, e.g. t0.s = t1.p
+          buddyalias = "t#{i/4}"
+          buddyjoin = "#{buddyalias}.#{SPOC[i%4]}"
+
+          # TODO: fix reuse of same table names as aliases, e.g.
+          # "from triple as t1 join triple as t2 on ... join t1 on ..."
+          # is not allowed as such by sqlite
+          # but on the other hand, restating the aliases gives ambiguity:
+          # "from triple as t1 join triple as t2 on ... join triple as t1 ..."
+          # is ambiguous
+          if join_stmt.include?(buddyalias)
+            join << "and #{termjoin} = #{buddyjoin}"
+          else
+            join << " join triple as #{buddyalias} on #{termjoin} = #{buddyjoin} "
+          end
+        end
+        join_stmt << join
+
+        # remove term from 'todo' list of still-considered terms
+        considering.delete(term)
+      end
+
+      if join_stmt == ''
+        return " from triple as t0 "
+      else
+        return " from #{join_stmt} "
+      end
+    end
+
+    # construct where clause
+    def construct_where(query)
+      # collecting where clauses, these will be added to the sql string later
+      where = []
+
+      # collecting all the right-hand sides of where clauses (e.g. where name = 
+      # 'abc'), to add to query string later using ?-notation, because then 
+      # sqlite will automatically encode quoted literals correctly
+      right_hand_sides = []
+
+      # convert each where clause to SQL:
+      # add where clause for each subclause, except if it's a variable
+      query.where_clauses.each_with_index do |clause,level|
+        raise ActiveRdfError, "where clause #{clause} is not a triple" unless clause.is_a?(Array)
+        clause.each_with_index do |subclause, i|
+          # dont add where clause for variables
+          unless subclause.is_a?(Symbol) || subclause.nil?
+            conditions = compute_where_condition(i, subclause, query.reasoning? && reasoning?)
+            if conditions.size == 1
+              where << "t#{level}.#{SPOC[i]} = ?"
+              right_hand_sides << conditions.first
+            else
+              conditions = conditions.collect {|c| "'#{c}'"}
+              where << "t#{level}.#{SPOC[i]} in (#{conditions.join(',')})"
+            end
+          end
+        end
+      end
+
+      # if keyword clause given, convert it using keyword index
+      if query.keyword? && keyword_search?
+        subjects = []
+        select_subject = query.keywords.collect {|subj,key| subj}.uniq
+        raise ActiveRdfError, "cannot do keyword search over multiple subjects" if select_subject.size > 1
+
+        keywords = query.keywords.collect {|subj,key| key}
+        @ferret.search_each("object:#{keywords}") do |idx,score|
+          subjects << @ferret[idx][:subject]
+        end
+        subjects.uniq! if query.distinct?
+        where << "#{variable_name(query,select_subject.first)} in (#{subjects.collect {'?'}.join(',')})"
+        right_hand_sides += subjects
+      end
+
+      if where.empty?
+        ['',[]]
+      else
+        ["where " + where.join(' and '), right_hand_sides]
+      end
+    end
+
+    def compute_where_condition(index, subclause, reasoning)
+      conditions = [subclause]
+
+      # expand conditions with rdfs rules if reasoning enabled
+      if reasoning
+        case index
+        when 0: ;
+          # no rule for subjects
+        when 1:
+          # expand properties to include all subproperties
+          conditions = subproperties(subclause) if subclause.respond_to?(:uri)
+        when 2:
+          # no rule for objects
+        when 3:
+          # no rule for contexts
+        end
+      end
+
+      # convert conditions into internal format
+      #conditions.collect { |c| c.respond_to?(:uri) ? "<#{c.uri}>" : c.to_s }
+      conditions.collect { |c| internalise(c) }
+    end
+
+    def subproperties(resource)
+      # compute and store subproperties of this resource 
+      # or use earlier computed value if available
+      unless @subprops[resource]
+        subproperty = Namespace.lookup(:rdfs,:subPropertyOf)
+        children_query = Query.new.distinct(:sub).where(:sub, subproperty, resource)
+        children_query.reasoning = false
+        children = children_query.execute
+
+        if children.empty?
+          @subprops[resource] = [resource]
+        else
+          @subprops[resource] = [resource] + children.collect{|c| subproperties(c)}.flatten.compact
+        end
+      end
+      @subprops[resource]
+    end
+
+    # returns sql variable name for a queryterm
+    def variable_name(query,term)
+      # look up the first occurence of this term in the where clauses, and compute 
+      # the level and s/p/o position of it
+      index = query.where_clauses.flatten.index(term)
+
+      if index.nil? 
+        # term does not appear in where clause
+        # but maybe it appears in a keyword clause
+
+        # index would not be nil if we had:
+        # select(:o).where(knud, knows, :o).where(:o, :keyword, 'eyal')
+        #
+        # the only possibility that index is nil is if we have:
+        # select(:o).where(:o, :keyword, :eyal) (selecting subject)
+        # or if we use a select clause that does not appear in any where clause
+
+        # so we check if we find the term in the keyword clauses, otherwise we throw 
+        # an error
+        if query.keywords.keys.include?(term)
+          return "t0.s"
+        else
+          raise ActiveRdfError, "unbound variable :#{term.to_s} in select of #{query.to_sp}"
+        end
+      end
+
+      termtable = "t#{index / 4}"
+      termspo = SPOC[index % 4]
+      return "#{termtable}.#{termspo}"
+    end
+
+    # wrap resources into ActiveRDF resources, literals into Strings. result_type
+    # is the type that should be used for ActiveRDF resources
+    def wrap(results, result_type)
+      results.collect do |row|
+        row.collect { |result| parse(result, result_type) }
+      end
+    end
+
+    # Return the result as a correct type. result_type is the type that should 
+    # be used for "resouce" elements
+    def parse(result, result_type = RDFS::Resource)
+      NTriplesParser.parse_node(result, result_type) || result
+      # case result
+      # when Literal
+      #   # replace special characters to allow string interpolation for e.g. 'test\nbreak'
+      #   $1.double_quote
+      # when Resource
+      #   result_type.new($1)
+      # else
+      #   # when we do a count(*) query we get a number, not a resource/literal
+      #   result
+      # end
+    end
+
+    def create_indices(params)
+      sidx = params[:sidx] || false
+      pidx = params[:pidx] || false
+      oidx = params[:oidx] || false
+      spidx = params[:spidx] || true
+      soidx = params[:soidx] || false
+      poidx = params[:poidx] || true
+      opidx = params[:opidx] || false
+
+      # creating lookup indices
+      @db.transaction do 
+        @db.execute('create index if not exists sidx on triple(s)') if sidx
+        @db.execute('create index if not exists pidx on triple(p)') if pidx
+        @db.execute('create index if not exists oidx on triple(o)') if oidx
+        @db.execute('create index if not exists spidx on triple(s,p)') if spidx
+        @db.execute('create index if not exists soidx on triple(s,o)') if soidx
+        @db.execute('create index if not exists poidx on triple(p,o)') if poidx
+        @db.execute('create index if not exists opidx on triple(o,p)') if opidx
+      end
+    end
+
+    # transform triple into internal format <uri> and "literal"
+    def internalise(r)
+      if r.nil? or r.is_a? Symbol
+        nil
+      else
+        r.to_ntriple
+      end
+    end
+
+    # transform resource/literal into ntriples format
+    def serialise(r)
+      # Fixme: r is most likely NOT a resource
+      if r.nil?
+        nil
+      else
+        r.to_ntriple
+      end
+    end
+
+    Resource = /<([^>]*)>/
+    Literal = /"((?:\\"|[^"])*)"/
+
+    public :subproperties
+  end
+
+  class String
+    def double_quote
+      Thread.new do
+        $SAFE = 12
+        begin
+          eval('"%s"' % self)
+        rescue Exception => e
+          self
+        end
+      end.value
+    end
+  end