xapian-fu 1.3.2 → 1.4.0

data/CHANGELOG.rdoc

@@ -1,3 +1,16 @@
+=== 1.4.0 (13th March 2012)
+
+* Support for indexing Arrays properly
+
+* Support returning all and no documents by using Xapian's special
+  queries `MatchAll` and `MatchNothing`.
+
+* Add boolean terms and faceted queries.
+
+  See [http://bit.ly/rMuA4M](http://bit.ly/rMuA4M).
+
+* Fix number range queries when no prefixes are given.
+
 === 1.3.2 (4th December 2011)
 
 * Number range queries (Damian Janowski)

data/README.rdoc

@@ -17,6 +17,14 @@ Hash with full text indexing (and ACID transactions).
 
   sudo gem install xapian-fu
 
+=== Xapian Bindings
+
+Xapian Fu requires the Xapian Ruby bindings to be available.  On
+Debian/Ubuntu, you can install the `libxapian-ruby1.8` package to get
+them.  Alternatively, you can install the `xapian-ruby` gem, which
+reportedly will also provide them.  You can also just get the
+upstream Xapian release and manually install it.
+
 == Documentation
 
 XapianFu::XapianDb is the corner-stone of XapianFu.  A XapianDb
@@ -97,7 +105,7 @@ type of object is instantiated when returning those stored values.
 And in the case of Fixnum and Bignum, allows you to order search
 results without worrying about leading zeros.
 
-  db = XapianDb.new(:fields => { 
+  db = XapianDb.new(:fields => {
                                  :title => { :store => true },
                                  :released => { :type => Date, :store => true },
                                  :votes => { :type => Fixnum, :store => true }
@@ -113,6 +121,13 @@ Find the document with the highest :year value
 
   db.documents.max(:year)
 
+== Special queries
+
+XapianFu supports Xapian's `MatchAll` and `MatchNothing` queries:
+
+  db.search(:all)
+  db.search(:nothing)
+
 == Search examples
 
 Search on particular fields
@@ -145,6 +160,54 @@ And any combinations of the above:
 
   db.search("(ruby OR sinatra) -rails xap*")
 
+== Boolean terms
+
+If you want to implement something like [this](http://getting-started-with-xapian.readthedocs.org/en/latest/howtos/boolean_filters.html#searching),
+then:
+
+  db = XapianDb.new(
+    fields: {
+      name:   {:index => true},
+      colors: {:boolean => true}
+    }
+  )
+
+  db << {name: "Foo", colors: ["red", "black"]}
+  db << {name: "Foo", colors: ["red", "green"]}
+  db << {name: "Foo", colors: ["blue", "yellow"]}
+
+  db.search("foo", filter: {:colors => ["red"]})
+
+The main thing here is that filtering by color doesn't affect the relevancy of the documents returned.
+
+== Facets
+
+Many times you want to allow users to narrow down the search results by restricting the query
+to specific values of a given category. This is called [faceted search](readthedocs.org/docs/getting-started-with-xapian/en/latest/xapian-core-rst/facets.html).
+
+To find out which values you can display to your users, you can do something like this:
+
+  results = db.search("foo", facets: [:colors, :year])
+
+  results.facets
+  # {
+  #   :colors => [
+  #     ["blue",  4]
+  #     ["red",   1]
+  #   ],
+  #
+  #   :year => [
+  #     [2010, 3],
+  #     [2011, 2],
+  #     [2012, 1]
+  #   ]
+  # }
+
+When filtering by one of these values, it's best to define the field as
+boolean (see section above) and then use `:filter`:
+
+  db.search("foo", filter: {colors: ["blue"], year: [2010]})
+
 == ActiveRecord Integration
 
 XapianFu always stores the :id field, so you can easily use it with
@@ -176,7 +239,7 @@ perhaps by reindexing once in a while.
 
   db = XapianDb.new(:dir => 'posts.db')
   deleted_posts = Post.find(:all, :conditions => 'deleted_at is not null')
-  deleted_posts.each do |post| 
+  deleted_posts.each do |post|
     db.documents.delete(post.id)
     post.destroy
   end
@@ -184,8 +247,8 @@ perhaps by reindexing once in a while.
 = More Info
 
 Author::  John Leach  (mailto:john@johnleach.co.uk)
-Copyright:: Copyright (c) 2009-2011 John Leach
+Copyright:: Copyright (c) 2009-2012 John Leach
 License:: MIT (The Xapian library is GPL)
 Mailing list:: http://rubyforge.org/mailman/listinfo/xapian-fu-discuss
 Web page:: http://johnleach.co.uk/documents/xapian-fu
-Github:: http://github.com/johnl/xapian-fu/tree/master
+Github:: http://github.com/johnl/xapian-fu/tree/master

data/lib/xapian_fu/query_parser.rb

@@ -49,15 +49,15 @@ module XapianFu #:nodoc:
   # or false.
   #
   class QueryParser #:notnew:
-    
+
     # The stemming strategy to use when generating terms from a query.
     # Defaults to <tt>:some</tt>
     attr_accessor :stemming_strategy
-    
+
     # The default operation when combining search terms.  Defaults to
     # <tt>:and</tt>
     attr_accessor :default_op
-    
+
     # The database that this query is agains, used for setting up
     # fields, stemming, stopping and spelling.
     attr_accessor :database
@@ -72,9 +72,17 @@ module XapianFu #:nodoc:
       self.database = @options[:database]
     end
 
-    # Parse the given query string and return a Xapian::Query object
+    # Parse the given query and return a Xapian::Query object
+    # Accepts either a string or a special query
     def parse_query(q)
-      query_parser.parse_query(q, xapian_flags)
+      case q
+      when :all
+        Xapian::Query.new("")
+      when :nothing
+        Xapian::Query.new()
+      else
+        query_parser.parse_query(q, xapian_flags)
+      end
     end
 
     # Return the query string with any spelling corrections made
@@ -93,16 +101,32 @@ module XapianFu #:nodoc:
         qp.stemmer = database.stemmer if database
         qp.default_op = xapian_default_op
         qp.stemming_strategy = xapian_stemming_strategy
+
         fields.each do |name, type|
+          next if database && database.boolean_fields.include?(name)
           qp.add_prefix(name.to_s.downcase, "X" + name.to_s.upcase)
         end
 
+        database.boolean_fields.each do |name|
+          qp.add_boolean_prefix(name.to_s.downcase, "X#{name.to_s.upcase}")
+        end if database
+
         database.sortable_fields.each do |field, opts|
-          if opts[:range_prefix]
-            qp.add_valuerangeprocessor(Xapian::NumberValueRangeProcessor.new(XapianDocValueAccessor.value_key(field), opts[:range_prefix], true))
+          prefix, string = nil
+
+          if opts[:range_postfix]
+            prefix = false
+            string = opts[:range_postfix]
           else
-            qp.add_valuerangeprocessor(Xapian::NumberValueRangeProcessor.new(XapianDocValueAccessor.value_key(field), opts[:range_postfix], false))
+            prefix = true
+            string = opts[:range_prefix] || "#{field.to_s.downcase}:"
           end
+
+          qp.add_valuerangeprocessor(Xapian::NumberValueRangeProcessor.new(
+            XapianDocValueAccessor.value_key(field),
+            string,
+            prefix
+          ))
         end if database
 
         @query_parser = qp

data/lib/xapian_fu/result_set.rb

@@ -12,12 +12,23 @@ module XapianFu
     # by :corrected_query, otherwise this is empty.
     attr_reader :corrected_query
 
+    attr_reader :facets
+
     # nodoc
     def initialize(options = { })
       @mset = options[:mset]
       @current_page = options[:current_page]
       @per_page = options[:per_page]
       @corrected_query = options[:corrected_query]
+      @facets = {}
+      @db = options[:xapian_db]
+
+      options[:spies].each do |name, spy|
+        @facets[name] = spy.values.map do |value|
+          [@db.unserialize_value(name, value.term), value.termfreq]
+        end
+      end if options[:spies]
+
       doc_options = {:xapian_db => options[:xapian_db] }
       concat mset.matches.collect { |m| XapianDoc.new(m, doc_options) }
     end

data/lib/xapian_fu/version.rb

@@ -1,3 +1,3 @@
 module XapianFu #:nodoc:
-  VERSION = "1.3.2"
+  VERSION = "1.4.0"
 end

data/lib/xapian_fu/xapian_db.rb

@@ -100,7 +100,7 @@ module XapianFu #:nodoc:
   #
   class XapianDb # :nonew:
     # Path to the on-disk database. Nil if in-memory database
-    attr_reader :dir 
+    attr_reader :dir
     attr_reader :db_flag #:nodoc:
     # An array of the fields that will be stored in the Xapian
     attr_reader :store_values
@@ -112,6 +112,8 @@ module XapianFu #:nodoc:
     attr_reader :fields
     # An array of fields that will not be indexed
     attr_reader :unindexed_fields
+    # An array of fields that will be treated as boolean terms
+    attr_reader :boolean_fields
     # Whether this db will generate a spelling dictionary during indexing
     attr_reader :spelling
     attr_reader :sortable_fields
@@ -130,7 +132,7 @@ module XapianFu #:nodoc:
       setup_fields(@options[:fields])
       @store_values << @options[:store]
       @store_values << @options[:sortable]
-      @store_values << @options[:collapsible]      
+      @store_values << @options[:collapsible]
       @store_values = @store_values.flatten.uniq.compact
       @spelling = @options[:spelling]
     end
@@ -195,10 +197,10 @@ module XapianFu #:nodoc:
     #
     # The <tt>:page</tt> option sets which page of results to return.
     # Defaults to 1.
-    # 
+    #
     # The <tt>:order</tt> option specifies the stored field to order
     # the results by (instead of the default search result weight).
-    # 
+    #
     # The <tt>:reverse</tt> option reverses the order of the results,
     # so lowest search weight first (or lowest stored field value
     # first).
@@ -213,9 +215,13 @@ module XapianFu #:nodoc:
     # enabled, spelling suggestions are available using the
     # XapianFu::ResultSet <tt>corrected_query</tt> method.
     #
+    # The first parameter can also be <tt>:all</tt> or
+    # <tt>:nothing</tt>, to match all documents or no documents
+    # respectively.
+    #
     # For additional options on how the query is parsed, see
     # XapianFu::QueryParser
-    
+
     def search(q, options = {})
       defaults = { :page => 1, :reverse => false,
         :boolean => true, :boolean_anycase => true, :wildcards => true,
@@ -227,15 +233,29 @@ module XapianFu #:nodoc:
       per_page = per_page.to_i rescue 10
       offset = page * per_page
       qp = XapianFu::QueryParser.new({ :database => self }.merge(options))
-      query = qp.parse_query(q.to_s)
+      query = qp.parse_query(q.is_a?(Symbol) ? q : q.to_s)
+      query = filter_query(query, options[:filter]) if options[:filter]
       enquiry = Xapian::Enquire.new(ro)
       setup_ordering(enquiry, options[:order], options[:reverse])
       if options[:collapse]
         enquiry.collapse_key = XapianDocValueAccessor.value_key(options[:collapse])
       end
+      if options[:facets]
+        spies = options[:facets].inject({}) do |accum, name|
+          accum[name] = spy = Xapian::ValueCountMatchSpy.new(XapianDocValueAccessor.value_key(name))
+          enquiry.add_matchspy(spy)
+          accum
+        end
+      end
       enquiry.query = query
-      ResultSet.new(:mset => enquiry.mset(offset, per_page), :current_page => page + 1,
-                    :per_page => per_page, :corrected_query => qp.corrected_query, :xapian_db => self)
+
+      ResultSet.new(:mset => enquiry.mset(offset, per_page),
+                    :current_page => page + 1,
+                    :per_page => per_page,
+                    :corrected_query => qp.corrected_query,
+                    :spies => spies,
+                    :xapian_db => self
+                   )
     end
 
     # Run the given block in a XapianDB transaction.  Any changes to the
@@ -272,6 +292,22 @@ module XapianFu #:nodoc:
       ro.reopen
     end
 
+    def serialize_value(field, value, type = nil)
+      if sortable_fields.include?(field)
+        Xapian.sortable_serialise(value)
+      else
+        (type || fields[field] || Object).to_xapian_fu_storage_value(value)
+      end
+    end
+
+    def unserialize_value(field, value, type = nil)
+      if sortable_fields.include?(field)
+        Xapian.sortable_unserialise(value)
+      else
+        (type || fields[field] || Object).from_xapian_fu_storage_value(value)
+      end
+    end
+
     private
 
     # Setup the writable database
@@ -317,12 +353,17 @@ module XapianFu #:nodoc:
       @unindexed_fields = []
       @store_values = []
       @sortable_fields = {}
+      @boolean_fields = []
       return nil if field_options.nil?
       default_opts = {
         :store => true,
         :index => true,
         :type => String
       }
+      boolean_default_opts = default_opts.merge(
+        :store => false,
+        :index => false
+      )
       # Convert array argument to hash, with String as default type
       if field_options.is_a? Array
         fohash = { }
@@ -332,16 +373,62 @@ module XapianFu #:nodoc:
       field_options.each do |name,opts|
         # Handle simple setup by type only
         opts = { :type => opts } unless opts.is_a? Hash
-        opts = default_opts.merge(opts)
+        if opts[:boolean]
+          opts = boolean_default_opts.merge(opts)
+        else
+          opts = default_opts.merge(opts)
+        end
         @store_values << name if opts[:store]
         @sortable_fields[name] = {:range_prefix => opts[:range_prefix], :range_postfix => opts[:range_postfix]} if opts[:sortable]
         @unindexed_fields << name if opts[:index] == false
+        @boolean_fields << name if opts[:boolean]
         @fields[name] = opts[:type]
       end
       @fields
     end
-    
+
+    def filter_query(query, filter)
+      subqueries = filter.map do |field, values|
+        values = Array(values)
+
+        if sortable_fields[field]
+          sortable_filter_query(field, values)
+        elsif boolean_fields.include?(field)
+          boolean_filter_query(field, values)
+        end
+      end
+
+      combined_subqueries = Xapian::Query.new(Xapian::Query::OP_AND, subqueries)
+
+      Xapian::Query.new(Xapian::Query::OP_FILTER, query, combined_subqueries)
+    end
+
+    def sortable_filter_query(field, values)
+      subqueries = values.map do |value|
+        from, to = value.split("..")
+        slot = XapianDocValueAccessor.value_key(field)
+
+        if from.empty?
+          Xapian::Query.new(Xapian::Query::OP_VALUE_LE, slot, Xapian.sortable_serialise(to.to_f))
+        elsif to.nil?
+          Xapian::Query.new(Xapian::Query::OP_VALUE_GE, slot, Xapian.sortable_serialise(from.to_f))
+        else
+          Xapian::Query.new(Xapian::Query::OP_VALUE_RANGE, slot, Xapian.sortable_serialise(from.to_f), Xapian.sortable_serialise(to.to_f))
+        end
+      end
+
+      Xapian::Query.new(Xapian::Query::OP_OR, subqueries)
+    end
+
+    def boolean_filter_query(field, values)
+      subqueries = values.map do |value|
+        Xapian::Query.new("X#{field.to_s.upcase}#{value.to_s.downcase}")
+      end
+
+      Xapian::Query.new(Xapian::Query::OP_OR, subqueries)
+    end
+
   end
-  
+
 end
 

data/lib/xapian_fu/xapian_doc.rb

@@ -18,9 +18,15 @@ class DateTime #:nodoc:
   end
 end
 
+class Array #:nodoc:
+  def to_xapian_fu_string
+    join(" ")
+  end
+end
+
 module XapianFu #:nodoc:
   require 'xapian_doc_value_accessor'
-  
+
   # Raised whenever a XapianDb is needed but has not been provided,
   # such as when retrieving the terms list for a document
   class XapianDbNotSet < XapianFuError ; end
@@ -33,26 +39,26 @@ module XapianFu #:nodoc:
   # documents to the database.  You usually don't need to instantiate
   # them yourself unless you're doing something a bit advanced.
   class XapianDoc
-    
+
     # A hash of the fields given to this object on initialize
     attr_reader :fields
-    
+
     # An abitrary blob of data stored alongside the document in the
     # Xapian database.
     attr_reader :data
-    
+
     # The search score of this document when returned as part of a
     # search result
     attr_reader :weight
-    
+
     # The Xapian::Match object for this document when returned as part
     # of a search result.
     attr_reader :match
-    
+
     # The unsigned integer "primary key" for this document in the
     # Xapian database.
     attr_accessor :id
-    
+
     # The XapianDb object that this document was retrieved from, or
     # should be stored in.
     attr_accessor :db
@@ -66,7 +72,7 @@ module XapianFu #:nodoc:
     # and term enumeration.
     def initialize(doc, options = {})
       @options = options
-      
+
       @fields = {}
       if doc.is_a? Xapian::Match
         match = doc
@@ -84,6 +90,9 @@ module XapianFu #:nodoc:
       elsif doc.respond_to?(:has_key?) and doc.respond_to?("[]")
         @fields = doc
         @id = doc[:id] if doc.has_key?(:id)
+      # Handle initialisation from an object with a to_xapian_fu_string method
+      elsif doc.respond_to?(:to_xapian_fu_string)
+        @fields = { :content => doc.to_xapian_fu_string }
       # Handle initialisation from anything else that can be coerced
       # into a string
       elsif doc.respond_to? :to_s
@@ -107,10 +116,10 @@ module XapianFu #:nodoc:
     def values
       @value_accessor ||= XapianDocValueAccessor.new(self)
     end
-    
+
     # Return a list of terms that the db has for this document.
     def terms
-      raise XapianFu::XapianDbNotSet unless db      
+      raise XapianFu::XapianDbNotSet unless db
       db.ro.termlist(id) if db.respond_to?(:ro) and db.ro and id
     end
 
@@ -123,7 +132,7 @@ module XapianFu #:nodoc:
       xapian_document.clear_values
       add_values_to_xapian_document
       # Clear and add terms
-      xapian_document.clear_terms      
+      xapian_document.clear_terms
       generate_terms
       xapian_document
     end
@@ -135,7 +144,7 @@ module XapianFu #:nodoc:
     def xapian_document
       @xapian_document ||= Xapian::Document.new
     end
-    
+
     # Compare IDs with another XapianDoc
     def ==(b)
       if b.is_a?(XapianDoc)
@@ -144,7 +153,7 @@ module XapianFu #:nodoc:
         super(b)
       end
     end
-    
+
     def inspect
       s = ["<#{self.class.to_s} id=#{id}"]
       s << "weight=%.5f" % weight if weight
@@ -167,7 +176,7 @@ module XapianFu #:nodoc:
     def update
       db.rw.replace_document(id, to_xapian_document)
     end
-    
+
     # Set the stemmer to use for this document.  Accepts any string
     # that the Xapian::Stem class accepts (Either the English name for
     # the language or the two letter ISO639 code). Can also be an
@@ -183,7 +192,7 @@ module XapianFu #:nodoc:
       if @stemmer
         @stemmer
       else
-        @stemmer = 
+        @stemmer =
           if ! @options[:stemmer].nil?
             @options[:stemmer]
           elsif @options[:language]
@@ -196,7 +205,7 @@ module XapianFu #:nodoc:
         @stemmer = StemFactory.stemmer_for(@stemmer)
       end
     end
-    
+
     # Return the stopper for this document.  If not set on initialize
     # by the :stopper or :language option, it will try the database's
     # stopper and otherwise default to an English stopper..
@@ -218,7 +227,7 @@ module XapianFu #:nodoc:
       end
     end
 
-    # Return this document's language which is set on initialize, inherited 
+    # Return this document's language which is set on initialize, inherited
     # from the database or defaults to :english
     def language
       if @language
@@ -234,14 +243,14 @@ module XapianFu #:nodoc:
           end
       end
     end
-    
+
     private
-    
+
     # Array of field names not to run through the TermGenerator
     def unindexed_fields
       db ? db.unindexed_fields : []
     end
-    
+
     # Add all the fields to be stored as XapianDb values
     def add_values_to_xapian_document
       db.store_values.collect do |key|
@@ -271,12 +280,19 @@ module XapianFu #:nodoc:
         # add value without field name
         tg.send(index_method, v)
       end
+
+      db.boolean_fields.each do |name|
+        Array(fields[name]).each do |value|
+          xapian_document.add_boolean_term("X#{name.to_s.upcase}#{value.to_s.downcase}")
+        end
+      end
+
       xapian_document
     end
-    
+
   end
-  
-  
+
+
   class StemFactory
     # Return a Xapian::Stem object for the given option. Accepts any
     # string that the Xapian::Stem class accepts (Either the English
@@ -296,5 +312,5 @@ module XapianFu #:nodoc:
       end
     end
   end
-  
+
 end

data/lib/xapian_fu/xapian_doc_value_accessor.rb

@@ -63,20 +63,31 @@ class Date #:nodoc:
   end
 end
 
+class Object
+  def self.to_xapian_fu_storage_value(value)
+    value.to_s
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    value
+  end
+end
+
 module XapianFu #:nodoc:
-  
+
   class ValueOutOfBounds < XapianFuError
   end
-  
+
   # A XapianDocValueAccessor is used to provide the XapianDoc#values
   # interface to read and write field values to a XapianDb.  It is
   # usually set up by a XapianDoc so you shouldn't need to set up your
   # own.
   class XapianDocValueAccessor
+
     def initialize(xapian_doc)
       @doc = xapian_doc
     end
-    
+
     # Add the given <tt>value</tt> with the given <tt>key</tt> to the
     # XapianDoc.  If the value has a
     # <tt>to_xapian_fu_storage_value</tt> method then it is used to
@@ -84,23 +95,12 @@ module XapianFu #:nodoc:
     # is used.  This is usually paired with a
     # <tt>from_xapian_fu_storage_value</tt> class method on retrieval.
     def store(key, value, type = nil)
-      type = @doc.db.fields[key] if type.nil? and @doc.db
-
-      if @doc.db && @doc.db.sortable_fields.include?(key)
-        converted_value = Xapian.sortable_serialise(value)
-      else
-        if type and type.respond_to?(:to_xapian_fu_storage_value)
-          converted_value = type.to_xapian_fu_storage_value(value)
-        else
-          converted_value = value.to_s
-        end
-      end
-
+      converted_value = @doc.db.serialize_value(key, value, type)
       @doc.xapian_document.add_value(XapianDocValueAccessor.value_key(key), converted_value)
       value
     end
     alias_method "[]=", :store
-    
+
     # Retrieve the value with the given <tt>key</tt> from the
     # XapianDoc. <tt>key</tt> can be a symbol or string, in which case
     # it's hashed to get an integer value number. Or you can give the
@@ -117,20 +117,15 @@ module XapianFu #:nodoc:
     # empty string is returned.
     def fetch(key, type = nil)
       value = @doc.xapian_document.value(XapianDocValueAccessor.value_key(key))
-      type = @doc.db.fields[key] if type.nil? and @doc.db
-      if type and type.respond_to?(:from_xapian_fu_storage_value)
-        type.from_xapian_fu_storage_value(value)
-      else
-        value
-      end
+      @doc.db.unserialize_value(key, value, type)
     end
     alias_method "[]", :fetch
-    
+
     # Count the values stored in the XapianDoc
     def size
       @doc.xapian_document.values_count
     end
-    
+
     # Remove the value with the given key from the XapianDoc and return it
     def delete(key)
       value = fetch(key)
@@ -142,6 +137,6 @@ module XapianFu #:nodoc:
     # value number
     def self.value_key(key)
       (key.is_a?(Integer) ? key : Zlib.crc32(key.to_s))
-    end    
+    end
   end
 end

data/spec/facets_spec.rb

@@ -0,0 +1,34 @@
+require File.expand_path('../lib/xapian_fu.rb', File.dirname(__FILE__))
+
+tmp_dir = '/tmp/xapian_fu_test.db'
+
+describe "Facets support" do
+
+  before do
+    @xdb = XapianFu::XapianDb.new(
+      :dir => tmp_dir, :create => true, :overwrite => true,
+      :fields => {
+        :name      => { :index => true },
+        :age       => { :type => Integer, :sortable => true },
+        :height    => { :type => Float, :sortable => true }
+      }
+    )
+
+    @xdb << {:name => "John A",   :age => 30, :height => 1.8}
+    @xdb << {:name => "John B",   :age => 35, :height => 1.8}
+    @xdb << {:name => "John C",   :age => 40, :height => 1.7}
+    @xdb << {:name => "John D",   :age => 40, :height => 1.7}
+    @xdb << {:name => "Markus",   :age => 35, :height => 1.7}
+    @xdb.flush
+  end
+
+  it "should expose facets when searching" do
+    results = @xdb.search("john", {:facets => [:age, :height]})
+
+    results.facets[:age].should == [[30, 1], [35, 1], [40, 2]]
+    results.facets[:height].should == [[1.7, 2], [1.8, 2]]
+
+    results.facets.keys.map(&:to_s).sort == %w(age height)
+  end
+
+end

data/spec/query_parser_spec.rb

@@ -37,6 +37,13 @@ describe QueryParser do
       terms.should_not include "john"
     end
 
+    it "should turn :all into a query with no terms" do
+      qp = QueryParser.new
+      qp.parse_query(:all).terms.should == []
+      qp.parse_query(:all).empty?.should be_false
+      qp.parse_query(:nothing).empty?.should be_true
+    end
+
   end
 
 end

data/spec/xapian_db_spec.rb

@@ -202,6 +202,46 @@ describe XapianDb do
       xdb.size.should == 2
     end
 
+    it "should generate boolean terms for multiple values" do
+      xdb = XapianDb.new(:dir => tmp_dir, :create => true,
+                         :fields => {
+                           :name => { :index => true },
+                           :colors => { :boolean => true }
+                         }
+                        )
+
+      xdb << {:name => "Foo", :colors => [:red, :black]}
+      xdb << {:name => "Foo", :colors => [:red, :green]}
+      xdb << {:name => "Foo", :colors => [:blue, :yellow]}
+
+      xdb.flush
+
+      xdb.search("foo", :filter => {:colors => [:red]}).map(&:id).should == [1, 2]
+      xdb.search("foo", :filter => {:colors => [:black, :green]}).map(&:id).should == [1, 2]
+
+      xdb.search("red").should be_empty
+    end
+
+    it "should index boolean terms if asked to" do
+      xdb = XapianDb.new(:dir => tmp_dir, :create => true,
+                         :fields => {
+                           :name   => { :index => true },
+                           :colors => { :index => true, :boolean => true }
+                         }
+                        )
+
+      xdb << {:name => "Foo", :colors => [:red, :black]}
+      xdb << {:name => "Foo", :colors => [:red, :green]}
+      xdb << {:name => "Foo", :colors => [:blue, :yellow]}
+
+      xdb.flush
+
+      xdb.search("foo", :filter => {:colors => [:red]}).map(&:id).should == [1, 2]
+      xdb.search("foo", :filter => {:colors => [:black, :green]}).map(&:id).should == [1, 2]
+
+      xdb.search("red").map(&:id).should == [1, 2]
+    end
+
   end
 
   describe "search" do
@@ -394,6 +434,85 @@ describe XapianDb do
       xdb.search("jon").should be_empty
       xdb.search("jon", :synonyms => true).should_not be_empty
     end
+
+    describe "with special queries" do
+      before do
+        @xdb = XapianDb.new
+        @xdb << "Doc 1"
+        @xdb << "Doc 2"
+      end
+
+      it "should return empty array on :nothing" do
+        @xdb.search(:nothing).should be_empty
+      end
+
+      it "should return all documents on :all" do
+        @xdb.search(:all).length.should eq 2
+      end
+    end
+
+    it "should allow to search by boolean terms" do
+      xdb = XapianDb.new(:dir => tmp_dir, :create => true,
+                         :fields => {
+                           :name => { :index => true },
+                           :age => { :boolean => true },
+                           :city => { :boolean => true }
+                         }
+                        )
+
+      xdb << {:name => "John A", :age => 10, :city => "London"}
+      xdb << {:name => "John B", :age => 11, :city => "Liverpool"}
+      xdb << {:name => "John C", :age => 12, :city => "Liverpool"}
+
+      xdb.flush
+
+      xdb.search("john").size.should == 3
+      xdb.search("john", :filter => {:age => 10}).map(&:id).should == [1]
+      xdb.search("john", :filter => {:age => [10, 12]}).map(&:id).should == [1, 3]
+
+      xdb.search("john", :filter => {:age => 10, :city => "Liverpool"}).map(&:id).should == []
+      xdb.search("john", :filter => {:city => "Liverpool"}).map(&:id).should == [2, 3]
+      xdb.search("john", :filter => {:age => 11..15, :city => "Liverpool"}).map(&:id).should == [2, 3]
+
+      xdb.search("liverpool").should be_empty
+      xdb.search("city:liverpool").map(&:id).should == [2, 3]
+    end
+  end
+
+  describe "filtering" do
+    before do
+      @xdb = XapianDb.new(
+        :dir => tmp_dir, :create => true, :overwrite => true,
+        :fields => {
+          :name      => { :index => true },
+          :age       => { :type => Integer, :sortable => true },
+          :height    => { :type => Float, :sortable => true }
+        }
+      )
+    end
+
+    it "should filter results using value ranges" do
+      @xdb << {:name => "John",   :age => 30, :height => 1.8}
+      @xdb << {:name => "John",   :age => 35, :height => 1.9}
+      @xdb << {:name => "John",   :age => 40, :height => 1.7}
+      @xdb << {:name => "Markus", :age => 35, :height => 1.7}
+      @xdb.flush
+
+      # Make sure we're combining queries using OP_FILTER by comparing
+      # the weights with and without filtering.
+      @xdb.search("markus")[0].weight.should == @xdb.search("markus", :filter => {:age => "35"})[0].weight
+
+      @xdb.search("john", :filter => {:age => "10..20"}).should be_empty
+
+      @xdb.search("john", :filter => {:age => "10..30"}).map(&:id).should == [1]
+      @xdb.search("john", :filter => {:age => "35.."}).map(&:id).should == [2, 3]
+      @xdb.search("john", :filter => {:age => "..35"}).map(&:id).should == [1, 2]
+      @xdb.search("john", :filter => {:age => ["..30", "40.."]}).map(&:id).should == [1, 3]
+
+      @xdb.search("john", :filter => {:age => "10..30", :height => "1.8"}).map(&:id).should == [1]
+      @xdb.search("john", :filter => {:age => "10..30", :height => "..1.8"}).map(&:id).should == [1]
+      @xdb.search("john", :filter => {:age => "10..30", :height => "1.9.."}).should be_empty
+    end
   end
 
   describe "add_doc" do
@@ -468,6 +587,21 @@ describe XapianDb do
       docs.map { |d| d.id }.should == [1, 3]
     end
 
+    it "should allow range queries without prefixes" do
+      xdb = XapianDb.new(:fields => {
+        :price => { :type => Integer, :sortable => true, :range_prefix => "$" },
+        :age => { :type => Integer, :sortable => true }
+      })
+
+      xdb << XapianDoc.new(:price => 10, :age => 40)
+      xdb << XapianDoc.new(:price => 20, :age => 35)
+      xdb << XapianDoc.new(:price => 45, :age => 30)
+
+      docs = xdb.search("$20..40 OR age:40..50")
+
+      docs.map { |d| d.id }.should == [1, 2]
+    end
+
     it "should store values declared as to be collapsible" do
       xdb = XapianDb.new(:collapsible => :group_id)
       xdb << XapianDoc.new(:group_id => "666", :author => "Jim Jones")

data/spec/xapian_doc_spec.rb

@@ -37,6 +37,30 @@ describe XapianDoc do
       xdb.ro.positionlist(doc.id, "once").first.should == nil
     end
 
+    it "should tokenize an array given as a field" do
+      xdb = XapianDb.new
+      xdoc = xdb.documents.new(:colors => [:red, :green, :blue]).to_xapian_document
+      xdoc.terms.should be_a_kind_of Array
+      xdoc.terms.last.should be_a_kind_of Xapian::Term
+      terms = xdoc.terms.collect { |t| t.term }
+      terms.should include "red"
+      terms.should include "green"
+      terms.should include "blue"
+      terms.should_not include "redgreenblue"
+    end
+
+    it "should tokenize an array given as the content" do
+      xdb = XapianDb.new
+      xdoc = xdb.documents.new([:red, :green, :blue]).to_xapian_document
+      xdoc.terms.should be_a_kind_of Array
+      xdoc.terms.last.should be_a_kind_of Xapian::Term
+      terms = xdoc.terms.collect { |t| t.term }
+      terms.should include "red"
+      terms.should include "green"
+      terms.should include "blue"
+      terms.should_not include "redgreenblue"
+    end
+
     it "should tokenize a hash" do
       xdb = XapianDb.new
       xdoc = xdb.documents.new(:title => 'once upon a time').to_xapian_document

data/spec/xapian_doc_value_accessor_spec.rb

@@ -12,8 +12,14 @@ describe XapianDocValueAccessor do
     end
   end
 
+  before do
+    @xdb = XapianDb.new(:fields => {:name => {:index => true}, :city => {:store => true}})
+    @xdb << {:name => "John"}
+    @xdb.flush
+  end
+
   it "should store and fetch values like a hash" do
-    values = XapianDocValueAccessor.new(XapianDoc.new(nil))
+    values = @xdb.documents.find(1).values
     values.store(:city, "Leeds").should == "Leeds"
     values.fetch(:city).should == "Leeds"
     values[:city] = "London"
@@ -21,7 +27,7 @@ describe XapianDocValueAccessor do
   end
 
   it "should add and retrieve values from the Xapian::Document" do
-    doc = XapianDoc.new(nil)
+    doc = @xdb.documents.find(1)
     values = XapianDocValueAccessor.new(doc)
     lambda { values[:city] = "London" }.should change(doc.xapian_document, :values_count).by(1)
   end
@@ -95,12 +101,13 @@ describe XapianDocValueAccessor do
   end
 
   it "should count the stored values when size is called" do
-    doc = XapianDoc.new(nil)
+    doc = @xdb.documents.find(1)
     lambda { doc.values[:city] = "London" }.should change(doc.values, :size).by(1)
   end
 
   it "should delete values from the Xapian::Document" do
-    doc = XapianDoc.new(nil)
+    doc = @xdb.documents.find(1)
+    values = doc.values
     doc.values[:city] = "Leeds"
     lambda { doc.values.delete(:city) }.should change(doc.values, :size).by(-1)
     doc.values[:city] = "London"

metadata

@@ -1,21 +1,22 @@
 --- !ruby/object:Gem::Specification 
 name: xapian-fu
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 7
   prerelease: 
   segments: 
   - 1
-  - 3
-  - 2
-  version: 1.3.2
+  - 4
+  - 0
+  version: 1.4.0
 platform: ruby
 authors: 
 - John Leach
+- Damian Janowski
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-12-04 00:00:00 Z
+date: 2012-03-13 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -107,6 +108,7 @@ files:
 - spec/xapian_doc_spec.rb
 - spec/xapian_db_spec.rb
 - spec/stopper_factory_spec.rb
+- spec/facets_spec.rb
 - spec/fixtures/film_data/x86_64-linux~1.8.7/value.baseA
 - spec/fixtures/film_data/x86_64-linux~1.8.7/position.baseA
 - spec/fixtures/film_data/x86_64-linux~1.8.7/record.baseA
@@ -203,6 +205,7 @@ test_files:
 - spec/xapian_doc_spec.rb
 - spec/xapian_db_spec.rb
 - spec/stopper_factory_spec.rb
+- spec/facets_spec.rb
 - spec/fixtures/film_data/x86_64-linux~1.8.7/value.baseA
 - spec/fixtures/film_data/x86_64-linux~1.8.7/position.baseA
 - spec/fixtures/film_data/x86_64-linux~1.8.7/record.baseA