xapit 0.2.7 → 0.3.0

data/lib/xapit/server/query.rb

@@ -0,0 +1,240 @@
+module Xapit
+  module Server
+    class Query
+      def initialize(clauses)
+        @clauses = clauses
+        @xapian_query = nil
+      end
+
+      def matches
+        enquire = Xapian::Enquire.new(Xapit.database.xapian_database)
+        enquire.query = xapian_query
+        enquire.set_sort_by_key_then_relevance(sorter, false) if sorter
+        enquire.mset((page.to_i-1)*per_page.to_i, per_page.to_i).matches
+      end
+
+      def records
+        matches.map do |match|
+          class_name, id = match.document.data.split('-')
+          {:class => class_name, :id => id, :relevance => match.percent}
+        end
+      end
+
+      def spelling_suggestion
+        text = @clauses.map { |clause| clause[:search] }.compact.join(" ")
+        if [text, *text.scan(/\w+/)].all? { |term| term_suggestion(term).nil? }
+          nil
+        else
+          return term_suggestion(text) unless term_suggestion(text).to_s.empty?
+          text.downcase.gsub(/\w+/) do |term|
+            term_suggestion(term) || term
+          end
+        end
+      end
+
+      def facets
+        facets = {}
+        enquire = Xapian::Enquire.new(Xapit.database.xapian_database)
+        enquire.query = xapian_query
+        spies = facet_spies
+        spies.values.each do |spy|
+          enquire.add_matchspy(spy)
+        end
+        enquire.mset(0, 200)
+        spies.each do |attribute, spy|
+          values = {}
+          spy.values.map do |spy_value|
+            spy_value.term.split("\3").each do |term| # used to support multiple facet values
+              values[term] ||= 0
+              values[term] += spy_value.termfreq.to_i
+            end
+          end
+          facets[attribute] = values.map { |value, count| {:value => value, :count => count} }
+        end
+        facets
+      end
+
+      def applied_facet_options
+        facet_options = []
+        @clauses.each do |clause|
+          if clause[:with_facets]
+            clause[:with_facets].each do |identifier|
+              facet_options << facet_option(identifier)
+            end
+          end
+        end
+        facet_options
+      end
+
+      def facet_option(identifier)
+        match = self.class.new([{:in_classes => ["FacetOption"]}, {:where => {:id => identifier}}]).matches.first
+        if match.nil?
+          raise "Unable to find facet option for #{identifier}."
+        else
+          name, value = match.document.data.split('|||')
+          {:id => identifier, :name => name, :value => value}
+        end
+      end
+
+      def total
+        enquire = Xapian::Enquire.new(Xapit.database.xapian_database)
+        enquire.query = xapian_query
+        enquire.mset(0, Xapit.database.xapian_database.doccount).matches_estimated
+      end
+
+      def data
+        {:records => records, :facets => facets, :applied_facet_options => applied_facet_options, :total => total}
+      end
+
+      private
+
+      def page
+        @clauses.map { |clause| clause[:page] }.compact.last || 1
+      end
+
+      def per_page
+        @clauses.map { |clause| clause[:per_page] }.compact.last || Xapit::Client::Collection::DEFAULT_PER_PAGE
+      end
+
+      def term_suggestion(term)
+        suggestion = Xapit.database.xapian_database.get_spelling_suggestion(term.downcase)
+        suggestion.to_s.empty? ? nil : suggestion
+      end
+
+      def sorter
+        if @clauses.any? { |c| c[:order] }
+          sorter = Xapian::MultiValueKeyMaker.new
+          @clauses.each do |clause|
+            if clause[:order]
+              attribute, direction = clause[:order]
+              sorter.add_value(Xapit.value_index(:sortable, attribute), direction.to_sym == :desc)
+            end
+          end
+          sorter
+        end
+      end
+
+      def facet_spies
+        spies = {}
+        @clauses.each do |clause|
+          if clause[:include_facets]
+            clause[:include_facets].each do |facet|
+              spies[facet] = Xapian::ValueCountMatchSpy.new(Xapit.value_index(:facet, facet))
+            end
+          end
+        end
+        spies
+      end
+
+      def xapian_query
+        build_xapian_query if @query.nil?
+        @xapian_query
+      end
+
+      def build_xapian_query
+        @xapian_query = Xapian::Query.new("")
+        @clauses.each do |clause|
+          clause.each do |type, options|
+            apply_clause(type, options)
+          end
+        end
+      end
+
+      def apply_clause(type, value)
+        case type
+        when :search
+          merge(:and, search_query(value))
+        when :where
+          merge(:and, where_query(value))
+        when :or_where
+          merge(:or, where_query(value))
+        when :not_where
+          merge(:not, where_query(value))
+        when :in_classes
+          merge(:and, value.map { |c| "C#{c}" })
+        when :not_in_classes
+          merge(:not, value.map { |c| "C#{c}" })
+        when :similar_to
+          similar_to(value)
+        when :with_facets
+          merge(:and, facet_terms(value))
+        end
+      end
+
+      def similar_to(data)
+        indexer = Indexer.new(data)
+        terms = (indexer.text_terms + indexer.field_terms).map { |a| a.first }
+        merge(:and, Xapian::Query.new(xapian_operator(:or), terms))
+        merge(:not, ["Q#{data[:class]}-#{data[:id]}"])
+      end
+
+      def where_query(conditions)
+        queries = []
+        terms = []
+        conditions.each do |name, value|
+          if value.kind_of?(Hash) && value[:from] && value[:to]
+            queries << Xapian::Query.new(xapian_operator(:range), Xapit.value_index(:field, name), Xapit.serialize_value(value[:from]), Xapit.serialize_value(value[:to]))
+          else
+            terms << "X#{name}-#{value.to_s.downcase}"
+          end
+        end
+        queries << Xapian::Query.new(xapian_operator(:and), terms) unless terms.empty?
+        queries.inject(queries.shift) do |merged_query, query|
+          Xapian::Query.new(xapian_operator(:and), merged_query, query)
+        end
+      end
+
+      def where_terms(conditions)
+        conditions.map do |name, value|
+          "X#{name}-#{value.to_s.downcase}"
+        end
+      end
+
+      def facet_terms(facets)
+        facets.map { |facet| "F#{facet}" }
+      end
+
+      def search_query(text)
+        clean_text = text.gsub(/\b([a-z])\*/i, "\\1").gsub(/[^\w\*\s:]/u, "")
+        xapian_parser.parse_query(clean_text, Xapian::QueryParser::FLAG_WILDCARD | Xapian::QueryParser::FLAG_BOOLEAN) # Xapian::QueryParser::FLAG_LOVEHATE
+      end
+
+      def merge(operator, query)
+        query = Xapian::Query.new(xapian_operator(:and), query) unless query.kind_of? Xapian::Query
+        @xapian_query = Xapian::Query.new(xapian_operator(operator), @xapian_query, query)
+      end
+
+      def xapian_operator(operator)
+        case operator
+        when :and then Xapian::Query::OP_AND
+        when :or then Xapian::Query::OP_OR
+        when :not then Xapian::Query::OP_AND_NOT
+        when :range then Xapian::Query::OP_VALUE_RANGE
+        else raise "Unknown Xapian operator #{operator}"
+        end
+      end
+
+      def xapian_parser
+        @xapian_parser ||= build_xapian_parser
+      end
+
+      def build_xapian_parser
+        parser = Xapian::QueryParser.new
+        parser.database = Xapit.database.xapian_database
+        if Xapit.config[:stemming]
+          parser.stemmer = Xapian::Stem.new(Xapit.config[:stemming])
+          parser.stemming_strategy = Xapian::QueryParser::STEM_SOME
+        end
+        parser.default_op = xapian_operator(:and)
+        @clauses.each do |clause|
+          if clause[:search]
+            clause[:search].scan(/([a-z0-9_]+)\:/i).each do
+              parser.add_prefix($1, "X#{$1}-")
+            end
+          end
+        end
+        parser
+      end
+    end
+  end
+end

data/spec/fixtures/blankdb/iamchert

@@ -0,0 +1 @@
+IAmChert�����pn�N3��gf�]

data/spec/fixtures/xapit.ru

@@ -0,0 +1,13 @@
+require "rubygems"
+
+# Add lib directory so we can include xapit, this isn't necessary when a gem is available
+root = File.expand_path('../../..', __FILE__)
+$:.unshift "#{root}/lib"
+require "xapit"
+
+
+FileUtils.rm_rf("#{root}/tmp/testdb") # quick hack to start with a fresh database every time
+Xapit.config[:database_path] = "#{root}/tmp/testdb"
+Xapit.config[:template_path] = "#{root}/spec/fixtures/blankdb"
+
+run Xapit::Server::App.new

data/spec/fixtures/xapit.yml

@@ -0,0 +1,4 @@
+development:
+  database_path: development_database
+production:
+  database_path: production_database

data/spec/spec_helper.rb

@@ -1,15 +1,14 @@
 require 'rubygems'
-require 'spec'
-require 'active_support'
-require 'fileutils'
-require File.dirname(__FILE__) + '/../lib/xapit'
-require File.dirname(__FILE__) + '/xapit_member'
+require 'bundler/setup'
+Bundler.require(:default)
 
-Spec::Runner.configure do |config|
-  config.mock_with :rr
+require "support/spec_macros"
+require "support/xapit_member"
+
+RSpec.configure do |config|
+  config.include SpecMacros
   config.before(:each) do
-    Xapit.setup(:database_path => File.dirname(__FILE__) + '/tmp/xapiandb')
-    Xapit.remove_database
+    Xapit.reset_config
     XapitMember.delete_all
   end
 end

data/spec/support/spec_macros.rb

@@ -0,0 +1,6 @@
+module SpecMacros
+  def load_xapit_database
+    Xapit.reset_config
+    Xapit.config[:spelling] = false
+  end
+end

data/spec/{xapit_member.rb → support/xapit_member.rb}

@@ -1,22 +1,17 @@
 class XapitMember
-  include Xapit::Membership
-  
+  include Xapit::Client::Membership
+  xapit { } # loads Xapit member methods
+
   attr_reader :id
-  
-  # Make it look like this inherits from ActiveRecord::Base
-  # so it will use the ActiveRecord adapter.
-  def self.ancestors
-    ["ActiveRecord::Base"]
-  end
-  
+
   def self.find_each(&block)
     @@records.each(&block) if @@records
   end
-  
+
   def self.delete_all
     @@records = []
   end
-  
+
   def self.find(ids)
     if ids.kind_of? Array
       # change the order to mimic database where we can't predict the order
@@ -25,18 +20,21 @@ class XapitMember
       @@records.detect { |r| r.id == ids.to_i }
     end
   end
-  
+
   def self.find_by_id(id)
     find(id)
   end
-  
+
   def initialize(attributes = {})
     @@records ||= []
     @id = @@records.size + 1
-    @attributes = attributes
+    @attributes = {}
+    attributes.each do |key, value|
+      @attributes[key.to_sym] = value
+    end
     @@records << self
   end
-  
+
   def method_missing(name, *args)
     if @attributes.has_key? name
       @attributes[name]
@@ -44,7 +42,7 @@ class XapitMember
       super
     end
   end
-  
+
   def update_attribute(name, value)
     @attributes[name] = value
   end

data/spec/xapit/client/collection_spec.rb

@@ -0,0 +1,63 @@
+require "spec_helper"
+
+describe Xapit::Client::Collection do
+  it "builds up clauses with in_classes, search, where, order calls" do
+    collection1 = Xapit::Client::Collection.new([:initial])
+    collection2 = collection1.in_classes(String).search("hello").where(:foo => "bar").order(:bar)
+    collection1.clauses.should eq([:initial])
+    collection2.clauses.should eq([:initial, {:in_classes => [String]}, {:search => "hello"}, {:where => {:foo => "bar"}}, {:order => [:bar, :asc]}])
+  end
+
+  it "returns same collection when searching nil or empty string" do
+    collection1 = Xapit::Client::Collection.new
+    collection1.search("").should eq(collection1)
+    collection1.search(nil).should eq(collection1)
+    collection1.search.should eq(collection1)
+  end
+
+  it "returns indexed records and delegates array methods to it" do
+    load_xapit_database
+    member = XapitMember.new
+    member.class.xapit_index_builder.add_document(member)
+    collection = Xapit::Client::Collection.new
+    collection.records.should eq([member])
+    collection.should respond_to(:flatten)
+    collection.flatten.should eq([member])
+  end
+
+  it "splits up matching facets into an array" do
+    collection = Xapit::Client::Collection.new.with_facets("foo-bar")
+    collection.clauses.should eq([{:with_facets => %w[foo bar]}])
+  end
+
+  it "splits range into from/to hash" do
+    collection = Xapit::Client::Collection.new.where(:priority => 3..5)
+    collection.clauses.should eq([{:where => {:priority => {:from => 3, :to => 5}}}])
+  end
+
+  it "does not raise an exception when passing nil to with_facets" do
+    lambda {
+      Xapit::Client::Collection.new.with_facets(nil).should be_kind_of(Xapit::Client::Collection)
+    }.should_not raise_exception
+  end
+
+  it "defaults to 20 per page and page 1" do
+    Xapit::Client::Collection.new.limit_value.should eq(20)
+    Xapit::Client::Collection.new.current_page.should eq(1)
+  end
+
+  it "supports kaminari pagination" do
+    collection = Xapit::Client::Collection.new.page("2").per("10")
+    collection.stub(:total_entries) { 29 }
+    collection.current_page.should eq(2)
+    collection.num_pages.should eq(3)
+    collection.limit_value.should eq(10)
+  end
+
+  it "supports will_paginate pagination" do
+    collection = Xapit::Client::Collection.new.page("2").per("10")
+    collection.stub(:total_entries) { 29 }
+    collection.current_page.should eq(2)
+    collection.total_pages.should eq(3)
+  end
+end

data/spec/xapit/client/facet_option_spec.rb

@@ -0,0 +1,26 @@
+require "spec_helper"
+
+describe Xapit::Client::FacetOption do
+  it "has an identifier using attribute and value" do
+    option = Xapit::Client::FacetOption.new("greeting", :value => "Hello")
+    option.identifier.should eq(Xapit.facet_identifier("greeting", "Hello"))
+  end
+
+  it "has a name and count matching passed options" do
+    option = Xapit::Client::FacetOption.new("greeting", :value => "Hello", :count => "3")
+    option.name.should eq("Hello")
+    option.count.should eq(3)
+  end
+
+  it "combines previous identifiers with current one on to_param" do
+    id = Xapit.facet_identifier("greeting", "Hello")
+    option = Xapit::Client::FacetOption.new("greeting", {:value => "Hello"}, %w[abc 123])
+    option.to_param.should == "abc-123-#{id}"
+  end
+
+  it "removes current identifier from previous identifiers if it exists" do
+    id = Xapit.facet_identifier("greeting", "Hello")
+    option = Xapit::Client::FacetOption.new("greeting", {:value => "Hello"}, %w[abc 123] + [id])
+    option.to_param.should == "abc-123"
+  end
+end