solrsam 0.4.2

data/lib/solrsam/search.rb

@@ -0,0 +1,195 @@
+module Solrsam
+    module Search
+        extend ActiveSupport::Concern
+        module ClassMethods
+
+            attr_accessor :solr_server
+
+            HL_START_TAG = "<mark>"
+            HL_END_TAG = "</mark>"
+
+            def class_name
+                to_s.underscore
+            end
+
+            def solr(method = :read)
+                @solr_server ||= :default
+                Solrsam::Config.instance.rsolr_object(@solr_server, method)
+            end
+
+            def perform_solr_command(method = :read)
+                yield(solr(method))
+                solr(method).commit
+            end
+
+            def search(search_params={})
+                start = search_params[:start] || 0
+                rows = search_params[:rows] || 20
+
+                solr_params = parse_params_for_solr(search_params)
+
+                begin
+                    solr_response = solr.select :data => solr_params, :method => :post
+                    parse_solr_response(solr_response)
+                rescue RSolr::Error::Http => e
+                    {:docs => [], 
+                        :metadata => 
+                    {:error => {:http_status_code => e.response[:status], 
+                        :http_message_status => RSolr::Error::Http::STATUS_CODES[e.response[:status].to_i],
+                        :full_message => e.message}}}
+                end
+            end
+
+            def parse_params_for_solr(search_params={})
+                solr_params = { :echoParams => 'explicit',
+                    :q => "*:*",
+                    :facet => "on",
+                    :'facet.mincount' => 1}.merge(search_params)
+
+                solr_params[:hl] = true unless search_params[:'hl.fl'].blank?
+                solr_params[:fq] =  parse_fq(search_params[:fq]) + ( search_params[:untyped] ? [] : ["type:#{class_name}"])
+                solr_params
+            end
+
+            def parse_solr_response(solr_response)
+                docs = solr_response['response']['docs']
+                parsed_facet_counts = parse_facet_counts(solr_response['facet_counts'])
+                highlighting = solr_response['highlighting']
+                suggestions = solr_response['spellcheck']['suggestions'] rescue {}
+
+                metadata = {
+                    :total_count => solr_response['response']['numFound'],
+                    :start => solr_response['response']['start'],
+                    :rows => solr_response['responseHeader']['params']['rows'],
+                    :time => solr_response['responseHeader']['QTime'],
+                    :status => solr_response['responseHeader']['status'],
+                    :sort => solr_response['responseHeader']['params']['sort'],
+                    :debug => solr_response['debug']
+                }
+                response = {:docs => docs, :metadata =>  metadata,
+                    :facet_counts => parsed_facet_counts, :highlighting => highlighting}
+                response[:stats] = solr_response['stats'] if solr_response['stats']
+                response[:suggestions] = suggestions
+
+                embed_highlighting(response)
+            end
+
+            def parse_fq(fq)
+                return [] if fq.nil?
+                if fq.is_a?(Hash)
+                    fq.map{|k,v| parse_element_in_fq({k => v})}.flatten
+                elsif fq.is_a?(Array)
+                    fq.map{|ele| parse_element_in_fq(ele)}.flatten
+                else
+                    raise "fq must be a hash or array"
+                end
+            end
+
+            def parse_element_in_fq(element)
+                if element.is_a?(String)
+                    element
+                elsif element.is_a?(Hash)
+                    element.map do |k,values|
+                        if values.is_a?(String)
+                            key_value_query(k,values)
+                        else
+                            values.map{|value| key_value_query(k,value) }
+                        end
+                    end
+                else
+                    raise "each fq parameter must be a string or hash"
+                end
+            end
+
+            def key_value_query(key, value)
+                if value.starts_with?("[")
+                    "#{key}:#{value}"
+                elsif value.starts_with?("(")
+                    # we don't want to escape double quotes
+                    "#{key}:" + value
+                else
+                    "#{key}:\"#{value}\""
+                end
+            end
+
+            def parse_facet_counts(facet_counts)
+                return {} unless facet_counts
+
+                if facet_counts['facet_fields']
+                    facet_counts['facet_fields'] = facet_counts['facet_fields'].reduce({}) do |acc, facet_collection|
+                        acc[facet_collection[0]] = map_facet_array_to_facet_hash(facet_collection[1])
+                        acc
+                    end
+                end
+
+                if facet_counts['facet_queries']
+                    facet_counts['facet_queries'] = facet_counts['facet_queries'].group_by{|k,v| k.split(":").first}.reduce({}) do |acc, facet_collection|
+                        facet_name = facet_collection[0]
+                        values = facet_collection[1]
+
+                        acc[facet_name] = values.reduce({}) do |inner_acc, tuple|
+                            range = tuple[0].split(":")[1]
+                            inner_acc[range] = tuple[1]
+                            inner_acc
+                        end
+                        acc
+                    end
+                end
+
+                facet_counts
+            end
+
+            # solr facet_fields comes in tuple array format []
+            def map_facet_array_to_facet_hash(facet_collection)
+                if facet_collection.is_a?(Array)
+                    facet_collection.each_slice(2).reduce({}){|acc, tuple| acc[tuple[0]] = tuple[1]; acc}
+                else
+                    facet_collection
+                end
+            end
+
+            def embed_highlighting(search_response)
+                return search_response unless search_response[:highlighting]
+
+                excluded_highlighting_fields = ['id', 'db_id', 'type']
+
+                highlighted_docs = search_response[:docs].map do |doc|
+                    hl_metadata = search_response[:highlighting][doc['id']]
+
+                    hl_metadata.drop_while{|k,v| excluded_highlighting_fields.include?(k) }.each do |k,v|
+                        new_value = if doc[k].is_a?(Array)
+                                        matched = v.map{|t| t.gsub(HL_START_TAG,"").gsub(HL_END_TAG,"") }
+                                        v.concat(doc[k].drop_while{|text| matched.include?(text) })
+                                    else
+                                        v
+                                    end
+                        doc[k] = new_value
+                    end if hl_metadata
+
+                    doc
+                end
+
+                search_response[:docs] = highlighted_docs
+
+                search_response[:docs].extend RSolr::Pagination::PaginatedDocSet
+
+                search_response
+            end
+
+        end
+    end
+end
+
+# namespace test documents
+if defined?(Rails) && Rails.env == "test"
+    module Solrsam
+        module Search
+            extend ActiveSupport::Concern
+            module ClassMethods
+                def class_name
+                    "#{to_s.underscore}_test"
+                end
+            end
+        end
+    end
+end

data/lib/solrsam/version.rb

@@ -0,0 +1,3 @@
+module Solrsam
+  VERSION = "0.4.2"
+end

data/lib/solrsam.rb

@@ -0,0 +1,12 @@
+require 'rsolr'
+require 'active_support/json'
+require 'active_support/concern'
+require 'active_support/core_ext'
+
+require 'singleton'
+require 'solrsam/config'
+require 'solrsam/search'
+require 'solrsam/indexer'
+
+module Solrsam
+end

data/lib/tasks/solr.rake

@@ -0,0 +1,71 @@
+require 'uri'
+require 'yaml'
+require 'open-uri'
+
+namespace :solr do
+  env =  "development"
+  base_dir = File.join(File.dirname(__FILE__), "..", "..")
+
+  if defined? Rails
+    env = Rails.env
+    base_dir = Rails.root
+  end
+
+  solr_config = YAML::load( File.open( File.join(base_dir, "config", "solr.yml") ) )
+  solr_home = File.join(base_dir, "config", "solr")
+  solr_data_dir = solr_config[env]['solr_data_dir']
+  solr_server_url = solr_config[env]['solr_server_url']
+  java_opts = solr_config[env]['java_opts'] || ""
+
+  jetty_port = URI.parse(solr_server_url).port
+  jetty_path = solr_config[env]['jetty_path']
+
+  solr_server_dir = "cd #{jetty_path};"
+  start_solr_cmd = "java #{java_opts} -jar start.jar"
+  logging_xml = "etc/jetty-logging.xml"
+  jetty_port_opt = "jetty.port=#{jetty_port}"
+  solr_params = "#{jetty_port_opt} -Dsolr.solr.home=#{solr_home} -Dsolr.data.dir=#{solr_data_dir}"
+
+  desc "Start solr"
+  task(:start) do
+    # -Dsolr.clustering.enabled=true
+    cmd = kill_matching_process_cmd(jetty_port_opt)
+    stop_exit_status = run_system_command(cmd)
+
+    sleep(1)
+
+    cmd = "#{solr_server_dir} #{start_solr_cmd} #{logging_xml} #{solr_params} &"
+    run_system_command(cmd)
+  end
+
+  desc "Stop solr"
+  task(:stop) do
+    cmd = kill_matching_process_cmd(jetty_port_opt)
+    run_system_command(cmd)
+  end
+
+  desc "update spelling dictionary"
+  task :build_dictionary do
+    open("#{solr_server_url}/select?q=facebok&qt=spell&spellcheck.build=true")
+  end
+
+  def run_system_command(cmd)
+    puts cmd
+    status = system(cmd)
+    $?.exitstatus
+  end
+
+  def kill_matching_process_cmd(process_name)
+      cmd = "echo `ps -ef | grep -v grep | grep \"#{process_name.gsub("-", "\\-")}\" | awk '{print $2}'` | xargs kill"
+  end
+
+#  #example of a task to index all items
+#  desc "index food"
+#  task(:import_foods, :needs => :environment) do
+#    FoodDescription.find_in_batches do |batch| 
+#      FoodIndexer.index(batch)
+#      puts "Done with batch of size: #{batch.size}"
+#    end
+#  end
+end
+

data/solrsam.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "solrsam/version"
+
+Gem::Specification.new do |s|
+  s.name        = "solrsam"
+  s.version     = Solrsam::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Tommy Chheng", "Sam Dalton"]
+  s.email       = ["tommy.chheng@gmail.com", "sam@quid.com"]
+  s.homepage    = "http://github.com/samdalton/solrsan"
+  s.summary = %q{Lightweight wrapper for using Apache Solr within a Ruby application including Rails and Sinatra.}
+  s.description = %q{solrsan is a lightweight wrapper for using Apache Solr within a Ruby application including Rails and Sinatra.}
+
+  s.rubyforge_project = "solrsam"
+
+  s.add_dependency 'activesupport'
+  s.add_dependency 'activemodel'
+  s.add_dependency 'rsolr', '1.0.0'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/test/models/document.rb

@@ -0,0 +1,11 @@
+require 'active_model/attribute_methods'
+
+class Document
+  include ActiveModel::AttributeMethods
+  include Solrsam::Search
+  attr_accessor :attributes
+
+  def initialize(attributes={})
+    @attributes = attributes
+  end
+end

data/test/search_test_helper.rb

@@ -0,0 +1,13 @@
+module SearchTestHelper
+  def document_mock
+    doc = Document.new({:id => 500,
+    :title => "Solr test document",
+    :author => "Tommy Chheng",
+    :content => "ruby scala search",
+    :review_count => 4,
+    :scores => [1,2,3,4],
+    :created_at => Date.parse("Dec 10 2010")})
+    doc
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,27 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+MODELS = File.join(File.dirname(__FILE__), "models")
+$LOAD_PATH.unshift(MODELS)
+
+#simulate rails env
+class Rails
+  def self.env
+    "test"
+  end
+end
+
+require "rubygems"
+require "test/unit"
+require 'solrsam'
+
+#test models
+require 'document'
+
+solr_config = YAML::load( File.open( File.join(File.dirname(__FILE__), "..", "config", "solr.yml") ) )
+solr_server_url = solr_config["test"]['solr_server_url']
+
+Solrsam::Config.instance.add_server(:example, 'example.com', :read)
+Solrsam::Config.instance.add_server(:example, 'example.com', :write)
+Solrsam::Config.instance.add_server(:default, solr_server_url, :read)
+Solrsam::Config.instance.add_server(:default, solr_server_url, :write)

data/test/unit/indexer_test.rb

@@ -0,0 +1,25 @@
+require 'test_helper'
+require 'search_test_helper'
+
+class IndexerTest < Test::Unit::TestCase
+  include SearchTestHelper
+
+  def setup
+    @document = document_mock
+  end
+
+  def teardown
+  end
+
+  def test_indexed_fields
+    created_doc = @document.indexed_fields
+    assert_equal @document.attributes[:id], created_doc[:db_id]
+    assert_equal @document.attributes[:title], created_doc[:title]
+    assert_equal @document.attributes[:author], created_doc[:author]
+    assert_equal @document.attributes[:content], created_doc[:content]
+    assert_equal @document.attributes[:review_count], created_doc[:review_count]
+    assert_equal @document.attributes[:scores], created_doc[:scores]
+    assert_equal @document.attributes[:created_at].to_time.utc.xmlschema, created_doc[:created_at]
+  end
+
+end

data/test/unit/search_test.rb

@@ -0,0 +1,251 @@
+require 'test_helper'
+require 'search_test_helper'
+
+class SearchTest < Test::Unit::TestCase
+    include SearchTestHelper
+
+    def setup
+        Document.destroy_all_index_documents!
+        @document = document_mock
+    end
+
+    def teardown
+        Document.destroy_all_index_documents!
+    end
+
+    def test_simple_query
+        Document.index(@document)
+        q = @document.attributes[:title]
+
+        response = Document.search(:q => q)
+        metadata = response[:metadata]
+        docs = response[:docs]
+        assert metadata[:total_count] > 0
+    end
+
+    def test_sort
+        Document.index(Document.new(:id => 3, :title => "solar city",:review_count => 10))
+        Document.index(Document.new(:id => 4, :title => "city solar", :review_count => 5))
+
+        q = "solar"
+        response = Document.search(:q => q, :sort => "review_count asc")
+        docs = response[:docs]
+
+        assert_not_nil docs[0], "Not enough docs for #{q} to test."
+        first_result_name = docs[0][:title]
+        assert_not_nil docs[1], "Not enough docs for #{q} to test."
+        second_result_name = docs[1][:title]
+
+        assert_equal [second_result_name, first_result_name].sort, [first_result_name, second_result_name]
+
+        response = Document.search(:q => q, :sort => "review_count desc")
+        docs = response[:docs]
+
+        assert_not_nil docs[0], "Not enough docs for #{q} to test."
+        first_result_name = docs[0][:title]
+        assert_not_nil docs[1], "Not enough docs for #{q} to test."
+        second_result_name = docs[1][:title]
+
+        assert_equal [second_result_name, first_result_name].sort.reverse, [first_result_name, second_result_name]
+    end
+
+    def test_invalid_query_should_return_error_message_in_metadata
+        response = Document.search(:q => "http://tommy.chheng.com")
+        docs = response[:docs]
+        metadata = response[:metadata]
+
+        assert_not_nil response[:metadata][:error]
+        assert_equal 400, response[:metadata][:error][:http_status_code]
+    end
+
+    def test_parse_facet_fields
+        facet_counts = {'facet_queries'=>{},
+            'facet_fields' => {'language' => ["Scala", 2, "Ruby", 1, "Java", 0]},
+            'facet_dates'=>{}}
+
+        facet_counts = Document.parse_facet_counts(facet_counts)
+
+        assert_equal ["Scala", "Ruby", "Java"], facet_counts['facet_fields']['language'].keys
+    end
+
+    def test_parse_facet_queries
+        facet_counts = {"facet_queries"=>{"funding:[0 TO 5000000]"=>1, "funding:[10000000 TO 50000000]"=>0}, "facet_fields"=>{}, "facet_dates"=>{}}
+
+        facet_counts = Document.parse_facet_counts(facet_counts)
+
+        expected = {"funding"=>{"[0 TO 5000000]"=>1, "[10000000 TO 50000000]"=>0}}
+        assert_equal expected, facet_counts['facet_queries']
+    end
+
+    def test_parse_fq_with_hash
+        params = {:fq => {:tags => ["ruby", "scala"]}}
+        filters = Document.parse_fq(params[:fq])
+
+        expected = ["tags:\"ruby\"", "tags:\"scala\""]
+        assert_equal expected, filters
+    end
+
+    def test_parse_fq_with_hash_multiple_entries
+        params = {:fq => {:tags => ["ruby", "scala"], :author => "Bert"}}
+        filters = Document.parse_fq(params[:fq])
+
+        expected = ["tags:\"ruby\"", "tags:\"scala\"", "author:\"Bert\""]
+        assert_equal expected, filters
+    end
+
+    def test_parse_fq_with_hash_array_args
+        params = {:fq => [{:tags => ["ruby", "scala"]}]}
+        filters = Document.parse_fq(params[:fq])
+
+        expected = ["tags:\"ruby\"", "tags:\"scala\""]
+        assert_equal expected, filters
+    end
+
+    def test_parse_fq_with_hash_string_args
+        params = {:fq => [{:tags => "ruby"}]}
+        filters = Document.parse_fq(params[:fq])
+
+        expected = ["tags:\"ruby\""]
+        assert_equal expected, filters
+    end
+
+    def test_parse_fq_with_string_args
+        params = {:fq => ["tags:ruby"]}
+        filters = Document.parse_fq(params[:fq])
+
+        expected = ["tags:ruby"]
+        assert_equal expected, filters
+    end
+
+    def test_parse_fq_with_empty
+        filters = Document.parse_fq([])
+        expected = []
+        assert_equal expected, filters
+    end
+
+
+    def test_filter_query_mulitple_filters
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10, :tags => ['ruby']))
+        Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5, :tags => ['ruby', 'scala']))
+
+        response = Document.search(:q => "solr", :fq => {:tags => ["scala"], :author => "Ernie"})
+        docs = response[:docs]
+        metadata = response[:metadata]
+
+        assert_equal 1, metadata[:total_count]
+
+        doc = docs.first
+        assert_not_nil doc['tags']
+        assert doc['tags'].include?("scala")
+    end
+
+    def test_filter_query
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10, :tags => ['ruby']))
+        Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5, :tags => ['ruby', 'scala']))
+
+        response = Document.search(:q => "solr", :fq => [{:tags => ["scala"]}])
+        docs = response[:docs]
+        metadata = response[:metadata]
+
+        assert_equal 1, metadata[:total_count]
+
+        doc = docs.first
+        assert_not_nil doc['tags']
+        assert doc['tags'].include?("scala")
+    end
+
+    def test_text_faceting
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10))
+        Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5))
+
+        response = Document.search(:q => "solr", :'facet.field' => ['author'])
+        docs = response[:docs]
+        facet_counts = response[:facet_counts]
+        assert_not_nil facet_counts["facet_fields"]["author"]
+
+        author_facet_entries = facet_counts["facet_fields"]["author"]
+        assert author_facet_entries.keys.include?("Bert") && author_facet_entries.keys.include?("Ernie")
+    end
+
+    def test_range_faceting
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10))
+        Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5))
+
+        response = Document.search(:q => "solr", :'facet.field' => ['author'], :'facet.query' => ["review_count:[1 TO 5]", "review_count:[6 TO 10]"])
+        docs = response[:docs]
+        facet_counts = response[:facet_counts]
+
+        assert_not_nil facet_counts["facet_fields"]["author"]
+        assert_not_nil facet_counts["facet_queries"]["review_count"]
+        assert_equal({"[1 TO 5]"=>1, "[6 TO 10]"=>1}, facet_counts["facet_queries"]["review_count"])
+    end
+
+    def test_highlighting_support
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10, :tags => ["solr"]))
+
+        response = Document.search(:q => "solr", 
+                                   :'hl.fl' => "*")
+        docs = response[:docs]
+        highlighting = response[:highlighting]
+
+        first_result = highlighting.first
+        assert first_result[1]['tags'].include?("<mark>solr</mark>")
+    end
+
+    def test_embed_highlighting
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10, :tags => ["solr", "sphinx"]))
+
+        response = Document.search(:q => "solr", 
+                                   :'hl.fl' => "*")
+        docs = response[:docs]
+        assert_equal ["<mark>solr</mark>","sphinx"], docs.first['tags']
+    end
+
+    def test_debug_response
+        Document.index(@document)
+        q = @document.attributes[:title]
+
+        response = Document.search({:q => q, :debugQuery => true})
+        metadata = response[:metadata]
+        assert_not_nil metadata[:debug]
+    end
+
+    def test_stats_response
+        Document.index(@document)
+        q = @document.attributes[:title]
+
+        response = Document.search({:q => q, :stats => true, :'stats.field' => 'review_count'})
+        assert_not_nil response[:stats]
+        assert_equal 1, response[:stats]['stats_fields']['review_count']['count']
+    end
+
+    def test_will_paginate_support
+        Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10))
+        Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5))
+
+        response = Document.search(:q => "solr")
+        docs = response[:docs]
+        assert_not_nil docs.start
+        assert_not_nil docs.per_page
+        assert_not_nil docs.current_page
+    end
+
+    def test_solr_server_attribute_writable
+        begin
+            Document.solr_server = :example
+            assert_equal :example, Document.solr_server        
+        ensure
+            Document.solr_server = nil
+        end
+    end
+
+    def test_rsolr_is_configured_with_the_set_solr_server
+        begin
+            Document.solr_server = :example
+            assert_equal "example.com/", Document.solr.options[:url]
+        ensure
+            Document.solr_server = nil
+        end
+    end
+end
+