solrsan 0.0.20

data/lib/solrsan/search.rb

@@ -0,0 +1,149 @@
+module Solrsan
+  module Search
+    extend ActiveSupport::Concern
+    module ClassMethods
+      def class_name
+        to_s.underscore
+      end
+        
+      def perform_solr_command
+        @rsolr = Solrsan::Config.instance.rsolr_object
+        yield(@rsolr)
+        @rsolr.commit
+      end
+
+      def search(search_params={})
+        @rsolr ||= Solrsan::Config.instance.rsolr_object
+
+        start = search_params[:start] || 0
+        rows = search_params[:rows] || 20
+
+        solr_params = parse_params_for_solr(search_params)
+
+        begin
+          solr_response = @rsolr.paginate(start, rows, 'select', :params => solr_params)
+          parse_solr_response(solr_response)
+        rescue RSolr::Error::Http => e
+          {:docs => [], 
+           :metadata => 
+            {:error => {:http_status_code => e.response[:status], 
+                        :http_message_status => RSolr::Error::Http::STATUS_CODES[e.response[:status].to_i],
+                        :full_message => e.message}}}
+        end
+      end
+
+      def parse_params_for_solr(search_params={})
+        solr_params = { :echoParams => 'explicit',
+          :q => "*:*",
+          :facet => "on",
+          :'facet.mincount' => 1}.merge(search_params)
+        solr_params[:hl] = true unless search_params[:'hl.fl'].blank?
+        solr_params[:fq] = ["type:#{class_name}"] + parse_fq(search_params[:fq])
+        solr_params
+      end
+
+      def parse_solr_response(solr_response)
+        docs = solr_response['response']['docs']
+        parsed_facet_counts = parse_facet_counts(solr_response['facet_counts'])
+        highlighting = solr_response['highlighting']
+
+        metadata = {
+          :total_count => solr_response['response']['numFound'],
+          :start => solr_response['response']['start'],
+          :rows => solr_response['responseHeader']['params']['rows'],
+          :time => solr_response['responseHeader']['QTime'],
+          :status => solr_response['responseHeader']['status']
+        }
+
+        {:docs => docs, :metadata =>  metadata,
+         :facet_counts => parsed_facet_counts, :highlighting => highlighting}
+      end
+
+      def parse_fq(fq)
+        return [] if fq.nil?
+        if fq.is_a?(Hash)
+          fq.map{|k,v| parse_element_in_fq({k => v})}.flatten
+        elsif fq.is_a?(Array)
+          fq.map{|ele| parse_element_in_fq(ele)}.flatten
+        else
+          raise "fq must be a hash or array"
+        end
+      end
+
+      def parse_element_in_fq(element)
+        if element.is_a?(String)
+          element
+        elsif element.is_a?(Hash)
+          element.map do |k,values|
+            if values.is_a?(String)
+              key_value_query(k,values)
+            else
+              values.map{|value| key_value_query(k,value) }
+            end
+          end
+        else
+          raise "each fq parameter must be a string or hash"
+        end
+      end
+
+      def key_value_query(key, value)
+        if value.starts_with?("[")
+          "#{key}:#{value}"
+        else
+          "#{key}:\"#{value}\""
+        end
+      end
+
+      def parse_facet_counts(facet_counts)
+        return {} unless facet_counts
+        
+        if facet_counts['facet_fields']
+          facet_counts['facet_fields'] = facet_counts['facet_fields'].reduce({}) do |acc, facet_collection|
+            acc[facet_collection[0]] = map_facet_array_to_facet_hash(facet_collection[1])
+            acc
+          end
+        end
+  
+        if facet_counts['facet_queries']
+          facet_counts['facet_queries'] = facet_counts['facet_queries'].group_by{|k,v| k.split(":").first}.reduce({}) do |acc, facet_collection|
+            facet_name = facet_collection[0]
+            values = facet_collection[1]
+
+            acc[facet_name] = values.reduce({}) do |inner_acc, tuple|
+              range = tuple[0].split(":")[1]
+              inner_acc[range] = tuple[1]
+              inner_acc
+            end
+            acc
+          end
+        end
+
+        facet_counts
+      end
+    
+      # solr facet_fields comes in tuple array format []
+      def map_facet_array_to_facet_hash(facet_collection)
+        if facet_collection.is_a?(Array)
+          facet_collection.each_slice(2).reduce({}){|acc, tuple| acc[tuple[0]] = tuple[1]; acc}
+        else
+          facet_collection
+        end
+      end
+
+    end
+  end
+end
+
+# namespace test documents
+if defined?(Rails) && Rails.env == "test"
+  module Solrsan
+    module Search
+      extend ActiveSupport::Concern
+      module ClassMethods
+        def class_name
+          "#{to_s.underscore}_test"
+        end
+      end
+    end
+  end
+end

data/lib/solrsan/version.rb

@@ -0,0 +1,3 @@
+module Solrsan
+  VERSION = "0.0.20"
+end

data/lib/solrsan.rb

@@ -0,0 +1,12 @@
+require 'rsolr'
+require 'active_support/json'
+require 'active_support/concern'
+require 'active_support/core_ext'
+
+require 'singleton'
+require 'solrsan/config'
+require 'solrsan/search'
+require 'solrsan/indexer'
+
+module Solrsan
+end

data/lib/tasks/solr.rake

@@ -0,0 +1,63 @@
+require 'uri'
+require 'yaml'
+
+namespace :solr do
+  env =  "development"
+  base_dir = File.join(File.dirname(__FILE__), "..", "..")
+
+  if defined? Rails
+    env = Rails.env
+    base_dir = Rails.root
+  end
+
+  solr_config = YAML::load( File.open( File.join(base_dir, "config", "solr.yml") ) )
+  solr_home = File.join(base_dir, "config", "solr")
+  solr_data_dir = solr_config[env]['solr_data_dir']
+  solr_server_url = solr_config[env]['solr_server_url']
+
+  jetty_port = URI.parse(solr_server_url).port
+  jetty_path = solr_config[env]['jetty_path']
+
+  solr_server_dir = "cd #{jetty_path};"
+  start_solr_cmd = "java -jar start.jar"
+  jetty_port_opt = "jetty.port=#{jetty_port}"
+  solr_params = "#{jetty_port_opt} -Dsolr.solr.home=#{solr_home} -Dsolr.data.dir=#{solr_data_dir}"
+
+  desc "Start solr"
+  task(:start) do
+    # -Dsolr.clustering.enabled=true
+    cmd = kill_matching_process_cmd(jetty_port_opt)
+    stop_exit_status = run_system_command(cmd)
+
+    sleep(1)
+
+    cmd = "#{solr_server_dir} #{start_solr_cmd } #{solr_params} &"
+    run_system_command(cmd)
+  end
+
+  desc "Stop solr"
+  task(:stop) do
+    cmd = kill_matching_process_cmd(jetty_port_opt)
+    run_system_command(cmd)
+  end
+
+  def run_system_command(cmd)
+    puts cmd
+    status = system(cmd)
+    $?.exitstatus
+  end
+
+  def kill_matching_process_cmd(process_name)
+      cmd = "echo `ps -ef | grep -v grep | grep \"#{process_name.gsub("-", "\\-")}\" | awk '{print $2}'` | xargs kill"
+  end
+
+#  #example of a task to index all items
+#  desc "index food"
+#  task(:import_foods, :needs => :environment) do
+#    FoodDescription.find_in_batches do |batch| 
+#      FoodIndexer.index(batch)
+#      puts "Done with batch of size: #{batch.size}"
+#    end
+#  end
+end
+

data/solrsan.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "solrsan/version"
+
+Gem::Specification.new do |s|
+  s.name        = "solrsan"
+  s.version     = Solrsan::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Tommy Chheng"]
+  s.email       = ["tommy.chheng@gmail.com"]
+  s.homepage    = "http://github.com/tc/solrsan"
+  s.summary = %q{Lightweight wrapper for using Apache Solr within a Ruby application including Rails and Sinatra.}
+  s.description = %q{solrsan is a lightweight wrapper for using Apache Solr within a Ruby application including Rails and Sinatra.}
+
+  s.add_dependency('rsolr', '~>1.0.0')
+  s.add_dependency('activemodel', '~>3.0.5')
+  s.add_dependency('activesupport', '~>3.0.5')
+  
+  s.rubyforge_project = "solrsan"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/test/models/document.rb

@@ -0,0 +1,11 @@
+require 'active_model/attribute_methods'
+
+class Document
+  include ActiveModel::AttributeMethods
+  include Solrsan::Search
+  attr_accessor :attributes
+
+  def initialize(attributes={})
+    @attributes = attributes
+  end
+end

data/test/search_test_helper.rb

@@ -0,0 +1,13 @@
+module SearchTestHelper
+  def document_mock
+    doc = Document.new({:id => 500,
+    :title => "Solr test document",
+    :author => "Tommy Chheng",
+    :content => "ruby scala search",
+    :review_count => 4,
+    :scores => [1,2,3,4],
+    :created_at => Date.parse("Dec 10 2010")})
+    doc
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,24 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+MODELS = File.join(File.dirname(__FILE__), "models")
+$LOAD_PATH.unshift(MODELS)
+
+#simulate rails env
+class Rails
+  def self.env
+    "test"
+  end
+end
+
+require "rubygems"
+require "test/unit"
+require 'solrsan'
+
+#test models
+require 'document'
+
+solr_config = YAML::load( File.open( File.join(File.dirname(__FILE__), "..", "config", "solr.yml") ) )
+solr_server_url = solr_config["test"]['solr_server_url']
+
+Solrsan::Config.instance.solr_server_url = solr_server_url

data/test/unit/indexer_test.rb

@@ -0,0 +1,25 @@
+require 'test_helper'
+require 'search_test_helper'
+
+class IndexerTest < Test::Unit::TestCase
+  include SearchTestHelper
+
+  def setup
+    @document = document_mock
+  end
+
+  def teardown
+  end
+
+  def test_indexed_fields
+    created_doc = @document.indexed_fields
+    assert_equal @document.attributes[:id], created_doc[:db_id]
+    assert_equal @document.attributes[:title], created_doc[:title]
+    assert_equal @document.attributes[:author], created_doc[:author]
+    assert_equal @document.attributes[:content], created_doc[:content]
+    assert_equal @document.attributes[:review_count], created_doc[:review_count]
+    assert_equal @document.attributes[:scores], created_doc[:scores]
+    assert_equal @document.attributes[:created_at].to_time.utc.xmlschema, created_doc[:created_at]
+  end
+
+end

data/test/unit/search_test.rb

@@ -0,0 +1,171 @@
+require 'test_helper'
+require 'search_test_helper'
+
+class SearchTest < Test::Unit::TestCase
+  include SearchTestHelper
+
+  def setup
+    Document.destroy_all_index_documents!
+    @document = document_mock
+  end
+  
+  def teardown
+    Document.destroy_all_index_documents!
+  end
+
+  def test_simple_query
+    Document.index(@document)
+    q = @document.attributes[:title]
+
+    response = Document.search(:q => q)
+    metadata = response[:metadata]
+    docs = response[:docs]
+    assert metadata[:total_count] > 0
+  end
+
+  def test_sort
+    Document.index(Document.new(:id => 3, :title => "solar city",:review_count => 10))
+    Document.index(Document.new(:id => 4, :title => "city solar", :review_count => 5))
+
+    q = "solar"
+    response = Document.search(:q => q, :sort => "review_count asc")
+    docs = response[:docs]
+
+    assert_not_nil docs[0], "Not enough docs for #{q} to test."
+    first_result_name = docs[0][:title]
+    assert_not_nil docs[1], "Not enough docs for #{q} to test."
+    second_result_name = docs[1][:title]
+
+    assert_equal [second_result_name, first_result_name].sort, [first_result_name, second_result_name]
+
+    response = Document.search(:q => q, :sort => "review_count desc")
+    docs = response[:docs]
+
+    assert_not_nil docs[0], "Not enough docs for #{q} to test."
+    first_result_name = docs[0][:title]
+    assert_not_nil docs[1], "Not enough docs for #{q} to test."
+    second_result_name = docs[1][:title]
+
+    assert_equal [second_result_name, first_result_name].sort.reverse, [first_result_name, second_result_name]
+  end
+  
+  def test_invalid_query_should_return_error_message_in_metadata
+    response = Document.search(:q => "http://tommy.chheng.com")
+    docs = response[:docs]
+    metadata = response[:metadata]
+
+    assert_not_nil response[:metadata][:error]
+    assert_equal 400, response[:metadata][:error][:http_status_code]
+  end
+
+  def test_parse_facet_fields
+    facet_counts = {'facet_queries'=>{},
+      'facet_fields' => {'language' => ["Scala", 2, "Ruby", 1, "Java", 0]},
+      'facet_dates'=>{}}
+    
+    facet_counts = Document.parse_facet_counts(facet_counts)
+    
+    assert_equal ["Scala", "Ruby", "Java"], facet_counts['facet_fields']['language'].keys
+  end
+
+  def test_parse_facet_queries
+    facet_counts = {"facet_queries"=>{"funding:[0 TO 5000000]"=>1, "funding:[10000000 TO 50000000]"=>0}, "facet_fields"=>{}, "facet_dates"=>{}}
+
+    facet_counts = Document.parse_facet_counts(facet_counts)
+
+    expected = {"funding"=>{"[0 TO 5000000]"=>1, "[10000000 TO 50000000]"=>0}}
+    assert_equal expected, facet_counts['facet_queries']
+  end
+  
+  def test_parse_fq_with_hash
+    params = {:fq => {:tags => ["ruby", "scala"]}}
+    filters = Document.parse_fq(params[:fq])
+
+    expected = ["tags:\"ruby\"", "tags:\"scala\""]
+    assert_equal expected, filters
+  end
+
+  def test_parse_fq_with_hash_array_args
+    params = {:fq => [{:tags => ["ruby", "scala"]}]}
+    filters = Document.parse_fq(params[:fq])
+
+    expected = ["tags:\"ruby\"", "tags:\"scala\""]
+    assert_equal expected, filters
+  end
+
+  def test_parse_fq_with_hash_string_args
+    params = {:fq => [{:tags => "ruby"}]}
+    filters = Document.parse_fq(params[:fq])
+
+    expected = ["tags:\"ruby\""]
+    assert_equal expected, filters
+  end
+
+  def test_parse_fq_with_string_args
+    params = {:fq => ["tags:ruby"]}
+    filters = Document.parse_fq(params[:fq])
+
+    expected = ["tags:ruby"]
+    assert_equal expected, filters
+  end
+
+  def test_parse_fq_with_empty
+    filters = Document.parse_fq([])
+    expected = []
+    assert_equal expected, filters
+  end
+
+  def test_filter_query
+    Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10, :tags => ['ruby']))
+    Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5, :tags => ['ruby', 'scala']))
+    
+    response = Document.search(:q => "solr", :fq => [{:tags => ["scala"]}])
+    docs = response[:docs]
+    metadata = response[:metadata]
+
+    assert_equal 1, metadata[:total_count]
+
+    doc = docs.first
+    assert_not_nil doc['tags']
+    assert doc['tags'].include?("scala")
+  end
+
+  def test_text_faceting
+    Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10))
+    Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5))
+    
+    response = Document.search(:q => "solr", :'facet.field' => ['author'])
+    docs = response[:docs]
+    facet_counts = response[:facet_counts]
+    assert_not_nil facet_counts["facet_fields"]["author"]
+
+    author_facet_entries = facet_counts["facet_fields"]["author"]
+    assert author_facet_entries.keys.include?("Bert") && author_facet_entries.keys.include?("Ernie")
+  end
+
+  def test_range_faceting
+    Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10))
+    Document.index(Document.new(:id => 4, :author => "Ernie", :title => "lucene solr", :review_count => 5))
+    
+    response = Document.search(:q => "solr", :'facet.field' => ['author'], :'facet.query' => ["review_count:[1 TO 5]", "review_count:[6 TO 10]"])
+    docs = response[:docs]
+    facet_counts = response[:facet_counts]
+
+    assert_not_nil facet_counts["facet_fields"]["author"]
+    assert_not_nil facet_counts["facet_queries"]["review_count"]
+    assert_equal({"[1 TO 5]"=>1, "[6 TO 10]"=>1}, facet_counts["facet_queries"]["review_count"])
+  end
+
+  def test_highlighting_support
+    Document.index(Document.new(:id => 3, :author => "Bert", :title => "solr lucene",:review_count => 10, :tags => ["solr"]))
+
+    response = Document.search(:q => "solr", 
+                               :'hl.fl' => "*")
+    docs = response[:docs]
+    highlighting = response[:highlighting]
+
+    first_result = highlighting.first
+    assert first_result[1]['tags'].include?("<em>solr</em>")
+  end
+end
+

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: solrsan
+version: !ruby/object:Gem::Version
+  version: 0.0.20
+  prerelease: 
+platform: ruby
+authors:
+- Tommy Chheng
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-03-18 00:00:00.000000000 -07:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rsolr
+  requirement: &2153183880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153183880
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: &2153183260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153183260
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &2153182780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153182780
+description: solrsan is a lightweight wrapper for using Apache Solr within a Ruby
+  application including Rails and Sinatra.
+email:
+- tommy.chheng@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.markdown
+- Rakefile
+- config/solr.yml
+- config/solr.yml.example
+- config/solr/conf/elevate.xml
+- config/solr/conf/mapping-ISOLatin1Accent.txt
+- config/solr/conf/protwords.txt
+- config/solr/conf/schema.xml
+- config/solr/conf/solrconfig.xml
+- config/solr/conf/spellings.txt
+- config/solr/conf/stopwords.txt
+- config/solr/conf/synonyms.txt
+- config/solr/conf/xslt/example.xsl
+- config/solr/conf/xslt/example_atom.xsl
+- config/solr/conf/xslt/example_rss.xsl
+- config/solr/conf/xslt/luke.xsl
+- lib/rails/generators/solrsan/config/config_generator.rb
+- lib/rails/generators/solrsan/config/templates/solr.yml
+- lib/rails/generators/solrsan/config/templates/solrsan.rb
+- lib/rails/generators/solrsan_generator.rb
+- lib/solrsan.rb
+- lib/solrsan/config.rb
+- lib/solrsan/indexer.rb
+- lib/solrsan/search.rb
+- lib/solrsan/version.rb
+- lib/tasks/solr.rake
+- solrsan.gemspec
+- test/models/document.rb
+- test/search_test_helper.rb
+- test/test_helper.rb
+- test/unit/indexer_test.rb
+- test/unit/search_test.rb
+has_rdoc: true
+homepage: http://github.com/tc/solrsan
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: solrsan
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Lightweight wrapper for using Apache Solr within a Ruby application including
+  Rails and Sinatra.
+test_files:
+- test/models/document.rb
+- test/search_test_helper.rb
+- test/test_helper.rb
+- test/unit/indexer_test.rb
+- test/unit/search_test.rb