rlibsphinxclient 0.2.2

data/lib/sphinx/fast_client.rb

@@ -0,0 +1,184 @@
+# = fast_client.rb - Wrapper for pure C Sphinx client API
+# 
+# Author::    Dmytro Shteflyuk <mailto:kpumuk@kpumuk.info>.
+# Copyright:: Copyright (c) 2009 Dmytro Shteflyuk
+# License::   Distributes under the MIT license.
+# Version::   0.2.2
+# Website::   http://kpumuk.info/projects/ror-plugins/sphinx
+#
+# This library is distributed under the terms of the MIT license.
+
+# == Sphinx Client API
+# 
+# The Sphinx Client API is used to communicate with <tt>searchd</tt>
+# daemon and get search results from Sphinx.
+# 
+# === Usage
+# 
+#   begin
+#     sphinx = Sphinx::FastClient.new
+#     result = sphinx.Query('test')
+#     ids = result['matches'].map { |match| match['id'] }.join(',')
+#     posts = Post.find :all, :conditions => "id IN (#{ids})"
+#   
+#     docs = posts.map(&:body)
+#     excerpts = sphinx.BuildExcerpts(docs, 'index', 'test')
+#   ensure
+#     sphinx.destroy
+#   end
+
+module Sphinx
+
+  # A wrapper for pure C Sphinx client API.
+  class FastClient
+    Lib = Rlibsphinxclient
+
+    # Constructs the <tt>Sphinx::FastClient</tt> object and sets options to their default values.
+    def initialize
+      @sphinx = Lib.sphinx_create(true)
+    end
+
+    # Destroys all internal memory buffers. Should be called when Sphinx API is no needed.
+    def destroy
+      Lib.sphinx_destroy(@sphinx)
+      @sphinx = nil
+    end
+    
+    # See Client.GetLastError for details.
+    def GetLastError
+      Lib.sphinx_error(@sphinx)
+    end
+
+    # See Client.GetLastWarning for details.
+    def GetLastWarning
+      Lib.sphinx_warning(@sphinx)
+    end
+    
+    # See Client.SetServer for details.
+    def SetServer(host, port)
+      Lib.sphinx_set_server(@sphinx, host, port)
+    end
+
+    def SetConnectionTimeout(seconds)
+      Lib.sphinx_set_connect_timeout(@sphinx, seconds)
+    end
+
+    # See Client.SetLimits for details.
+    def SetLimits(offset, limit, max_matches = 0, cutoff = 0)
+      Lib.sphinx_set_limits(@sphinx, offset, limit, max_matches, cutoff)
+    end
+
+    # See Client.SetMaxQueryTime for details.
+    def SetMaxQueryTime(max_query_time)
+      Lib.sphinx_set_max_query_time(@sphinx, max_query_time)
+    end
+
+    # See Client.SetMatchMode for details.
+    def SetMatchMode(mode)
+      mode = Lib.const_get("SPH_MATCH_#{mode.to_s.upcase}") if mode.is_a? Symbol
+      Lib.sphinx_set_match_mode(@sphinx, mode)
+    end
+    
+    # See Client.SetRankingMode for details.
+    def SetRankingMode(ranker)
+      ranker = Lib.const_get("SPH_RANK_#{ranker.to_s.upcase}") if ranker.is_a? Symbol
+      Lib.sphinx_set_ranking_mode(@sphinx, ranker)
+    end
+    
+    # See Client.SetSortMode for details.
+    def SetSortMode(mode, sortby = '')
+      mode = Lib.const_get("SPH_SORT_#{mode.to_s.upcase}") if mode.is_a? Symbol
+      Lib.sphinx_set_sort_mode(@sphinx, mode, sortby)
+    end
+    
+    # See Client.SetFieldWeights for details.
+    def SetFieldWeights(weights)
+      Lib.sphinx_set_field_weights(@sphinx, weights.size, weights.keys, weights.values)
+    end
+    
+    # See Client.SetIndexWeights for details.
+    def SetIndexWeights(weights)
+      Lib.sphinx_set_index_weights(@sphinx, weights.size, weights.keys, weights.values)
+    end
+    
+    # See Client.SetIDRange for details.
+    def SetIDRange(min, max)
+      Lib.sphinx_set_id_range(@sphinx, min, max)
+    end
+    
+    # See Client.SetFilter for details.
+    def SetFilter(attribute, values, exclude = false)
+      Lib.sphinx_add_filter(@sphinx, attribute, values.length, values, exclude)
+    end
+
+    # See Client.SetFilterRange for details.
+    def SetFilterRange(attribute, min, max, exclude = false)
+      Lib.sphinx_add_filter_range(@sphinx, attribute, min, max, exclude)
+    end
+    
+    # See Client.SetFilterFloatRange for details.
+    def SetFilterFloatRange(attribute, min, max, exclude = false)
+      Lib.sphinx_add_filter_float_range(@sphinx, attribute, min, max, exclude)
+    end
+    
+    # See Client.SetGeoAnchor for details.
+    def SetGeoAnchor(attrlat, attrlong, lat, long)
+      Lib.sphinx_set_geoanchor(@sphinx, attrlat, attrlong, lat, long)
+    end
+    
+    # See Client.SetGroupBy for details.
+    def SetGroupBy(attribute, func, groupsort = '@group desc')
+      Lib.sphinx_set_groupby(@sphinx, attribute, func, groupsort)
+    end
+    
+    # See Client.SetGroupDistinct for details.
+    def SetGroupDistinct(attribute)
+      Lib.sphinx_set_groupby_distinct(@sphinx, attribute)
+    end
+    
+    # See Client.SetRetries for details.
+    def SetRetries(count, delay = 0)
+      Lib.sphinx_set_retries(@sphinx, count, delay)
+    end
+    
+    # See Client.ResetFilters for details.
+    def ResetFilters
+      Lib.sphinx_reset_filters(@sphinx)
+    end
+    
+    # See Client.ResetGroupBy for details.
+    def ResetGroupBy
+      Lib.sphinx_reset_groupby(@sphinx)
+    end
+    
+    # See Client.Query for details.
+    def Query(query, index = '*', comment = '')
+      Lib.sphinx_query(@sphinx, query, index, comment)
+    end
+    
+    # See Client.AddQuery for details.
+    def AddQuery(query, index = '*', comment = '')
+      Lib.sphinx_add_query(@sphinx, query, index, comment)
+    end
+    
+    # See Client.RunQueries for details.
+    def RunQueries
+      Lib.sphinx_run_queries(@sphinx)
+    end
+    
+    # See Client.BuildExcerpts for details.
+    def BuildExcerpts(docs, index, words, opts = {})
+      Lib.sphinx_build_excerpts(@sphinx, docs.size, docs, index, words, opts)
+    end
+    
+    # See Client.BuildKeywords for details.
+    def BuildKeywords(query, index, hits)
+      Lib.sphinx_build_keywords(@sphinx, query, index, hits)
+    end
+    
+    # See Client.UpdateAttributes for details.
+    def UpdateAttributes(index, attrs, values, mva = false)
+      Lib.sphinx_update_attributes(@sphinx, index, attrs.size, attrs, values.size, values.keys, values.values.flatten)
+    end
+  end
+end

data/lib/sphinx/request.rb

@@ -0,0 +1,49 @@
+module Sphinx
+  # Pack ints, floats, strings, and arrays to internal representation
+  # needed by Sphinx search engine.
+  class Request
+    # Initialize new request.
+    def initialize
+      @request = ''
+    end
+    
+    # Put int(s) to request.
+    def put_int(*ints)
+      ints.each { |i| @request << [i].pack('N') }
+    end
+
+    # Put 64-bit int(s) to request.
+    def put_int64(*ints)
+      ints.each { |i| @request << [i >> 32, i & ((1 << 32) - 1)].pack('NN') }
+    end
+    
+    # Put 64-bit int(s) to request (Sphinx 0.9.9 compatible version).
+    def put_int64_new(*ints)
+      ints.each { |i| @request << [i].pack('q').reverse }
+    end
+
+    # Put string(s) to request (first length, then the string itself).
+    def put_string(*strings)
+      strings.each { |s| @request << [s.length].pack('N') + s }
+    end
+    
+    # Put float(s) to request.
+    def put_float(*floats)
+      floats.each do |f|
+        t1 = [f].pack('f') # machine order
+        t2 = t1.unpack('L*').first # int in machine order
+        @request << [t2].pack('N')
+      end
+    end
+    
+    # Put array of ints to request (first length, then the array itself)
+    def put_int_array(arr)
+      put_int arr.length, *arr
+    end
+    
+    # Returns the entire message
+    def to_s
+      @request
+    end
+  end
+end

data/lib/sphinx/response.rb

@@ -0,0 +1,69 @@
+module Sphinx
+  # Unpack internal Sphinx representation of ints, floats, strings, and arrays.
+  # needed by Sphinx search engine.
+  class Response
+    # Initialize new request.
+    def initialize(response)
+      @response = response
+      @position = 0
+      @size = response.length
+    end
+    
+    # Gets current stream position.
+    def position
+      @position
+    end
+    
+    # Gets response size.
+    def size
+      @size
+    end
+    
+    # Returns <tt>true</tt> when response stream is out.
+    def eof?
+      @position >= @size
+    end
+
+    # Get int from stream.
+    def get_int
+      raise EOFError if @position + 4 > @size
+      value = @response[@position, 4].unpack('N*').first
+      @position += 4
+      return value
+    end
+
+    # Get 64-bit int from stream.
+    def get_int64
+      raise EOFError if @position + 8 > @size
+      hi, lo = @response[@position, 8].unpack('N*N*')
+      @position += 8
+      return (hi << 32) + lo
+    end
+
+    # Get array of <tt>count</tt> ints from stream.
+    def get_ints(count)
+      length = 4 * count
+      raise EOFError if @position + length > @size
+      values = @response[@position, length].unpack('N*' * count)
+      @position += length
+      return values
+    end
+    
+    # Get string from stream.
+    def get_string
+      length = get_int
+      raise EOFError if @position + length > @size
+      value = length > 0 ? @response[@position, length] : ''
+      @position += length
+      return value
+    end
+    
+    # Get float from stream.
+    def get_float
+      raise EOFError if @position + 4 > @size
+      uval = @response[@position, 4].unpack('N*').first;
+      @position += 4
+      return ([uval].pack('L')).unpack('f*').first
+    end
+  end
+end

data/lib/sphinx/safe_executor.rb

@@ -0,0 +1,11 @@
+module Sphinx
+  class SafeExecutor
+    def self.execute(timeout = 5, attempts = 3, &block)
+      Sphinx::Timeout.timeout(timeout, &block)
+    rescue ::Timeout::Error
+      attempts -= 1
+      raise if attempts <= 0
+      retry
+    end
+  end
+end

data/lib/sphinx/timeout.rb

@@ -0,0 +1,9 @@
+module Sphinx
+  begin
+    require 'system_timer'
+    Timeout = ::SystemTimer
+  rescue LoadError
+    require 'timeout'
+    Timeout = ::Timeout
+  end
+end

data/rlibsphinxclient.gemspec

@@ -0,0 +1,117 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{rlibsphinxclient}
+  s.version = "0.2.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Dmytro Shteflyuk"]
+  s.date = %q{2009-10-15}
+  s.email = %q{kpumuk@kpumuk.info}
+  s.extensions = ["ext/extconf.rb"]
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "CHANGELOG.rdoc",
+     "MIT-LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "ext/extconf.rb",
+     "ext/rlibsphinxclient.i",
+     "ext/rlibsphinxclient_wrap.c",
+     "init.rb",
+     "lib/sphinx.rb",
+     "lib/sphinx/client.rb",
+     "lib/sphinx/fast_client.rb",
+     "lib/sphinx/request.rb",
+     "lib/sphinx/response.rb",
+     "lib/sphinx/safe_executor.rb",
+     "lib/sphinx/timeout.rb",
+     "rlibsphinxclient.gemspec",
+     "spec/client_response_spec.rb",
+     "spec/client_spec.rb",
+     "spec/fixtures/default_search.php",
+     "spec/fixtures/default_search_index.php",
+     "spec/fixtures/excerpt_custom.php",
+     "spec/fixtures/excerpt_default.php",
+     "spec/fixtures/excerpt_flags.php",
+     "spec/fixtures/field_weights.php",
+     "spec/fixtures/filter.php",
+     "spec/fixtures/filter_exclude.php",
+     "spec/fixtures/filter_float_range.php",
+     "spec/fixtures/filter_float_range_exclude.php",
+     "spec/fixtures/filter_range.php",
+     "spec/fixtures/filter_range_exclude.php",
+     "spec/fixtures/filter_ranges.php",
+     "spec/fixtures/filters.php",
+     "spec/fixtures/filters_different.php",
+     "spec/fixtures/geo_anchor.php",
+     "spec/fixtures/group_by_attr.php",
+     "spec/fixtures/group_by_attrpair.php",
+     "spec/fixtures/group_by_day.php",
+     "spec/fixtures/group_by_day_sort.php",
+     "spec/fixtures/group_by_month.php",
+     "spec/fixtures/group_by_week.php",
+     "spec/fixtures/group_by_year.php",
+     "spec/fixtures/group_distinct.php",
+     "spec/fixtures/id_range.php",
+     "spec/fixtures/id_range64.php",
+     "spec/fixtures/index_weights.php",
+     "spec/fixtures/keywords.php",
+     "spec/fixtures/limits.php",
+     "spec/fixtures/limits_cutoff.php",
+     "spec/fixtures/limits_max.php",
+     "spec/fixtures/limits_max_cutoff.php",
+     "spec/fixtures/match_all.php",
+     "spec/fixtures/match_any.php",
+     "spec/fixtures/match_boolean.php",
+     "spec/fixtures/match_extended.php",
+     "spec/fixtures/match_extended2.php",
+     "spec/fixtures/match_fullscan.php",
+     "spec/fixtures/match_phrase.php",
+     "spec/fixtures/max_query_time.php",
+     "spec/fixtures/miltiple_queries.php",
+     "spec/fixtures/ranking_bm25.php",
+     "spec/fixtures/ranking_none.php",
+     "spec/fixtures/ranking_proximity_bm25.php",
+     "spec/fixtures/ranking_wordcount.php",
+     "spec/fixtures/retries.php",
+     "spec/fixtures/retries_delay.php",
+     "spec/fixtures/sort_attr_asc.php",
+     "spec/fixtures/sort_attr_desc.php",
+     "spec/fixtures/sort_expr.php",
+     "spec/fixtures/sort_extended.php",
+     "spec/fixtures/sort_relevance.php",
+     "spec/fixtures/sort_time_segments.php",
+     "spec/fixtures/sphinxapi.php",
+     "spec/fixtures/update_attributes.php",
+     "spec/fixtures/weights.php",
+     "spec/sphinx/sphinx.conf",
+     "spec/sphinx/sphinx_test.sql"
+  ]
+  s.homepage = %q{http://github.com/kpumuk/rlibsphinxclient}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{A Ruby wrapper for pure C searchd client API library.}
+  s.test_files = [
+    "spec/client_response_spec.rb",
+     "spec/client_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/spec/client_response_spec.rb

@@ -0,0 +1,135 @@
+require File.dirname(__FILE__) + '/../init'
+
+describe 'client API library', :shared => true do
+  it 'should have all filters working' do
+    @sphinx.SetServer('localhost', 3312)
+    @sphinx.SetLimits(1, 100, 20, 30)
+    @sphinx.SetMaxQueryTime(5)
+    @sphinx.SetMatchMode(Sphinx::Client::SPH_MATCH_EXTENDED2)
+    @sphinx.SetRankingMode(Sphinx::Client::SPH_RANK_BM25)
+    @sphinx.SetSortMode(Sphinx::Client::SPH_SORT_RELEVANCE)
+    @sphinx.SetFieldWeights('group_id' => 10, 'rating' => 20)
+    @sphinx.SetIndexWeights('test1' => 20, 'test2' => 30)
+    @sphinx.SetIDRange(1, 100)
+    @sphinx.SetFilter('group_id', [1], true)
+    @sphinx.SetFilterRange('group_id', 1, 2, true)
+    @sphinx.SetFilterFloatRange('rating', 1, 3, true)
+    @sphinx.SetGroupBy('created_at', Sphinx::Client::SPH_GROUPBY_DAY)
+    @sphinx.SetGroupDistinct('group_id')
+    @sphinx.SetRetries(5, 10)
+    result = @sphinx.Query('wifi', 'test1')
+
+    result.should_not be_nil
+  end
+  
+  it 'should parse response in Query method' do
+    result = @sphinx.Query('wifi', 'test1')
+    validate_results_wifi(result)
+  end
+
+  it 'should process 64-bit keys in Query method' do
+    result = @sphinx.Query('wifi', 'test2')
+    result['total_found'].should == 3
+    result['matches'].length.should == 3
+    result['matches'][0]['id'].should == 4294967298
+    result['matches'][1]['id'].should == 4294967299
+    result['matches'][2]['id'].should == 4294967297
+  end
+  
+  it 'should parse batch-query responce in RunQueries method' do
+    @sphinx.AddQuery('wifi', 'test1')
+    @sphinx.AddQuery('gprs', 'test1')
+    results = @sphinx.RunQueries
+    results.should be_an(Array)
+    results.length.should == 2
+    validate_results_wifi(results[0])
+  end
+  
+  it 'should parse response in BuildExcerpts method' do
+    result = @sphinx.BuildExcerpts(['what the world', 'London is the capital of Great Britain'], 'test1', 'the')
+    result.should == ['what <b>the</b> world', 'London is <b>the</b> capital of Great Britain']
+  end
+
+  it 'should parse response in BuildKeywords method' do
+    result = @sphinx.BuildKeywords('wifi gprs', 'test1', true)
+    result.should == [
+      { 'normalized' => 'wifi', 'tokenized' => 'wifi', 'hits' => 6, 'docs' => 3 },
+      { 'normalized' => 'gprs', 'tokenized' => 'gprs', 'hits' => 1, 'docs' => 1 }
+    ]
+  end
+
+  it 'should parse response in UpdateAttributes method' do
+    @sphinx.UpdateAttributes('test1', ['group_id'], { 2 => [1] }).should == 1
+    result = @sphinx.Query('wifi', 'test1')
+    result['matches'][0]['attrs']['group_id'].should == 1
+    @sphinx.UpdateAttributes('test1', ['group_id'], { 2 => [2] }).should == 1
+    result = @sphinx.Query('wifi', 'test1')
+    result['matches'][0]['attrs']['group_id'].should == 2
+  end
+  
+  it 'should process errors in Query method' do
+  	@sphinx.Query('wifi', 'fakeindex').should be_false
+  	@sphinx.GetLastError.length.should_not == 0
+  end
+
+  it 'should process errors in RunQueries method' do
+  	@sphinx.AddQuery('wifi', 'fakeindex')
+  	r = @sphinx.RunQueries
+  	r.should be_an(Array)
+  	r[0]['error'].length.should_not == 0
+  end
+  
+  def validate_results_wifi(result)
+    result['total_found'].should == 3
+    result['matches'].length.should == 3
+    result['time'].should_not be_nil
+    result['attrs'].should == {
+      'group_id' => Sphinx::Client::SPH_ATTR_INTEGER,
+      'created_at' => Sphinx::Client::SPH_ATTR_TIMESTAMP,
+      'rating' => Sphinx::Client::SPH_ATTR_FLOAT,
+      'tags' => Sphinx::Client::SPH_ATTR_MULTI | Sphinx::Client::SPH_ATTR_INTEGER
+    }
+    result['fields'].should == [ 'name', 'description' ]
+    result['total'].should == 3
+    result['matches'].should be_an(Array)
+    
+    result['matches'][0]['id'].should == 2
+    result['matches'][0]['weight'].should == 2
+    result['matches'][0]['attrs']['group_id'].should == 2
+    result['matches'][0]['attrs']['created_at'].should == 1175658555
+    result['matches'][0]['attrs']['tags'].should == [5, 6, 7, 8]
+    ('%0.2f' % result['matches'][0]['attrs']['rating']).should == '54.85'
+    
+    result['matches'][1]['id'].should == 3
+    result['matches'][1]['weight'].should == 2
+    result['matches'][1]['attrs']['group_id'].should == 1
+    result['matches'][1]['attrs']['created_at'].should == 1175658647
+    result['matches'][1]['attrs']['tags'].should == [1, 7, 9, 10]
+    ('%0.2f' % result['matches'][1]['attrs']['rating']).should == '16.25'
+
+    result['matches'][2]['id'].should == 1
+    result['matches'][2]['weight'].should == 1
+    result['matches'][2]['attrs']['group_id'].should == 1
+    result['matches'][2]['attrs']['created_at'].should == 1175658490
+    result['matches'][2]['attrs']['tags'].should == [1, 2, 3, 4]
+    ('%0.2f' % result['matches'][2]['attrs']['rating']).should == '13.32'
+    
+    result['words'].should == { 'wifi' => { 'hits' => 6, 'docs' => 3 } }
+  end
+end
+
+# To execute these tests you need to execute sphinx_test.sql and configure sphinx using sphinx.conf
+# (both files are placed under sphinx directory)
+describe 'The Sphinx::Client connected to Sphinx' do
+  before(:each) { @sphinx = Sphinx::Client.new }
+  after (:each) { @sphinx.destroy }
+  
+  it_should_behave_like 'client API library'
+end
+
+describe 'The Sphinx::FastClient connected to Sphinx' do
+  before(:each) { @sphinx = Sphinx::FastClient.new }
+  after (:each) { @sphinx.destroy }
+  
+  it_should_behave_like 'client API library'
+end