kpumuk-rlibsphinxclient 0.2.0

data/Manifest

@@ -0,0 +1,75 @@
+CHANGELOG
+ext/extconf.rb
+ext/rlibsphinxclient.i
+ext/rlibsphinxclient_wrap.c
+init.rb
+lib/sphinx/client.rb
+lib/sphinx/fast_client.rb
+lib/sphinx/request.rb
+lib/sphinx/response.rb
+lib/sphinx/safe_executor.rb
+lib/sphinx/timeout.rb
+lib/sphinx.rb
+Manifest
+Rakefile
+README.rdoc
+spec/client_response_spec.rb
+spec/client_spec.rb
+spec/fixtures/default_search.php
+spec/fixtures/default_search_index.php
+spec/fixtures/excerpt_custom.php
+spec/fixtures/excerpt_default.php
+spec/fixtures/excerpt_flags.php
+spec/fixtures/field_weights.php
+spec/fixtures/filter.php
+spec/fixtures/filter_exclude.php
+spec/fixtures/filter_float_range.php
+spec/fixtures/filter_float_range_exclude.php
+spec/fixtures/filter_range.php
+spec/fixtures/filter_range_exclude.php
+spec/fixtures/filter_ranges.php
+spec/fixtures/filters.php
+spec/fixtures/filters_different.php
+spec/fixtures/geo_anchor.php
+spec/fixtures/group_by_attr.php
+spec/fixtures/group_by_attrpair.php
+spec/fixtures/group_by_day.php
+spec/fixtures/group_by_day_sort.php
+spec/fixtures/group_by_month.php
+spec/fixtures/group_by_week.php
+spec/fixtures/group_by_year.php
+spec/fixtures/group_distinct.php
+spec/fixtures/id_range.php
+spec/fixtures/id_range64.php
+spec/fixtures/index_weights.php
+spec/fixtures/keywords.php
+spec/fixtures/limits.php
+spec/fixtures/limits_cutoff.php
+spec/fixtures/limits_max.php
+spec/fixtures/limits_max_cutoff.php
+spec/fixtures/match_all.php
+spec/fixtures/match_any.php
+spec/fixtures/match_boolean.php
+spec/fixtures/match_extended.php
+spec/fixtures/match_extended2.php
+spec/fixtures/match_fullscan.php
+spec/fixtures/match_phrase.php
+spec/fixtures/max_query_time.php
+spec/fixtures/miltiple_queries.php
+spec/fixtures/ranking_bm25.php
+spec/fixtures/ranking_none.php
+spec/fixtures/ranking_proximity_bm25.php
+spec/fixtures/ranking_wordcount.php
+spec/fixtures/retries.php
+spec/fixtures/retries_delay.php
+spec/fixtures/sort_attr_asc.php
+spec/fixtures/sort_attr_desc.php
+spec/fixtures/sort_expr.php
+spec/fixtures/sort_extended.php
+spec/fixtures/sort_relevance.php
+spec/fixtures/sort_time_segments.php
+spec/fixtures/sphinxapi.php
+spec/fixtures/update_attributes.php
+spec/fixtures/weights.php
+spec/sphinx/sphinx.conf
+spec/sphinx/sphinx_test.sql

data/README.rdoc

@@ -0,0 +1,149 @@
+= rlibsphinxclient
+
+A Ruby wrapper for pure C searchd client API library. This is *highly experimental* library
+so use it at your own risk.
+
+== Installing the rlibsphinxclient gem
+
+This gem can be more difficult to install than the typical Ruby extension. First you have to
+install Sphinx and Sphinx pure C searchd client API library.
+
+=== Step 1: Install Sphinx and client API
+
+Go to http://sphinxsearch.com/downloads.html and download the latest stable release.
+Install it to your preferred folder (I like /opt/sphinx):
+
+    ./configure --prefix=/opt/sphinx
+    make
+    sudo make install
+
+Then go to <tt>api/libsphinxclient</tt> directory and install client API library to the same
+folder:
+
+    cd api/libsphinxclient
+    ./configure --prefix=/opt/sphinx
+    make
+    sudo make install
+
+On Max OS X you may get following error:
+
+    configure: error: C++ preprocessor "/lib/cpp" fails sanity check
+
+In this case you should specify environment variable for <tt>./configure</tt> script:
+
+    CXXCPP="gcc -E" ./configure --prefix=/opt/sphinx
+
+=== Step 2: Install rlibsphinxclient gem
+
+If you have installed the Sphinx to <tt>/opt/sphinx</tt>, just run:
+
+    sudo gem install rlibsphinxclient --no-ri --no-rdoc
+
+Otherwise, specify where sphinx has been installed to:
+
+    sudo gem install rlibsphinxclient --no-ri --no-rdoc -- --with-libsphinxclient-dir=/opt/sphinx-0.9.9
+
+On Mac OS X you should specify <tt>ARCHFLAGS</tt> environment variable:
+
+    env ARCHFLAGS="-arch i386" sudo gem install rlibsphinxclient --no-rdoc --no-ri -- --with-libsphinxclient-dir=/opt/sphinx-0.9.9
+
+== Using the rlibsphinxclient gem
+
+The gem includes two versions of the client API: pure Ruby and wrapper for pure C client API.
+They are 100% equivalent in use, so you can switch to any of them. To use pure Ruby client,
+instantiate the <tt>Sphinx::Client</tt>, for pure C wrapper use <tt>Sphinx::FastClient</tt>.
+
+Important note: you should call <tt>destroy</tt> method when you do not need client API any more.
+The reason for that is the C wrapper saves all query results in memory, and frees them in
+the <tt>destroy</tt> method call. You can omit this call in pure Ruby library, but I'd like
+to do call in any case just for consistence (to be able to switch to another client).
+
+Important note #2: to ensure that <tt>destroy</tt> method will be called, use <tt>ensure</tt>
+block:
+
+    begin
+      @sphinx = Sphinx::FastClient.new
+      @sphinx.Query('test')
+    ensure
+      @sphinx.destroy
+    end
+
+== Examples of usage
+
+Ok, let's take a look at the examples. First, here is the search example with all possible
+filters and options set:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    @sphinx.SetServer('localhost', 3312)
+    @sphinx.SetLimits(1, 100, 20, 30)
+    @sphinx.SetMaxQueryTime(5)
+    @sphinx.SetMatchMode(Sphinx::Client::SPH_MATCH_EXTENDED2)
+    @sphinx.SetRankingMode(Sphinx::Client::SPH_RANK_BM25)
+    @sphinx.SetSortMode(Sphinx::Client::SPH_SORT_RELEVANCE)
+    @sphinx.SetFieldWeights('group_id' => 10, 'rating' => 20)
+    @sphinx.SetIndexWeights('test1' => 20, 'test2' => 30)
+    @sphinx.SetIDRange(1, 100)
+    @sphinx.SetFilter('group_id', [1], true)
+    @sphinx.SetFilterRange('group_id', 1, 2, true)
+    @sphinx.SetFilterFloatRange('rating', 1, 3, true)
+    @sphinx.SetGroupBy('created_at', Sphinx::Client::SPH_GROUPBY_DAY)
+    @sphinx.SetGroupDistinct('group_id')
+    @sphinx.SetRetries(5, 10)
+    results = @sphinx.Query('test')
+    @sphinx.destroy
+
+<tt>BuildKeywords</tt> example:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    results = @sphinx.BuildKeywords('wifi gprs', 'test1', true)
+    @sphinx.destroy
+
+<tt>BuildExcerpts</tt> example:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    results = @sphinx.BuildExcerpts(['what the world', 'London is the capital of Great Britain'], 'test1', 'the')
+    @sphinx.destroy
+
+<tt>UpdateAttributes</tt> example:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    results = @sphinx.UpdateAttributes('test1', ['group_id'], { 2 => [1] })
+    @sphinx.destroy
+
+== Benchmarks
+
+The reason to write this gem was to investigate why we keep getting timeout errors
+when using Sphinx (occur rarely, but they are annoying me.) But the side effect of
+this library was the slight search performance improvement: Ruby library is slower
+when generating Sphinx request and parsing its results.
+
+    require 'sphinx'
+    require 'benchmark'
+    
+    def run_test(klass)
+      sphinx = klass.new
+      sphinx.Query('test hello')
+    ensure
+      sphinx.destroy
+    end
+    
+    Benchmark.bm do |x|
+      x.report('pure ruby') { 1000.times { run_test(Sphinx::Client) } }
+      x.report('c wrapper') { 1000.times { run_test(Sphinx::FastClient) } }
+    end
+
+On my MBP I got the following results:
+
+               user       system     total    real
+    pure ruby  0.420000   0.230000   0.650000 ( 14.721659)
+    c wrapper  0.060000   0.090000   0.150000 (  2.248645)
+
+== Who are the authors?
+
+This plugin has been created in Scribd.com for our internal use and then the sources were opened 
+for other people to use. All the code in this package has been developed by Dmytro Shteflyuk
+for Scribd.com and is released under the GPLv2 license. For more details, see LICENSE file.

data/Rakefile

@@ -0,0 +1,33 @@
+require 'rubygems'
+require 'rake'
+require 'echoe'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+Echoe.new('rlibsphinxclient') do |p|
+  p.author = 'Dmytro Shteflyuk'
+  p.email = 'kpumuk@kpumuk.info'
+  p.summary = 'A Ruby wrapper for pure C searchd client API library'
+  p.url = 'http://github.com/kpumuk/rlibsphinxclient'
+  p.version = '0.2.0'
+end
+
+desc 'Update SWIG wrapper for pure C searchd client API library'
+task :swig do
+  system 'cd ext && swig -I/opt/local/include -I/opt/sphinx-0.9.9/include -ruby -autorename rlibsphinxclient.i'
+end
+
+desc 'Generate documentation for the rlibsphinxclient gem.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'rlibsphinxclient'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc', 'CHANGELOG', 'LICENSE')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc 'Builds a gem and installs it to ~/.gem folder. Used only for development purposes.'
+task :dev do
+  system 'rake package'
+  system 'cd pkg && env ARCHFLAGS="-arch i386" DEBUG=1 gem install rlibsphinxclient --no-rdoc --no-ri -- --with-libsphinxclient-dir=/opt/sphinx-0.9.9'
+end

data/ext/extconf.rb

@@ -0,0 +1,20 @@
+require 'mkmf'
+
+$CFLAGS.gsub! /-O\d/, ''
+
+if ENV['DEBUG']
+  puts "setting debug flags"
+  $CFLAGS << " -O0 -ggdb -DHAVE_DEBUG"
+else
+  $CFLAGS << " -O3"
+end
+
+find_library(*['sphinxclient', 'sphinx_create', '/opt/sphinx'].compact) or
+find_library(*['sphinxclient', 'sphinx_create', dir_config('libsphinxclient').last].compact) or
+  raise "shared library 'libsphinxclient' not found"
+
+find_header(*['sphinxclient.h', '/opt/sphinx'].compact) or
+find_header(*['sphinxclient.h', dir_config('libsphinxclient').first].compact) or
+  raise "header file 'sphinxclient.h' not found"
+
+create_makefile 'rlibsphinxclient'

data/ext/rlibsphinxclient.i

@@ -0,0 +1,314 @@
+%module rlibsphinxclient
+%{
+#include <sphinxclient.h>
+%}
+
+%include "typemaps.i"
+
+%newobject sphinx_create;
+
+/* -----------------------------------------------------------------------------
+ *  This section contains generic input parameter type mappings.
+ * ----------------------------------------------------------------------------- */
+
+// Processing (sphinx_bool) params
+%typemap(in) sphinx_bool {
+  switch(TYPE($input)) {
+    case T_TRUE:
+    case T_FALSE:
+      $1 = $input == Qtrue ? SPH_TRUE : SPH_FALSE;
+      break;
+    default:
+      SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument " "$argnum"" of type '" "$1_type""'");
+      break;
+  }
+}
+
+// Processing (sphinx_uint64_t *) params.
+%typemap(in) sphinx_uint64_t * {
+  int size = RARRAY_LEN($input), i;
+
+  $1 = (sphinx_uint64_t *) malloc(size * sizeof(sphinx_uint64_t));
+  VALUE *ptr = RARRAY_PTR($input); 
+  for (i = 0; i < size; i++, ptr++) {
+    $1[i] = NUM2ULL(*ptr);
+  }
+}
+
+// Cleaning up (sphinx_uint64_t *) params.
+%typemap(freearg) sphinx_uint64_t * {
+  free((char *) $1);
+}
+
+// Processing (int *) params.
+%typemap(in) int * {
+  int size = RARRAY_LEN($input), i;
+
+  $1 = (int *) malloc(size * sizeof(int));
+  VALUE *ptr = RARRAY_PTR($input);
+  for (i = 0; i < size; i++, ptr++) {
+    $1[i] = NUM2INT(*ptr);
+  }
+}
+
+// Cleaning up (int *) params.
+%typemap(freearg) int * {
+  free((char *) $1);
+}
+
+// Processing (char **) params.
+%typemap(in) char ** {
+  int size = RARRAY_LEN($input), i;
+
+  $1 = (char **) malloc(size * sizeof(char *));
+  VALUE *ptr = RARRAY_PTR($input);
+  for (i = 0; i < size; i++, ptr++) {
+    $1[i] = STR2CSTR(*ptr);
+  }
+}
+
+// Cleaning up (char **) params.
+%typemap(freearg) char ** {
+  free((char *) $1);
+}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains sphinx_result return value type mappings
+ *  (functions sphinx_query and sphinx_run_queries.)
+ * ----------------------------------------------------------------------------- */
+
+// Processing sphinx_run_queries return value (array of sphinx_result).
+%typemap(out) (sphinx_result *) {
+  int num_results = sphinx_get_num_results(arg1), i;
+
+  $result = rb_ary_new();
+  for (i = 0; i < num_results; i++) {
+    rb_ary_store($result, i, convert_sphinx_result(arg1, $1 + i));
+  }
+}
+sphinx_result * sphinx_run_queries(sphinx_client * client);
+
+// Processing sphinx_query return value (single instance of sphinx_result).
+%typemap(out) (sphinx_result *) {
+  $result = convert_sphinx_result(arg1, $1);
+}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains type mappings for sphinx_build_excerpts function.
+ * ----------------------------------------------------------------------------- */
+
+// Processing sphinx_excerpt_options input parameter.
+%typemap(in) sphinx_excerpt_options * (sphinx_excerpt_options opts, VALUE val) {
+  Check_Type($input, T_HASH);
+  sphinx_init_excerpt_options(&opts);
+  
+  // before_match
+  val = rb_hash_aref($input, rb_str_new2("before_match"));
+  if (val != Qnil) {
+    Check_Type($input, T_STRING);
+    opts.before_match = STR2CSTR(val);
+  }
+
+  // after_match
+  val = rb_hash_aref($input, rb_str_new2("after_match"));
+  if (val != Qnil) {
+    Check_Type($input, T_STRING);
+    opts.after_match = STR2CSTR(val);
+  }
+
+  // chunk_separator
+  val = rb_hash_aref($input, rb_str_new2("chunk_separator"));
+  if (val != Qnil) {
+    Check_Type($input, T_STRING);
+    opts.chunk_separator = STR2CSTR(val);
+  }
+  
+  // limit
+  val = rb_hash_aref($input, rb_str_new2("limit"));
+  if (val != Qnil) {
+    Check_Type($input, T_FIXNUM);
+    opts.limit = NUM2INT(val);
+  }
+  
+  // around
+  val = rb_hash_aref($input, rb_str_new2("around"));
+  if (val != Qnil) {
+    Check_Type($input, T_FIXNUM);
+    opts.around = NUM2INT(val);
+  }
+
+  // exact_phrase
+  val = rb_hash_aref($input, rb_str_new2("exact_phrase"));
+  if (val != Qnil) {
+    opts.around = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+
+  // single_passage
+  val = rb_hash_aref($input, rb_str_new2("single_passage"));
+  if (val != Qnil) {
+    opts.single_passage = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+
+  // use_boundaries
+  val = rb_hash_aref($input, rb_str_new2("use_boundaries"));
+  if (val != Qnil) {
+    opts.use_boundaries = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+
+  // weight_order
+  val = rb_hash_aref($input, rb_str_new2("weight_order"));
+  if (val != Qnil) {
+    opts.weight_order = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+  
+  $1 = &opts;
+}
+
+// Processing char ** output parameter.
+%typemap(out) char ** {
+  int num_docs, i;
+  
+  SWIG_AsVal_int(argv[1], &num_docs);
+  
+  if ($1) {
+    $result = rb_ary_new();
+    for (i = 0; i < num_docs; i++) {
+      rb_ary_store($result, i, rb_str_new2($1[i]));
+    
+      free((char *) $1[i]);
+    }
+    free((char *) $1);
+  } else {
+    $result = Qfalse;
+  }
+}
+char ** sphinx_build_excerpts(sphinx_client * client, int num_docs, const char ** docs, const char * index, const char * words, sphinx_excerpt_options * opts);
+%typemap(in) sphinx_excerpt_options * {}
+%typemap(out) char ** {}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains type mappings for sphinx_build_keywords function.
+ * ----------------------------------------------------------------------------- */
+
+// Defining out_num_keywords input parameter (will contain a number of keywords).
+%typemap(in, numinputs = 0) int * out_num_keywords (int out_num_keywords) {
+  $1 = &out_num_keywords;
+}
+
+// Doing nothing because out_num_keywords is a local variable.
+%typemap(freearg) int * out_num_keywords {
+}
+
+// Processing array of sphinx_keyword_info values.
+%typemap(out) sphinx_keyword_info * {
+  int i;
+  VALUE keyword = Qnil;
+  $result = rb_ary_new();
+  for (i = 0; i < out_num_keywords5; i++) {
+    keyword = rb_hash_new();
+    rb_hash_aset(keyword, rb_str_new2("tokenized"), rb_str_new2($1[i].tokenized));
+    rb_hash_aset(keyword, rb_str_new2("normalized"), rb_str_new2($1[i].normalized));
+    rb_hash_aset(keyword, rb_str_new2("docs"), INT2FIX($1[i].num_docs));
+    rb_hash_aset(keyword, rb_str_new2("hits"), INT2FIX($1[i].num_hits));
+    rb_ary_store($result, i, keyword);
+    
+    free((char *) $1[i].tokenized);
+    free((char *) $1[i].normalized);
+  }
+  
+  free((char *) $1);
+}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains some useful helpers for type mappers.
+ * ----------------------------------------------------------------------------- */
+
+%{
+static VALUE convert_sphinx_result(sphinx_client *client, sphinx_result *input) {
+  int i, j, k;
+  VALUE result = Qnil, var1 = Qnil, var2 = Qnil, var3 = Qnil, var4 = Qnil;
+  char *msg = (char *) 0;
+  unsigned int *mva = 0;
+  char *time = 0;
+  
+  if (!input) {
+    result = Qfalse;
+  } else {
+    result = rb_hash_new();
+    rb_hash_aset(result, rb_str_new2("error"), input->error ? rb_str_new2(input->error) : Qnil);
+    rb_hash_aset(result, rb_str_new2("warning"), input->warning ? rb_str_new2(input->warning) : Qnil);
+    rb_hash_aset(result, rb_str_new2("status"), INT2FIX(input->status));
+    if (input->status != SEARCHD_OK && input->status != SEARCHD_WARNING) {
+      return result;
+    }
+    
+    rb_hash_aset(result, rb_str_new2("total"), INT2FIX(input->total));
+    rb_hash_aset(result, rb_str_new2("total_found"), INT2FIX(input->total_found));
+    time = (char *) malloc(20);
+    sprintf(time, "%.3f", input->time_msec / 1000.);
+    rb_hash_aset(result, rb_str_new2("time"), rb_str_new2(time));
+    free(time);
+  
+    // fields
+    var1 = rb_ary_new();
+    for (i = 0; i < input->num_fields; i++) {
+      rb_ary_store(var1, i, rb_str_new2(input->fields[i]));
+    }
+    rb_hash_aset(result, rb_str_new2("fields"), var1);
+  
+    // attrs
+    var1 = rb_hash_new();
+    for (i = 0; i < input->num_attrs; i++) {
+      rb_hash_aset(var1, rb_str_new2(input->attr_names[i]), INT2NUM(input->attr_types[i]));    
+    }
+    rb_hash_aset(result, rb_str_new2("attrs"), var1);    
+  
+    // words
+    var1 = rb_hash_new();
+    for (i = 0; i < input->num_words; i++) {
+      var2 = rb_hash_new();
+      rb_hash_aset(var2, rb_str_new2("docs"), INT2FIX(input->words[i].docs));
+      rb_hash_aset(var2, rb_str_new2("hits"), INT2FIX(input->words[i].hits));
+  
+      rb_hash_aset(var1, rb_str_new2(input->words[i].word), var2);
+    }
+    rb_hash_aset(result, rb_str_new2("words"), var1);
+
+    // matches
+    var1 = rb_ary_new();
+    for (i = 0; i < input->num_matches; i++) {
+      var2 = rb_hash_new();
+      rb_hash_aset(var2, rb_str_new2("id"), ULL2NUM(sphinx_get_id(input, i)));
+      rb_hash_aset(var2, rb_str_new2("weight"), INT2FIX(sphinx_get_weight(input, i)));
+      
+      var3 = rb_hash_new();
+      for (j = 0; j < input->num_attrs; j++) {
+        switch (input->attr_types[j]) {
+          case SPH_ATTR_MULTI | SPH_ATTR_INTEGER:
+            mva = (unsigned int *) sphinx_get_mva(input, i, j);
+            var4 = rb_ary_new();
+            for (k = 0; k < (int) mva[0]; k++) {
+              rb_ary_store(var4, k, INT2NUM(mva[k + 1]));
+            }
+            break;
+          case SPH_ATTR_FLOAT:
+            var4 = rb_float_new(sphinx_get_float(input, i, j));
+            break;
+          default:
+            var4 = ULL2NUM(sphinx_get_int(input, i, j));
+            break;
+        }
+        rb_hash_aset(var3, rb_str_new2(input->attr_names[j]), var4);
+      }
+      rb_hash_aset(var2, rb_str_new2("attrs"), var3);
+      
+      rb_ary_store(var1, i, var2);
+    }
+    rb_hash_aset(result, rb_str_new2("matches"), var1);
+  }
+  
+  return result;
+}
+%}
+
+%include "/opt/sphinx-0.9.9/include/sphinxclient.h"