rlibsphinxclient 0.2.2

data/.gitignore

@@ -0,0 +1,3 @@
+mkmf.log
+pkg/
+rdoc

data/CHANGELOG.rdoc

@@ -0,0 +1,18 @@
+=== Version 0.2.2 / 2009-10-15
+
+* Moved from Echoe to Jeweler, gem is hosted on Gemcutter now.
+
+=== Version 0.2.1 / 2009-01-27
+
+* 4 minor bug fixes
+
+  * Fixed extconf.rb to use /opt/sphinx as default library search path.
+  * Updated Mac OS X installation steps and prerequisites section in README.rdoc.
+  * Updated Manifest to include LICENSE file.
+  * Added CHANGELOG file.
+
+=== Version 0.2.0 / 2009-01-27
+
+This is the first preview release of rsphinxclientapi, a Ruby wrapper for pure C searchd client API library.
+
+* Implemented wrappers for all API functions.

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Dmytro Shteflyuk
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,151 @@
+= rlibsphinxclient
+
+A Ruby wrapper for pure C searchd client API library. This is *highly experimental* library
+so use it at your own risk.
+
+== Installing the rlibsphinxclient gem
+
+This gem can be more difficult to install than the typical Ruby extension. First you have to
+install Sphinx and Sphinx pure C searchd client API library.
+
+=== Step 1: Install pure C Sphinx client API
+
+Go to http://sphinxsearch.com/downloads.html and download the latest stable release.
+Then go to <tt>api/libsphinxclient</tt> directory and install client API to your
+preferred folder (I like /opt/sphinx):
+
+  cd api/libsphinxclient
+  ./configure --prefix=/opt/sphinx
+  make
+  sudo make install
+
+On Max OS X you may get the following error:
+
+  configure: error: C++ preprocessor "/lib/cpp" fails sanity check
+
+In this case you should specify environment variable for <tt>./configure</tt> script:
+
+  CXXCPP="gcc -E" ./configure --prefix=/opt/sphinx
+
+=== Step 2: Install rlibsphinxclient gem
+
+If you have installed the Sphinx to <tt>/opt/sphinx</tt>, just run:
+
+  sudo gem install kpumuk-rlibsphinxclient --no-ri --no-rdoc
+
+Otherwise, specify where sphinx has been installed to:
+
+  sudo gem install kpumuk-rlibsphinxclient --no-ri --no-rdoc -- --with-libsphinxclient-dir=/opt/sphinx-0.9.9
+
+On Mac OS X with MacPorts you should specify <tt>ARCHFLAGS</tt> environment variable:
+
+  sudo env ARCHFLAGS="-arch i386" gem install kpumuk-rlibsphinxclient --no-rdoc --no-ri -- --with-libsphinxclient-dir=/opt/sphinx-0.9.9
+
+If you are working on Ruby on Rails application, you can add gem dependency to your
+<tt>config/environment.rb</tt>:
+
+  config.gem 'kpumuk-rlibsphinxclient', :lib => 'sphinx'
+
+Also don't forget to remove the <tt>sphinx</tt> plugin, because it's functionality
+is completely covered by this gem.
+
+== Using the rlibsphinxclient gem
+
+The gem includes two versions of the client API: pure Ruby and wrapper for pure C client API.
+They are 100% equivalent in use, so you can switch to any of them. To use pure Ruby client,
+instantiate the <tt>Sphinx::Client</tt>, for pure C wrapper use <tt>Sphinx::FastClient</tt>.
+
+Important note: you should call <tt>destroy</tt> method when you do not need client API any more.
+The reason for that is the C wrapper saves all query results in memory, and frees them in
+the <tt>destroy</tt> method call. You can omit this call in pure Ruby library, but I'd like
+to do call in any case just for consistence (to be able to switch to another client).
+
+Important note #2: to ensure that <tt>destroy</tt> method will be called, use <tt>ensure</tt>
+block:
+
+    begin
+      @sphinx = Sphinx::FastClient.new
+      @sphinx.Query('test')
+    ensure
+      @sphinx.destroy
+    end
+
+== Examples of usage
+
+Ok, let's take a look at the examples. First, here is the search example with all possible
+filters and options set:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    @sphinx.SetServer('localhost', 3312)
+    @sphinx.SetLimits(1, 100, 20, 30)
+    @sphinx.SetMaxQueryTime(5)
+    @sphinx.SetMatchMode(Sphinx::Client::SPH_MATCH_EXTENDED2)
+    @sphinx.SetRankingMode(Sphinx::Client::SPH_RANK_BM25)
+    @sphinx.SetSortMode(Sphinx::Client::SPH_SORT_RELEVANCE)
+    @sphinx.SetFieldWeights('group_id' => 10, 'rating' => 20)
+    @sphinx.SetIndexWeights('test1' => 20, 'test2' => 30)
+    @sphinx.SetIDRange(1, 100)
+    @sphinx.SetFilter('group_id', [1], true)
+    @sphinx.SetFilterRange('group_id', 1, 2, true)
+    @sphinx.SetFilterFloatRange('rating', 1, 3, true)
+    @sphinx.SetGroupBy('created_at', Sphinx::Client::SPH_GROUPBY_DAY)
+    @sphinx.SetGroupDistinct('group_id')
+    @sphinx.SetRetries(5, 10)
+    results = @sphinx.Query('test')
+    @sphinx.destroy
+
+<tt>BuildKeywords</tt> example:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    results = @sphinx.BuildKeywords('wifi gprs', 'test1', true)
+    @sphinx.destroy
+
+<tt>BuildExcerpts</tt> example:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    results = @sphinx.BuildExcerpts(['what the world', 'London is the capital of Great Britain'], 'test1', 'the')
+    @sphinx.destroy
+
+<tt>UpdateAttributes</tt> example:
+
+    require 'sphinx'
+    @sphinx = Sphinx::FastClient.new
+    results = @sphinx.UpdateAttributes('test1', ['group_id'], { 2 => [1] })
+    @sphinx.destroy
+
+== Benchmarks
+
+The reason to write this gem was to investigate why we keep getting timeout errors
+when using Sphinx (occur rarely, but they are annoying me.) But the side effect of
+this library was the slight search performance improvement: Ruby library is slower
+when generating Sphinx request and parsing its results.
+
+    require 'sphinx'
+    require 'benchmark'
+    
+    def run_test(klass)
+      sphinx = klass.new
+      sphinx.Query('test hello')
+    ensure
+      sphinx.destroy
+    end
+    
+    Benchmark.bm do |x|
+      x.report('pure ruby') { 1000.times { run_test(Sphinx::Client) } }
+      x.report('c wrapper') { 1000.times { run_test(Sphinx::FastClient) } }
+    end
+
+On my MBP I got the following results:
+
+               user       system     total    real
+    pure ruby  0.420000   0.230000   0.650000 ( 14.721659)
+    c wrapper  0.060000   0.090000   0.150000 (  2.248645)
+
+== Who are the authors?
+
+This plugin has been created in Scribd.com for our internal use and then the sources were opened 
+for other people to use. All the code in this package has been developed by Dmytro Shteflyuk
+for Scribd.com and is released under the MIT license. For more details, see MIT-LICENSE file.

data/Rakefile

@@ -0,0 +1,39 @@
+require 'rubygems'
+require 'rake'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name           = 'rlibsphinxclient'
+    gemspec.summary        = 'A Ruby wrapper for pure C searchd client API library.'
+    gemspec.email          = 'kpumuk@kpumuk.info'
+    gemspec.homepage       = 'http://github.com/kpumuk/rlibsphinxclient'
+    gemspec.author         = 'Dmytro Shteflyuk'
+    gemspec.extensions     = ['ext/extconf.rb']
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts 'Jeweler not available. Install it with: sudo gem install jeweler -s http://gemcutter.org'
+end
+
+desc 'Update SWIG wrapper for pure C searchd client API library'
+task :swig do
+  system 'cd ext && swig -I/opt/local/include -I/opt/sphinx-0.9.9/include -ruby -autorename rlibsphinxclient.i'
+end
+
+desc 'Generate documentation for the rlibsphinxclient gem.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'rlibsphinxclient'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc', 'CHANGELOG.rdoc', 'MIT-LICENSE')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc 'Builds a gem and installs it to ~/.gem folder. Used only for development purposes.'
+task :dev do
+  system 'rake package'
+  system 'cd pkg && env ARCHFLAGS="-arch i386" DEBUG=1 gem install rlibsphinxclient --no-rdoc --no-ri -- --with-libsphinxclient-dir=/opt/sphinx-0.9.9'
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/ext/extconf.rb

@@ -0,0 +1,20 @@
+require 'mkmf'
+
+$CFLAGS.gsub! /-O\d/, ''
+
+if ENV['DEBUG']
+  puts "setting debug flags"
+  $CFLAGS << " -O0 -ggdb -DHAVE_DEBUG"
+else
+  $CFLAGS << " -O3"
+end
+
+find_library(*['sphinxclient', 'sphinx_create', '/opt/sphinx/lib'].compact) or
+find_library(*['sphinxclient', 'sphinx_create', dir_config('libsphinxclient').last].compact) or
+  raise "shared library 'libsphinxclient' not found"
+
+find_header(*['sphinxclient.h', '/opt/sphinx/include'].compact) or
+find_header(*['sphinxclient.h', dir_config('libsphinxclient').first].compact) or
+  raise "header file 'sphinxclient.h' not found"
+
+create_makefile 'rlibsphinxclient'

data/ext/rlibsphinxclient.i

@@ -0,0 +1,314 @@
+%module rlibsphinxclient
+%{
+#include <sphinxclient.h>
+%}
+
+%include "typemaps.i"
+
+%newobject sphinx_create;
+
+/* -----------------------------------------------------------------------------
+ *  This section contains generic input parameter type mappings.
+ * ----------------------------------------------------------------------------- */
+
+// Processing (sphinx_bool) params
+%typemap(in) sphinx_bool {
+  switch(TYPE($input)) {
+    case T_TRUE:
+    case T_FALSE:
+      $1 = $input == Qtrue ? SPH_TRUE : SPH_FALSE;
+      break;
+    default:
+      SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument " "$argnum"" of type '" "$1_type""'");
+      break;
+  }
+}
+
+// Processing (sphinx_uint64_t *) params.
+%typemap(in) sphinx_uint64_t * {
+  int size = RARRAY_LEN($input), i;
+
+  $1 = (sphinx_uint64_t *) malloc(size * sizeof(sphinx_uint64_t));
+  VALUE *ptr = RARRAY_PTR($input); 
+  for (i = 0; i < size; i++, ptr++) {
+    $1[i] = NUM2ULL(*ptr);
+  }
+}
+
+// Cleaning up (sphinx_uint64_t *) params.
+%typemap(freearg) sphinx_uint64_t * {
+  free((char *) $1);
+}
+
+// Processing (int *) params.
+%typemap(in) int * {
+  int size = RARRAY_LEN($input), i;
+
+  $1 = (int *) malloc(size * sizeof(int));
+  VALUE *ptr = RARRAY_PTR($input);
+  for (i = 0; i < size; i++, ptr++) {
+    $1[i] = NUM2INT(*ptr);
+  }
+}
+
+// Cleaning up (int *) params.
+%typemap(freearg) int * {
+  free((char *) $1);
+}
+
+// Processing (char **) params.
+%typemap(in) char ** {
+  int size = RARRAY_LEN($input), i;
+
+  $1 = (char **) malloc(size * sizeof(char *));
+  VALUE *ptr = RARRAY_PTR($input);
+  for (i = 0; i < size; i++, ptr++) {
+    $1[i] = STR2CSTR(*ptr);
+  }
+}
+
+// Cleaning up (char **) params.
+%typemap(freearg) char ** {
+  free((char *) $1);
+}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains sphinx_result return value type mappings
+ *  (functions sphinx_query and sphinx_run_queries.)
+ * ----------------------------------------------------------------------------- */
+
+// Processing sphinx_run_queries return value (array of sphinx_result).
+%typemap(out) (sphinx_result *) {
+  int num_results = sphinx_get_num_results(arg1), i;
+
+  $result = rb_ary_new();
+  for (i = 0; i < num_results; i++) {
+    rb_ary_store($result, i, convert_sphinx_result(arg1, $1 + i));
+  }
+}
+sphinx_result * sphinx_run_queries(sphinx_client * client);
+
+// Processing sphinx_query return value (single instance of sphinx_result).
+%typemap(out) (sphinx_result *) {
+  $result = convert_sphinx_result(arg1, $1);
+}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains type mappings for sphinx_build_excerpts function.
+ * ----------------------------------------------------------------------------- */
+
+// Processing sphinx_excerpt_options input parameter.
+%typemap(in) sphinx_excerpt_options * (sphinx_excerpt_options opts, VALUE val) {
+  Check_Type($input, T_HASH);
+  sphinx_init_excerpt_options(&opts);
+  
+  // before_match
+  val = rb_hash_aref($input, rb_str_new2("before_match"));
+  if (val != Qnil) {
+    Check_Type($input, T_STRING);
+    opts.before_match = STR2CSTR(val);
+  }
+
+  // after_match
+  val = rb_hash_aref($input, rb_str_new2("after_match"));
+  if (val != Qnil) {
+    Check_Type($input, T_STRING);
+    opts.after_match = STR2CSTR(val);
+  }
+
+  // chunk_separator
+  val = rb_hash_aref($input, rb_str_new2("chunk_separator"));
+  if (val != Qnil) {
+    Check_Type($input, T_STRING);
+    opts.chunk_separator = STR2CSTR(val);
+  }
+  
+  // limit
+  val = rb_hash_aref($input, rb_str_new2("limit"));
+  if (val != Qnil) {
+    Check_Type($input, T_FIXNUM);
+    opts.limit = NUM2INT(val);
+  }
+  
+  // around
+  val = rb_hash_aref($input, rb_str_new2("around"));
+  if (val != Qnil) {
+    Check_Type($input, T_FIXNUM);
+    opts.around = NUM2INT(val);
+  }
+
+  // exact_phrase
+  val = rb_hash_aref($input, rb_str_new2("exact_phrase"));
+  if (val != Qnil) {
+    opts.around = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+
+  // single_passage
+  val = rb_hash_aref($input, rb_str_new2("single_passage"));
+  if (val != Qnil) {
+    opts.single_passage = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+
+  // use_boundaries
+  val = rb_hash_aref($input, rb_str_new2("use_boundaries"));
+  if (val != Qnil) {
+    opts.use_boundaries = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+
+  // weight_order
+  val = rb_hash_aref($input, rb_str_new2("weight_order"));
+  if (val != Qnil) {
+    opts.weight_order = val == Qtrue ? SPH_TRUE : SPH_FALSE;
+  }
+  
+  $1 = &opts;
+}
+
+// Processing char ** output parameter.
+%typemap(out) char ** {
+  int num_docs, i;
+  
+  SWIG_AsVal_int(argv[1], &num_docs);
+  
+  if ($1) {
+    $result = rb_ary_new();
+    for (i = 0; i < num_docs; i++) {
+      rb_ary_store($result, i, rb_str_new2($1[i]));
+    
+      free((char *) $1[i]);
+    }
+    free((char *) $1);
+  } else {
+    $result = Qfalse;
+  }
+}
+char ** sphinx_build_excerpts(sphinx_client * client, int num_docs, const char ** docs, const char * index, const char * words, sphinx_excerpt_options * opts);
+%typemap(in) sphinx_excerpt_options * {}
+%typemap(out) char ** {}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains type mappings for sphinx_build_keywords function.
+ * ----------------------------------------------------------------------------- */
+
+// Defining out_num_keywords input parameter (will contain a number of keywords).
+%typemap(in, numinputs = 0) int * out_num_keywords (int out_num_keywords) {
+  $1 = &out_num_keywords;
+}
+
+// Doing nothing because out_num_keywords is a local variable.
+%typemap(freearg) int * out_num_keywords {
+}
+
+// Processing array of sphinx_keyword_info values.
+%typemap(out) sphinx_keyword_info * {
+  int i;
+  VALUE keyword = Qnil;
+  $result = rb_ary_new();
+  for (i = 0; i < out_num_keywords5; i++) {
+    keyword = rb_hash_new();
+    rb_hash_aset(keyword, rb_str_new2("tokenized"), rb_str_new2($1[i].tokenized));
+    rb_hash_aset(keyword, rb_str_new2("normalized"), rb_str_new2($1[i].normalized));
+    rb_hash_aset(keyword, rb_str_new2("docs"), INT2FIX($1[i].num_docs));
+    rb_hash_aset(keyword, rb_str_new2("hits"), INT2FIX($1[i].num_hits));
+    rb_ary_store($result, i, keyword);
+    
+    free((char *) $1[i].tokenized);
+    free((char *) $1[i].normalized);
+  }
+  
+  free((char *) $1);
+}
+
+/* -----------------------------------------------------------------------------
+ *  This section contains some useful helpers for type mappers.
+ * ----------------------------------------------------------------------------- */
+
+%{
+static VALUE convert_sphinx_result(sphinx_client *client, sphinx_result *input) {
+  int i, j, k;
+  VALUE result = Qnil, var1 = Qnil, var2 = Qnil, var3 = Qnil, var4 = Qnil;
+  char *msg = (char *) 0;
+  unsigned int *mva = 0;
+  char *time = 0;
+  
+  if (!input) {
+    result = Qfalse;
+  } else {
+    result = rb_hash_new();
+    rb_hash_aset(result, rb_str_new2("error"), input->error ? rb_str_new2(input->error) : Qnil);
+    rb_hash_aset(result, rb_str_new2("warning"), input->warning ? rb_str_new2(input->warning) : Qnil);
+    rb_hash_aset(result, rb_str_new2("status"), INT2FIX(input->status));
+    if (input->status != SEARCHD_OK && input->status != SEARCHD_WARNING) {
+      return result;
+    }
+    
+    rb_hash_aset(result, rb_str_new2("total"), INT2FIX(input->total));
+    rb_hash_aset(result, rb_str_new2("total_found"), INT2FIX(input->total_found));
+    time = (char *) malloc(20);
+    sprintf(time, "%.3f", input->time_msec / 1000.);
+    rb_hash_aset(result, rb_str_new2("time"), rb_str_new2(time));
+    free(time);
+  
+    // fields
+    var1 = rb_ary_new();
+    for (i = 0; i < input->num_fields; i++) {
+      rb_ary_store(var1, i, rb_str_new2(input->fields[i]));
+    }
+    rb_hash_aset(result, rb_str_new2("fields"), var1);
+  
+    // attrs
+    var1 = rb_hash_new();
+    for (i = 0; i < input->num_attrs; i++) {
+      rb_hash_aset(var1, rb_str_new2(input->attr_names[i]), INT2NUM(input->attr_types[i]));    
+    }
+    rb_hash_aset(result, rb_str_new2("attrs"), var1);    
+  
+    // words
+    var1 = rb_hash_new();
+    for (i = 0; i < input->num_words; i++) {
+      var2 = rb_hash_new();
+      rb_hash_aset(var2, rb_str_new2("docs"), INT2FIX(input->words[i].docs));
+      rb_hash_aset(var2, rb_str_new2("hits"), INT2FIX(input->words[i].hits));
+  
+      rb_hash_aset(var1, rb_str_new2(input->words[i].word), var2);
+    }
+    rb_hash_aset(result, rb_str_new2("words"), var1);
+
+    // matches
+    var1 = rb_ary_new();
+    for (i = 0; i < input->num_matches; i++) {
+      var2 = rb_hash_new();
+      rb_hash_aset(var2, rb_str_new2("id"), ULL2NUM(sphinx_get_id(input, i)));
+      rb_hash_aset(var2, rb_str_new2("weight"), INT2FIX(sphinx_get_weight(input, i)));
+      
+      var3 = rb_hash_new();
+      for (j = 0; j < input->num_attrs; j++) {
+        switch (input->attr_types[j]) {
+          case SPH_ATTR_MULTI | SPH_ATTR_INTEGER:
+            mva = (unsigned int *) sphinx_get_mva(input, i, j);
+            var4 = rb_ary_new();
+            for (k = 0; k < (int) mva[0]; k++) {
+              rb_ary_store(var4, k, INT2NUM(mva[k + 1]));
+            }
+            break;
+          case SPH_ATTR_FLOAT:
+            var4 = rb_float_new(sphinx_get_float(input, i, j));
+            break;
+          default:
+            var4 = ULL2NUM(sphinx_get_int(input, i, j));
+            break;
+        }
+        rb_hash_aset(var3, rb_str_new2(input->attr_names[j]), var4);
+      }
+      rb_hash_aset(var2, rb_str_new2("attrs"), var3);
+      
+      rb_ary_store(var1, i, var2);
+    }
+    rb_hash_aset(result, rb_str_new2("matches"), var1);
+  }
+  
+  return result;
+}
+%}
+
+%include "/opt/sphinx-0.9.9/include/sphinxclient.h"