riddle 0.9.8.1533.10 → 1.0.0

data/lib/riddle/client/message.rb

@@ -35,6 +35,10 @@ module Riddle
         @message << [float].pack('f').unpack('L*').pack("N")
       end
       
+      def append_boolean(bool)
+        append_int(bool ? 1 : 0)
+      end
+      
       # Append multiple integers
       def append_ints(*ints)
         ints.each { |int| append_int(int) }

data/lib/riddle/client/response.rb

@@ -75,6 +75,16 @@ module Riddle
         return items
       end
       
+      def next_64bit_int_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_64bit_int
+        end
+        
+        return items
+      end
+      
       # Returns the length of the streamed data
       def length
         @str.length

data/lib/riddle/configuration/distributed_index.rb

@@ -1,16 +1,17 @@
 module Riddle
   class Configuration
     class DistributedIndex < Riddle::Configuration::Section
-      self.settings = [:type, :local, :agent, :agent_connect_timeout,
-        :agent_query_timeout]
+      self.settings = [:type, :local, :agent, :agent_blackhole,
+        :agent_connect_timeout, :agent_query_timeout]
       
-      attr_accessor :name, :local_indexes, :remote_indexes,
+      attr_accessor :name, :local_indexes, :remote_indexes, :agent_blackhole,
         :agent_connect_timeout, :agent_query_timeout
       
       def initialize(name)
-        @name           = name
-        @local_indexes  = []
-        @remote_indexes = []
+        @name             = name
+        @local_indexes    = []
+        @remote_indexes   = []
+        @agent_blackhole  = []
       end
       
       def type
@@ -29,7 +30,7 @@ module Riddle
           }.collect { |index| index.name }.join(",")
         }
       end
-      
+            
       def render
         raise ConfigurationError unless valid?
         

data/lib/riddle/configuration/index.rb

@@ -2,19 +2,23 @@ module Riddle
   class Configuration
     class Index < Riddle::Configuration::Section
       self.settings = [:source, :path, :docinfo, :mlock, :morphology,
-        :stopwords, :wordforms, :exceptions, :min_word_len, :charset_type,
-        :charset_table, :ignore_chars, :min_prefix_len, :min_infix_len,
-        :prefix_fields, :infix_fields, :enable_star, :ngram_len, :ngram_chars,
-        :phrase_boundary, :phrase_boundary_step, :html_strip,
-        :html_index_attrs, :html_remove_elements, :preopen]
+        :min_stemming_len, :stopwords, :wordforms, :exceptions, :min_word_len,
+        :charset_type, :charset_table, :ignore_chars, :min_prefix_len,
+        :min_infix_len, :prefix_fields, :infix_fields, :enable_star,
+        :ngram_len, :ngram_chars, :phrase_boundary, :phrase_boundary_step,
+        :html_strip, :html_index_attrs, :html_remove_elements, :preopen,
+        :ondisk_dict, :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap,
+        :inplace_reloc_factor, :inplace_write_factor, :index_exact_words]
       
       attr_accessor :name, :parent, :sources, :path, :docinfo, :mlock,
-        :morphologies, :stopword_files, :wordform_files, :exception_files,
-        :min_word_len, :charset_type, :charset_table, :ignore_characters,
-        :min_prefix_len, :min_infix_len, :prefix_field_names,
-        :infix_field_names, :enable_star, :ngram_len, :ngram_characters,
-        :phrase_boundaries, :phrase_boundary_step, :html_strip,
-        :html_index_attrs, :html_remove_element_tags, :preopen
+        :morphologies, :min_stemming_len, :stopword_files, :wordform_files,
+        :exception_files, :min_word_len, :charset_type, :charset_table,
+        :ignore_characters, :min_prefix_len, :min_infix_len,
+        :prefix_field_names, :infix_field_names, :enable_star, :ngram_len,
+        :ngram_characters, :phrase_boundaries, :phrase_boundary_step,
+        :html_strip, :html_index_attrs, :html_remove_element_tags, :preopen,
+        :ondisk_dict, :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap,
+        :inplace_reloc_factor, :inplace_write_factor, :index_exact_words
       
       def initialize(name, *sources)
         @name                     = name

data/lib/riddle/configuration/searchd.rb

@@ -1,9 +1,11 @@
 module Riddle
   class Configuration
     class Searchd < Riddle::Configuration::Section
-      self.settings = [:address, :port, :log, :query_log, :read_timeout,
-        :max_children, :pid_file, :max_matches, :seamless_rotate,
-        :preopen_indexes, :unlink_old]
+      self.settings = [:listen, :address, :port, :log, :query_log,
+        :read_timeout, :client_timeout, :max_children, :pid_file, :max_matches,
+        :seamless_rotate, :preopen_indexes, :unlink_old, :attr_flush_period,
+        :ondisk_dict_default, :max_packet_size, :mva_updates_pool,
+        :crash_log_path, :max_filters, :max_filter_values]
       
       attr_accessor *self.settings
             
@@ -22,4 +24,4 @@ module Riddle
       end
     end
   end
-end
+end

data/lib/riddle/configuration/sql_source.rb

@@ -3,10 +3,12 @@ module Riddle
     class SQLSource < Riddle::Configuration::Source
       self.settings = [:type, :sql_host, :sql_user, :sql_pass, :sql_db,
         :sql_port, :sql_sock, :mysql_connect_flags, :sql_query_pre, :sql_query,
-        :sql_query_range, :sql_range_step, :sql_attr_uint, :sql_attr_bool,
-        :sql_attr_timestamp, :sql_attr_str2ordinal, :sql_attr_float,
-        :sql_attr_multi, :sql_query_post, :sql_query_post_index,
-        :sql_ranged_throttle, :sql_query_info]
+        :sql_query_range, :sql_range_step, :sql_query_killlist, :sql_attr_uint,
+        :sql_attr_bool, :sql_attr_bigint, :sql_attr_timestamp,
+        :sql_attr_str2ordinal, :sql_attr_float, :sql_attr_multi,
+        :sql_query_post, :sql_query_post_index, :sql_ranged_throttle,
+        :sql_query_info, :mssql_winauth, :mssql_unicode, :unpack_zlib,
+        :unpack_mysqlcompress, :unpack_mysqlcompress_maxsize]
       
       attr_accessor *self.settings
       
@@ -17,12 +19,15 @@ module Riddle
         @sql_query_pre        = []
         @sql_attr_uint        = []
         @sql_attr_bool        = []
+        @sql_attr_bigint      = []
         @sql_attr_timestamp   = []
         @sql_attr_str2ordinal = []
         @sql_attr_float       = []
         @sql_attr_multi       = []
         @sql_query_post       = []
         @sql_query_post_index = []
+        @unpack_zlib          = []
+        @unpack_mysqlcompress = []
       end
             
       def valid?

data/lib/riddle/controller.rb

@@ -5,8 +5,10 @@ module Riddle
       @path           = path
     end
     
-    def index
-      cmd = "indexer --config #{@path} --all"
+    def index(*indexes)
+      indexes << '--all' if indexes.empty?
+      
+      cmd = "indexer --config #{@path} #{indexes.join(' ')}"
       cmd << " --rotate" if running?
       `#{cmd}`
     end
@@ -50,4 +52,4 @@ module Riddle
       false
     end
   end
-end
+end

data/spec/fixtures/data_generator.0.9.8.php

@@ -0,0 +1,208 @@
+<?php
+include 'sphinxapi.0.9.8.php';
+
+$client = new SphinxClient();
+$client->SetServer("localhost", 3312);
+
+// simple
+$file = fopen("spec/fixtures/data/simple.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+// index
+$file = fopen("spec/fixtures/data/index.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ", "edition")]);
+fclose($file);
+
+// any
+$client->SetMatchMode(SPH_MATCH_ANY);
+
+$file = fopen("spec/fixtures/data/any.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test this ")]);
+fclose($file);
+
+$client->SetMatchMode(SPH_MATCH_ALL);
+
+// sort
+$client->SetSortMode(SPH_SORT_EXTENDED, "id");
+
+$file = fopen("spec/fixtures/data/sort.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("testing ")]);
+fclose($file);
+
+$client->SetSortMode(SPH_SORT_RELEVANCE, "");
+
+// boolean
+$client->SetMatchMode(SPH_MATCH_BOOLEAN);
+
+$file = fopen("spec/fixtures/data/boolean.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetMatchMode(SPH_MATCH_ALL);
+
+// phrase
+$client->SetMatchMode(SPH_MATCH_PHRASE);
+
+$file = fopen("spec/fixtures/data/phrase.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("testing this ")]);
+fclose($file);
+
+$client->SetMatchMode(SPH_MATCH_ALL);
+
+// filter
+$client->SetFilter("id", array(10, 100, 1000));
+
+$file = fopen("spec/fixtures/data/filter.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetFilters();
+
+// group
+$client->SetGroupBy("id", SPH_GROUPBY_ATTR, "id");
+
+$file = fopen("spec/fixtures/data/group.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetGroupBy();
+
+// distinct
+$client->SetGroupDistinct("id");
+
+$file = fopen("spec/fixtures/data/distinct.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetGroupBy();
+
+// weights
+$client->SetWeights(array(100, 1));
+
+$file = fopen("spec/fixtures/data/weights.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetWeights(array());
+
+// anchor
+$client->SetGeoAnchor("latitude", "longitude", 10.0, 95.0);
+
+$file = fopen("spec/fixtures/data/anchor.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetFilters();
+
+// rank_mode
+$client->SetRankingMode(SPH_RANK_WORDCOUNT);
+
+$file = fopen("spec/fixtures/data/rank_mode.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetRankingMode(SPH_RANK_PROXIMITY_BM25);
+
+// index_weights
+$client->SetIndexWeights(array("people" => 101));
+
+$file = fopen("spec/fixtures/data/index_weights.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetIndexWeights(array());
+
+// index_weights
+$client->SetFieldWeights(array("city" => 101));
+
+$file = fopen("spec/fixtures/data/field_weights.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetFieldWeights(array());
+
+// comment
+$file = fopen("spec/fixtures/data/comment.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ", "*", "commenting")]);
+fclose($file);
+
+// update_simple
+$file = fopen("spec/fixtures/data/update_simple.bin", "w");
+fwrite($file, $client->UpdateAttributes("people", array("birthday"), array(1 => array(191163600))));
+fclose($file);
+
+// keywords_without_hits
+$file = fopen("spec/fixtures/data/keywords_without_hits.bin", "w");
+fwrite($file, $client->BuildKeywords("pat", "people", false));
+fclose($file);
+
+// keywords_with_hits
+$file = fopen("spec/fixtures/data/keywords_with_hits.bin", "w");
+fwrite($file, $client->BuildKeywords("pat", "people", true));
+fclose($file);
+
+// filter_array
+$client->SetFilter("field", array(1, 2, 3));
+
+$file = fopen("spec/fixtures/data/filter_array.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_array_exclude
+$client->SetFilter("field", array(1, 2, 3), true);
+
+$file = fopen("spec/fixtures/data/filter_array_exclude.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_range
+$client->SetFilterRange("field", 1, 3);
+
+$file = fopen("spec/fixtures/data/filter_range.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_range_exclude
+$client->SetFilterRange("field", 1, 3, true);
+
+$file = fopen("spec/fixtures/data/filter_range_exclude.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_floats
+$client->SetFilterFloatRange("field", 5.4, 13.5);
+
+$file = fopen("spec/fixtures/data/filter_floats.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_floats_exclude
+$client->SetFilterFloatRange("field", 5.4, 13.5, true);
+
+$file = fopen("spec/fixtures/data/filter_floats_exclude.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_boolean
+$client->SetFilter("field", array(0, 1));
+
+$file = fopen("spec/fixtures/data/filter_boolean.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+?>

data/spec/fixtures/data_generator.0.9.9.php

@@ -0,0 +1,225 @@
+<?php
+include 'sphinxapi.0.9.9.php';
+
+$client = new SphinxClient();
+$client->SetServer("localhost", 3312);
+
+// simple
+$file = fopen("spec/fixtures/data/simple.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+// index
+$file = fopen("spec/fixtures/data/index.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ", "edition")]);
+fclose($file);
+
+// any
+$client->SetMatchMode(SPH_MATCH_ANY);
+
+$file = fopen("spec/fixtures/data/any.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test this ")]);
+fclose($file);
+
+$client->SetMatchMode(SPH_MATCH_ALL);
+
+// sort
+$client->SetSortMode(SPH_SORT_EXTENDED, "id");
+
+$file = fopen("spec/fixtures/data/sort.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("testing ")]);
+fclose($file);
+
+$client->SetSortMode(SPH_SORT_RELEVANCE, "");
+
+// boolean
+$client->SetMatchMode(SPH_MATCH_BOOLEAN);
+
+$file = fopen("spec/fixtures/data/boolean.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetMatchMode(SPH_MATCH_ALL);
+
+// phrase
+$client->SetMatchMode(SPH_MATCH_PHRASE);
+
+$file = fopen("spec/fixtures/data/phrase.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("testing this ")]);
+fclose($file);
+
+$client->SetMatchMode(SPH_MATCH_ALL);
+
+// filter
+$client->SetFilter("id", array(10, 100, 1000));
+
+$file = fopen("spec/fixtures/data/filter.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetFilters();
+
+// group
+$client->SetGroupBy("id", SPH_GROUPBY_ATTR, "id");
+
+$file = fopen("spec/fixtures/data/group.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetGroupBy();
+
+// distinct
+$client->SetGroupDistinct("id");
+
+$file = fopen("spec/fixtures/data/distinct.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetGroupBy();
+
+// weights
+$client->SetWeights(array(100, 1));
+
+$file = fopen("spec/fixtures/data/weights.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetWeights(array());
+
+// anchor
+$client->SetGeoAnchor("latitude", "longitude", 10.0, 95.0);
+
+$file = fopen("spec/fixtures/data/anchor.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetFilters();
+
+// rank_mode
+$client->SetRankingMode(SPH_RANK_WORDCOUNT);
+
+$file = fopen("spec/fixtures/data/rank_mode.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetRankingMode(SPH_RANK_PROXIMITY_BM25);
+
+// index_weights
+$client->SetIndexWeights(array("people" => 101));
+
+$file = fopen("spec/fixtures/data/index_weights.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetIndexWeights(array());
+
+// index_weights
+$client->SetFieldWeights(array("city" => 101));
+
+$file = fopen("spec/fixtures/data/field_weights.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetFieldWeights(array());
+
+// comment
+$file = fopen("spec/fixtures/data/comment.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ", "*", "commenting")]);
+fclose($file);
+
+// update_simple
+$file = fopen("spec/fixtures/data/update_simple.bin", "w");
+fwrite($file, $client->UpdateAttributes("people", array("birthday"), array(1 => array(191163600))));
+fclose($file);
+
+// keywords_without_hits
+$file = fopen("spec/fixtures/data/keywords_without_hits.bin", "w");
+fwrite($file, $client->BuildKeywords("pat", "people", false));
+fclose($file);
+
+// keywords_with_hits
+$file = fopen("spec/fixtures/data/keywords_with_hits.bin", "w");
+fwrite($file, $client->BuildKeywords("pat", "people", true));
+fclose($file);
+
+// overrides
+$client->SetOverride("rating", SPH_ATTR_FLOAT, array(1 => 10.0));
+
+$file = fopen("spec/fixtures/data/overrides.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->ResetOverrides();
+
+// select
+$client->SetSelect("selecting");
+
+$file = fopen("spec/fixtures/data/select.bin", "w");
+fwrite($file, $client->_reqs[$client->AddQuery("test ")]);
+fclose($file);
+
+$client->SetSelect("*");
+
+// filter_array
+$client->SetFilter("field", array(1, 2, 3));
+
+$file = fopen("spec/fixtures/data/filter_array.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_array_exclude
+$client->SetFilter("field", array(1, 2, 3), true);
+
+$file = fopen("spec/fixtures/data/filter_array_exclude.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_range
+$client->SetFilterRange("field", 1, 3);
+
+$file = fopen("spec/fixtures/data/filter_range.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_range_exclude
+$client->SetFilterRange("field", 1, 3, true);
+
+$file = fopen("spec/fixtures/data/filter_range_exclude.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_floats
+$client->SetFilterFloatRange("field", 5.4, 13.5);
+
+$file = fopen("spec/fixtures/data/filter_floats.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_floats_exclude
+$client->SetFilterFloatRange("field", 5.4, 13.5, true);
+
+$file = fopen("spec/fixtures/data/filter_floats_exclude.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+
+// filter_boolean
+$client->SetFilter("field", array(0, 1));
+
+$file = fopen("spec/fixtures/data/filter_boolean.bin", "w");
+fwrite($file, $client->FilterOutput());
+fclose($file);
+
+$client->ResetFilters();
+?>