ultrasphinx 1.6 → 1.6.7

data/CHANGELOG

@@ -1,5 +1,9 @@
 
-vTODO. Use Pat Allan's association configurator.
+v1.6.7. Fix GROUP_CONCAT aggregate problem. Discourage enable_star in default.base. Allow faceting on includes.
+
+v1.6.6. Only use DISTINCT when necessary to improve indexing speed.
+
+v1.6.5. Many PostgreSQL improvements.
 
 v1.6. API changes! Drop Sphinx 0.9.7 compatibility; switch to Pat Allan's 0.9.8 client plugin; remove legacy keynames; fix string sorting bug; improve error handling.
 

data/Manifest

@@ -6,8 +6,13 @@ lib/ultrasphinx/autoload.rb
 lib/ultrasphinx/configure.rb
 lib/ultrasphinx/core_extensions.rb
 lib/ultrasphinx/fields.rb
-lib/ultrasphinx/hex_to_int.sql
 lib/ultrasphinx/is_indexed.rb
+lib/ultrasphinx/postgresql/concat_ws.sql
+lib/ultrasphinx/postgresql/crc32.sql
+lib/ultrasphinx/postgresql/group_concat.sql
+lib/ultrasphinx/postgresql/hex_to_int.sql
+lib/ultrasphinx/postgresql/language.sql
+lib/ultrasphinx/postgresql/unix_timestamp.sql
 lib/ultrasphinx/search/internals.rb
 lib/ultrasphinx/search/parser.rb
 lib/ultrasphinx/search.rb
@@ -124,12 +129,47 @@ test/integration/spell_test.rb
 test/setup.rb
 test/test_all.rb
 test/test_helper.rb
+test/ts.multi
 test/unit/parser_test.rb
 TODO
-vendor/riddle/MIT-LICENSE
-vendor/riddle/riddle/client/filter.rb
-vendor/riddle/riddle/client/message.rb
-vendor/riddle/riddle/client/response.rb
-vendor/riddle/riddle/client.rb
-vendor/riddle/riddle.rb
+vendor/riddle/lib/riddle/client/filter.rb
+vendor/riddle/lib/riddle/client/message.rb
+vendor/riddle/lib/riddle/client/response.rb
+vendor/riddle/lib/riddle/client.rb
+vendor/riddle/lib/riddle.rb
+vendor/riddle/MIT-LICENCE
+vendor/riddle/Rakefile
+vendor/riddle/README
+vendor/riddle/spec/fixtures/data/anchor.bin
+vendor/riddle/spec/fixtures/data/any.bin
+vendor/riddle/spec/fixtures/data/boolean.bin
+vendor/riddle/spec/fixtures/data/distinct.bin
+vendor/riddle/spec/fixtures/data/filter.bin
+vendor/riddle/spec/fixtures/data/filter_array.bin
+vendor/riddle/spec/fixtures/data/filter_array_exclude.bin
+vendor/riddle/spec/fixtures/data/filter_floats.bin
+vendor/riddle/spec/fixtures/data/filter_floats_exclude.bin
+vendor/riddle/spec/fixtures/data/filter_floats_range.bin
+vendor/riddle/spec/fixtures/data/filter_range.bin
+vendor/riddle/spec/fixtures/data/filter_range_exclude.bin
+vendor/riddle/spec/fixtures/data/group.bin
+vendor/riddle/spec/fixtures/data/index.bin
+vendor/riddle/spec/fixtures/data/phrase.bin
+vendor/riddle/spec/fixtures/data/simple.bin
+vendor/riddle/spec/fixtures/data/sort.bin
+vendor/riddle/spec/fixtures/data/update_simple.bin
+vendor/riddle/spec/fixtures/data/weights.bin
+vendor/riddle/spec/fixtures/sphinx/configuration.erb
+vendor/riddle/spec/fixtures/sql/conf.example.yml
+vendor/riddle/spec/fixtures/sql/data.sql
+vendor/riddle/spec/fixtures/sql/structure.sql
+vendor/riddle/spec/functional/excerpt_spec.rb
+vendor/riddle/spec/functional/search_spec.rb
+vendor/riddle/spec/functional/update_spec.rb
+vendor/riddle/spec/spec_helper.rb
+vendor/riddle/spec/sphinx_helper.rb
+vendor/riddle/spec/unit/client_spec.rb
+vendor/riddle/spec/unit/filter_spec.rb
+vendor/riddle/spec/unit/message_spec.rb
+vendor/riddle/spec/unit/response_spec.rb
 vendor/will_paginate/LICENSE

data/README

@@ -5,14 +5,14 @@ Ruby on Rails configurator and client to the Sphinx full text search engine.
 
 == License
 
-Copyright 2007 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file. Some portions copyright Pat Allan, distributed under the MIT license, and used with permission. Some portions copyright PJ Hyett and Mislav Marohnić, distributed under the MIT license, and used with permission.
+Copyright 2007 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file. Some portions copyright Pat Allan, distributed under the MIT license, and used with permission. Some portions copyright PJ Hyett and Mislav Marohnić, distributed under the MIT license, and used with permission. 
 
 The public certificate for the gem is at http://rubyforge.org/frs/download.php/25331/evan_weaver-original-public_cert.pem.
 
 == Requirements
 
 * MySQL (or Postgres, experimental)
-* Sphinx 0.9.8-dev r877 or greater
+* Sphinx 0.9.8-dev r871 or greater
 * Rails 1.2.3 or greater
 
 == Features
@@ -33,7 +33,7 @@ Good Rails integration:
 * <tt>will_paginate</tt> compatibility
 * query spellcheck
 * Google-style query parser
-* temporary error recovery
+* error recovery
 * multiple deployment environments
 * comprehensive Rake tasks
 
@@ -43,7 +43,7 @@ And some other things.
 
 == Installation
   
-First, compile and install Sphinx itself using the 0.9.8 development snapshot (http://www.sphinxsearch.com). 
+First, install Sphinx itself. Get the 0.9.8 development snapshot (http://www.sphinxsearch.com), then run <tt>./configure</tt>, <tt>make</tt>, and <tt>sudo make install</tt>. Make sure to set your <tt>./configure</tt> flags : <tt>--prefix</tt> if necessary, and also <tt>--with-pgsql</tt> if you need Postgres support.
 
 You also need the <tt>chronic</tt> gem:
   sudo gem install chronic

data/TODO

@@ -1,4 +1,5 @@
 
+* Use Pat Allan's association configurator, possibly with an API change to avoid the message-passing DSL
 * Finish unifying filters and textfields
 * Support exclude filters
 * Make sure filters can be set to nil

data/examples/default.base

@@ -39,7 +39,7 @@ client
 source
 {
   # Individual SQL source options
-  sql_range_step = 20000   
+  sql_range_step = 5000   
   strip_html = 0
   index_html_attrs =
   sql_query_post =
@@ -53,7 +53,11 @@ index
   morphology = stem_en
   stopwords = # /path/to/stopwords.txt
   min_word_len = 1
-  enable_star = 1
+  
+  # Enable these if you need wildcard searching. They will slow down indexing significantly.
+  #   min_infix_len = 1    
+  #   enable_star = 1 
+    
   charset_type = utf-8 # or sbcs (Single Byte Character Set)
   charset_table = 0..9, A..Z->a..z, -, _, ., &, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F,U+C5->U+E5, U+E5, U+C4->U+E4, U+E4, U+D6->U+F6, U+F6, U+16B, U+0c1->a, U+0c4->a, U+0c9->e, U+0cd->i, U+0d3->o, U+0d4->o, U+0da->u, U+0dd->y, U+0e1->a, U+0e4->a, U+0e9->e, U+0ed->i, U+0f3->o, U+0f4->o, U+0fa->u, U+0fd->y, U+104->U+105, U+105, U+106->U+107, U+10c->c, U+10d->c, U+10e->d, U+10f->d, U+116->U+117, U+117, U+118->U+119, U+11a->e, U+11b->e, U+12E->U+12F, U+12F, U+139->l, U+13a->l, U+13d->l, U+13e->l, U+141->U+142, U+142, U+143->U+144, U+144,U+147->n, U+148->n, U+154->r, U+155->r, U+158->r, U+159->r, U+15A->U+15B, U+15B, U+160->s, U+160->U+161, U+161->s, U+164->t, U+165->t, U+16A->U+16B, U+16B, U+16e->u, U+16f->u, U+172->U+173, U+173, U+179->U+17A, U+17A, U+17B->U+17C, U+17C, U+17d->z, U+17e->z,
 }

data/lib/ultrasphinx.rb

@@ -10,7 +10,7 @@ if defined? RAILS_ENV and RAILS_ENV == "development"
   end
 end
 
-$LOAD_PATH << "#{File.dirname(__FILE__)}/../vendor/riddle/"
+$LOAD_PATH << "#{File.dirname(__FILE__)}/../vendor/riddle/lib"
 require 'riddle'
 
 require 'ultrasphinx/ultrasphinx'

data/lib/ultrasphinx/configure.rb

@@ -72,7 +72,7 @@ module Ultrasphinx
       
       
       def setup_source_database(klass)
-        # Tentatively supporting Postgres now
+        # Supporting Postgres now
         connection_settings = klass.connection.instance_variable_get("@config")
 
         adapter_defaults = ADAPTER_DEFAULTS[ADAPTER]
@@ -95,7 +95,7 @@ module Ultrasphinx
           "(#{klass.table_name}.#{klass.primary_key} * #{MODEL_CONFIGURATION.size} + #{class_id}) AS id", 
           "#{class_id} AS class_id", "'#{klass.name}' AS class"]
         remaining_columns = fields.types.keys - ["class", "class_id"]        
-        [column_strings, [], condition_strings, remaining_columns]
+        [column_strings, [], condition_strings, [], false, remaining_columns]
       end
       
       
@@ -111,15 +111,21 @@ module Ultrasphinx
             
       def build_source(fields, model, options, class_id, klass, source, groups)
                 
-        column_strings, join_strings, condition_strings, remaining_columns = 
-          setup_source_arrays(klass, fields, class_id, options['conditions'])
+        column_strings, join_strings, condition_strings, group_bys, use_distinct, remaining_columns = 
+          setup_source_arrays(
+            klass, fields, class_id, options['conditions'])
 
-        column_strings, join_strings, remaining_columns = 
-          build_regular_fields(klass, fields, options['fields'], column_strings, join_strings, remaining_columns)
-        column_strings, join_strings, remaining_columns = 
-          build_includes(klass, fields, options['include'], column_strings, join_strings, remaining_columns)
-        column_strings, join_strings, remaining_columns = 
-          build_concatenations(klass, fields, options['concatenate'], column_strings, join_strings, remaining_columns)
+        column_strings, join_strings, group_bys, remaining_columns = 
+          build_regular_fields(
+            klass, fields, options['fields'], column_strings, join_strings, group_bys, remaining_columns)
+            
+        column_strings, join_strings, group_bys, remaining_columns = 
+          build_includes(
+            klass, fields, options['include'], column_strings, join_strings, group_bys, remaining_columns)
+            
+        column_strings, join_strings, group_bys, use_distinct, remaining_columns = 
+          build_concatenations(
+            klass, fields, options['concatenate'], column_strings, join_strings, group_bys, use_distinct, remaining_columns)
         
         column_strings = add_missing_columns(fields, remaining_columns, column_strings)
        
@@ -128,26 +134,37 @@ module Ultrasphinx
           SOURCE_SETTINGS._to_conf_string,
           setup_source_database(klass),
           range_select_string(klass),
-          build_query(klass, column_strings, join_strings, condition_strings),
+          build_query(klass, column_strings, join_strings, condition_strings, use_distinct, group_bys),
           "\n" + groups,
           query_info_string(klass, class_id),
           "}\n\n"]
       end
       
       
-      def build_query(klass, column_strings, join_strings, condition_strings)
+      def build_query(klass, column_strings, join_strings, condition_strings, use_distinct, group_bys)
+        
+        primary_key = "#{klass.table_name}.#{klass.primary_key}"
+        group_bys = case ADAPTER
+          when 'mysql'
+            primary_key
+          when 'postgresql'
+            # Postgres is very fussy about GROUP_BY 
+            ([primary_key] + group_bys.reject {|s| s == primary_key}.uniq.sort).join(', ')
+          end
+        
         ["sql_query =", 
-          "SELECT", 
+          "SELECT",
+          # Avoid DISTINCT; it destroys performance
           column_strings.sort_by do |string| 
-            # sphinx wants them always in the same order, but "id" must be first
+            # Sphinx wants them always in the same order, but "id" must be first
             (field = string[/.*AS (.*)/, 1]) == "id" ? "*" : field
           end.join(", "),
           "FROM #{klass.table_name}",
           join_strings.uniq,
-          "WHERE #{klass.table_name}.#{klass.primary_key} >= $start AND #{klass.table_name}.#{klass.primary_key} <= $end",
+          "WHERE #{primary_key} >= $start AND #{primary_key} <= $end",
           condition_strings.uniq.map {|condition| "AND #{condition}" },
-          ADAPTER_SQL_FUNCTIONS[ADAPTER]['group_by']
-        ].flatten.join(" ")
+          "GROUP BY #{group_bys}"
+        ].flatten.compact.join(" ")
       end
       
       
@@ -159,17 +176,18 @@ module Ultrasphinx
       end
       
 
-      def build_regular_fields(klass, fields, entries, column_strings, join_strings, remaining_columns)          
+      def build_regular_fields(klass, fields, entries, column_strings, join_strings, group_bys, remaining_columns)          
         entries.to_a.each do |entry|
           source_string = "#{entry['table']}.#{entry['field']}"
+          group_bys << source_string
           column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)
         end
         
-        [column_strings, join_strings, remaining_columns]
+        [column_strings, join_strings, group_bys, remaining_columns]
       end
       
 
-      def build_includes(klass, fields, entries, column_strings, join_strings, remaining_columns)                  
+      def build_includes(klass, fields, entries, column_strings, join_strings, group_bys, remaining_columns)                  
         entries.to_a.each do |entry|
         
           join_klass = entry['class_name'].constantize
@@ -189,18 +207,19 @@ module Ultrasphinx
           end
           
           source_string = "#{entry['table']}.#{entry['field']}"
+          group_bys << source_string
           column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)                         
         end
         
-        [column_strings, join_strings, remaining_columns]
+        [column_strings, join_strings, group_bys, remaining_columns]
       end
       
         
-      def build_concatenations(klass, fields, entries, column_strings, join_strings, remaining_columns)
+      def build_concatenations(klass, fields, entries, column_strings, join_strings, group_bys, use_distinct, remaining_columns)
         entries.to_a.each do |entry|
           if entry['class_name'] and entry['field']
-            # group concats
-            # only has_many's or explicit sql right now
+            # Group concats
+            # Only has_many's or explicit sql right now
             join_klass = entry['class_name'].constantize
         
             join_strings = install_join_unless_association_sql(entry['association_sql'], nil, join_strings) do 
@@ -210,13 +229,19 @@ module Ultrasphinx
                 (entry['conditions'] ? " AND (#{entry['conditions']})" : "")
             end
             
-            source_string = "GROUP_CONCAT(DISTINCT #{entry['table']}.#{entry['field']} SEPARATOR ' ')"
+            source_string = "#{entry['table']}.#{entry['field']}"
+            # We are using the field in an aggregate, so we don't want to add it to group_bys
+            source_string = ADAPTER_SQL_FUNCTIONS[ADAPTER]['group_concat']._interpolate(source_string)
+            use_distinct = true
+            
             column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)
             
           elsif entry['fields']
-            # regular concats
+            # Regular concats
             source_string = "CONCAT_WS(' ', " + entry['fields'].map do |subfield| 
               "#{entry['table']}.#{subfield}"
+            end.each do |subsource_string|
+              group_bys << subsource_string
             end.join(', ') + ")"
             
             column_strings, remaining_columns = install_field(fields, source_string, entry['as'], entry['function_sql'], entry['facet'], column_strings, remaining_columns)              
@@ -226,7 +251,7 @@ module Ultrasphinx
           end
         end
         
-        [column_strings, join_strings, remaining_columns]
+        [column_strings, join_strings, group_bys, use_distinct, remaining_columns]
       end
       
     
@@ -249,7 +274,7 @@ module Ultrasphinx
         
         # Generate hashed integer fields for text grouping
         if with_facet
-          column_strings << "#{ADAPTER_SQL_FUNCTIONS[ADAPTER]['hash']._interpolate(source_string)} AS #{as}_facet"
+          column_strings << "CRC32(#{source_string}) AS #{as}_facet"
           remaining_columns.delete("#{as}_facet")
         end
         [column_strings, remaining_columns]

data/lib/ultrasphinx/fields.rb

@@ -60,11 +60,9 @@ This is a special singleton configuration class that stores the index field conf
     
     def cast(source_string, field)
       if types[field] == "date"
-        "#{ADAPTER_SQL_FUNCTIONS[ADAPTER]['timestamp']._interpolate(source_string)}"
+        "UNIX_TIMESTAMP(#{source_string})"
       elsif types[field] == "integer"
         source_string # "CAST(#{source_string} AS UNSIGNED)"
-      elsif source_string =~ /GROUP_CONCAT/
-        "CAST(#{source_string} AS CHAR)"
       else
         source_string              
       end + " AS #{field}"
@@ -77,7 +75,7 @@ This is a special singleton configuration class that stores the index field conf
         when 'integer', 'float'
           "0"
         when 'date'
-          "UNIX_TIMESTAMP('1970-01-01 00:00:00')"
+          "18000" # Midnight on 1/1/1970
         when nil
           raise "Field #{field} is missing"
         else
@@ -104,17 +102,12 @@ This is a special singleton configuration class that stores the index field conf
             extract_field_alias!(entry, klass)
             
             unless klass.columns_hash[entry['field']]
+              # XXX I think this is here for migrations
               Ultrasphinx.say "warning: field #{entry['field']} is not present in #{model}"
             else
               save_and_verify_type(entry['as'], klass.columns_hash[entry['field']].type, entry['sortable'], klass)
-            end
-            
-            if entry['facet']
-              save_and_verify_type(entry['as'], 'text', nil, klass) # source must be a string
-              save_and_verify_type("#{entry['as']}_facet", 'integer', nil, klass)
-            end
-            
-            entry
+              install_facets!(entry, klass)
+            end            
           end  
           
           # Joins are whatever they are in the target       
@@ -122,13 +115,15 @@ This is a special singleton configuration class that stores the index field conf
             extract_table_alias!(entry, klass)
             extract_field_alias!(entry, klass)
 
-            save_and_verify_type(entry['as'] || entry['field'], entry['class_name'].constantize.columns_hash[entry['field']].type, entry['sortable'], klass)
+            save_and_verify_type(entry['as'] || entry['field'], entry['class_name'].constantize.columns_hash[entry['field']].type, entry['sortable'], klass)            
+            install_facets!(entry, klass)
           end  
           
           # Regular concats are CHAR, group_concats are BLOB and need to be cast to CHAR
           options['concatenate'].to_a.each do |entry|
             extract_table_alias!(entry, klass) # XXX Doesn't actually do anything useful
             save_and_verify_type(entry['as'], 'text', entry['sortable'], klass) 
+            install_facets!(entry, klass)
           end          
           
         rescue ActiveRecord::StatementInvalid
@@ -139,6 +134,14 @@ This is a special singleton configuration class that stores the index field conf
       self
     end
     
+    def install_facets!(entry, klass)
+      if entry['facet']
+        save_and_verify_type(entry['as'], 'text', nil, klass) # source must be a string
+        save_and_verify_type("#{entry['as']}_facet", 'integer', nil, klass)
+      end
+      entry
+    end
+    
     def extract_field_alias!(entry, klass)
       unless entry['as']    
         entry['as'] = entry['field'] 

data/lib/ultrasphinx/postgresql/concat_ws.sql

@@ -0,0 +1,35 @@
+
+/* http://osdir.com/ml/db.postgresql.admIN/2003-08/msg00057.html */
+
+CREATE OR REPLACE FUNCTION MAKE_CONCAT_WS() RETURNS text AS '
+declare
+  v_args int := 32;
+  v_first text := ''CREATE OR REPLACE FUNCTION CONCAT_WS(text,text,text) RETURNS text AS ''''SELECT CASE WHEN $1 IS NULL THEN NULL WHEN $3 IS NULL THEN $2 ELSE $2 || $1 || $3 END'''' LANGUAGE sql IMMUTABLE'';
+  v_part1 text := ''CREATE OR REPLACE FUNCTION CONCAT_WS(text,text'';
+  v_part2 text := '') RETURNS text AS ''''SELECT CONCAT_WS($1,CONCAT_WS($1,$2'';
+  v_part3 text := '')'''' LANGUAGE sql IMMUTABLE'';  
+  v_sql text;
+  
+BEGIN
+  EXECUTE v_first;
+  FOR i IN 4 .. v_args loop
+    v_sql := v_part1;
+    FOR j IN 3 .. i loop
+      v_sql := v_sql || '',text'';
+    END loop;
+
+    v_sql := v_sql || v_part2;
+
+    FOR j IN 3 .. i - 1 loop
+      v_sql := v_sql || '',$'' || j::text;
+    END loop;
+    v_sql := v_sql || ''),$'' || i::text;
+
+    v_sql := v_sql || v_part3;
+    EXECUTE v_sql;
+  END loop;
+  RETURN ''OK'';
+END;
+' LANGUAGE 'plpgsql';
+
+SELECT MAKE_CONCAT_WS();

data/lib/ultrasphinx/postgresql/crc32.sql

@@ -0,0 +1,7 @@
+
+/* Fake CRC32 */
+
+CREATE OR REPLACE FUNCTION crc32(text)
+RETURNS int AS $$
+  SELECT hex_to_int(SUBSTRING(MD5($1) FROM 1 FOR 8))::int
+$$ VOLATILE LANGUAGE SQL;