riddle 1.5.3 → 1.5.4

data/HISTORY

@@ -1,3 +1,12 @@
+1.5.4 - January 2nd 2013
+- RT indices get most of the same settings as SQL indices.
+- Escape single quotes in SphinxQL match queries, given we're wrapping them in single quotes.
+- Remove unnecessary characters from string values for SphinxQL inserts.
+- Convert time objects to integers for SphinxQL inserts.
+- Include 'orphan' sources (which aren't used directly by indices, but could be parents of sources that are used) when generating configuration files.
+- Use parent source type if child source has no type supplied.
+- Ignore comments when parsing Sphinx configuration files.
+
 1.5.3 - August 10th 2012
 - Sphinx 2.0.5 support.
 - :with_all and :without_all support for SphinxQL.

data/README.textile

@@ -1,6 +1,6 @@
 h1. Riddle
 
-"!https://secure.travis-ci.org/freelancing-god/riddle.png!":http://travis-ci.org/freelancing-god/riddle
+"!https://secure.travis-ci.org/pat/riddle.png!":http://travis-ci.org/pat/riddle
 
 This client has been written to interface with "Sphinx":http://sphinxsearch.com/. It is written by "Pat Allan":http://freelancing-gods.com, and has been influenced by both Dmytro Shteflyuk's Ruby client and the original PHP client - credit where credit's due, after all.
 

data/lib/riddle/configuration.rb

@@ -1,4 +1,5 @@
 require 'riddle/configuration/section'
+require 'riddle/configuration/index_settings'
 
 require 'riddle/configuration/distributed_index'
 require 'riddle/configuration/index'
@@ -17,7 +18,7 @@ module Riddle
     class ConfigurationError < StandardError #:nodoc:
     end
 
-    attr_reader :indices, :searchd
+    attr_reader :indices, :searchd, :sources
     attr_accessor :indexer
 
     def self.parse!(input)
@@ -28,11 +29,13 @@ module Riddle
       @indexer = Riddle::Configuration::Indexer.new
       @searchd = Riddle::Configuration::Searchd.new
       @indices = []
+      @sources = []
     end
 
     def render
       (
         [@indexer.render, @searchd.render] +
+        @sources.collect { |source| source.render } +
         @indices.collect { |index| index.render }
       ).join("\n")
     end

data/lib/riddle/configuration/index.rb

@@ -1,142 +1,25 @@
 module Riddle
   class Configuration
     class Index < Riddle::Configuration::Section
+      include Riddle::Configuration::IndexSettings
+
       def self.settings
-        [
-          :type, :source, :path, :docinfo, :mlock, :morphology,
-          :dict, :index_sp, :index_zones, :min_stemming_len, :stopwords,
-          :wordforms, :exceptions, :min_word_len, :charset_dictpath,
-          :charset_type, :charset_table, :ignore_chars, :min_prefix_len,
-          :min_infix_len, :prefix_fields, :infix_fields, :enable_star,
-          :expand_keywords, :ngram_len, :ngram_chars, :phrase_boundary,
-          :phrase_boundary_step, :blend_chars, :blend_mode, :html_strip,
-          :html_index_attrs, :html_remove_elements, :preopen, :ondisk_dict,
-          :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap,
-          :inplace_reloc_factor, :inplace_write_factor, :index_exact_words,
-          :overshort_step, :stopwords_step, :hitless_words
-        ]
+        Riddle::Configuration::IndexSettings.settings + [:source]
       end
 
-      attr_accessor :name, :parent, :type, :sources, :path, :docinfo, :mlock,
-        :morphologies, :dict, :index_sp, :index_zones, :min_stemming_len,
-        :stopword_files, :wordform_files, :exception_files, :min_word_len,
-        :charset_dictpath, :charset_type, :charset_table, :ignore_characters,
-        :min_prefix_len, :min_infix_len, :prefix_field_names,
-        :infix_field_names, :enable_star, :expand_keywords, :ngram_len,
-        :ngram_characters, :phrase_boundaries, :phrase_boundary_step,
-        :blend_chars, :blend_mode, :html_strip, :html_index_attrs,
-        :html_remove_element_tags, :preopen, :ondisk_dict, :inplace_enable,
-        :inplace_hit_gap, :inplace_docinfo_gap, :inplace_reloc_factor,
-        :inplace_write_factor, :index_exact_words, :overshort_step,
-        :stopwords_step, :hitless_words
+      attr_accessor :parent, :sources
 
       def initialize(name, *sources)
         @name                     = name
         @sources                  = sources
-        @morphologies             = []
-        @stopword_files           = []
-        @wordform_files           = []
-        @exception_files          = []
-        @ignore_characters        = []
-        @prefix_field_names       = []
-        @infix_field_names        = []
-        @ngram_characters         = []
-        @phrase_boundaries        = []
-        @html_remove_element_tags = []
+
+        initialize_settings
       end
 
       def source
         @sources.collect { |s| s.name }
       end
 
-      def morphology
-        nil_join @morphologies, ", "
-      end
-
-      def morphology=(morphology)
-        @morphologies = nil_split morphology, /,\s?/
-      end
-
-      def stopwords
-        nil_join @stopword_files, " "
-      end
-
-      def stopwords=(stopwords)
-        @stopword_files = nil_split stopwords, ' '
-      end
-
-      def wordforms
-        nil_join @wordform_files, " "
-      end
-
-      def wordforms=(wordforms)
-        @wordform_files = nil_split wordforms, ' '
-      end
-
-      def exceptions
-        nil_join @exception_files, " "
-      end
-
-      def exceptions=(exceptions)
-        @exception_files = nil_split exceptions, ' '
-      end
-
-      def ignore_chars
-        nil_join @ignore_characters, ", "
-      end
-
-      def ignore_chars=(ignore_chars)
-        @ignore_characters = nil_split ignore_chars, /,\s?/
-      end
-
-      def prefix_fields
-        nil_join @prefix_field_names, ", "
-      end
-
-      def prefix_fields=(fields)
-        if fields.is_a?(Array)
-          @prefix_field_names = fields
-        else
-          @prefix_field_names = fields.split(/,\s*/)
-        end
-      end
-
-      def infix_fields
-        nil_join @infix_field_names, ", "
-      end
-
-      def infix_fields=(fields)
-        if fields.is_a?(Array)
-          @infix_field_names = fields
-        else
-          @infix_field_names = fields.split(/,\s*/)
-        end
-      end
-
-      def ngram_chars
-        nil_join @ngram_characters, ", "
-      end
-
-      def ngram_chars=(ngram_chars)
-        @ngram_characters = nil_split ngram_chars, /,\s?/
-      end
-
-      def phrase_boundary
-        nil_join @phrase_boundaries, ", "
-      end
-
-      def phrase_boundary=(phrase_boundary)
-        @phrase_boundaries = nil_split phrase_boundary, /,\s?/
-      end
-
-      def html_remove_elements
-        nil_join @html_remove_element_tags, ", "
-      end
-
-      def html_remove_elements=(html_remove_elements)
-        @html_remove_element_tags = nil_split html_remove_elements, /,\s?/
-      end
-
       def render
         raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid?
 
@@ -153,20 +36,6 @@ module Riddle
       def valid?
         (!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?)
       end
-
-      private
-
-      def nil_split(string, pattern)
-        (string || "").split(pattern)
-      end
-
-      def nil_join(array, delimiter)
-        if array.length == 0
-          nil
-        else
-          array.join(delimiter)
-        end
-      end
     end
   end
 end

data/lib/riddle/configuration/index_settings.rb

@@ -0,0 +1,149 @@
+module Riddle
+  class Configuration
+    module IndexSettings
+      def self.settings
+        [
+          :type, :path, :docinfo, :mlock, :morphology,
+          :dict, :index_sp, :index_zones, :min_stemming_len, :stopwords,
+          :wordforms, :exceptions, :min_word_len, :charset_dictpath,
+          :charset_type, :charset_table, :ignore_chars, :min_prefix_len,
+          :min_infix_len, :prefix_fields, :infix_fields, :enable_star,
+          :expand_keywords, :ngram_len, :ngram_chars, :phrase_boundary,
+          :phrase_boundary_step, :blend_chars, :blend_mode, :html_strip,
+          :html_index_attrs, :html_remove_elements, :preopen, :ondisk_dict,
+          :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap,
+          :inplace_reloc_factor, :inplace_write_factor, :index_exact_words,
+          :overshort_step, :stopwords_step, :hitless_words
+        ]
+      end
+
+      attr_accessor :name, :type, :path, :docinfo, :mlock,
+        :morphologies, :dict, :index_sp, :index_zones, :min_stemming_len,
+        :stopword_files, :wordform_files, :exception_files, :min_word_len,
+        :charset_dictpath, :charset_type, :charset_table, :ignore_characters,
+        :min_prefix_len, :min_infix_len, :prefix_field_names,
+        :infix_field_names, :enable_star, :expand_keywords, :ngram_len,
+        :ngram_characters, :phrase_boundaries, :phrase_boundary_step,
+        :blend_chars, :blend_mode, :html_strip, :html_index_attrs,
+        :html_remove_element_tags, :preopen, :ondisk_dict, :inplace_enable,
+        :inplace_hit_gap, :inplace_docinfo_gap, :inplace_reloc_factor,
+        :inplace_write_factor, :index_exact_words, :overshort_step,
+        :stopwords_step, :hitless_words
+
+      def initialize_settings
+        @morphologies             = []
+        @stopword_files           = []
+        @wordform_files           = []
+        @exception_files          = []
+        @ignore_characters        = []
+        @prefix_field_names       = []
+        @infix_field_names        = []
+        @ngram_characters         = []
+        @phrase_boundaries        = []
+        @html_remove_element_tags = []
+      end
+
+      def morphology
+        nil_join @morphologies, ", "
+      end
+
+      def morphology=(morphology)
+        @morphologies = nil_split morphology, /,\s?/
+      end
+
+      def stopwords
+        nil_join @stopword_files, " "
+      end
+
+      def stopwords=(stopwords)
+        @stopword_files = nil_split stopwords, ' '
+      end
+
+      def wordforms
+        nil_join @wordform_files, " "
+      end
+
+      def wordforms=(wordforms)
+        @wordform_files = nil_split wordforms, ' '
+      end
+
+      def exceptions
+        nil_join @exception_files, " "
+      end
+
+      def exceptions=(exceptions)
+        @exception_files = nil_split exceptions, ' '
+      end
+
+      def ignore_chars
+        nil_join @ignore_characters, ", "
+      end
+
+      def ignore_chars=(ignore_chars)
+        @ignore_characters = nil_split ignore_chars, /,\s?/
+      end
+
+      def prefix_fields
+        nil_join @prefix_field_names, ", "
+      end
+
+      def prefix_fields=(fields)
+        if fields.is_a?(Array)
+          @prefix_field_names = fields
+        else
+          @prefix_field_names = fields.split(/,\s*/)
+        end
+      end
+
+      def infix_fields
+        nil_join @infix_field_names, ", "
+      end
+
+      def infix_fields=(fields)
+        if fields.is_a?(Array)
+          @infix_field_names = fields
+        else
+          @infix_field_names = fields.split(/,\s*/)
+        end
+      end
+
+      def ngram_chars
+        nil_join @ngram_characters, ", "
+      end
+
+      def ngram_chars=(ngram_chars)
+        @ngram_characters = nil_split ngram_chars, /,\s?/
+      end
+
+      def phrase_boundary
+        nil_join @phrase_boundaries, ", "
+      end
+
+      def phrase_boundary=(phrase_boundary)
+        @phrase_boundaries = nil_split phrase_boundary, /,\s?/
+      end
+
+      def html_remove_elements
+        nil_join @html_remove_element_tags, ", "
+      end
+
+      def html_remove_elements=(html_remove_elements)
+        @html_remove_element_tags = nil_split html_remove_elements, /,\s?/
+      end
+
+      private
+
+      def nil_split(string, pattern)
+        (string || "").split(pattern)
+      end
+
+      def nil_join(array, delimiter)
+        if array.length == 0
+          nil
+        else
+          array.join(delimiter)
+        end
+      end
+    end
+  end
+end

data/lib/riddle/configuration/parser.rb

@@ -27,11 +27,24 @@ class Riddle::Configuration::Parser
     set_sources
     set_indices
 
+    add_orphan_sources
+
     configuration
   end
 
   private
 
+  def add_orphan_sources
+    all_names      = sources.keys
+    attached_names = configuration.indices.collect { |index|
+      index.respond_to?(:sources) ? index.sources.collect(&:name) : []
+    }.flatten
+
+    (all_names - attached_names).each do |name|
+      configuration.sources << sources[name]
+    end
+  end
+
   def inner
     @inner ||= InnerParser.new(@input).parse!
   end
@@ -46,7 +59,7 @@ class Riddle::Configuration::Parser
 
   def each_with_prefix(prefix)
     inner.keys.select { |key| key[/^#{prefix}\s+/] }.each do |key|
-      yield key.gsub(/^#{prefix}\s+/, ''), inner[key]
+      yield key.gsub(/^#{prefix}\s+/, '').gsub(/\s*{$/, ''), inner[key]
     end
   end
 
@@ -60,10 +73,14 @@ class Riddle::Configuration::Parser
 
   def set_sources
     each_with_prefix 'source' do |name, settings|
-      names         = name.split(/\s*:\s*/)
-      type          = settings.delete('type').first
+      names   = name.split(/\s*:\s*/)
+      types   = settings.delete('type')
+      parent  = names.length > 1 ? names.last : nil
+      types ||= [sources[parent].type] if parent
+      type    = types.first
+
       source        = SOURCE_CLASSES[type].new names.first, type
-      source.parent = names.last if names.length > 1
+      source.parent = parent
 
       set_settings source, settings
 
@@ -139,11 +156,14 @@ class Riddle::Configuration::Parser
 
       while line != '}' do
         begin
-          key, value = *SETTING_PATTERN.match(line).captures
-          settings[key] << value
-          while value[/\\$/] do
-            value = next_line
-            settings[key].last << "\n" << value
+          match = SETTING_PATTERN.match(line)
+          unless match.nil?
+            key, value = *match.captures
+            settings[key] << value
+            while value[/\\$/] do
+              value = next_line
+              settings[key].last << "\n" << value
+            end
           end
         rescue => error
           raise error, "Error handling line '#{line}': #{error.message}",

data/lib/riddle/configuration/realtime_index.rb

@@ -1,16 +1,18 @@
 module Riddle
   class Configuration
     class RealtimeIndex < Riddle::Configuration::Section
+      include Riddle::Configuration::IndexSettings
+
       def self.settings
-        [
-          :type, :path, :rt_mem_limit, :rt_field, :rt_attr_uint,
-          :rt_attr_bigint, :rt_attr_float, :rt_attr_timestamp, :rt_attr_string
+        Riddle::Configuration::IndexSettings.settings + [
+          :rt_mem_limit, :rt_field, :rt_attr_uint, :rt_attr_bigint,
+          :rt_attr_float, :rt_attr_timestamp, :rt_attr_string
         ]
       end
-      
-      attr_accessor :name
-      attr_accessor *self.settings
-      
+
+      attr_accessor :rt_mem_limit, :rt_field, :rt_attr_uint, :rt_attr_bigint,
+        :rt_attr_float, :rt_attr_timestamp, :rt_attr_string
+
       def initialize(name)
         @name               = name
         @rt_field           = []
@@ -19,19 +21,21 @@ module Riddle
         @rt_attr_float      = []
         @rt_attr_timestamp  = []
         @rt_attr_string     = []
+
+        initialize_settings
       end
-      
+
       def type
         "rt"
       end
-      
+
       def valid?
         !(@name.nil? || @path.nil?)
       end
-      
+
       def render
         raise ConfigurationError unless valid?
-        
+
         (
           ["index #{name}", "{"] +
           settings_body +

data/lib/riddle/query/insert.rb

@@ -38,9 +38,11 @@ class Riddle::Query::Insert
   def translated_value(value)
     case value
     when String
-      "'#{value}'"
+      "'#{value.gsub(/['\\]/, '').gsub(/\s+/, ' ')}'"
     when TrueClass, FalseClass
       value ? 1 : 0
+    when Time
+      value.to_i
     else
       value
     end

data/lib/riddle/query/select.rb

@@ -26,7 +26,7 @@ class Riddle::Query::Select
   end
 
   def matching(match)
-    @matching = match
+    @matching = match.gsub("'") { |x| "\\'" }
     self
   end
 

data/lib/riddle/version.rb

@@ -1,3 +1,3 @@
 module Riddle
-  Version = '1.5.3'
+  Version = '1.5.4'
 end

data/riddle.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.platform    = Gem::Platform::RUBY
   s.authors     = ['Pat Allan']
   s.email       = ['pat@freelancing-gods.com']
-  s.homepage    = 'http://freelancing-god.github.com/riddle/'
+  s.homepage    = 'http://pat.github.com/riddle/'
   s.summary     = %q{An API for Sphinx, written in and for Ruby.}
   s.description = %q{A Ruby API and configuration helper for the Sphinx search service.}
 

data/spec/riddle/query/select_spec.rb

@@ -26,6 +26,11 @@ describe Riddle::Query::Select do
     query.from('foo_core').matching('foo').to_sql.
       should == "SELECT * FROM foo_core WHERE MATCH('foo')"
   end
+  
+  it "escapes single quotes in the search terms" do
+    query.from('foo_core').matching("fo'o").to_sql.
+      should == "SELECT * FROM foo_core WHERE MATCH('fo\\'o')"
+  end
 
   it 'handles filters with integers' do
     query.from('foo_core').matching('foo').where(:bar_id => 10).to_sql.

data/spec/unit/configuration/index_spec.rb

@@ -4,44 +4,44 @@ describe Riddle::Configuration::DistributedIndex do
   it "should be invalid without a name, sources or path if there's no parent" do
     index = Riddle::Configuration::Index.new(nil)
     index.should_not be_valid
-    
+
     index.name = "test1"
     index.should_not be_valid
-    
+
     index.sources << Riddle::Configuration::SQLSource.new("source", "mysql")
     index.should_not be_valid
-    
+
     index.path = "a/path"
     index.should be_valid
-    
+
     index.name = nil
     index.should_not be_valid
-    
+
     index.name = "test1"
     index.sources.clear
     index.should_not be_valid
   end
-  
+
   it "should be invalid without a name but not sources or path if it has a parent" do
     index = Riddle::Configuration::Index.new(nil)
     index.should_not be_valid
-    
+
     index.name = "test1stemmed"
     index.should_not be_valid
-    
+
     index.parent = "test1"
     index.should be_valid
   end
-  
+
   it "should raise a ConfigurationError if rendering when not valid" do
     index = Riddle::Configuration::Index.new("test1")
     lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError)
   end
-  
+
   it "should render correctly if supplied settings are valid" do
     source = Riddle::Configuration::XMLSource.new("src1", "xmlpipe")
     source.xmlpipe_command = "ls /dev/null"
-    
+
     index = Riddle::Configuration::Index.new("test1", source)
     index.path                      = "/var/data/test1"
     index.docinfo                   = "extern"
@@ -75,7 +75,7 @@ describe Riddle::Configuration::DistributedIndex do
     index.inplace_reloc_factor      = 0.1
     index.inplace_write_factor      = 0.1
     index.index_exact_words         = 1
-    
+
     index.render.should == <<-INDEX
 source src1
 {
@@ -85,7 +85,6 @@ source src1
 
 index test1
 {
-  source = src1
   path = /var/data/test1
   docinfo = extern
   mlock = 0
@@ -118,6 +117,7 @@ index test1
   inplace_reloc_factor = 0.1
   inplace_write_factor = 0.1
   index_exact_words = 1
+  source = src1
 }
     INDEX
   end

data/spec/unit/configuration/realtime_index_spec.rb

@@ -2,49 +2,49 @@ require 'spec_helper'
 
 describe Riddle::Configuration::RealtimeIndex do
   let(:index) { Riddle::Configuration::RealtimeIndex.new('rt1') }
-  
+
   describe '#valid?' do
     it "should not be valid without a name" do
       index.name = nil
       index.path = 'foo'
       index.should_not be_valid
     end
-    
+
     it "should not be valid without a path" do
       index.path = nil
       index.should_not be_valid
     end
-    
+
     it "should be valid with a name and path" do
       index.path = 'foo'
       index.should be_valid
     end
   end
-  
+
   describe '#type' do
     it "should be 'rt'" do
       index.type.should == 'rt'
     end
   end
-  
+
   describe '#render' do
     it "should raise a ConfigurationError if rendering when not valid" do
       lambda {
         index.render
       }.should raise_error(Riddle::Configuration::ConfigurationError)
     end
-  
+
     it "should render correctly if supplied settings are valid" do
       index.path = '/var/data/rt'
       index.rt_mem_limit = '512M'
       index.rt_field << 'title' << 'content'
-      
+
       index.rt_attr_uint      << 'gid'
       index.rt_attr_bigint    << 'guid'
       index.rt_attr_float     << 'gpa'
       index.rt_attr_timestamp << 'ts_added'
       index.rt_attr_string    << 'author'
-          
+
       index.render.should == <<-RTINDEX
 index rt1
 {

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: riddle
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 11
   prerelease: 
   segments: 
   - 1
   - 5
-  - 3
-  version: 1.5.3
+  - 4
+  version: 1.5.4
 platform: ruby
 authors: 
 - Pat Allan
@@ -15,10 +15,10 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-08-10 00:00:00 Z
+date: 2013-01-02 00:00:00 +11:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  type: :development
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -31,10 +31,10 @@ dependencies:
         - 2
         version: 0.9.2
   version_requirements: *id001
-  prerelease: false
   name: rake
-- !ruby/object:Gem::Dependency 
+  prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency 
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -47,10 +47,10 @@ dependencies:
         - 0
         version: 2.5.0
   version_requirements: *id002
-  prerelease: false
   name: rspec
-- !ruby/object:Gem::Dependency 
+  prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency 
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -63,8 +63,9 @@ dependencies:
         - 2
         version: 0.7.2
   version_requirements: *id003
-  prerelease: false
   name: yard
+  prerelease: false
+  type: :development
 description: A Ruby API and configuration helper for the Sphinx search service.
 email: 
 - pat@freelancing-gods.com
@@ -101,6 +102,7 @@ files:
 - lib/riddle/configuration.rb
 - lib/riddle/configuration/distributed_index.rb
 - lib/riddle/configuration/index.rb
+- lib/riddle/configuration/index_settings.rb
 - lib/riddle/configuration/indexer.rb
 - lib/riddle/configuration/parser.rb
 - lib/riddle/configuration/realtime_index.rb
@@ -275,7 +277,8 @@ files:
 - spec/unit/message_spec.rb
 - spec/unit/response_spec.rb
 - spec/unit/riddle_spec.rb
-homepage: http://freelancing-god.github.com/riddle/
+has_rdoc: true
+homepage: http://pat.github.com/riddle/
 licenses: []
 
 post_install_message: 
@@ -304,7 +307,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: riddle
-rubygems_version: 1.8.16
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: An API for Sphinx, written in and for Ruby.
@@ -467,4 +470,3 @@ test_files:
 - spec/unit/message_spec.rb
 - spec/unit/response_spec.rb
 - spec/unit/riddle_spec.rb
-has_rdoc: