riddle 0.9.8.1231.0 → 0.9.8.1533.10

data/lib/riddle/configuration/indexer.rb

@@ -0,0 +1,19 @@
+module Riddle
+  class Configuration
+    class Indexer < Riddle::Configuration::Section
+      self.settings = [:mem_limit, :max_iops, :max_iosize]
+      
+      attr_accessor *self.settings
+            
+      def render
+        raise ConfigurationError unless valid?
+        
+        (
+          ["indexer", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+    end
+  end
+end

data/lib/riddle/configuration/remote_index.rb

@@ -0,0 +1,17 @@
+module Riddle
+  class Configuration
+    class RemoteIndex
+      attr_accessor :address, :port, :name
+      
+      def initialize(address, port, name)
+        @address  = address
+        @port     = port
+        @name     = name
+      end
+      
+      def remote
+        "#{address}:#{port}"
+      end
+    end
+  end
+end

data/lib/riddle/configuration/searchd.rb

@@ -0,0 +1,25 @@
+module Riddle
+  class Configuration
+    class Searchd < Riddle::Configuration::Section
+      self.settings = [:address, :port, :log, :query_log, :read_timeout,
+        :max_children, :pid_file, :max_matches, :seamless_rotate,
+        :preopen_indexes, :unlink_old]
+      
+      attr_accessor *self.settings
+            
+      def render
+        raise ConfigurationError unless valid?
+        
+        (
+          ["searchd", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+      
+      def valid?
+        !( @port.nil? || @pid_file.nil? )
+      end
+    end
+  end
+end

data/lib/riddle/configuration/section.rb

@@ -0,0 +1,43 @@
+module Riddle
+  class Configuration
+    class Section
+      class << self
+        attr_accessor :settings
+      end
+      
+      settings = []
+      
+      def valid?
+        true
+      end
+      
+      private
+      
+      def settings_body
+        self.class.settings.select { |setting|
+          !send(setting).nil?
+        }.collect { |setting|
+          if send(setting) == ""
+            conf = "  #{setting} = "
+          else
+            conf = setting_to_array(setting).collect { |set|
+              "  #{setting} = #{set}"
+            }
+          end
+          conf.length == 0 ? nil : conf
+        }.flatten.compact
+      end
+      
+      def setting_to_array(setting)
+        value = send(setting)
+        case value
+        when Array      then value
+        when TrueClass  then [1]
+        when FalseClass then [0]
+        else
+          [value]
+        end
+      end
+    end
+  end
+end

data/lib/riddle/configuration/source.rb

@@ -0,0 +1,23 @@
+module Riddle
+  class Configuration
+    class Source < Riddle::Configuration::Section
+      attr_accessor :name, :parent, :type
+            
+      def render
+        raise ConfigurationError unless valid?
+        
+        inherited_name = "#{name}"
+        inherited_name << " : #{parent}" if parent
+        (
+          ["source #{inherited_name}", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+      
+      def valid?
+        !( @name.nil? || @type.nil? )
+      end
+    end
+  end
+end

data/lib/riddle/configuration/sql_source.rb

@@ -0,0 +1,34 @@
+module Riddle
+  class Configuration
+    class SQLSource < Riddle::Configuration::Source
+      self.settings = [:type, :sql_host, :sql_user, :sql_pass, :sql_db,
+        :sql_port, :sql_sock, :mysql_connect_flags, :sql_query_pre, :sql_query,
+        :sql_query_range, :sql_range_step, :sql_attr_uint, :sql_attr_bool,
+        :sql_attr_timestamp, :sql_attr_str2ordinal, :sql_attr_float,
+        :sql_attr_multi, :sql_query_post, :sql_query_post_index,
+        :sql_ranged_throttle, :sql_query_info]
+      
+      attr_accessor *self.settings
+      
+      def initialize(name, type)
+        @name = name
+        @type = type
+        
+        @sql_query_pre        = []
+        @sql_attr_uint        = []
+        @sql_attr_bool        = []
+        @sql_attr_timestamp   = []
+        @sql_attr_str2ordinal = []
+        @sql_attr_float       = []
+        @sql_attr_multi       = []
+        @sql_query_post       = []
+        @sql_query_post_index = []
+      end
+            
+      def valid?
+        super && (!( @sql_host.nil? || @sql_user.nil? || @sql_db.nil? ||
+          @sql_query.nil? ) || !@parent.nil?)
+      end
+    end
+  end
+end

data/lib/riddle/configuration/xml_source.rb

@@ -0,0 +1,28 @@
+module Riddle
+  class Configuration
+    class XMLSource < Riddle::Configuration::Source
+      self.settings = [:type, :xmlpipe_command, :xmlpipe_field,
+        :xmlpipe_attr_uint, :xmlpipe_attr_bool, :xmlpipe_attr_timestamp,
+        :xmlpipe_attr_str2ordinal, :xmlpipe_attr_float, :xmlpipe_attr_multi]
+      
+      attr_accessor *self.settings
+      
+      def initialize(name, type)
+        @name = name
+        @type = type
+        
+        @xmlpipe_field            = []
+        @xmlpipe_attr_uint        = []
+        @xmlpipe_attr_bool        = []
+        @xmlpipe_attr_timestamp   = []
+        @xmlpipe_attr_str2ordinal = []
+        @xmlpipe_attr_float       = []
+        @xmlpipe_attr_multi       = []
+      end
+            
+      def valid?
+        super && ( !@xmlpipe_command.nil? || !parent.nil? )
+      end
+    end
+  end
+end

data/lib/riddle/controller.rb

@@ -0,0 +1,53 @@
+module Riddle
+  class Controller
+    def initialize(configuration, path)
+      @configuration  = configuration
+      @path           = path
+    end
+    
+    def index
+      cmd = "indexer --config #{@path} --all"
+      cmd << " --rotate" if running?
+      `#{cmd}`
+    end
+    
+    def start
+      return if running?
+      
+      cmd = "searchd --pidfile --config #{@path}"
+      
+      if RUBY_PLATFORM =~ /mswin/
+        system("start /B #{cmd} 1> NUL 2>&1")
+      else
+        `#{cmd}`
+      end
+      
+      sleep(1)
+      
+      unless running?
+        puts "Failed to start searchd daemon. Check #{@configuration.searchd.log}."
+      end
+    end
+    
+    def stop
+      return unless running?
+      Process.kill('SIGTERM', pid.to_i)
+    rescue Errno::EINVAL
+      Process.kill('SIGKILL', pid.to_i)
+    end
+    
+    def pid
+      if File.exists?(@configuration.searchd.pid_file)
+        File.read(@configuration.searchd.pid_file)[/\d+/]
+      else
+        nil
+      end
+    end
+    
+    def running?
+      !!pid && !!Process.kill(0, pid.to_i)
+    rescue
+      false
+    end
+  end
+end

data/lib/tabtab_definitions.rb

@@ -0,0 +1,15 @@
+TabTab::Definition.register("searchd") do |c|
+  c.flag :help, :h
+  c.flag :config, :c
+  c.flag :stop
+  c.flag :iostats
+  c.flag :console
+  c.flag :port, :p
+  c.flag :index, :i
+end
+
+TabTab::Definition.register("indexer") do |c|
+  c.flag :config, :c
+  c.flag :all
+  c.flag :rotate
+end

data/spec/unit/client_spec.rb

@@ -177,4 +177,32 @@ describe Riddle::Client do
       true
     ).should == query_contents(:keywords_with_hits)
   end
-end
+  
+  it "should timeout after a specified time" do
+    client = Riddle::Client.new
+    client.port     = 3314
+    client.timeout  = 1
+    
+    server = TCPServer.new "localhost", 3314
+    
+    lambda {
+      client.send(:connect) { |socket| }
+    }.should raise_error(Riddle::ConnectionError)
+    
+    server.close
+  end
+
+  context "connection retrying" do
+    it "should try fives time when connection refused" do
+      client = Riddle::Client.new
+      client.port    = 3314
+
+      TCPSocket.should_receive(:new).with('localhost', 3314).exactly(5).times.
+        and_raise(Errno::ECONNREFUSED)
+
+      lambda {
+        client.send(:connect) { |socket| }
+      }.should raise_error(Riddle::ConnectionError)
+    end
+  end
+end

data/spec/unit/configuration/distributed_index_spec.rb

@@ -0,0 +1,56 @@
+require 'spec/spec_helper'
+
+describe Riddle::Configuration::DistributedIndex do
+  it "should not be valid without any indexes" do
+    index = Riddle::Configuration::DistributedIndex.new("dist1")
+    index.should_not be_valid
+  end
+  
+  it "should be valid with just local indexes" do
+    index = Riddle::Configuration::DistributedIndex.new("dist1")
+    index.local_indexes << "local_one"
+    index.should be_valid
+  end
+  
+  it "should be valid with just remote indexes" do
+    index = Riddle::Configuration::DistributedIndex.new("dist1")
+    index.remote_indexes << Riddle::Configuration::RemoteIndex.new("local", 3312, "remote_one")
+    index.should be_valid
+  end
+  
+  it "should be of type 'distributed'" do
+    index = Riddle::Configuration::DistributedIndex.new("dist1")
+    index.type.should == 'distributed'
+  end
+  
+  it "should raise a ConfigurationError if rendering when not valid" do
+    index = Riddle::Configuration::DistributedIndex.new("dist1")
+    lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError)
+  end
+  
+  it "should render correctly if supplied settings are valid" do
+    index = Riddle::Configuration::DistributedIndex.new("dist1")
+    
+    index.local_indexes << "test1" << "test1stemmed"
+    index.remote_indexes <<
+      Riddle::Configuration::RemoteIndex.new("localhost", 3313, "remote1") <<
+      Riddle::Configuration::RemoteIndex.new("localhost", 3314, "remote2") <<
+      Riddle::Configuration::RemoteIndex.new("localhost", 3314, "remote3")
+    
+    index.agent_connect_timeout = 1000
+    index.agent_query_timeout   = 3000
+    
+    index.render.should == <<-DISTINDEX
+index dist1
+{
+  type = distributed
+  local = test1
+  local = test1stemmed
+  agent = localhost:3313:remote1
+  agent = localhost:3314:remote2,remote3
+  agent_connect_timeout = 1000
+  agent_query_timeout = 3000
+}
+    DISTINDEX
+  end
+end

data/spec/unit/configuration/index_spec.rb

@@ -0,0 +1,108 @@
+require 'spec/spec_helper'
+
+describe Riddle::Configuration::DistributedIndex do
+  it "should be invalid without a name, sources or path if there's no parent" do
+    index = Riddle::Configuration::Index.new(nil)
+    index.should_not be_valid
+    
+    index.name = "test1"
+    index.should_not be_valid
+    
+    index.sources << Riddle::Configuration::SQLSource.new("source", "mysql")
+    index.should_not be_valid
+    
+    index.path = "a/path"
+    index.should be_valid
+    
+    index.name = nil
+    index.should_not be_valid
+    
+    index.name = "test1"
+    index.sources.clear
+    index.should_not be_valid
+  end
+  
+  it "should be invalid without a name but not sources or path if it has a parent" do
+    index = Riddle::Configuration::Index.new(nil)
+    index.should_not be_valid
+    
+    index.name = "test1stemmed"
+    index.should_not be_valid
+    
+    index.parent = "test1"
+    index.should be_valid
+  end
+  
+  it "should raise a ConfigurationError if rendering when not valid" do
+    index = Riddle::Configuration::Index.new("test1")
+    lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError)
+  end
+  
+  it "should render correctly if supplied settings are valid" do
+    source = Riddle::Configuration::XMLSource.new("src1", "xmlpipe")
+    source.xmlpipe_command = "ls /dev/null"
+    
+    index = Riddle::Configuration::Index.new("test1", source)
+    index.path                      = "/var/data/test1"
+    index.docinfo                   = "extern"
+    index.mlock                     = 0
+    index.morphologies             << "stem_en" << "stem_ru" << "soundex"
+    index.stopword_files           << "/var/data/stopwords.txt" << "/var/data/stopwords2.txt"
+    index.wordform_files           << "/var/data/wordforms.txt"
+    index.exception_files          << "/var/data/exceptions.txt"
+    index.min_word_len              = 1
+    index.charset_type              = "utf-8"
+    index.charset_table             = "0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F"
+    index.ignore_characters        << "U+00AD"
+    index.min_prefix_len            = 0
+    index.min_infix_len             = 0
+    index.prefix_field_names       << "filename"
+    index.infix_field_names        << "url" << "domain"
+    index.enable_star               = true
+    index.ngram_len                 = 1
+    index.ngram_characters         << "U+3000..U+2FA1F"
+    index.phrase_boundaries        << "." << "?" << "!" << "U+2026"
+    index.phrase_boundary_step      = 100
+    index.html_strip                = 0
+    index.html_index_attrs          = "img=alt,title; a=title"
+    index.html_remove_element_tags << "style" << "script"
+    index.preopen                   = 1
+    
+    index.render.should == <<-INDEX
+source src1
+{
+  type = xmlpipe
+  xmlpipe_command = ls /dev/null
+}
+
+index test1
+{
+  source = src1
+  path = /var/data/test1
+  docinfo = extern
+  mlock = 0
+  morphology = stem_en, stem_ru, soundex
+  stopwords = /var/data/stopwords.txt /var/data/stopwords2.txt
+  wordforms = /var/data/wordforms.txt
+  exceptions = /var/data/exceptions.txt
+  min_word_len = 1
+  charset_type = utf-8
+  charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F
+  ignore_chars = U+00AD
+  min_prefix_len = 0
+  min_infix_len = 0
+  prefix_fields = filename
+  infix_fields = url, domain
+  enable_star = 1
+  ngram_len = 1
+  ngram_chars = U+3000..U+2FA1F
+  phrase_boundary = ., ?, !, U+2026
+  phrase_boundary_step = 100
+  html_strip = 0
+  html_index_attrs = img=alt,title; a=title
+  html_remove_elements = style, script
+  preopen = 1
+}
+    INDEX
+  end
+end