rt_sphinx 0.0.1

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.ruby-version
+coverage
+InstalledFiles
+Gemfile.lock
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor
+tags
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,7 @@
+source 'http://rubygems.org'
+
+gemspec
+
+gem 'connection_pool'
+gem 'mysql2',     '0.3.2',  :platform => :ruby
+gem 'jdbc-mysql', '5.1.13', :platform => :jruby

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Dmitry Koprov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,7 @@
+rt_sphinx
+=========
+
+A tool for work with sphinx search realtime indexes.
+
+Requires Sphinx version 2.1.1beta and later.
+Can work through jruby and regular MRI Ruby edition.

data/lib/rt_sphinx.rb

@@ -0,0 +1,6 @@
+require 'rt_sphinx/manager'
+require 'rt_sphinx/connection'
+require 'rt_sphinx/query'
+
+module RtSphinx
+end

data/lib/rt_sphinx/connection.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+require 'singleton'
+require 'connection_pool'
+
+require 'date'
+require 'bigdecimal'
+
+module RtSphinx
+  class Connection
+    include Singleton
+
+    def self.instance
+      @@instance ||= new
+    end
+
+    def initialize
+      @pool = ConnectionPool.new(:size => 5, :timeout => 5) do
+        if RUBY_PLATFORM =~ /java/
+          require 'jdbc/mysql'
+          Jdbc::MySQL.load_driver(:require) if Jdbc::MySQL.respond_to?(:load_driver)
+          url = "jdbc:mysql://127.0.0.1:9327?characterEncoding=utf8&maxAllowedPacket=512000"
+          conn = java.sql.DriverManager.getConnection(url, "", "")
+        else
+          require 'mysql2'
+          Mysql2::Client.new(
+            :host => '127.0.0.1',
+            :port => 9327,
+            :encoding => 'utf8'
+          )
+        end
+      end
+    end
+
+    def execute_sql(sql, mode)
+      puts sql
+      res = {}
+      @pool.with do |connection|
+        if RUBY_PLATFORM =~ /java/
+          st = connection.create_statement
+          res = case mode
+                when 'select'
+                  resultset_to_hash(st.execute_query(sql))
+                when 'insert'
+                  st.execute_update(sql)
+                end
+          st.close
+        else
+          res = connection.query(insert_sql)
+        end
+      end
+
+      res
+    end
+
+    private
+    def resultset_to_hash(resultset)
+      meta = resultset.meta_data
+      rows = []
+
+      while resultset.next
+        row = {}
+
+        (1..meta.column_count).each do |i|
+          name = meta.column_name i
+          row[name]  =  case meta.column_type(i)
+                        when -6, -5, 5, 4
+                          # TINYINT, BIGINT, INTEGER
+                          resultset.get_int(i).to_i
+                        when 41
+                          # Date
+                          resultset.get_date(i)
+                        when 92
+                          # Time
+                          resultset.get_time(i).to_i
+                        when 93
+                          # Timestamp
+                          resultset.get_timestamp(i)
+                        when 2, 3, 6
+                          # NUMERIC, DECIMAL, FLOAT
+                          case meta.scale(i)
+                          when 0
+                            resultset.get_long(i).to_i
+                          else
+                            BigDecimal.new(resultset.get_string(i).to_s)
+                          end
+                        when 1, -15, -9, 12
+                          # CHAR, NCHAR, NVARCHAR, VARCHAR
+                          resultset.get_string(i).to_s
+                        else
+                          resultset.get_string(i).to_s
+                        end
+        end
+
+        rows << row
+      end
+      rows
+    end
+  end
+end

data/lib/rt_sphinx/manager.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+module RtSphinx
+  class Manager
+    def self.start
+      `killall searchd`
+      sleep(1)
+      res = `searchd`
+      puts res
+    end
+
+    # TODO: create configurator method. Template should be like this:
+    # <<INDEX_TEMPLATE
+    # index #{config.index} {
+    #   type = rt
+    #   ...
+    # }
+    # INDEX_TEMPLATE
+  end
+end

data/lib/rt_sphinx/query.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+module RtSphinx
+  class Query
+
+    def initialize
+      @connection = RtSphinx::Connection.instance
+    end
+
+    def select(index, clause)
+      select_sql = "SELECT * FROM #{index} WHERE #{clause}"
+      @connection.execute_sql(select_sql, 'select')
+    end
+
+    def insert(index, *values)
+      item_id = rand(10_000)*rand(9_888)
+      values_str = values.map{|v| "'#{v}'"}.join(', ')
+      insert_sql = "INSERT INTO #{index} (id, timestamp, message, logjson) VALUES (#{item_id}, #{values_str})"
+      @connection.execute_sql(insert_sql, 'insert')
+    end
+  end
+end

data/rt_sphinx.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'rt_sphinx'
+  s.version     = '0.0.1'
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Dmitry Koprov']
+  s.email       = ['dmitry.koprov@gmail.com']
+  s.homepage    = 'http://dkoprov.github.com/rt_sphinx/'
+  s.summary     = %q{A tool to work with Sphinx search real time indexes}
+  s.description = %q{A tool for populating Sphinx search service through real time indexes.}
+
+  s.rubyforge_project = 'rt_sphinx'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.add_development_dependency 'rake',   '>= 0.9.2'
+  s.add_development_dependency 'rspec',  '>= 2.5.0'
+  s.add_development_dependency 'yard',   '>= 0.7.2'
+end

data/sphinx.conf

@@ -0,0 +1,30 @@
+index rt_logs
+{
+  type = rt
+  path = /usr/local/var/data/rt_logs
+  charset_type = utf-8
+  min_word_len = 1
+  html_strip   = 1
+  enable_star  = 1
+  rt_mem_limit = 256M
+
+  rt_field = timestamp
+  rt_field = message
+  rt_attr_string = timestamp
+  rt_attr_string = message
+  rt_attr_json = logjson
+}
+
+searchd
+{
+  listen = 127.0.0.1:9327:mysql41
+  pid_file = /usr/local/var/data/searchd.pid
+  max_children = 30
+  max_matches = 1000
+  log = /usr/local/var/data/searchd.log
+  query_log = /usr/local/var/data/query.log
+  seamless_rotate = 1
+  preopen_indexes = 1
+  unlink_old = 1
+  workers = threads
+}

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: rt_sphinx
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 0.0.1
+platform: ruby
+authors:
+- Dmitry Koprov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: yard
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+    none: false
+  prerelease: false
+  type: :development
+description: A tool for populating Sphinx search service through real time indexes.
+email:
+- dmitry.koprov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- lib/rt_sphinx.rb
+- lib/rt_sphinx/connection.rb
+- lib/rt_sphinx/manager.rb
+- lib/rt_sphinx/query.rb
+- rt_sphinx.gemspec
+- sphinx.conf
+homepage: http://dkoprov.github.com/rt_sphinx/
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project: rt_sphinx
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
+summary: A tool to work with Sphinx search real time indexes
+test_files: []