rt_sphinx 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +22 -0
- data/Gemfile +7 -0
- data/LICENSE +20 -0
- data/README.md +7 -0
- data/lib/rt_sphinx.rb +6 -0
- data/lib/rt_sphinx/connection.rb +99 -0
- data/lib/rt_sphinx/manager.rb +20 -0
- data/lib/rt_sphinx/query.rb +22 -0
- data/rt_sphinx.gemspec +24 -0
- data/sphinx.conf +30 -0
- metadata +103 -0
data/.gitignore
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
.ruby-version
|
6
|
+
coverage
|
7
|
+
InstalledFiles
|
8
|
+
Gemfile.lock
|
9
|
+
lib/bundler/man
|
10
|
+
pkg
|
11
|
+
rdoc
|
12
|
+
spec/reports
|
13
|
+
test/tmp
|
14
|
+
test/version_tmp
|
15
|
+
tmp
|
16
|
+
vendor
|
17
|
+
tags
|
18
|
+
|
19
|
+
# YARD artifacts
|
20
|
+
.yardoc
|
21
|
+
_yardoc
|
22
|
+
doc/
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2013 Dmitry Koprov
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
data/lib/rt_sphinx.rb
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'singleton'
|
3
|
+
require 'connection_pool'
|
4
|
+
|
5
|
+
require 'date'
|
6
|
+
require 'bigdecimal'
|
7
|
+
|
8
|
+
module RtSphinx
|
9
|
+
class Connection
|
10
|
+
include Singleton
|
11
|
+
|
12
|
+
def self.instance
|
13
|
+
@@instance ||= new
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize
|
17
|
+
@pool = ConnectionPool.new(:size => 5, :timeout => 5) do
|
18
|
+
if RUBY_PLATFORM =~ /java/
|
19
|
+
require 'jdbc/mysql'
|
20
|
+
Jdbc::MySQL.load_driver(:require) if Jdbc::MySQL.respond_to?(:load_driver)
|
21
|
+
url = "jdbc:mysql://127.0.0.1:9327?characterEncoding=utf8&maxAllowedPacket=512000"
|
22
|
+
conn = java.sql.DriverManager.getConnection(url, "", "")
|
23
|
+
else
|
24
|
+
require 'mysql2'
|
25
|
+
Mysql2::Client.new(
|
26
|
+
:host => '127.0.0.1',
|
27
|
+
:port => 9327,
|
28
|
+
:encoding => 'utf8'
|
29
|
+
)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def execute_sql(sql, mode)
|
35
|
+
puts sql
|
36
|
+
res = {}
|
37
|
+
@pool.with do |connection|
|
38
|
+
if RUBY_PLATFORM =~ /java/
|
39
|
+
st = connection.create_statement
|
40
|
+
res = case mode
|
41
|
+
when 'select'
|
42
|
+
resultset_to_hash(st.execute_query(sql))
|
43
|
+
when 'insert'
|
44
|
+
st.execute_update(sql)
|
45
|
+
end
|
46
|
+
st.close
|
47
|
+
else
|
48
|
+
res = connection.query(insert_sql)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
res
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
def resultset_to_hash(resultset)
|
57
|
+
meta = resultset.meta_data
|
58
|
+
rows = []
|
59
|
+
|
60
|
+
while resultset.next
|
61
|
+
row = {}
|
62
|
+
|
63
|
+
(1..meta.column_count).each do |i|
|
64
|
+
name = meta.column_name i
|
65
|
+
row[name] = case meta.column_type(i)
|
66
|
+
when -6, -5, 5, 4
|
67
|
+
# TINYINT, BIGINT, INTEGER
|
68
|
+
resultset.get_int(i).to_i
|
69
|
+
when 41
|
70
|
+
# Date
|
71
|
+
resultset.get_date(i)
|
72
|
+
when 92
|
73
|
+
# Time
|
74
|
+
resultset.get_time(i).to_i
|
75
|
+
when 93
|
76
|
+
# Timestamp
|
77
|
+
resultset.get_timestamp(i)
|
78
|
+
when 2, 3, 6
|
79
|
+
# NUMERIC, DECIMAL, FLOAT
|
80
|
+
case meta.scale(i)
|
81
|
+
when 0
|
82
|
+
resultset.get_long(i).to_i
|
83
|
+
else
|
84
|
+
BigDecimal.new(resultset.get_string(i).to_s)
|
85
|
+
end
|
86
|
+
when 1, -15, -9, 12
|
87
|
+
# CHAR, NCHAR, NVARCHAR, VARCHAR
|
88
|
+
resultset.get_string(i).to_s
|
89
|
+
else
|
90
|
+
resultset.get_string(i).to_s
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
rows << row
|
95
|
+
end
|
96
|
+
rows
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module RtSphinx
|
4
|
+
class Manager
|
5
|
+
def self.start
|
6
|
+
`killall searchd`
|
7
|
+
sleep(1)
|
8
|
+
res = `searchd`
|
9
|
+
puts res
|
10
|
+
end
|
11
|
+
|
12
|
+
# TODO: create configurator method. Template should be like this:
|
13
|
+
# <<INDEX_TEMPLATE
|
14
|
+
# index #{config.index} {
|
15
|
+
# type = rt
|
16
|
+
# ...
|
17
|
+
# }
|
18
|
+
# INDEX_TEMPLATE
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module RtSphinx
|
4
|
+
class Query
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@connection = RtSphinx::Connection.instance
|
8
|
+
end
|
9
|
+
|
10
|
+
def select(index, clause)
|
11
|
+
select_sql = "SELECT * FROM #{index} WHERE #{clause}"
|
12
|
+
@connection.execute_sql(select_sql, 'select')
|
13
|
+
end
|
14
|
+
|
15
|
+
def insert(index, *values)
|
16
|
+
item_id = rand(10_000)*rand(9_888)
|
17
|
+
values_str = values.map{|v| "'#{v}'"}.join(', ')
|
18
|
+
insert_sql = "INSERT INTO #{index} (id, timestamp, message, logjson) VALUES (#{item_id}, #{values_str})"
|
19
|
+
@connection.execute_sql(insert_sql, 'insert')
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/rt_sphinx.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = 'rt_sphinx'
|
6
|
+
s.version = '0.0.1'
|
7
|
+
s.platform = Gem::Platform::RUBY
|
8
|
+
s.authors = ['Dmitry Koprov']
|
9
|
+
s.email = ['dmitry.koprov@gmail.com']
|
10
|
+
s.homepage = 'http://dkoprov.github.com/rt_sphinx/'
|
11
|
+
s.summary = %q{A tool to work with Sphinx search real time indexes}
|
12
|
+
s.description = %q{A tool for populating Sphinx search service through real time indexes.}
|
13
|
+
|
14
|
+
s.rubyforge_project = 'rt_sphinx'
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ['lib']
|
20
|
+
|
21
|
+
s.add_development_dependency 'rake', '>= 0.9.2'
|
22
|
+
s.add_development_dependency 'rspec', '>= 2.5.0'
|
23
|
+
s.add_development_dependency 'yard', '>= 0.7.2'
|
24
|
+
end
|
data/sphinx.conf
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
index rt_logs
|
2
|
+
{
|
3
|
+
type = rt
|
4
|
+
path = /usr/local/var/data/rt_logs
|
5
|
+
charset_type = utf-8
|
6
|
+
min_word_len = 1
|
7
|
+
html_strip = 1
|
8
|
+
enable_star = 1
|
9
|
+
rt_mem_limit = 256M
|
10
|
+
|
11
|
+
rt_field = timestamp
|
12
|
+
rt_field = message
|
13
|
+
rt_attr_string = timestamp
|
14
|
+
rt_attr_string = message
|
15
|
+
rt_attr_json = logjson
|
16
|
+
}
|
17
|
+
|
18
|
+
searchd
|
19
|
+
{
|
20
|
+
listen = 127.0.0.1:9327:mysql41
|
21
|
+
pid_file = /usr/local/var/data/searchd.pid
|
22
|
+
max_children = 30
|
23
|
+
max_matches = 1000
|
24
|
+
log = /usr/local/var/data/searchd.log
|
25
|
+
query_log = /usr/local/var/data/query.log
|
26
|
+
seamless_rotate = 1
|
27
|
+
preopen_indexes = 1
|
28
|
+
unlink_old = 1
|
29
|
+
workers = threads
|
30
|
+
}
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rt_sphinx
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.1
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Dmitry Koprov
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-07-22 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rake
|
16
|
+
version_requirements: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - '>='
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: 0.9.2
|
21
|
+
none: false
|
22
|
+
requirement: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.9.2
|
27
|
+
none: false
|
28
|
+
prerelease: false
|
29
|
+
type: :development
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rspec
|
32
|
+
version_requirements: !ruby/object:Gem::Requirement
|
33
|
+
requirements:
|
34
|
+
- - '>='
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 2.5.0
|
37
|
+
none: false
|
38
|
+
requirement: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - '>='
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 2.5.0
|
43
|
+
none: false
|
44
|
+
prerelease: false
|
45
|
+
type: :development
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: yard
|
48
|
+
version_requirements: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - '>='
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 0.7.2
|
53
|
+
none: false
|
54
|
+
requirement: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - '>='
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: 0.7.2
|
59
|
+
none: false
|
60
|
+
prerelease: false
|
61
|
+
type: :development
|
62
|
+
description: A tool for populating Sphinx search service through real time indexes.
|
63
|
+
email:
|
64
|
+
- dmitry.koprov@gmail.com
|
65
|
+
executables: []
|
66
|
+
extensions: []
|
67
|
+
extra_rdoc_files: []
|
68
|
+
files:
|
69
|
+
- .gitignore
|
70
|
+
- Gemfile
|
71
|
+
- LICENSE
|
72
|
+
- README.md
|
73
|
+
- lib/rt_sphinx.rb
|
74
|
+
- lib/rt_sphinx/connection.rb
|
75
|
+
- lib/rt_sphinx/manager.rb
|
76
|
+
- lib/rt_sphinx/query.rb
|
77
|
+
- rt_sphinx.gemspec
|
78
|
+
- sphinx.conf
|
79
|
+
homepage: http://dkoprov.github.com/rt_sphinx/
|
80
|
+
licenses: []
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options: []
|
83
|
+
require_paths:
|
84
|
+
- lib
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
none: false
|
91
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - '>='
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
none: false
|
97
|
+
requirements: []
|
98
|
+
rubyforge_project: rt_sphinx
|
99
|
+
rubygems_version: 1.8.24
|
100
|
+
signing_key:
|
101
|
+
specification_version: 3
|
102
|
+
summary: A tool to work with Sphinx search real time indexes
|
103
|
+
test_files: []
|