oedipus 0.0.1.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +8 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/LICENSE +20 -0
- data/README.md +149 -0
- data/Rakefile +1 -0
- data/lib/oedipus/connection.rb +50 -0
- data/lib/oedipus/index.rb +103 -0
- data/lib/oedipus/mysql/client.rb +136 -0
- data/lib/oedipus/query_builder.rb +41 -0
- data/lib/oedipus/version.rb +12 -0
- data/lib/oedipus.rb +41 -0
- data/oedipus.gemspec +35 -0
- data/spec/data/.gitkeep +0 -0
- data/spec/spec_helper.rb +39 -0
- data/spec/support/test_harness.rb +143 -0
- data/spec/unit/connection_spec.rb +36 -0
- data/spec/unit/index_spec.rb +85 -0
- metadata +89 -0
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--colour
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright © 2011 Chris Corbyn
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,149 @@
|
|
1
|
+
# Oedipus: Sphinx 2 Search Client for Ruby
|
2
|
+
|
3
|
+
Oedipus is a client for the Sphinx search engine (>= 2.0.2), with support for
|
4
|
+
real-time indexes and multi and/or faceted searches.
|
5
|
+
|
6
|
+
It is not a clone of the PHP API, rather it is written from the ground up,
|
7
|
+
wrapping the SphinxQL API offered by searchd. Nor is it a plugin for
|
8
|
+
ActiveRecord or DataMapper... though this is planned in separate gems.
|
9
|
+
|
10
|
+
It will provide some higher level of abstraction in terms of the ease with
|
11
|
+
which faceted search may be implemented, though it will remain light and simple.
|
12
|
+
|
13
|
+
## Current Status
|
14
|
+
|
15
|
+
This gem is in development. It is not ready for production use. I work for
|
16
|
+
a company called Flippa.com, which currently implements faceted search in a PHP
|
17
|
+
part of the website, using a slightly older version of Sphinx with lesser
|
18
|
+
support for SphinxQL. Once a month the developers at Flippa are given three days
|
19
|
+
to work on a project of their own choice. This is my 'Triple Time' project.
|
20
|
+
|
21
|
+
I anticipate another week or two of development before I can consider this project
|
22
|
+
production-ready.
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Not all of the following features are currently implemented, but the interface
|
27
|
+
style is as follows.
|
28
|
+
|
29
|
+
``` ruby
|
30
|
+
require "oedipus"
|
31
|
+
|
32
|
+
sphinx = Oedipus.connect('localhost:9306') # sphinxql host
|
33
|
+
|
34
|
+
# insert a record into the 'articles' real-time index
|
35
|
+
record = sphinx[:articles].insert(
|
36
|
+
7,
|
37
|
+
title: "Badgers in the wild",
|
38
|
+
body: "A big long wodge of text",
|
39
|
+
author_id: 4,
|
40
|
+
views: 102
|
41
|
+
)
|
42
|
+
# The attributes (but not the indexed fields) are returned
|
43
|
+
# => { id: 7, author_id: 4, views: 102 }
|
44
|
+
|
45
|
+
# updating a record
|
46
|
+
record = sphinx[:articles].update(7, views: 103)
|
47
|
+
# The new attributes (but not the indexed fields) are returned
|
48
|
+
# => { id: 7, author_id: 4, views: 103 }
|
49
|
+
|
50
|
+
# deleting a record
|
51
|
+
sphinx[:articles].delete(7)
|
52
|
+
# => true
|
53
|
+
|
54
|
+
# searching the index
|
55
|
+
results = sphinx[:articles].search("badgers", limit: 2)
|
56
|
+
|
57
|
+
# Meta deta indicates the overall number of matched records, while the ':records'
|
58
|
+
# array contains the actual data returned.
|
59
|
+
#
|
60
|
+
# => {
|
61
|
+
# total_found: 987,
|
62
|
+
# time: 0.000,
|
63
|
+
# keyword[0]: "badgers",
|
64
|
+
# docs[0]: 987,
|
65
|
+
# records: [
|
66
|
+
# { id: 7, author_id: 4, views: 102 },
|
67
|
+
# { id: 11, author_id: 6, views: 23 }
|
68
|
+
# ]
|
69
|
+
# }
|
70
|
+
|
71
|
+
# using attribute filters
|
72
|
+
results = sphinx[:articles].search(
|
73
|
+
"example",
|
74
|
+
author_id: 7
|
75
|
+
)
|
76
|
+
# => (the same results, filtered by author)
|
77
|
+
|
78
|
+
# performing a faceted search
|
79
|
+
results = sphinx[:articles].facted_search(
|
80
|
+
"badgers",
|
81
|
+
facets: {
|
82
|
+
popular: { views: 100..10000 },
|
83
|
+
farming: "farming",
|
84
|
+
popular_farming: ["farming", { views: 100..10000 } ]
|
85
|
+
}
|
86
|
+
)
|
87
|
+
# The main results are returned in the ':records' array, and all the facets in
|
88
|
+
# the ':facets' Hash.
|
89
|
+
# => {
|
90
|
+
# total_found: 987,
|
91
|
+
# time: 0.000,
|
92
|
+
# records: [ ... ],
|
93
|
+
# facets: {
|
94
|
+
# popular: {
|
95
|
+
# total_found: 25,
|
96
|
+
# time: 0.000,
|
97
|
+
# records: [ ... ]
|
98
|
+
# },
|
99
|
+
# farming: {
|
100
|
+
# total_found: 123,
|
101
|
+
# time: 0.000,
|
102
|
+
# records: [ ... ]
|
103
|
+
# },
|
104
|
+
# popular_farming: {
|
105
|
+
# total_found: 2,
|
106
|
+
# time: 0.000,
|
107
|
+
# records: [ ... ]
|
108
|
+
# }
|
109
|
+
# }
|
110
|
+
# }
|
111
|
+
#
|
112
|
+
# When performing a faceted search, the primary search is used as the basis for
|
113
|
+
# each facet, so they can be considered refinements.
|
114
|
+
|
115
|
+
# performing a mutli-search
|
116
|
+
results = sphinx[:articles].multi_search(
|
117
|
+
badgers: ["badgers", { limit: 30 }],
|
118
|
+
frogs: "frogs AND wetlands",
|
119
|
+
rabbits: ["rabbits OR burrows", { view_count: 20..100 }]
|
120
|
+
)
|
121
|
+
# The results are returned in a 2-dimensional Hash, keyed as sent in the query
|
122
|
+
# => {
|
123
|
+
# badgers: {
|
124
|
+
# ...
|
125
|
+
# },
|
126
|
+
# frogs: {
|
127
|
+
# ...
|
128
|
+
# },
|
129
|
+
# rabbits: {
|
130
|
+
# ...
|
131
|
+
# }
|
132
|
+
# }
|
133
|
+
#
|
134
|
+
# Unlike with a faceted search, the queries in a multi-search do not have to be
|
135
|
+
# related to one another.
|
136
|
+
```
|
137
|
+
|
138
|
+
## Future Plans
|
139
|
+
|
140
|
+
I plan to release gems for integration with DataMapper and ActiveRecord. DataMapper
|
141
|
+
first, since that's what we use.
|
142
|
+
|
143
|
+
I also intend to remove ruby-mysql from the dependencies, as it doesn't perfectly fit
|
144
|
+
the needs of SphinxQL. I will be implementing the limited subset of the MySQL protocol
|
145
|
+
by hand (which is not as big a deal as it sounds).
|
146
|
+
|
147
|
+
## Copyright and Licensing
|
148
|
+
|
149
|
+
Refer to the LICENSE file for details.
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
require "mysql"
|
11
|
+
|
12
|
+
# SphinxQL greets with charset number 0
|
13
|
+
Mysql::Charset::NUMBER_TO_CHARSET[0] = Mysql::Charset::COLLATION_TO_CHARSET["utf8_general_ci"]
|
14
|
+
|
15
|
+
module Oedipus
|
16
|
+
# Provides an interface for talking to SphinxQL.
|
17
|
+
class Connection
|
18
|
+
# Instantiate a new Connection to a SphinxQL host.
|
19
|
+
#
|
20
|
+
# @param [Hash]
|
21
|
+
# a Hash containing :host and :port
|
22
|
+
#
|
23
|
+
# The connection will be established on initialization.
|
24
|
+
def initialize(options)
|
25
|
+
@conn = ::Mysql.new(options[:host], nil, nil, nil, options[:port], nil, ::Mysql::CLIENT_MULTI_STATEMENTS | ::Mysql::CLIENT_MULTI_RESULTS)
|
26
|
+
end
|
27
|
+
|
28
|
+
# Acess a specific index for querying.
|
29
|
+
#
|
30
|
+
# @param [String] index_name
|
31
|
+
# the name of an existing index in Sphinx
|
32
|
+
#
|
33
|
+
# @return [Index]
|
34
|
+
# an index that can be queried
|
35
|
+
def [](index_name)
|
36
|
+
Index.new(index_name, self)
|
37
|
+
end
|
38
|
+
|
39
|
+
def execute(sql)
|
40
|
+
@conn.query(sql)
|
41
|
+
end
|
42
|
+
|
43
|
+
def quote(v)
|
44
|
+
case v
|
45
|
+
when Numeric then v
|
46
|
+
else "'#{::Mysql.quote(v.to_s)}'"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
module Oedipus
|
11
|
+
# Representation of a search index for querying.
|
12
|
+
class Index
|
13
|
+
attr_reader :name
|
14
|
+
attr_reader :attributes
|
15
|
+
|
16
|
+
# Initialize the index named +name+ on the connection +conn+.
|
17
|
+
#
|
18
|
+
# @param [Symbol] name
|
19
|
+
# the name of an existing index in sphinx
|
20
|
+
#
|
21
|
+
# @param [Connection] conn
|
22
|
+
# an instance of Oedipus::Connection for querying
|
23
|
+
def initialize(name, conn)
|
24
|
+
@name = name.to_sym
|
25
|
+
@conn = conn
|
26
|
+
@attributes = reflect_attributes
|
27
|
+
@builder = QueryBuilder.new(name, conn)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Insert the record with the ID +id+.
|
31
|
+
#
|
32
|
+
# @example
|
33
|
+
# index.insert(42, :title => "example", :views => 22)
|
34
|
+
#
|
35
|
+
# @param [Integer] id
|
36
|
+
# the unique ID of the document in the index
|
37
|
+
#
|
38
|
+
# @param [Hash] hash
|
39
|
+
# a symbol-keyed hash of data to insert
|
40
|
+
#
|
41
|
+
# @return [Hash]
|
42
|
+
# a copy of the inserted record
|
43
|
+
def insert(id, hash)
|
44
|
+
data = Hash[
|
45
|
+
[:id, *hash.keys.map(&:to_sym)].zip \
|
46
|
+
[id, *hash.values.map { |v| String === v ? @conn.quote(v) : v }]
|
47
|
+
]
|
48
|
+
@conn.execute("INSERT INTO #{name} (#{data.keys.join(', ')}) VALUES (#{data.values.join(', ')})")
|
49
|
+
attributes.merge(data.select { |k, _| attributes.key?(k) })
|
50
|
+
end
|
51
|
+
|
52
|
+
def search(*args)
|
53
|
+
raise ArgumentError, "Wrong number of arguments (#{args.size} for 1..2)" unless (1..2) === args.size
|
54
|
+
|
55
|
+
query, attrs = case args.first
|
56
|
+
when String then [args[0], args[1] || {}]
|
57
|
+
when Hash then ["", args[0]]
|
58
|
+
else raise ArgumentError, "Invalid argument type #{args.first.class} for argument 0"
|
59
|
+
end
|
60
|
+
|
61
|
+
results = @conn.execute(@builder.sql(query, attrs))
|
62
|
+
meta = @conn.execute("SHOW META")
|
63
|
+
|
64
|
+
# FIXME: This needs optimizing
|
65
|
+
{}.tap do |r|
|
66
|
+
meta.each do |k, v|
|
67
|
+
r[k.to_sym] = case k
|
68
|
+
when 'total', 'total_found' then Integer(v)
|
69
|
+
when 'time' then Float(v)
|
70
|
+
else v
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
r[:records] = []
|
75
|
+
|
76
|
+
results.each_hash do |hash|
|
77
|
+
r[:records] << Hash[hash.keys.map(&:to_sym).zip(hash.map { |k, v| cast(k, v) })]
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def cast(key, value)
|
85
|
+
case attributes[key.to_sym]
|
86
|
+
when Fixnum then Integer(value)
|
87
|
+
else value
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def reflect_attributes
|
92
|
+
rs = @conn.execute("DESC #{name}")
|
93
|
+
{}.tap do |attrs|
|
94
|
+
rs.each_hash do |row|
|
95
|
+
case row['Type']
|
96
|
+
when 'uint', 'integer' then attrs[row['Field'].to_sym] = 0
|
97
|
+
when 'string' then attrs[row['Field'].to_sym] = ""
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
require "socket"
|
11
|
+
require "net/protocol"
|
12
|
+
|
13
|
+
module Oedipus
|
14
|
+
module Mysql
|
15
|
+
# Limited subset of MySQL protocol for communication with SphinxQL.
|
16
|
+
#
|
17
|
+
# This needs to exist since other ruby MySQL clients do not provide the relevant features.
|
18
|
+
class Client
|
19
|
+
# Connect to the SphinxQL server.
|
20
|
+
#
|
21
|
+
# @param [Hash]
|
22
|
+
# a Hash containing :host and :port
|
23
|
+
def initialize(options)
|
24
|
+
@sock = TCPSocket.new(options[:host], options[:port])
|
25
|
+
@seq = 0
|
26
|
+
perform_handshake
|
27
|
+
end
|
28
|
+
|
29
|
+
def execute(sql)
|
30
|
+
p send_packet(query_packet(sql))
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def perform_handshake
|
36
|
+
auth_pkt = clnt_authentication_packet(serv_initialization_packet)
|
37
|
+
raise "Some sort of error" unless send_packet(auth_pkt)[:type] == :ok
|
38
|
+
end
|
39
|
+
|
40
|
+
def incr_seq(seq)
|
41
|
+
raise ProtocolError, "Invalid packet sequence value #{seq} != #{@seq}" unless seq == @seq
|
42
|
+
@seq = (seq + 1) % 256
|
43
|
+
end
|
44
|
+
|
45
|
+
def recv_packet
|
46
|
+
a, b, seq = @sock.read(4).unpack("CvC")
|
47
|
+
incr_seq(seq)
|
48
|
+
@sock.read(a | (b << 8))
|
49
|
+
end
|
50
|
+
|
51
|
+
def send_packet(pkt)
|
52
|
+
while chunk = pkt.read(2**24 - 1)
|
53
|
+
@sock.write([chunk.length % 256, chunk.length / 256, @seq, chunk].pack("CvCZ*"))
|
54
|
+
incr_seq(@seq)
|
55
|
+
end
|
56
|
+
|
57
|
+
serv_result_packet
|
58
|
+
end
|
59
|
+
|
60
|
+
def serv_result_packet
|
61
|
+
pkt = recv_packet
|
62
|
+
case pkt[0]
|
63
|
+
when "\x00" then ok_packet(pkt)
|
64
|
+
else raise ProtocolError, "Unknown packet type #{pkt[0]}"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def scan_lcb(str)
|
69
|
+
case v = str.slice!(0)
|
70
|
+
when "\xFB" then nil
|
71
|
+
when "\xFC" then str.slice!(0, 2).unpack("v").first
|
72
|
+
when "\xFD"
|
73
|
+
a, b = str.slice!(0, 3).unpack("Cv")
|
74
|
+
a | (b << 8)
|
75
|
+
when "\xFE"
|
76
|
+
a, b = str.slice!(0, 8).unpack("VV")
|
77
|
+
a | (b << 32)
|
78
|
+
else v.ord
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def ok_packet(str)
|
83
|
+
Hash[
|
84
|
+
[
|
85
|
+
:field_count,
|
86
|
+
:affected_rows,
|
87
|
+
:insert_id,
|
88
|
+
:serv_stat,
|
89
|
+
:warning_count,
|
90
|
+
:message
|
91
|
+
].zip([scan_lcb(str), scan_lcb(str), scan_lcb(str)] + str.unpack("vva*"))
|
92
|
+
].tap { |pkt| pkt[:type] = :ok }
|
93
|
+
end
|
94
|
+
|
95
|
+
def serv_initialization_packet
|
96
|
+
Hash[
|
97
|
+
[
|
98
|
+
:prot_ver,
|
99
|
+
:serv_ver,
|
100
|
+
:thread_id,
|
101
|
+
:scramble_buf_a,
|
102
|
+
:filler_a,
|
103
|
+
:serv_cap_a,
|
104
|
+
:serv_enc,
|
105
|
+
:serv_stat,
|
106
|
+
:serv_cap_b,
|
107
|
+
:scramble_len,
|
108
|
+
:filler_b,
|
109
|
+
:scramble_buf_b
|
110
|
+
].zip(recv_packet.unpack("CZ*Va8CvCvvCa10Z*"))
|
111
|
+
].tap do |pkt|
|
112
|
+
raise ProtocolError, "Unsupported MySQL protocol version #{pkt[:prot_ver]}" unless pkt[:prot_ver] == 0x0A
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def clnt_authentication_packet(init_pkt)
|
117
|
+
StringIO.new [
|
118
|
+
197120, # clnt_cap (prot 4.1, multi-stmt, multi-rs)
|
119
|
+
1024**3, # max pkt size
|
120
|
+
0, # charset not used
|
121
|
+
'', # filler 23 bytes
|
122
|
+
'', # username not used
|
123
|
+
'', # scramble buf (no password)
|
124
|
+
'' # dbname not used
|
125
|
+
].pack("VVCa23Z*A*Z*")
|
126
|
+
end
|
127
|
+
|
128
|
+
def query_packet(sql)
|
129
|
+
StringIO.new [
|
130
|
+
0x03, # COM_QUERY type
|
131
|
+
sql
|
132
|
+
].pack("Ca*")
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
module Oedipus
|
11
|
+
class QueryBuilder
|
12
|
+
def initialize(index_name, conn)
|
13
|
+
@index_name = index_name
|
14
|
+
@conn = conn
|
15
|
+
end
|
16
|
+
|
17
|
+
def sql(query, filters)
|
18
|
+
[
|
19
|
+
from,
|
20
|
+
conditions(query, filters),
|
21
|
+
limits(filters)
|
22
|
+
].join(" ")
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def from
|
28
|
+
"SELECT * FROM #{@index_name}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def conditions(query, filters)
|
32
|
+
exprs = []
|
33
|
+
exprs << "MATCH(#{@conn.quote(query)})" unless query.empty?
|
34
|
+
"WHERE " << exprs.join(" AND ") if exprs.any?
|
35
|
+
end
|
36
|
+
|
37
|
+
def limits(filters)
|
38
|
+
"LIMIT #{filters[:offset].to_i}, #{filters[:limit].to_i}" if filters.key?(:limit)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/oedipus.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
require "oedipus/version"
|
11
|
+
require "oedipus/query_builder"
|
12
|
+
require "oedipus/connection"
|
13
|
+
require "oedipus/index"
|
14
|
+
require "oedipus/mysql/client"
|
15
|
+
|
16
|
+
module Oedipus
|
17
|
+
class << self
|
18
|
+
# Connect to Sphinx running SphinxQL.
|
19
|
+
#
|
20
|
+
# @example
|
21
|
+
# c = Oedipus.connect("localhost:9306")
|
22
|
+
# c = Oedipus.connect(:host => "localhost", :port => 9306)
|
23
|
+
#
|
24
|
+
# @param [String] server
|
25
|
+
# a 'hostname:port' string
|
26
|
+
#
|
27
|
+
# @param [Hash] options
|
28
|
+
# a Hash with :host and :port keys
|
29
|
+
#
|
30
|
+
# @return [Connection]
|
31
|
+
# a client connected to SphinxQL
|
32
|
+
def connect(options)
|
33
|
+
# TODO: Add pooling
|
34
|
+
Connection.new(
|
35
|
+
options.kind_of?(String) ?
|
36
|
+
Hash[ [:host, :port].zip(options.split(":")) ] :
|
37
|
+
options
|
38
|
+
)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/oedipus.gemspec
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "oedipus/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "oedipus"
|
7
|
+
s.version = Oedipus::VERSION
|
8
|
+
s.authors = ["d11wtq"]
|
9
|
+
s.email = ["chris@w3style.co.uk"]
|
10
|
+
s.homepage = "https://github.com/d11wtq/oedipus"
|
11
|
+
s.summary = "Sphinx 2 Search Client for Ruby"
|
12
|
+
s.description = <<-DESC.strip
|
13
|
+
Oedipus brings full support for Sphinx 2 to Ruby:
|
14
|
+
|
15
|
+
* real-time indexes
|
16
|
+
* faceted search
|
17
|
+
* multi-queries
|
18
|
+
* full attribute support
|
19
|
+
* optional model-style interaction
|
20
|
+
|
21
|
+
It works with 'stable' versions of Sphinx 2 (>= 2.0.2). All
|
22
|
+
features are implemented entirely through the SphinxQL interface.
|
23
|
+
DESC
|
24
|
+
|
25
|
+
s.rubyforge_project = "oedipus"
|
26
|
+
|
27
|
+
s.files = `git ls-files`.split("\n")
|
28
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
29
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
30
|
+
s.require_paths = ["lib"]
|
31
|
+
|
32
|
+
s.add_runtime_dependency "ruby-mysql"
|
33
|
+
|
34
|
+
s.add_development_dependency "rspec"
|
35
|
+
end
|
data/spec/data/.gitkeep
ADDED
File without changes
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
require "rspec"
|
11
|
+
require "bundler/setup"
|
12
|
+
require "oedipus"
|
13
|
+
|
14
|
+
Dir[File.expand_path("../support/**/*rb", __FILE__)].each { |f| require f }
|
15
|
+
|
16
|
+
RSpec.configure do |config|
|
17
|
+
include Oedipus::TestHarness
|
18
|
+
|
19
|
+
config.before(:suite) do
|
20
|
+
unless ENV.key?("SEARCHD")
|
21
|
+
raise "You must specify a path to the Sphinx 'searchd' executable (>= 2.0.2)"
|
22
|
+
end
|
23
|
+
|
24
|
+
set_data_dir File.expand_path("../data", __FILE__)
|
25
|
+
set_searchd ENV["SEARCHD"]
|
26
|
+
|
27
|
+
prepare_data_dirs
|
28
|
+
write_sphinx_conf
|
29
|
+
start_searchd
|
30
|
+
end
|
31
|
+
|
32
|
+
config.before(:each) do
|
33
|
+
empty_indexes
|
34
|
+
end
|
35
|
+
|
36
|
+
config.after(:suite) do
|
37
|
+
stop_searchd
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
module Oedipus
|
11
|
+
# Mixed into RSpec suites to manage starting/stopping Sphinx.
|
12
|
+
module TestHarness
|
13
|
+
# Set the path to the searchd executable.
|
14
|
+
#
|
15
|
+
# The version of Sphinx must be >= 2.0.2.
|
16
|
+
#
|
17
|
+
# @param [String] path
|
18
|
+
# the absolute path to searchd
|
19
|
+
def set_searchd(path)
|
20
|
+
@searchd = path
|
21
|
+
end
|
22
|
+
|
23
|
+
# Set the path to a temporary directory for writing test data to.
|
24
|
+
#
|
25
|
+
# @param [String] path
|
26
|
+
# the path to a writable directory whose contents may be completely deleted
|
27
|
+
def set_data_dir(path)
|
28
|
+
@data_dir = path
|
29
|
+
end
|
30
|
+
|
31
|
+
# Ensure that the temporary data directories exist and are clean.
|
32
|
+
def prepare_data_dirs
|
33
|
+
Dir.mkdir("#{data_dir}/index") unless Dir.exist?("#{data_dir}/index")
|
34
|
+
Dir.mkdir("#{data_dir}/binlog") unless Dir.exist?("#{data_dir}/binlog")
|
35
|
+
|
36
|
+
clean_data_dirs
|
37
|
+
end
|
38
|
+
|
39
|
+
def empty_indexes
|
40
|
+
require "mysql"
|
41
|
+
@conn ||= ::Mysql.new(searchd_host[:host], nil, nil, nil, searchd_host[:port])
|
42
|
+
|
43
|
+
idxs = @conn.query("SHOW TABLES")
|
44
|
+
|
45
|
+
while idx = idxs.fetch_hash
|
46
|
+
docs = @conn.query("SELECT id FROM #{idx['Index']}")
|
47
|
+
while hash = docs.fetch_hash
|
48
|
+
@conn.query("DELETE FROM #{idx['Index']} WHERE id = #{hash['id']}")
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# Write the sphinx.conf file, using #index_definiton.
|
54
|
+
#
|
55
|
+
# Sphinx will listen on localhost port 9399.
|
56
|
+
#
|
57
|
+
# Any string returned from #index_definition will be used to define one or more indexes.
|
58
|
+
def write_sphinx_conf
|
59
|
+
File.open(searchd_config, "wb") do |f|
|
60
|
+
f <<
|
61
|
+
<<-CONF.gsub(/^ {10}/m, "")
|
62
|
+
##
|
63
|
+
# This file is automatically generated during tests
|
64
|
+
##
|
65
|
+
|
66
|
+
index posts_rt
|
67
|
+
{
|
68
|
+
type = rt
|
69
|
+
path = #{data_dir}/index/posts_rt
|
70
|
+
|
71
|
+
rt_field = title
|
72
|
+
rt_field = body
|
73
|
+
|
74
|
+
rt_attr_uint = user_id
|
75
|
+
rt_attr_uint = views
|
76
|
+
rt_attr_string = status
|
77
|
+
}
|
78
|
+
|
79
|
+
searchd
|
80
|
+
{
|
81
|
+
compat_sphinxql_magics = 0
|
82
|
+
|
83
|
+
max_matches = 2000
|
84
|
+
pid_file = #{data_dir}/searchd.pid
|
85
|
+
listen = #{searchd_host[:host]}:#{searchd_host[:port]}:mysql41
|
86
|
+
workers = threads
|
87
|
+
log = #{data_dir}/searchd.log
|
88
|
+
query_log = #{data_dir}/searchd.log
|
89
|
+
binlog_path = #{data_dir}/binlog
|
90
|
+
}
|
91
|
+
CONF
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# Start the sphinx daemon in a child process and return the PID.
|
96
|
+
#
|
97
|
+
# Output is redirected to searchd.out and searchd.err in the data dir.
|
98
|
+
def start_searchd
|
99
|
+
@searchd_pid = Process.spawn(
|
100
|
+
searchd, "--console", "-c", searchd_config,
|
101
|
+
out: "#{data_dir}/searchd.out",
|
102
|
+
err: "#{data_dir}/searchd.err"
|
103
|
+
)
|
104
|
+
sleep 1
|
105
|
+
end
|
106
|
+
|
107
|
+
# Stop an already running sphinx daemon and wait for it to shutdown.
|
108
|
+
def stop_searchd
|
109
|
+
Process.kill("TERM", @searchd_pid) && Process.wait
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
def clean_data_dirs
|
115
|
+
clean_files("#{data_dir}/index/**/*")
|
116
|
+
clean_files("#{data_dir}/binlog/**/*")
|
117
|
+
clean_files("#{data_dir}/searchd.pid")
|
118
|
+
clean_files("#{data_dir}/searchd.out")
|
119
|
+
clean_files("#{data_dir}/searchd.err")
|
120
|
+
clean_files("#{data_dir}/sphinx.conf")
|
121
|
+
end
|
122
|
+
|
123
|
+
def searchd_host
|
124
|
+
{ host: "127.0.0.1", port: 9399 }
|
125
|
+
end
|
126
|
+
|
127
|
+
def searchd
|
128
|
+
@searchd ||= "searchd"
|
129
|
+
end
|
130
|
+
|
131
|
+
def searchd_config
|
132
|
+
"#{data_dir}/sphinx.conf"
|
133
|
+
end
|
134
|
+
|
135
|
+
def clean_files(path)
|
136
|
+
Dir[path].each { |f| File.delete(f) unless File.directory?(f) }
|
137
|
+
end
|
138
|
+
|
139
|
+
def data_dir
|
140
|
+
@data_dir ||= "./data"
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
require "spec_helper"
|
11
|
+
|
12
|
+
describe Oedipus::Connection do
|
13
|
+
let(:conn) { Oedipus::Connection.new(searchd_host) }
|
14
|
+
|
15
|
+
describe "#initialize" do
|
16
|
+
context "on successful connection" do
|
17
|
+
it "returns the connection" do
|
18
|
+
Oedipus::Connection.new(searchd_host).should be_a_kind_of(Oedipus::Connection)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
context "on failed connection" do
|
23
|
+
it "raises an error" do
|
24
|
+
expect {
|
25
|
+
Oedipus::Connection.new(:host => "127.0.0.1", :port => 45346138)
|
26
|
+
}.to raise_error
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
describe "#[]" do
|
32
|
+
it "returns an index" do
|
33
|
+
conn[:posts_rt].should be_a_kind_of(Oedipus::Index)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Oedipus Sphinx 2 Search.
|
5
|
+
# Copyright © 2012 Chris Corbyn.
|
6
|
+
#
|
7
|
+
# See LICENSE file for details.
|
8
|
+
##
|
9
|
+
|
10
|
+
require "spec_helper"
|
11
|
+
|
12
|
+
describe Oedipus::Index do
|
13
|
+
let(:conn) { Oedipus::Connection.new(searchd_host) }
|
14
|
+
let(:index) { Oedipus::Index.new(:posts_rt, conn) }
|
15
|
+
|
16
|
+
describe "#insert" do
|
17
|
+
context "with valid data" do
|
18
|
+
it "returns the inserted attributes as a Hash" do
|
19
|
+
index.insert(
|
20
|
+
10,
|
21
|
+
title: "Badgers",
|
22
|
+
body: "They live in setts, do badgers.",
|
23
|
+
views: 721,
|
24
|
+
user_id: 7
|
25
|
+
).should == { id: 10, views: 721, user_id: 7, status: "" }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context "with invalid data" do
|
30
|
+
it "raises an error" do
|
31
|
+
expect {
|
32
|
+
index.insert(
|
33
|
+
10,
|
34
|
+
bad_field: "Invalid",
|
35
|
+
body: "They live in setts, do badgers.",
|
36
|
+
views: 721,
|
37
|
+
user_id: 7
|
38
|
+
)
|
39
|
+
}.to raise_error
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe "#search" do
|
45
|
+
before(:each) do
|
46
|
+
index.insert(1, title: "Badgers and foxes", views: 150)
|
47
|
+
index.insert(2, title: "Rabbits and hares", views: 87)
|
48
|
+
index.insert(3, title: "Badgers in the wild", views: 41)
|
49
|
+
index.insert(4, title: "Badgers for all!", views: 3003)
|
50
|
+
end
|
51
|
+
|
52
|
+
context "by fulltext matching" do
|
53
|
+
it "indicates the number of records found" do
|
54
|
+
index.search("badgers")[:total_found].should == 3
|
55
|
+
end
|
56
|
+
|
57
|
+
it "includes the matches records" do
|
58
|
+
index.search("badgers")[:records].should == [
|
59
|
+
{ id: 1, views: 150, user_id: 0, status: "" },
|
60
|
+
{ id: 3, views: 41, user_id: 0, status: "" },
|
61
|
+
{ id: 4, views: 3003, user_id: 0, status: "" }
|
62
|
+
]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context "with limits" do
|
67
|
+
it "still indicates the number of records found" do
|
68
|
+
index.search("badgers", limit: 2)[:total_found].should == 3
|
69
|
+
end
|
70
|
+
|
71
|
+
it "returns the limited subset of the results" do
|
72
|
+
index.search("badgers", limit: 2)[:records].should == [
|
73
|
+
{ id: 1, views: 150, user_id: 0, status: "" },
|
74
|
+
{ id: 3, views: 41, user_id: 0, status: "" }
|
75
|
+
]
|
76
|
+
end
|
77
|
+
|
78
|
+
it "can use an offset" do
|
79
|
+
index.search("badgers", limit: 1, offset: 1)[:records].should == [
|
80
|
+
{ id: 3, views: 41, user_id: 0, status: "" }
|
81
|
+
]
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
metadata
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: oedipus
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.pre1
|
5
|
+
prerelease: 6
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- d11wtq
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-03-30 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: ruby-mysql
|
16
|
+
requirement: &20332040 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *20332040
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: rspec
|
27
|
+
requirement: &20331620 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :development
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *20331620
|
36
|
+
description: ! "Oedipus brings full support for Sphinx 2 to Ruby:\n\n * real-time
|
37
|
+
indexes\n * faceted search\n * multi-queries\n * full attribute support\n
|
38
|
+
\ * optional model-style interaction\n\n It works with 'stable' versions
|
39
|
+
of Sphinx 2 (>= 2.0.2). All\n features are implemented entirely through the SphinxQL
|
40
|
+
interface."
|
41
|
+
email:
|
42
|
+
- chris@w3style.co.uk
|
43
|
+
executables: []
|
44
|
+
extensions: []
|
45
|
+
extra_rdoc_files: []
|
46
|
+
files:
|
47
|
+
- .gitignore
|
48
|
+
- .rspec
|
49
|
+
- Gemfile
|
50
|
+
- LICENSE
|
51
|
+
- README.md
|
52
|
+
- Rakefile
|
53
|
+
- lib/oedipus.rb
|
54
|
+
- lib/oedipus/connection.rb
|
55
|
+
- lib/oedipus/index.rb
|
56
|
+
- lib/oedipus/mysql/client.rb
|
57
|
+
- lib/oedipus/query_builder.rb
|
58
|
+
- lib/oedipus/version.rb
|
59
|
+
- oedipus.gemspec
|
60
|
+
- spec/data/.gitkeep
|
61
|
+
- spec/spec_helper.rb
|
62
|
+
- spec/support/test_harness.rb
|
63
|
+
- spec/unit/connection_spec.rb
|
64
|
+
- spec/unit/index_spec.rb
|
65
|
+
homepage: https://github.com/d11wtq/oedipus
|
66
|
+
licenses: []
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ! '>'
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 1.3.1
|
83
|
+
requirements: []
|
84
|
+
rubyforge_project: oedipus
|
85
|
+
rubygems_version: 1.8.11
|
86
|
+
signing_key:
|
87
|
+
specification_version: 3
|
88
|
+
summary: Sphinx 2 Search Client for Ruby
|
89
|
+
test_files: []
|