airblade-Sphincter 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +15 -0
- data/LICENSE.txt +27 -0
- data/Manifest.txt +16 -0
- data/README.txt +153 -0
- data/Rakefile +21 -0
- data/lib/sphincter.rb +107 -0
- data/lib/sphincter/association_searcher.rb +22 -0
- data/lib/sphincter/configure.rb +499 -0
- data/lib/sphincter/search.rb +199 -0
- data/lib/sphincter/search_stub.rb +60 -0
- data/lib/sphincter/tasks.rb +114 -0
- data/test/sphincter_test_case.rb +220 -0
- data/test/test_sphincter_association_searcher.rb +39 -0
- data/test/test_sphincter_configure.rb +386 -0
- data/test/test_sphincter_search.rb +100 -0
- data/test/test_sphincter_search_stub.rb +50 -0
- metadata +101 -0
data/History.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
== 1.1.0 / 2007-08-13
|
|
2
|
+
|
|
3
|
+
* 2 major enhancements:
|
|
4
|
+
* Fields across relationships may be included via add_index.
|
|
5
|
+
* Sphincter now automatically configures Dmytro Shteflyuk's sphinx API. Run
|
|
6
|
+
`rake sphincter:setup_sphinx` and check in vendor/plugins/sphinx.
|
|
7
|
+
* 1 bug fix:
|
|
8
|
+
* `rake sphincter:index` task didn't correctly run reindex. Bug submitted
|
|
9
|
+
by Lee O'Mara.
|
|
10
|
+
|
|
11
|
+
== 1.0.0 / 2007-07-26
|
|
12
|
+
|
|
13
|
+
* 1 major enhancement:
|
|
14
|
+
* Birthday!
|
|
15
|
+
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Copyright 2007 Eric Hodel. All rights reserved.
|
|
2
|
+
|
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
|
4
|
+
modification, are permitted provided that the following conditions
|
|
5
|
+
are met:
|
|
6
|
+
|
|
7
|
+
1. Redistributions of source code must retain the above copyright
|
|
8
|
+
notice, this list of conditions and the following disclaimer.
|
|
9
|
+
2. Redistributions in binary form must reproduce the above copyright
|
|
10
|
+
notice, this list of conditions and the following disclaimer in the
|
|
11
|
+
documentation and/or other materials provided with the distribution.
|
|
12
|
+
3. Neither the names of the authors nor the names of their contributors
|
|
13
|
+
may be used to endorse or promote products derived from this software
|
|
14
|
+
without specific prior written permission.
|
|
15
|
+
|
|
16
|
+
THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
|
|
17
|
+
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
18
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
19
|
+
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
|
|
20
|
+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
|
21
|
+
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
|
22
|
+
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
|
23
|
+
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
|
24
|
+
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
25
|
+
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
|
26
|
+
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
27
|
+
|
data/Manifest.txt
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
History.txt
|
|
2
|
+
LICENSE.txt
|
|
3
|
+
Manifest.txt
|
|
4
|
+
README.txt
|
|
5
|
+
Rakefile
|
|
6
|
+
lib/sphincter.rb
|
|
7
|
+
lib/sphincter/association_searcher.rb
|
|
8
|
+
lib/sphincter/configure.rb
|
|
9
|
+
lib/sphincter/search.rb
|
|
10
|
+
lib/sphincter/search_stub.rb
|
|
11
|
+
lib/sphincter/tasks.rb
|
|
12
|
+
test/sphincter_test_case.rb
|
|
13
|
+
test/test_sphincter_association_searcher.rb
|
|
14
|
+
test/test_sphincter_configure.rb
|
|
15
|
+
test/test_sphincter_search.rb
|
|
16
|
+
test/test_sphincter_search_stub.rb
|
data/README.txt
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Sphincter
|
|
2
|
+
|
|
3
|
+
Eric Hodel <drbrain@segment7.net>
|
|
4
|
+
|
|
5
|
+
http://seattlerb.org/Sphincter
|
|
6
|
+
|
|
7
|
+
File bugs:
|
|
8
|
+
|
|
9
|
+
http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
|
|
10
|
+
|
|
11
|
+
Sphincter was named by David Yeu.
|
|
12
|
+
|
|
13
|
+
== DESCRIPTION:
|
|
14
|
+
|
|
15
|
+
Sphincter is an ActiveRecord extension for full-text searching with Sphinx.
|
|
16
|
+
|
|
17
|
+
Sphincter uses Dmytro Shteflyuk's sphinx Ruby API and automatic
|
|
18
|
+
configuration to make totally rad ActiveRecord searching. Well, you
|
|
19
|
+
still have to tell Sphincter what models you want to search. It
|
|
20
|
+
doesn't read your mind.
|
|
21
|
+
|
|
22
|
+
For complete documentation:
|
|
23
|
+
|
|
24
|
+
ri Sphincter
|
|
25
|
+
|
|
26
|
+
== FEATURES:
|
|
27
|
+
|
|
28
|
+
* Automatically configures itself.
|
|
29
|
+
* Handy set of rake tasks for easy, automatic management.
|
|
30
|
+
* Automatically adds has_many metadata for searching across the
|
|
31
|
+
association.
|
|
32
|
+
* Stub for testing without connecting to searchd, Sphincter::SearchStub.
|
|
33
|
+
* Easy pagination support.
|
|
34
|
+
* Filtering by index metadata and ranges, including dates.
|
|
35
|
+
|
|
36
|
+
== PROBLEMS:
|
|
37
|
+
|
|
38
|
+
* Setting match mode not supported.
|
|
39
|
+
* Setting sort mode not supported.
|
|
40
|
+
* Setting per-field weights not supported.
|
|
41
|
+
* Setting id range not supported.
|
|
42
|
+
* Setting group-by not supported.
|
|
43
|
+
|
|
44
|
+
== QUICK-START:
|
|
45
|
+
|
|
46
|
+
Download and install Sphinx from http://www.sphinxsearch.com/downloads.html
|
|
47
|
+
|
|
48
|
+
Install Sphincter:
|
|
49
|
+
|
|
50
|
+
$ gem install Sphincter
|
|
51
|
+
|
|
52
|
+
Load Sphincter tasks in Rakefile:
|
|
53
|
+
|
|
54
|
+
require 'sphincter/tasks'
|
|
55
|
+
|
|
56
|
+
Setup the Dmytro Shteflyuk's Sphinx client:
|
|
57
|
+
|
|
58
|
+
$ rake sphincter:setup_sphinx
|
|
59
|
+
|
|
60
|
+
Add vendor/plugins/sphinx to your SCM system.
|
|
61
|
+
|
|
62
|
+
Load Sphincter in config/environment.rb:
|
|
63
|
+
|
|
64
|
+
require 'sphincter'
|
|
65
|
+
|
|
66
|
+
Add indexes to models:
|
|
67
|
+
|
|
68
|
+
class Post < ActiveRecord::Base
|
|
69
|
+
belongs_to :blog
|
|
70
|
+
add_index :fields => %w[title body published]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
Add searching UI:
|
|
74
|
+
|
|
75
|
+
class BlogController < ApplicationController
|
|
76
|
+
def search
|
|
77
|
+
@blog = Blog.find params[:id]
|
|
78
|
+
|
|
79
|
+
@results = @blog.posts.search params[:q]
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
<ol>
|
|
84
|
+
<% @results.records.each do |post| -%>
|
|
85
|
+
<li>
|
|
86
|
+
<div><%= link_to post.title, post_path(post) %></div>
|
|
87
|
+
<div><%= truncate post.body, 250 %></div>
|
|
88
|
+
</li>
|
|
89
|
+
<% end -%>
|
|
90
|
+
</ol>
|
|
91
|
+
|
|
92
|
+
Start searchd:
|
|
93
|
+
|
|
94
|
+
$ rake sphincter:start_searchd
|
|
95
|
+
|
|
96
|
+
Then test it out in your browser.
|
|
97
|
+
|
|
98
|
+
NOTE: By default, Sphincter will run searchd on the same port for all
|
|
99
|
+
environments. See Sphincter::Configure for how to configure different
|
|
100
|
+
environments to use different ports.
|
|
101
|
+
|
|
102
|
+
== TESTING QUICK-START:
|
|
103
|
+
|
|
104
|
+
See Sphinx::SearchStub.
|
|
105
|
+
|
|
106
|
+
== EXAMPLES:
|
|
107
|
+
|
|
108
|
+
See Sphincter::Search#search for full documentation.
|
|
109
|
+
|
|
110
|
+
Example ActiveRecord model:
|
|
111
|
+
|
|
112
|
+
class Post < ActiveRecord::Base
|
|
113
|
+
belongs_to :blog
|
|
114
|
+
belongs_to :user
|
|
115
|
+
|
|
116
|
+
# published is a boolean and title and body are string or text fields
|
|
117
|
+
# user.name is automatically fetched via the user association
|
|
118
|
+
add_index :fields => %w[title body published]
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
Simple search:
|
|
122
|
+
|
|
123
|
+
Post.search 'words'
|
|
124
|
+
|
|
125
|
+
Only search published posts:
|
|
126
|
+
|
|
127
|
+
Post.search 'words', :conditions => { :published => 1 }
|
|
128
|
+
|
|
129
|
+
Only search posts created in the last week:
|
|
130
|
+
|
|
131
|
+
now = Time.now
|
|
132
|
+
ago = now - 1.weeks
|
|
133
|
+
Post.search 'words', :between => { :created_on => [ago, now] }
|
|
134
|
+
|
|
135
|
+
Pagination (defaults to ten records/page):
|
|
136
|
+
|
|
137
|
+
Post.search 'words', :page => 2
|
|
138
|
+
|
|
139
|
+
Pagination with custom page size:
|
|
140
|
+
|
|
141
|
+
Post.search 'words', :page => 2, :per_page => 20
|
|
142
|
+
|
|
143
|
+
Pagination with custom page size (better):
|
|
144
|
+
|
|
145
|
+
Add to config/sphincter.yml:
|
|
146
|
+
|
|
147
|
+
sphincter:
|
|
148
|
+
per_page: 20
|
|
149
|
+
|
|
150
|
+
Then search:
|
|
151
|
+
|
|
152
|
+
Post.search 'words', :page => 2
|
|
153
|
+
|
data/Rakefile
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# -*- ruby -*-
|
|
2
|
+
|
|
3
|
+
require 'rubygems'
|
|
4
|
+
require 'hoe'
|
|
5
|
+
$:.unshift 'lib'
|
|
6
|
+
require 'sphincter'
|
|
7
|
+
|
|
8
|
+
Hoe.new('Sphincter', Sphincter::VERSION) do |p|
|
|
9
|
+
p.rubyforge_name = 'seattlerb'
|
|
10
|
+
p.author = 'Eric Hodel'
|
|
11
|
+
p.email = 'drbrain@segment7.net'
|
|
12
|
+
p.summary = p.paragraphs_of('README.txt', 7).first
|
|
13
|
+
p.description = p.paragraphs_of('README.txt', 8).first
|
|
14
|
+
p.url = p.paragraphs_of('README.txt', 2).first
|
|
15
|
+
p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
|
|
16
|
+
|
|
17
|
+
p.extra_deps << ['rake', '>= 0.7.3']
|
|
18
|
+
p.extra_deps << ['rails', '>= 1.2.3']
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# vim: syntax=Ruby
|
data/lib/sphincter.rb
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
$TESTING = defined?($TESTING) && $TESTING
|
|
2
|
+
|
|
3
|
+
##
|
|
4
|
+
# Sphincter is a ActiveRecord extension for full-text searching using the
|
|
5
|
+
# Sphinx library.
|
|
6
|
+
#
|
|
7
|
+
# For the quick-start guide and some examples, see README.txt.
|
|
8
|
+
#
|
|
9
|
+
# == Installing
|
|
10
|
+
#
|
|
11
|
+
# Download and install Sphinx from http://www.sphinxsearch.com/downloads.html
|
|
12
|
+
#
|
|
13
|
+
# Download Sphinx Ruby API from
|
|
14
|
+
# http://rubyforge.org/frs/?group_id=2604&release_id=11049
|
|
15
|
+
#
|
|
16
|
+
# Unpack Sphinx Ruby API into vendor/plugins/.
|
|
17
|
+
#
|
|
18
|
+
# Install the gem:
|
|
19
|
+
#
|
|
20
|
+
# gem install Sphincter
|
|
21
|
+
#
|
|
22
|
+
# Require Sphincter in config/environment.rb:
|
|
23
|
+
#
|
|
24
|
+
# require 'sphincter'
|
|
25
|
+
#
|
|
26
|
+
# Require the Sphincter rake tasks in Rakefile:
|
|
27
|
+
#
|
|
28
|
+
# require 'sphincter/tasks'
|
|
29
|
+
#
|
|
30
|
+
# == Setup
|
|
31
|
+
#
|
|
32
|
+
# At best, you don't do anything to setup Sphincter. It has sensible built-in
|
|
33
|
+
# defaults.
|
|
34
|
+
#
|
|
35
|
+
# If you're running Sphinx's searchd for multiple environments on the same
|
|
36
|
+
# machine, you'll want to add a config file to change the port that searchd
|
|
37
|
+
# and the RAILS_ENV will comminicate across. Do that in a per-environment
|
|
38
|
+
# configuration file.
|
|
39
|
+
#
|
|
40
|
+
# If you have multiple machines, you'll want to change which address searchd
|
|
41
|
+
# will run on. Do that in the global configuration file.
|
|
42
|
+
#
|
|
43
|
+
# See Sphincter::Configure for full information on how to setup these and
|
|
44
|
+
# other options for Sphincter.
|
|
45
|
+
#
|
|
46
|
+
# When you're done, run:
|
|
47
|
+
#
|
|
48
|
+
# $ rake sphincter:configure
|
|
49
|
+
#
|
|
50
|
+
# == Indexing
|
|
51
|
+
#
|
|
52
|
+
# Sphincter automatically extends ActiveRecord::Base with Sphincter::Search, so
|
|
53
|
+
# you only have to call add_index in the models you want indexed:
|
|
54
|
+
#
|
|
55
|
+
# class Model < ActiveRecord::Base
|
|
56
|
+
# belongs_to :other
|
|
57
|
+
#
|
|
58
|
+
# add_index :fields => %w[title body]
|
|
59
|
+
# end
|
|
60
|
+
#
|
|
61
|
+
# class Other < ActiveRecord::Base
|
|
62
|
+
# has_many :models
|
|
63
|
+
# end
|
|
64
|
+
#
|
|
65
|
+
# add_index automatically adds a #search method to has_many associations
|
|
66
|
+
# referencing this model, so you could:
|
|
67
|
+
#
|
|
68
|
+
# Other.find(id).models.search 'some query'
|
|
69
|
+
#
|
|
70
|
+
# See Sphincter::Search for details.
|
|
71
|
+
#
|
|
72
|
+
# When you're done, run:
|
|
73
|
+
#
|
|
74
|
+
# rake sphincter:index
|
|
75
|
+
#
|
|
76
|
+
# == Tasks
|
|
77
|
+
#
|
|
78
|
+
# You can get a set of Sphincter tasks by requiring 'sphincter/tasks' in your
|
|
79
|
+
# Rakefile. These tasks are all in the 'sphincter' namespace:
|
|
80
|
+
#
|
|
81
|
+
# configure:: Creates sphinx.conf if it doesn't exist
|
|
82
|
+
# reconfigure:: Creates sphinx.conf, replacing the existing one.
|
|
83
|
+
# index:: Runs the sphinx indexer if the index doesn't exist.
|
|
84
|
+
# reindex:: Runs the sphinx indexer. Rotates the index if searchd is running.
|
|
85
|
+
# reset:: Stops searchd, reconfigures and reindexes
|
|
86
|
+
# restart_searchd:: Restarts the searchd sphinx daemon
|
|
87
|
+
# start_searchd:: Starts the searchd sphinx daemon
|
|
88
|
+
# stop_searchd:: Stops the searchd daemon
|
|
89
|
+
|
|
90
|
+
module Sphincter
|
|
91
|
+
|
|
92
|
+
##
|
|
93
|
+
# This is the version of Sphincter you are using.
|
|
94
|
+
|
|
95
|
+
VERSION = '1.1.0'
|
|
96
|
+
|
|
97
|
+
##
|
|
98
|
+
# Sphincter error base class.
|
|
99
|
+
|
|
100
|
+
class Error < RuntimeError; end
|
|
101
|
+
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
require 'sphincter/configure'
|
|
105
|
+
require 'sphincter/association_searcher'
|
|
106
|
+
require 'sphincter/search'
|
|
107
|
+
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
require 'sphincter'
|
|
2
|
+
|
|
3
|
+
##
|
|
4
|
+
# ActiveRecord::Associations::ClassMethods#has_many extension for searching
|
|
5
|
+
# the items of an ActiveRecord::Associations::AssociationProxy.
|
|
6
|
+
|
|
7
|
+
module Sphincter::AssociationSearcher
|
|
8
|
+
|
|
9
|
+
##
|
|
10
|
+
# Searches for +query+ with +options+. Adds a condition so only the
|
|
11
|
+
# proxy_owner's records are matched.
|
|
12
|
+
|
|
13
|
+
def search(query, options = {})
|
|
14
|
+
pkey = proxy_reflection.primary_key_name
|
|
15
|
+
options[:conditions] ||= {}
|
|
16
|
+
options[:conditions][pkey] = proxy_owner.id
|
|
17
|
+
|
|
18
|
+
proxy_reflection.klass.search query, options
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
end
|
|
22
|
+
|
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
require 'fileutils'
|
|
2
|
+
require 'yaml'
|
|
3
|
+
|
|
4
|
+
require 'sphincter'
|
|
5
|
+
|
|
6
|
+
##
|
|
7
|
+
# Configuration module for Sphincter.
|
|
8
|
+
#
|
|
9
|
+
# DEFAULT_CONF contains the default options. They can be overridden in both a
|
|
10
|
+
# global config/sphincter.yml and in a per-environment
|
|
11
|
+
# config/environments/sphincter.RAILS_ENV.yml.
|
|
12
|
+
#
|
|
13
|
+
# The only option you should need to override is the port option of
|
|
14
|
+
# sphincter, so a config file for separate test and development indexes would
|
|
15
|
+
# look like:
|
|
16
|
+
#
|
|
17
|
+
# config/environments/sphincter.development.yml:
|
|
18
|
+
#
|
|
19
|
+
# sphincter:
|
|
20
|
+
# port: 3313
|
|
21
|
+
#
|
|
22
|
+
# config/environments/sphincter.test.yml:
|
|
23
|
+
#
|
|
24
|
+
# sphincter:
|
|
25
|
+
# port: 3314
|
|
26
|
+
#
|
|
27
|
+
# Configuration options:
|
|
28
|
+
#
|
|
29
|
+
# sphincter:: Options for serachd's and Sphinx's port and address, and
|
|
30
|
+
# paths for index files.
|
|
31
|
+
# index:: Options for a sphinx index conf section
|
|
32
|
+
# indexer:: Options for the sphinx indexer
|
|
33
|
+
# mysql:: Options for the sphinx indexer's mysql database connection. The
|
|
34
|
+
# important ones are filled from config/database.yml
|
|
35
|
+
# searchd:: Options for a sphinx searchd conf section
|
|
36
|
+
# source:: Options for a sphinx source conf section
|
|
37
|
+
#
|
|
38
|
+
# The sphincter entry contains:
|
|
39
|
+
#
|
|
40
|
+
# address:: Which host searchd will run on, and which host Sphincter will
|
|
41
|
+
# connect to.
|
|
42
|
+
# port:: Which port searchd and Sphincter will connect to.
|
|
43
|
+
# path:: Location of searchd indexes, relative to RAILS_ROOT.
|
|
44
|
+
# per_page:: How many items to include in a search by default.
|
|
45
|
+
#
|
|
46
|
+
# All other entries are from Sphinx.
|
|
47
|
+
#
|
|
48
|
+
# See http://www.sphinxsearch.com/doc.html#reference for details on sphinx
|
|
49
|
+
# conf file settings.
|
|
50
|
+
|
|
51
|
+
module Sphincter::Configure
|
|
52
|
+
|
|
53
|
+
##
|
|
54
|
+
# A class for building sphinx.conf source/index sections.
|
|
55
|
+
|
|
56
|
+
class Index
|
|
57
|
+
|
|
58
|
+
attr_reader :source_conf
|
|
59
|
+
|
|
60
|
+
attr_reader :name
|
|
61
|
+
|
|
62
|
+
##
|
|
63
|
+
# Creates a new Index for +klass+ and +options+.
|
|
64
|
+
|
|
65
|
+
def initialize(klass, options)
|
|
66
|
+
@fields = []
|
|
67
|
+
@where = []
|
|
68
|
+
@group = false
|
|
69
|
+
|
|
70
|
+
@source_conf = {}
|
|
71
|
+
@source_conf['sql_date_column'] = []
|
|
72
|
+
@source_conf['sql_group_column'] = %w[sphincter_index_id]
|
|
73
|
+
|
|
74
|
+
@klass = klass
|
|
75
|
+
@table = @klass.table_name
|
|
76
|
+
@conn = @klass.connection
|
|
77
|
+
@tables = @table.dup
|
|
78
|
+
|
|
79
|
+
defaults = {
|
|
80
|
+
:conditions => [],
|
|
81
|
+
:fields => [],
|
|
82
|
+
:name => @table,
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
@options = defaults.merge options
|
|
86
|
+
|
|
87
|
+
@name = @options[:name] || @table
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
##
|
|
91
|
+
# Adds plain field +field+ to the index from class +klass+ using
|
|
92
|
+
# +as_table+ as the table name.
|
|
93
|
+
|
|
94
|
+
def add_field(field, klass = @klass, as_table = nil)
|
|
95
|
+
table = klass.table_name
|
|
96
|
+
quoted_field = @conn.quote_column_name field
|
|
97
|
+
|
|
98
|
+
column_type = klass.columns_hash[field].type
|
|
99
|
+
expr = case column_type
|
|
100
|
+
when :date, :datetime, :time, :timestamp then
|
|
101
|
+
@source_conf['sql_date_column'] << field
|
|
102
|
+
"UNIX_TIMESTAMP(#{table}.#{quoted_field})"
|
|
103
|
+
when :boolean, :integer then
|
|
104
|
+
@source_conf['sql_group_column'] << field
|
|
105
|
+
"#{table}.#{quoted_field}"
|
|
106
|
+
when :string, :text then
|
|
107
|
+
"#{table}.#{quoted_field}"
|
|
108
|
+
else
|
|
109
|
+
raise Sphincter::Error, "unknown column type #{column_type}"
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
as_name = [as_table, field].compact.join '_'
|
|
113
|
+
as_name = @conn.quote_column_name as_name
|
|
114
|
+
|
|
115
|
+
"#{expr} AS #{as_name}"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
##
|
|
119
|
+
# Includes field +as_field+ from association +as_name+ in the index.
|
|
120
|
+
|
|
121
|
+
def add_include(as_name, as_field)
|
|
122
|
+
as_assoc = @klass.reflect_on_all_associations.find do |assoc|
|
|
123
|
+
assoc.name == as_name.intern
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
if as_assoc.nil? then
|
|
127
|
+
raise Sphincter::Error,
|
|
128
|
+
"could not find association \"#{as_name}\" in #{@klass.name}"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
as_klass = as_assoc.class_name.constantize
|
|
132
|
+
as_table = as_klass.table_name
|
|
133
|
+
|
|
134
|
+
as_klass_key = @conn.quote_column_name as_klass.primary_key.to_s
|
|
135
|
+
as_assoc_key = @conn.quote_column_name as_assoc.primary_key_name.to_s
|
|
136
|
+
|
|
137
|
+
case as_assoc.macro
|
|
138
|
+
when :belongs_to then
|
|
139
|
+
@fields << add_field(as_field, as_klass, as_table)
|
|
140
|
+
@tables << " LEFT JOIN #{as_table} ON" \
|
|
141
|
+
" #{@table}.#{as_assoc_key} = #{as_table}.#{as_klass_key}"
|
|
142
|
+
|
|
143
|
+
when :has_many then
|
|
144
|
+
if as_assoc.options.include? :through then
|
|
145
|
+
raise Sphincter::Error,
|
|
146
|
+
"unsupported macro has_many :through for \"#{as_name}\" " \
|
|
147
|
+
"in #{klass.name}.add_index"
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
as_pkey = @conn.quote_column_name as_klass.primary_key.to_s
|
|
151
|
+
as_fkey = @conn.quote_column_name as_assoc.primary_key_name.to_s
|
|
152
|
+
|
|
153
|
+
as_name = [as_table, as_field].compact.join '_'
|
|
154
|
+
as_name = @conn.quote_column_name as_name
|
|
155
|
+
|
|
156
|
+
field = @conn.quote_column_name as_field
|
|
157
|
+
|
|
158
|
+
@fields << "GROUP_CONCAT(#{as_table}.#{field} SEPARATOR ' ') AS #{as_name}"
|
|
159
|
+
|
|
160
|
+
if as_assoc.options.include? :as then
|
|
161
|
+
poly_name = as_assoc.options[:as]
|
|
162
|
+
id_col = @conn.quote_column_name "#{poly_name}_id"
|
|
163
|
+
type_col = @conn.quote_column_name "#{poly_name}_type"
|
|
164
|
+
|
|
165
|
+
@tables << " LEFT JOIN #{as_table} ON"\
|
|
166
|
+
" #{@table}.#{as_klass_key} = #{as_table}.#{id_col} AND" \
|
|
167
|
+
" #{@conn.quote @klass.name} = #{as_table}.#{type_col}"
|
|
168
|
+
else
|
|
169
|
+
@tables << " LEFT JOIN #{as_table} ON" \
|
|
170
|
+
" #{@table}.#{as_klass_key} = #{as_table}.#{as_assoc_key}"
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
@group = true
|
|
174
|
+
else
|
|
175
|
+
raise Sphincter::Error,
|
|
176
|
+
"unsupported macro #{as_assoc.macro} for \"#{as_name}\" " \
|
|
177
|
+
"in #{klass.name}.add_index"
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def configure
|
|
182
|
+
conn = @klass.connection
|
|
183
|
+
pk = conn.quote_column_name @klass.primary_key
|
|
184
|
+
index_id = @options[:index_id]
|
|
185
|
+
|
|
186
|
+
index_count = Sphincter::Configure.index_count
|
|
187
|
+
|
|
188
|
+
@fields << "(#{@table}.#{pk} * #{index_count} + #{index_id}) AS #{pk}"
|
|
189
|
+
@fields << "#{index_id} AS sphincter_index_id"
|
|
190
|
+
@fields << "'#{@klass.name}' AS sphincter_klass"
|
|
191
|
+
|
|
192
|
+
@options[:fields].each do |field|
|
|
193
|
+
case field
|
|
194
|
+
when /\./ then add_include(*field.split('.', 2))
|
|
195
|
+
else @fields << add_field(field)
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
@fields = @fields.join ', '
|
|
200
|
+
|
|
201
|
+
@where << "#{@table}.#{pk} >= $start"
|
|
202
|
+
@where << "#{@table}.#{pk} <= $end"
|
|
203
|
+
@where.push(*@options[:conditions])
|
|
204
|
+
@where = @where.compact.join ' AND '
|
|
205
|
+
|
|
206
|
+
query = "SELECT #{@fields} FROM #{@tables} WHERE #{@where}"
|
|
207
|
+
query << " GROUP BY #{@table}.#{pk}" if @group
|
|
208
|
+
|
|
209
|
+
@source_conf['sql_query'] = query
|
|
210
|
+
@source_conf['sql_query_info'] =
|
|
211
|
+
"SELECT * FROM #{@table} " \
|
|
212
|
+
"WHERE #{@table}.#{pk} = (($id - #{index_id}) / #{index_count})"
|
|
213
|
+
@source_conf['sql_query_range'] =
|
|
214
|
+
"SELECT MIN(#{pk}), MAX(#{pk}) FROM #{@table}"
|
|
215
|
+
@source_conf['strip_html'] = @options[:strip_html] ? 1 : 0
|
|
216
|
+
|
|
217
|
+
@source_conf
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
@env_conf = nil
|
|
223
|
+
@index_count = nil
|
|
224
|
+
|
|
225
|
+
rails_env = defined?(RAILS_ENV) ? RAILS_ENV : 'RAILS_ENV'
|
|
226
|
+
|
|
227
|
+
##
|
|
228
|
+
# Default Sphincter configuration.
|
|
229
|
+
|
|
230
|
+
DEFAULT_CONF = {
|
|
231
|
+
'sphincter' => {
|
|
232
|
+
'address' => '127.0.0.1',
|
|
233
|
+
'path' => "sphinx/#{rails_env}",
|
|
234
|
+
'per_page' => 10,
|
|
235
|
+
'port' => 3312,
|
|
236
|
+
},
|
|
237
|
+
|
|
238
|
+
'index' => {
|
|
239
|
+
'charset_type' => 'utf-8',
|
|
240
|
+
'docinfo' => 'extern',
|
|
241
|
+
'min_word_len' => 1,
|
|
242
|
+
'morphology' => 'stem_en',
|
|
243
|
+
'stopwords' => '',
|
|
244
|
+
},
|
|
245
|
+
|
|
246
|
+
'indexer' => {
|
|
247
|
+
'mem_limit' => '32M',
|
|
248
|
+
},
|
|
249
|
+
|
|
250
|
+
'mysql' => {
|
|
251
|
+
'sql_query_pre' => [
|
|
252
|
+
'SET NAMES utf8',
|
|
253
|
+
],
|
|
254
|
+
},
|
|
255
|
+
|
|
256
|
+
'searchd' => {
|
|
257
|
+
'log' => "log/sphinx/searchd.#{rails_env}.log",
|
|
258
|
+
'max_children' => 30,
|
|
259
|
+
'max_matches' => 1000,
|
|
260
|
+
'query_log' => "log/sphinx/query.#{rails_env}.log",
|
|
261
|
+
'read_timeout' => 5,
|
|
262
|
+
},
|
|
263
|
+
|
|
264
|
+
'source' => {
|
|
265
|
+
'index_html_attrs' => '',
|
|
266
|
+
'sql_query_post' => '',
|
|
267
|
+
'sql_range_step' => 20000,
|
|
268
|
+
'strip_html' => 0,
|
|
269
|
+
},
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
##
|
|
273
|
+
# Builds and writes out a sphinx.conf file.
|
|
274
|
+
|
|
275
|
+
def self.configure
|
|
276
|
+
conf = get_conf
|
|
277
|
+
db_conf = get_db_conf
|
|
278
|
+
|
|
279
|
+
db_conf = conf[db_conf['type']].merge db_conf
|
|
280
|
+
|
|
281
|
+
sources = get_sources
|
|
282
|
+
|
|
283
|
+
sources.each do |name, source_conf|
|
|
284
|
+
sources[name] = db_conf.merge source_conf
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
write_configuration conf, sources
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
##
|
|
291
|
+
# Merges Hashes of Hashes +mergee+ and +hash+.
|
|
292
|
+
|
|
293
|
+
def self.deep_merge(mergee, hash)
|
|
294
|
+
mergee = mergee.dup
|
|
295
|
+
hash.keys.each do |key| mergee[key] ||= hash[key] end
|
|
296
|
+
mergee.each do |key, value|
|
|
297
|
+
next unless hash[key]
|
|
298
|
+
mergee[key] = value.merge hash[key]
|
|
299
|
+
end
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
##
|
|
303
|
+
# Builds the Sphincter configuration.
|
|
304
|
+
#
|
|
305
|
+
# Automatically fills in searchd address, port and pid_file from 'sphincter'
|
|
306
|
+
# section.
|
|
307
|
+
|
|
308
|
+
def self.get_conf
|
|
309
|
+
return @env_conf unless @env_conf.nil?
|
|
310
|
+
|
|
311
|
+
base_file = File.expand_path File.join(RAILS_ROOT, 'config', 'sphincter.yml')
|
|
312
|
+
base_conf = deep_merge DEFAULT_CONF, get_conf_from(base_file)
|
|
313
|
+
|
|
314
|
+
env_file = File.expand_path File.join(RAILS_ROOT, 'config', 'environments',
|
|
315
|
+
"sphincter.#{RAILS_ENV}.yml")
|
|
316
|
+
env_conf = deep_merge base_conf, get_conf_from(env_file)
|
|
317
|
+
|
|
318
|
+
env_conf['searchd']['address'] = env_conf['sphincter']['address']
|
|
319
|
+
env_conf['searchd']['port'] = env_conf['sphincter']['port']
|
|
320
|
+
env_conf['searchd']['pid_file'] = File.join(env_conf['sphincter']['path'],
|
|
321
|
+
'searchd.pid')
|
|
322
|
+
|
|
323
|
+
@env_conf = env_conf
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
##
|
|
327
|
+
# Reads configuration file +file+. Returns {} if the file does not exist.
|
|
328
|
+
|
|
329
|
+
def self.get_conf_from(file)
|
|
330
|
+
if File.exist? file then
|
|
331
|
+
YAML.load File.read(file)
|
|
332
|
+
else
|
|
333
|
+
{}
|
|
334
|
+
end
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
##
|
|
338
|
+
# Builds a sphinx.conf source configuration for each index.
|
|
339
|
+
|
|
340
|
+
def self.get_sources
|
|
341
|
+
load_models
|
|
342
|
+
|
|
343
|
+
indexes = Sphincter::Search.indexes
|
|
344
|
+
index_count # HACK necessary to set options[:index_id] per-index
|
|
345
|
+
|
|
346
|
+
sources = {}
|
|
347
|
+
|
|
348
|
+
indexes.each do |klass, model_indexes|
|
|
349
|
+
model_indexes.each do |options|
|
|
350
|
+
index = Index.new klass, options
|
|
351
|
+
index.configure
|
|
352
|
+
|
|
353
|
+
sources[index.name] = index.source_conf
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
sources
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
##
|
|
361
|
+
# Builds a field for a source's sql_query sphinx.conf setting.
|
|
362
|
+
#
|
|
363
|
+
# get_sources_field only understands :datetime, :boolean, :integer, :string
|
|
364
|
+
# and :text column types.
|
|
365
|
+
|
|
366
|
+
##
|
|
367
|
+
# Retrieves the database configuration for ActiveRecord::Base and adapts it
|
|
368
|
+
# for a sphinx.conf file.
|
|
369
|
+
|
|
370
|
+
def self.get_db_conf
|
|
371
|
+
conf = {}
|
|
372
|
+
ar_conf = ActiveRecord::Base.configurations[::RAILS_ENV]
|
|
373
|
+
|
|
374
|
+
conf['type'] = ar_conf['adapter']
|
|
375
|
+
conf['sql_host'] = ar_conf['host'] if ar_conf.include? 'host'
|
|
376
|
+
conf['sql_user'] = ar_conf['username'] if ar_conf.include? 'username'
|
|
377
|
+
conf['sql_pass'] = ar_conf['password'] if ar_conf.include? 'password'
|
|
378
|
+
conf['sql_db'] = ar_conf['database'] if ar_conf.include? 'database'
|
|
379
|
+
conf['sql_sock'] = ar_conf['socket'] if ar_conf.include? 'socket'
|
|
380
|
+
|
|
381
|
+
conf
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
##
|
|
385
|
+
# Iterates over the searchable ActiveRecord::Base classes and assigns an
|
|
386
|
+
# index to each one. Returns the total number of indexes found.
|
|
387
|
+
|
|
388
|
+
def self.index_count
|
|
389
|
+
return @index_count unless @index_count.nil?
|
|
390
|
+
|
|
391
|
+
@index_count = 0
|
|
392
|
+
|
|
393
|
+
load_models
|
|
394
|
+
|
|
395
|
+
Sphincter::Search.indexes.each do |model, model_indexes|
|
|
396
|
+
model_indexes.each do |options|
|
|
397
|
+
options[:index_id] = @index_count
|
|
398
|
+
@index_count += 1
|
|
399
|
+
end
|
|
400
|
+
end
|
|
401
|
+
@index_count
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
##
|
|
405
|
+
# Loads ActiveRecord::Base models from app/models.
|
|
406
|
+
|
|
407
|
+
def self.load_models
|
|
408
|
+
model_files = Dir[File.join(RAILS_ROOT, 'app', 'models', '*.rb')]
|
|
409
|
+
model_names = model_files.map { |name| File.basename name, '.rb' }
|
|
410
|
+
model_names.each { |name| name.camelize.constantize }
|
|
411
|
+
end
|
|
412
|
+
|
|
413
|
+
##
|
|
414
|
+
# Returns the pid of searchd if searchd is running, otherwise false.
|
|
415
|
+
|
|
416
|
+
def self.searchd_running?
|
|
417
|
+
pid_file = Sphincter::Configure.get_conf['searchd']['pid_file']
|
|
418
|
+
return false unless File.exist? pid_file
|
|
419
|
+
|
|
420
|
+
pid = File.read(pid_file).chomp
|
|
421
|
+
return false if pid.empty?
|
|
422
|
+
|
|
423
|
+
running = `ps -p #{pid}` =~ /#{pid}.*searchd/
|
|
424
|
+
running ? pid : false
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
##
|
|
428
|
+
# Outputs a sphinx.conf configuration section titled +heading+ using the
|
|
429
|
+
# Hash +data+. Values in +data+ may be a String or Array. For an Array,
|
|
430
|
+
# the Hash key is printed multiple times.
|
|
431
|
+
|
|
432
|
+
def self.section(heading, data)
|
|
433
|
+
section = []
|
|
434
|
+
section << heading
|
|
435
|
+
section << '{'
|
|
436
|
+
data.sort_by { |k,| k }.each do |key, value|
|
|
437
|
+
case value
|
|
438
|
+
when Array then
|
|
439
|
+
next if value.empty?
|
|
440
|
+
value.each do |v|
|
|
441
|
+
section << " #{key} = #{v}"
|
|
442
|
+
end
|
|
443
|
+
else
|
|
444
|
+
section << " #{key} = #{value}"
|
|
445
|
+
end
|
|
446
|
+
end
|
|
447
|
+
section << '}'
|
|
448
|
+
section.join "\n"
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
##
|
|
452
|
+
# The path to sphinx.conf.
|
|
453
|
+
|
|
454
|
+
def self.sphinx_conf
|
|
455
|
+
@sphinx_conf ||= File.join sphinx_dir, 'sphinx.conf'
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
##
|
|
459
|
+
# The directory where sphinx's files live.
|
|
460
|
+
|
|
461
|
+
def self.sphinx_dir
|
|
462
|
+
@sphinx_dir ||= File.join(RAILS_ROOT,
|
|
463
|
+
Sphincter::Configure.get_conf['sphincter']['path'])
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
##
|
|
467
|
+
# Writes out a sphinx.conf configuration using +conf+ and +sources+.
|
|
468
|
+
|
|
469
|
+
def self.write_configuration(conf, sources)
|
|
470
|
+
FileUtils.mkdir_p sphinx_dir
|
|
471
|
+
|
|
472
|
+
out = []
|
|
473
|
+
|
|
474
|
+
out << section('indexer', conf['indexer'])
|
|
475
|
+
out << nil
|
|
476
|
+
|
|
477
|
+
out << section('searchd', conf['searchd'])
|
|
478
|
+
out << nil
|
|
479
|
+
|
|
480
|
+
sources.each do |index_name, values|
|
|
481
|
+
source_data = conf['source'].merge values
|
|
482
|
+
out << section("source #{index_name}", source_data)
|
|
483
|
+
out << nil
|
|
484
|
+
|
|
485
|
+
index_path = File.join sphinx_dir, index_name
|
|
486
|
+
index_data = conf['index'].merge 'source' => index_name,
|
|
487
|
+
'path' => index_path
|
|
488
|
+
|
|
489
|
+
out << section("index #{index_name}", index_data)
|
|
490
|
+
out << nil
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
File.open sphinx_conf, 'w' do |fp|
|
|
494
|
+
fp.write out.join("\n")
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
end
|
|
499
|
+
|