dnif 0.0.1.alpha.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +57 -0
- data/Rakefile +38 -0
- data/dnif.gemspec +77 -0
- data/lib/dnif.rb +52 -0
- data/lib/dnif/configuration.rb +30 -0
- data/lib/dnif/index_builder.rb +30 -0
- data/lib/dnif/indexer.rb +92 -0
- data/lib/dnif/multi_attribute.rb +52 -0
- data/lib/dnif/search.rb +38 -0
- data/lib/dnif/tasks.rb +76 -0
- data/test/fixtures/db/schema.rb +33 -0
- data/test/fixtures/log/searchd.pid +0 -0
- data/test/fixtures/models.rb +19 -0
- data/test/fixtures/templates/config.erb +38 -0
- data/test/test_helper.rb +69 -0
- data/test/unit/test_configuration.rb +32 -0
- data/test/unit/test_dnif.rb +21 -0
- data/test/unit/test_index_builder.rb +33 -0
- data/test/unit/test_indexer.rb +35 -0
- data/test/unit/test_multi_attribute.rb +17 -0
- data/test/unit/test_search.rb +32 -0
- metadata +134 -0
data/README.rdoc
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
= dnif
|
2
|
+
|
3
|
+
dnif is the new find... for sphinx
|
4
|
+
|
5
|
+
dnif is a gem to index data using ActiveRecord finders, letting you index your custom methods and not your table fields
|
6
|
+
|
7
|
+
== Usage
|
8
|
+
|
9
|
+
require "dnif"
|
10
|
+
|
11
|
+
class Post < ActiveRecord::Base
|
12
|
+
|
13
|
+
define_index do
|
14
|
+
field :title
|
15
|
+
field :body
|
16
|
+
attribute :published_at, :type => :datetime
|
17
|
+
|
18
|
+
where ["draft = ?", false]
|
19
|
+
end
|
20
|
+
|
21
|
+
def slug
|
22
|
+
self.title.parameterize
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
Post.search("dnif")
|
27
|
+
|
28
|
+
== TODO
|
29
|
+
|
30
|
+
- Improve the test suite
|
31
|
+
|
32
|
+
== Maintainer
|
33
|
+
|
34
|
+
* Rafael Souza - http://rafaelss.com/
|
35
|
+
|
36
|
+
== License
|
37
|
+
|
38
|
+
(The MIT License)
|
39
|
+
|
40
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
41
|
+
a copy of this software and associated documentation files (the
|
42
|
+
'Software'), to deal in the Software without restriction, including
|
43
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
44
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
45
|
+
permit persons to whom the Software is furnished to do so, subject to
|
46
|
+
the following conditions:
|
47
|
+
|
48
|
+
The above copyright notice and this permission notice shall be
|
49
|
+
included in all copies or substantial portions of the Software.
|
50
|
+
|
51
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
52
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
53
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
54
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
55
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
56
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
57
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
begin
|
2
|
+
require 'jeweler'
|
3
|
+
|
4
|
+
Jeweler::Tasks.new do |gemspec|
|
5
|
+
gemspec.name = "dnif"
|
6
|
+
gemspec.summary = "dnif is the new find... for sphinx"
|
7
|
+
gemspec.description = "dnif is a gem to index data using ActiveRecord finders, letting you index your custom methods and not your table fields "
|
8
|
+
gemspec.email = "me@rafaelss.com"
|
9
|
+
gemspec.homepage = "http://github.com/rafaelss/dnif"
|
10
|
+
gemspec.authors = ["Rafael Souza"]
|
11
|
+
|
12
|
+
gemspec.has_rdoc = false
|
13
|
+
gemspec.files = %w(Rakefile dnif.gemspec README.rdoc) + Dir["{lib,test}/**/*"]
|
14
|
+
|
15
|
+
gemspec.add_dependency "activerecord"
|
16
|
+
gemspec.add_dependency "activesupport"
|
17
|
+
gemspec.add_dependency "riddle"
|
18
|
+
end
|
19
|
+
rescue LoadError
|
20
|
+
puts "Jeweler not available. Install it with: gem install jeweler"
|
21
|
+
end
|
22
|
+
|
23
|
+
desc "Generate gemspec and build gem"
|
24
|
+
task :build_gem do
|
25
|
+
Rake::Task["gemspec"].invoke
|
26
|
+
Rake::Task["build"].invoke
|
27
|
+
end
|
28
|
+
|
29
|
+
Jeweler::GemcutterTasks.new
|
30
|
+
|
31
|
+
require 'rake/testtask'
|
32
|
+
Rake::TestTask.new(:test) do |test|
|
33
|
+
test.test_files = FileList.new('test/**/test_*.rb') do |list|
|
34
|
+
list.exclude 'test/test_helper.rb'
|
35
|
+
end
|
36
|
+
test.libs << 'test'
|
37
|
+
test.verbose = true
|
38
|
+
end
|
data/dnif.gemspec
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{dnif}
|
8
|
+
s.version = "0.0.1.alpha.2"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Rafael Souza"]
|
12
|
+
s.date = %q{2010-07-01}
|
13
|
+
s.description = %q{dnif is a gem to index data using ActiveRecord finders, letting you index your custom methods and not your table fields }
|
14
|
+
s.email = %q{me@rafaelss.com}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"README.rdoc"
|
17
|
+
]
|
18
|
+
s.files = [
|
19
|
+
"README.rdoc",
|
20
|
+
"Rakefile",
|
21
|
+
"dnif.gemspec",
|
22
|
+
"lib/dnif.rb",
|
23
|
+
"lib/dnif/configuration.rb",
|
24
|
+
"lib/dnif/index_builder.rb",
|
25
|
+
"lib/dnif/indexer.rb",
|
26
|
+
"lib/dnif/multi_attribute.rb",
|
27
|
+
"lib/dnif/search.rb",
|
28
|
+
"lib/dnif/tasks.rb",
|
29
|
+
"test/fixtures/db/schema.rb",
|
30
|
+
"test/fixtures/log/searchd.pid",
|
31
|
+
"test/fixtures/models.rb",
|
32
|
+
"test/fixtures/templates/config.erb",
|
33
|
+
"test/test_helper.rb",
|
34
|
+
"test/unit/test_configuration.rb",
|
35
|
+
"test/unit/test_dnif.rb",
|
36
|
+
"test/unit/test_index_builder.rb",
|
37
|
+
"test/unit/test_indexer.rb",
|
38
|
+
"test/unit/test_multi_attribute.rb",
|
39
|
+
"test/unit/test_search.rb"
|
40
|
+
]
|
41
|
+
s.homepage = %q{http://github.com/rafaelss/dnif}
|
42
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
43
|
+
s.require_paths = ["lib"]
|
44
|
+
s.rubygems_version = %q{1.3.7}
|
45
|
+
s.summary = %q{dnif is the new find... for sphinx}
|
46
|
+
s.test_files = [
|
47
|
+
"test/fixtures/db/schema.rb",
|
48
|
+
"test/fixtures/models.rb",
|
49
|
+
"test/test_helper.rb",
|
50
|
+
"test/unit/test_configuration.rb",
|
51
|
+
"test/unit/test_dnif.rb",
|
52
|
+
"test/unit/test_index_builder.rb",
|
53
|
+
"test/unit/test_indexer.rb",
|
54
|
+
"test/unit/test_multi_attribute.rb",
|
55
|
+
"test/unit/test_search.rb"
|
56
|
+
]
|
57
|
+
|
58
|
+
if s.respond_to? :specification_version then
|
59
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
60
|
+
s.specification_version = 3
|
61
|
+
|
62
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
63
|
+
s.add_runtime_dependency(%q<activerecord>, [">= 0"])
|
64
|
+
s.add_runtime_dependency(%q<activesupport>, [">= 0"])
|
65
|
+
s.add_runtime_dependency(%q<riddle>, [">= 0"])
|
66
|
+
else
|
67
|
+
s.add_dependency(%q<activerecord>, [">= 0"])
|
68
|
+
s.add_dependency(%q<activesupport>, [">= 0"])
|
69
|
+
s.add_dependency(%q<riddle>, [">= 0"])
|
70
|
+
end
|
71
|
+
else
|
72
|
+
s.add_dependency(%q<activerecord>, [">= 0"])
|
73
|
+
s.add_dependency(%q<activesupport>, [">= 0"])
|
74
|
+
s.add_dependency(%q<riddle>, [">= 0"])
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
data/lib/dnif.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'active_record'
|
4
|
+
require 'active_support'
|
5
|
+
require 'riddle'
|
6
|
+
|
7
|
+
require "dnif/configuration"
|
8
|
+
require "dnif/index_builder"
|
9
|
+
require "dnif/indexer"
|
10
|
+
require "dnif/multi_attribute"
|
11
|
+
require "dnif/search"
|
12
|
+
|
13
|
+
module Dnif
|
14
|
+
|
15
|
+
def self.root_path
|
16
|
+
@root_path
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.root_path=(value)
|
20
|
+
@root_path = value
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.environment
|
24
|
+
@environment || 'development'
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.environment=(value)
|
28
|
+
@environment = value
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.models_path
|
32
|
+
@models_path
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.models_path=(value)
|
36
|
+
@models_path = value
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.load_models
|
40
|
+
models = Dir["#{self.models_path}/*.rb"]
|
41
|
+
models.map! do |filename|
|
42
|
+
filename = File.basename(filename, '.rb')
|
43
|
+
filename.classify.constantize
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
if defined?(Rails)
|
49
|
+
Dnif.root_path = RAILS_ROOT
|
50
|
+
Dnif.environment = RAILS_ENV
|
51
|
+
Dnif.models_path = File.join(RAILS_ROOT, "app", "models")
|
52
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'tilt'
|
3
|
+
require 'dnif'
|
4
|
+
|
5
|
+
module Dnif
|
6
|
+
|
7
|
+
class Configuration
|
8
|
+
|
9
|
+
def self.generate(config_path)
|
10
|
+
Tilt.register 'erb', Tilt::ErubisTemplate
|
11
|
+
|
12
|
+
template = Tilt.new(config_path)
|
13
|
+
output = template.render(self)
|
14
|
+
|
15
|
+
# TODO turn "db/sphinx" and "config/sphinx" configurable
|
16
|
+
FileUtils.mkdir_p(File.join(Dnif.root_path, "db", "sphinx", Dnif.environment))
|
17
|
+
File.open(Dnif.root_path + "/config/sphinx/" + Dnif.environment + ".conf", "w") do |f|
|
18
|
+
f.puts output
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.sources
|
23
|
+
classes = ActiveRecord::Base.classes.keys
|
24
|
+
classes.each do |class_name|
|
25
|
+
name = class_name.underscore.pluralize + "_main"
|
26
|
+
yield name, class_name
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Dnif
|
2
|
+
class IndexBuilder
|
3
|
+
|
4
|
+
attr_reader :fields
|
5
|
+
attr_reader :attributes
|
6
|
+
attr_reader :conditions
|
7
|
+
|
8
|
+
def initialize(object, &block)
|
9
|
+
@fields = []
|
10
|
+
@attributes = {}
|
11
|
+
|
12
|
+
@object = object
|
13
|
+
self.instance_eval(&block)
|
14
|
+
end
|
15
|
+
|
16
|
+
def field(name)
|
17
|
+
@fields << name
|
18
|
+
end
|
19
|
+
|
20
|
+
def attribute(name, options)
|
21
|
+
raise "You must specify the attribute type (:integer, :datetime, :date, :boolean, :float)" if options[:type].nil?
|
22
|
+
|
23
|
+
@attributes[name] = options[:type]
|
24
|
+
end
|
25
|
+
|
26
|
+
def where(conditions)
|
27
|
+
@conditions = conditions
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/dnif/indexer.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
module Dnif
|
2
|
+
module Indexer
|
3
|
+
|
4
|
+
def define_index(&block)
|
5
|
+
classes[self.name] = IndexBuilder.new(self, &block)
|
6
|
+
classes[self.name]
|
7
|
+
|
8
|
+
include InstanceMethods
|
9
|
+
end
|
10
|
+
|
11
|
+
def classes
|
12
|
+
@@classes ||= ActiveSupport::OrderedHash.new
|
13
|
+
end
|
14
|
+
|
15
|
+
def to_sphinx
|
16
|
+
return nil if classes.blank?
|
17
|
+
|
18
|
+
returning('') do |xml|
|
19
|
+
builder = classes[self.name]
|
20
|
+
results = all(:conditions => builder.conditions)
|
21
|
+
|
22
|
+
xml << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<sphinx:docset>\n"
|
23
|
+
|
24
|
+
xml << "<sphinx:schema>\n"
|
25
|
+
builder.fields.each do |name|
|
26
|
+
xml << " <sphinx:field name=\"#{name}\"/>\n"
|
27
|
+
end
|
28
|
+
|
29
|
+
xml << " <sphinx:attr name=\"class_id\" type=\"multi\"/>\n"
|
30
|
+
builder.attributes.each do |name, type|
|
31
|
+
xml << " <sphinx:attr name=\"#{name}\" "
|
32
|
+
|
33
|
+
case type
|
34
|
+
when :integer
|
35
|
+
xml << "type=\"int\""
|
36
|
+
when :date, :datetime
|
37
|
+
xml << "type=\"timestamp\""
|
38
|
+
when :boolean
|
39
|
+
xml << "type=\"bool\""
|
40
|
+
when :float
|
41
|
+
xml << "type=\"float\""
|
42
|
+
end
|
43
|
+
|
44
|
+
xml << "/>\n"
|
45
|
+
end
|
46
|
+
|
47
|
+
xml << "</sphinx:schema>\n"
|
48
|
+
|
49
|
+
results.each do |object|
|
50
|
+
xml << object.to_sphinx
|
51
|
+
end
|
52
|
+
xml << "</sphinx:docset>"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
module InstanceMethods
|
57
|
+
|
58
|
+
def to_sphinx
|
59
|
+
builder = ActiveRecord::Base.classes[self.class.name]
|
60
|
+
if not builder.nil?
|
61
|
+
class_id = Dnif::MultiAttribute.encode(self.class.name)
|
62
|
+
sphinx_id = id + class_id.split(',').sum { |c| c.to_i }
|
63
|
+
xml = "<sphinx:document id=\"#{sphinx_id}\">\n"
|
64
|
+
|
65
|
+
builder.fields.each do |field|
|
66
|
+
xml << " <#{field}><![CDATA[[#{send(field)}]]></#{field}>\n"
|
67
|
+
end
|
68
|
+
|
69
|
+
xml << " <class_id>#{class_id}</class_id>\n"
|
70
|
+
|
71
|
+
builder.attributes.each do |name, type|
|
72
|
+
value = send(name)
|
73
|
+
|
74
|
+
if [:date, :datetime].include?(builder.attributes[name])
|
75
|
+
if value.is_a?(Date)
|
76
|
+
value = value.to_datetime
|
77
|
+
end
|
78
|
+
|
79
|
+
value = value.to_i
|
80
|
+
end
|
81
|
+
|
82
|
+
xml << " <#{name}>#{value}</#{name}>\n"
|
83
|
+
end
|
84
|
+
|
85
|
+
xml << "</sphinx:document>\n"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
ActiveRecord::Base.extend(Dnif::Indexer)
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# Code extracted from MongoSphinx
|
2
|
+
# http://github.com/burke/mongosphinx/blob/master/lib/multi_attribute.rb
|
3
|
+
module Dnif
|
4
|
+
|
5
|
+
# Module MultiAttribute implements helpers to translate back and
|
6
|
+
# forth between Ruby Strings and an array of integers suitable for Sphinx
|
7
|
+
# attributes of type "multi".
|
8
|
+
#
|
9
|
+
# Background: Getting an ID as result for a query is OK, but for example to
|
10
|
+
# allow cast safety, we need an aditional attribute. Sphinx supports
|
11
|
+
# attributes which are returned together with the ID, but they behave a
|
12
|
+
# little different than expected: Instead we can use arrays of integers with
|
13
|
+
# ASCII character codes. These values are returned in ascending (!) order of
|
14
|
+
# value (yes, sounds funny but is reasonable from an internal view to
|
15
|
+
# Sphinx). So we mask each byte with 0x0100++ to keep the order...
|
16
|
+
#
|
17
|
+
# Sample:
|
18
|
+
#
|
19
|
+
# MongoSphinx::MultiAttribute.encode('Hello')
|
20
|
+
# => "328,613,876,1132,1391"
|
21
|
+
# MongoSphinx::MultiAttribute.decode('328,613,876,1132,1391')
|
22
|
+
# => "Hello"
|
23
|
+
|
24
|
+
module MultiAttribute
|
25
|
+
|
26
|
+
# Returns an numeric representation of a Ruby String suitable for "multi"
|
27
|
+
# attributes of Sphinx.
|
28
|
+
#
|
29
|
+
# Parameters:
|
30
|
+
#
|
31
|
+
# [str] String to translate
|
32
|
+
|
33
|
+
def self.encode(str)
|
34
|
+
offset = 0
|
35
|
+
return str.bytes.collect { |c| (offset+= 0x0100) + c }.join(',')
|
36
|
+
end
|
37
|
+
|
38
|
+
# Returns the original MongoDB ID created from a Sphinx ID. Only works if
|
39
|
+
# the ID was created from a MongoDB ID before!
|
40
|
+
#
|
41
|
+
# Parameters:
|
42
|
+
#
|
43
|
+
# [multi] Sphinx "multi" attribute to translate back
|
44
|
+
|
45
|
+
def self.decode(multi)
|
46
|
+
multi = multi.split(',') if not multi.is_a?(Array)
|
47
|
+
|
48
|
+
offset = 0
|
49
|
+
return multi.collect { |x| (x.to_i - (offset += 0x0100)).chr }.join("")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
data/lib/dnif/search.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
module Dnif
|
2
|
+
|
3
|
+
def self.search(query, options = {})
|
4
|
+
options.reverse_merge!(:index => '*')
|
5
|
+
|
6
|
+
client = Riddle::Client.new("127.0.0.1", 3313)
|
7
|
+
|
8
|
+
if not options[:class].nil?
|
9
|
+
filter_value = Dnif::MultiAttribute.encode(options[:class]).split(",").map(&:to_i)
|
10
|
+
client.filters << Riddle::Client::Filter.new("class_id", filter_value)
|
11
|
+
end
|
12
|
+
|
13
|
+
results = client.query(query, options[:index])
|
14
|
+
raise results[:error] if results[:error]
|
15
|
+
|
16
|
+
models = results[:matches].inject({}) do |memo, match|
|
17
|
+
class_id = match[:attributes]["class_id"].split(',')
|
18
|
+
class_name = Dnif::MultiAttribute.decode(class_id)
|
19
|
+
|
20
|
+
memo[class_name] ||= []
|
21
|
+
memo[class_name] << (match[:doc] - class_id.sum { |c| c.to_i })
|
22
|
+
memo
|
23
|
+
end
|
24
|
+
|
25
|
+
models.map do |class_name, ids|
|
26
|
+
class_name.constantize.find_all_by_id(ids)
|
27
|
+
end.flatten
|
28
|
+
end
|
29
|
+
|
30
|
+
module Search
|
31
|
+
|
32
|
+
def search(query)
|
33
|
+
Dnif.search(query, :class => self.name)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
ActiveRecord::Base.extend(Dnif::Search)
|
data/lib/dnif/tasks.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'dnif'
|
4
|
+
|
5
|
+
def controller
|
6
|
+
require 'riddle'
|
7
|
+
|
8
|
+
root_path = Dnif.root_path || "."
|
9
|
+
|
10
|
+
configuration = Riddle::Configuration.new
|
11
|
+
configuration.searchd.pid_file = "#{root_path}/log/searchd.#{Dnif.environment}.pid"
|
12
|
+
configuration.searchd.log = "#{root_path}/log/searchd.log"
|
13
|
+
configuration.searchd.query_log = "#{root_path}/log/searchd.query.log"
|
14
|
+
|
15
|
+
Riddle::Controller.new(configuration, "#{root_path}/config/sphinx/#{Dnif.environment}.conf")
|
16
|
+
end
|
17
|
+
|
18
|
+
namespace :dnif do
|
19
|
+
|
20
|
+
task :environment do
|
21
|
+
if task = Rake::Task[:environment]
|
22
|
+
task.invoke
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
desc "Generates the configuration file needed for sphinx"
|
27
|
+
task :configure => :environment do
|
28
|
+
if Dnif.models_path.nil?
|
29
|
+
puts "You need to specify where are your models (ex: Dnif.models_path = \"path/for/your/models\")"
|
30
|
+
exit
|
31
|
+
end
|
32
|
+
|
33
|
+
Dnif.root_path ||= File.expand_path(File.dirname("."))
|
34
|
+
Dnif.environment ||= "development"
|
35
|
+
|
36
|
+
config_path = File.join(Dnif.root_path, "config/sphinx")
|
37
|
+
if not File.exist?(config_path)
|
38
|
+
FileUtils.mkdir_p(config_path)
|
39
|
+
end
|
40
|
+
|
41
|
+
base_path = File.join(config_path, Dnif.environment + ".erb")
|
42
|
+
if not File.exist?(base_path)
|
43
|
+
FileUtils.cp(File.dirname(__FILE__) + "/../../test/fixtures/templates/config.erb", base_path) # TODO change this path. find out how this kind of stuff is handle in others gems
|
44
|
+
end
|
45
|
+
|
46
|
+
Dnif.load_models
|
47
|
+
Dnif::Configuration.generate(base_path)
|
48
|
+
end
|
49
|
+
|
50
|
+
desc "Generates the XML used by sphinx to create indexes"
|
51
|
+
task :xml => :environment do
|
52
|
+
::ActiveRecord::Base.logger = Logger.new(StringIO.new)
|
53
|
+
Dnif.load_models
|
54
|
+
|
55
|
+
klass = ENV['MODEL'].constantize
|
56
|
+
puts klass.to_sphinx
|
57
|
+
end
|
58
|
+
|
59
|
+
desc "Index data for sphinx"
|
60
|
+
task :index => :environment do
|
61
|
+
controller.index(:verbose => true)
|
62
|
+
end
|
63
|
+
|
64
|
+
desc "Stop sphinx daemon"
|
65
|
+
task :stop => :environment do
|
66
|
+
controller.stop
|
67
|
+
end
|
68
|
+
|
69
|
+
desc "Start sphinx daemon"
|
70
|
+
task :start => :environment do
|
71
|
+
controller.start
|
72
|
+
end
|
73
|
+
|
74
|
+
desc "Rebuild sphinx index"
|
75
|
+
task :rebuild => [:index, :stop, :start]
|
76
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
ActiveRecord::Schema.define(:version => 1) do
|
2
|
+
|
3
|
+
create_table "comments", :force => true do |t|
|
4
|
+
t.string :author
|
5
|
+
end
|
6
|
+
|
7
|
+
create_table "users", :force => true do |t|
|
8
|
+
t.string :name
|
9
|
+
end
|
10
|
+
|
11
|
+
create_table "people", :force => true do |t|
|
12
|
+
t.string :first_name
|
13
|
+
t.string :last_name
|
14
|
+
end
|
15
|
+
|
16
|
+
create_table "posts", :force => true do |t|
|
17
|
+
t.string :title
|
18
|
+
t.datetime :published_at
|
19
|
+
t.boolean :draft, :default => true
|
20
|
+
end
|
21
|
+
|
22
|
+
create_table "sales", :force => true do |t|
|
23
|
+
t.datetime :ordered_at
|
24
|
+
end
|
25
|
+
|
26
|
+
create_table "notes", :force => true do |t|
|
27
|
+
t.integer :clicked
|
28
|
+
t.datetime :published_at
|
29
|
+
t.date :expire_at
|
30
|
+
t.boolean :active
|
31
|
+
t.float :points
|
32
|
+
end
|
33
|
+
end
|
File without changes
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class Post < ActiveRecord::Base
|
2
|
+
|
3
|
+
define_index do
|
4
|
+
field :title
|
5
|
+
attribute :published_at, :type => :datetime
|
6
|
+
attribute :draft, :type => :boolean
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
class Comment < ActiveRecord::Base
|
11
|
+
|
12
|
+
define_index do
|
13
|
+
field :full_author
|
14
|
+
end
|
15
|
+
|
16
|
+
def full_author
|
17
|
+
"full author name"
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# Hand modifications will be overwritten.
|
2
|
+
indexer {
|
3
|
+
mem_limit = 512M
|
4
|
+
}
|
5
|
+
|
6
|
+
searchd {
|
7
|
+
listen = 127.0.0.1:3313
|
8
|
+
seamless_rotate = 1
|
9
|
+
log = <%= Dnif.root_path %>/log/searchd.<%= Dnif.environment %>.log
|
10
|
+
query_log = <%= Dnif.root_path %>/log/query.<%= Dnif.environment %>.log
|
11
|
+
read_timeout = 5
|
12
|
+
max_children = 300
|
13
|
+
pid_file = <%= Dnif.root_path %>/log/searchd.<%= Dnif.environment %>.pid
|
14
|
+
max_matches = 1000
|
15
|
+
}
|
16
|
+
|
17
|
+
<% sources do |name, model| %>
|
18
|
+
source <%= name %>
|
19
|
+
{
|
20
|
+
type = xmlpipe2
|
21
|
+
xmlpipe_command = MODEL=<%= model %> rake dnif:xml --silent
|
22
|
+
}
|
23
|
+
<% end %>
|
24
|
+
|
25
|
+
index main
|
26
|
+
{
|
27
|
+
<% sources do |name| %>
|
28
|
+
source = <%= name %>
|
29
|
+
<% end %>
|
30
|
+
path = <%= Dnif.root_path %>/db/sphinx/<%= Dnif.environment %>/index_main
|
31
|
+
docinfo = extern
|
32
|
+
morphology = none
|
33
|
+
min_word_len = 1
|
34
|
+
html_strip = 0
|
35
|
+
html_index_attrs =
|
36
|
+
charset_type = utf-8
|
37
|
+
charset_table = 0..9, A..Z->a..z, -, _, ., &, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F,U+C5->U+E5, U+E5, U+C4->U+E4, U+E4, U+D6->U+F6, U+F6, U+16B, U+0c1->a, U+0c4->a, U+0c9->e, U+0cd->i, U+0d3->o, U+0d4->o, U+0da->u, U+0dd->y, U+0e1->a, U+0e4->a, U+0e9->e, U+0ed->i, U+0f3->o, U+0f4->o, U+0fa->u, U+0fd->y, U+104->U+105, U+105, U+106->U+107, U+10c->c, U+10d->c, U+10e->d, U+10f->d, U+116->U+117, U+117, U+118->U+119, U+11a->e, U+11b->e, U+12E->U+12F, U+12F, U+139->l, U+13a->l, U+13d->l, U+13e->l, U+141->U+142, U+142, U+143->U+144, U+144,U+147->n, U+148->n, U+154->r, U+155->r, U+158->r, U+159->r, U+15A->U+15B, U+15B, U+160->s, U+160->U+161, U+161->s, U+164->t, U+165->t, U+16A->U+16B, U+16B, U+16e->u, U+16f->u, U+172->U+173, U+173, U+179->U+17A, U+17A, U+17B->U+17C, U+17C, U+17d->z, U+17e->z,
|
38
|
+
}
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# $:.unshift(File.dirname(__FILE__) + "/../../lib/")
|
4
|
+
|
5
|
+
require "test/unit"
|
6
|
+
require "mocha"
|
7
|
+
require "active_record"
|
8
|
+
|
9
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
10
|
+
require "dnif"
|
11
|
+
|
12
|
+
Dnif.root_path = File.expand_path(File.dirname(__FILE__) + "/fixtures")
|
13
|
+
|
14
|
+
ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
|
15
|
+
silence_stream(STDOUT) { load "fixtures/db/schema.rb" }
|
16
|
+
|
17
|
+
class Post
|
18
|
+
|
19
|
+
extend Dnif::Search
|
20
|
+
end
|
21
|
+
|
22
|
+
class Comment
|
23
|
+
|
24
|
+
extend Dnif::Search
|
25
|
+
end
|
26
|
+
|
27
|
+
class User < ActiveRecord::Base
|
28
|
+
|
29
|
+
define_index do
|
30
|
+
field :name
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class Category < ActiveRecord::Base
|
35
|
+
end
|
36
|
+
|
37
|
+
class Property < ActiveRecord::Base
|
38
|
+
end
|
39
|
+
|
40
|
+
class Person < ActiveRecord::Base
|
41
|
+
|
42
|
+
define_index do
|
43
|
+
field :full_name
|
44
|
+
end
|
45
|
+
|
46
|
+
def full_name
|
47
|
+
"#{self.first_name} #{self.last_name}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class Order < ActiveRecord::Base
|
52
|
+
|
53
|
+
define_index do
|
54
|
+
field :buyer
|
55
|
+
|
56
|
+
where ["ordered_at >= ?", 2.months.ago]
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
class Note < ActiveRecord::Base
|
61
|
+
|
62
|
+
define_index do
|
63
|
+
attribute :clicked, :type => :integer
|
64
|
+
attribute :published_at, :type => :datetime
|
65
|
+
attribute :expire_at, :type => :date
|
66
|
+
attribute :active, :type => :boolean
|
67
|
+
attribute :points, :type => :float
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'test_helper'
|
3
|
+
require 'erb'
|
4
|
+
|
5
|
+
class TestConfiguration < Test::Unit::TestCase
|
6
|
+
|
7
|
+
test "generate configuration from default template" do
|
8
|
+
Tilt.expects(:register).with('erb', Tilt::ErubisTemplate)
|
9
|
+
|
10
|
+
template = mock("Template")
|
11
|
+
template.expects(:render).with(Dnif::Configuration).returns('output')
|
12
|
+
Tilt.expects(:new).with("config/path.erb").returns(template)
|
13
|
+
|
14
|
+
file = mock
|
15
|
+
file.expects(:puts).with('output')
|
16
|
+
File.expects(:open).with(Dnif.root_path + "/config/sphinx/development.conf", "w").yields(file)
|
17
|
+
|
18
|
+
Dnif::Configuration.generate("config/path.erb")
|
19
|
+
end
|
20
|
+
|
21
|
+
test "sources should iterate over all indexed classes" do
|
22
|
+
names = []
|
23
|
+
classes = []
|
24
|
+
Dnif::Configuration.sources do |name, class_name|
|
25
|
+
names << name
|
26
|
+
classes << class_name
|
27
|
+
end
|
28
|
+
|
29
|
+
assert_equal ["users_main", "people_main", "orders_main", "notes_main"], names
|
30
|
+
assert_equal ["User", "Person", "Order", "Note"], classes
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
class TestDnif < Test::Unit::TestCase
|
5
|
+
|
6
|
+
test ".root_path" do
|
7
|
+
Dnif.root_path = "/root/path"
|
8
|
+
assert_equal "/root/path", Dnif.root_path
|
9
|
+
end
|
10
|
+
|
11
|
+
test ".environment" do
|
12
|
+
assert_equal "development", Dnif.environment
|
13
|
+
Dnif.environment = "production"
|
14
|
+
assert_equal "production", Dnif.environment
|
15
|
+
end
|
16
|
+
|
17
|
+
test ".models_path" do
|
18
|
+
Dnif.models_path = "/models/path"
|
19
|
+
assert_equal "/models/path", Dnif.models_path
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
class TestIndexBuilder < Test::Unit::TestCase
|
5
|
+
|
6
|
+
test "object" do
|
7
|
+
instance = Object.new
|
8
|
+
assert_equal instance, Dnif::IndexBuilder.new(instance, & proc {}).instance_variable_get("@object")
|
9
|
+
end
|
10
|
+
|
11
|
+
test "fields" do
|
12
|
+
builder = Dnif::IndexBuilder.new(Object.new, & proc {})
|
13
|
+
builder.field(:first_name)
|
14
|
+
builder.field(:last_name)
|
15
|
+
|
16
|
+
assert_equal [:first_name, :last_name], builder.fields
|
17
|
+
end
|
18
|
+
|
19
|
+
test "attribute" do
|
20
|
+
builder = Dnif::IndexBuilder.new(Object.new, & proc {})
|
21
|
+
builder.attribute(:ordered_at, :type => :datetime)
|
22
|
+
|
23
|
+
expected = { :ordered_at => :datetime }
|
24
|
+
assert_equal expected, builder.attributes
|
25
|
+
end
|
26
|
+
|
27
|
+
test "where" do
|
28
|
+
builder = Dnif::IndexBuilder.new(Object.new, & proc {})
|
29
|
+
builder.where("status = 'active'")
|
30
|
+
|
31
|
+
assert_equal "status = 'active'", builder.conditions
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
class TestIndexer < Test::Unit::TestCase
|
5
|
+
|
6
|
+
test "objects without index should not have dnif included" do
|
7
|
+
assert_false Post.new.respond_to?(:to_sphinx)
|
8
|
+
assert_false Post.respond_to?(:to_sphinx)
|
9
|
+
end
|
10
|
+
|
11
|
+
test "to_sphinx returns a string with sphinx document" do
|
12
|
+
comment = Person.create!(:first_name => "Rafael", :last_name => "Souza")
|
13
|
+
|
14
|
+
expected = "<sphinx:document id=\"6009\">\n <full_name><![CDATA[[Rafael Souza]]></full_name>\n <class_id>336,613,882,1139,1391,1646</class_id>\n</sphinx:document>\n"
|
15
|
+
assert_equal expected, comment.to_sphinx
|
16
|
+
end
|
17
|
+
|
18
|
+
test "attributes" do
|
19
|
+
note = Note.create!(:clicked => 10, :published_at => (now = DateTime.now), :expire_at => (expire = Date.today + 2.days), :active => true, :points => 1000)
|
20
|
+
|
21
|
+
expected = "<sphinx:document id=\"2967\">\n <class_id>334,623,884,1125</class_id>\n <clicked>10</clicked>\n <published_at>#{now.to_i}</published_at>\n <expire_at>#{expire.to_datetime.to_i}</expire_at>\n <active>true</active>\n <points>1000.0</points>\n</sphinx:document>\n"
|
22
|
+
assert_equal expected, note.to_sphinx
|
23
|
+
end
|
24
|
+
|
25
|
+
test ".to_sphinx should generate a full sphinx xml" do
|
26
|
+
comment = Person.create!(:first_name => "Rafael", :last_name => "Souza")
|
27
|
+
|
28
|
+
expected = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<sphinx:docset>\n<sphinx:schema>\n <sphinx:field name=\"full_name\"/>\n <sphinx:attr name=\"class_id\" type=\"multi\"/>\n</sphinx:schema>\n#{comment.to_sphinx}</sphinx:docset>"
|
29
|
+
assert_equal expected, Person.to_sphinx
|
30
|
+
end
|
31
|
+
|
32
|
+
test "return all indexed classes" do
|
33
|
+
assert_equal ["User", "Person", "Order", "Note"], ActiveRecord::Base.classes.keys
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
class TestMultiAttribute < Test::Unit::TestCase
|
5
|
+
|
6
|
+
test ".encode" do
|
7
|
+
assert_equal "324,622,873,1126", Dnif::MultiAttribute.encode("Dnif")
|
8
|
+
end
|
9
|
+
|
10
|
+
test ".decode" do
|
11
|
+
assert_equal "Dnif", Dnif::MultiAttribute.decode("324,622,873,1126")
|
12
|
+
end
|
13
|
+
|
14
|
+
test ".decode as array" do
|
15
|
+
assert_equal "Dnif", Dnif::MultiAttribute.decode(["324", "622", "873", "1126"])
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
class TestSearch < Test::Unit::TestCase
|
5
|
+
|
6
|
+
test "search" do
|
7
|
+
results_for_post = {
|
8
|
+
:matches => [{
|
9
|
+
:doc => 2983,
|
10
|
+
:attributes => {
|
11
|
+
"class_id" => "336,623,883,1140"
|
12
|
+
}
|
13
|
+
}]
|
14
|
+
}
|
15
|
+
results_for_comment = {
|
16
|
+
:matches => [{
|
17
|
+
:doc => 7893,
|
18
|
+
:attributes => {
|
19
|
+
"class_id" => "323,623,877,1133,1381,1646,1908"
|
20
|
+
}
|
21
|
+
}]
|
22
|
+
}
|
23
|
+
|
24
|
+
Riddle::Client.any_instance.expects(:query).times(2).with("post", "*").returns(results_for_post, results_for_comment)
|
25
|
+
|
26
|
+
Post.expects(:find_all_by_id).once.with([1])
|
27
|
+
Comment.expects(:find_all_by_id).once.with([2])
|
28
|
+
|
29
|
+
Post.search("post")
|
30
|
+
Comment.search("post")
|
31
|
+
end
|
32
|
+
end
|
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: dnif
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: true
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- alpha
|
10
|
+
- 2
|
11
|
+
version: 0.0.1.alpha.2
|
12
|
+
platform: ruby
|
13
|
+
authors:
|
14
|
+
- Rafael Souza
|
15
|
+
autorequire:
|
16
|
+
bindir: bin
|
17
|
+
cert_chain: []
|
18
|
+
|
19
|
+
date: 2010-07-01 00:00:00 -03:00
|
20
|
+
default_executable:
|
21
|
+
dependencies:
|
22
|
+
- !ruby/object:Gem::Dependency
|
23
|
+
name: activerecord
|
24
|
+
prerelease: false
|
25
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
26
|
+
none: false
|
27
|
+
requirements:
|
28
|
+
- - ">="
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: activesupport
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
segments:
|
44
|
+
- 0
|
45
|
+
version: "0"
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: riddle
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
segments:
|
57
|
+
- 0
|
58
|
+
version: "0"
|
59
|
+
type: :runtime
|
60
|
+
version_requirements: *id003
|
61
|
+
description: "dnif is a gem to index data using ActiveRecord finders, letting you index your custom methods and not your table fields "
|
62
|
+
email: me@rafaelss.com
|
63
|
+
executables: []
|
64
|
+
|
65
|
+
extensions: []
|
66
|
+
|
67
|
+
extra_rdoc_files:
|
68
|
+
- README.rdoc
|
69
|
+
files:
|
70
|
+
- README.rdoc
|
71
|
+
- Rakefile
|
72
|
+
- dnif.gemspec
|
73
|
+
- lib/dnif.rb
|
74
|
+
- lib/dnif/configuration.rb
|
75
|
+
- lib/dnif/index_builder.rb
|
76
|
+
- lib/dnif/indexer.rb
|
77
|
+
- lib/dnif/multi_attribute.rb
|
78
|
+
- lib/dnif/search.rb
|
79
|
+
- lib/dnif/tasks.rb
|
80
|
+
- test/fixtures/db/schema.rb
|
81
|
+
- test/fixtures/log/searchd.pid
|
82
|
+
- test/fixtures/models.rb
|
83
|
+
- test/fixtures/templates/config.erb
|
84
|
+
- test/test_helper.rb
|
85
|
+
- test/unit/test_configuration.rb
|
86
|
+
- test/unit/test_dnif.rb
|
87
|
+
- test/unit/test_index_builder.rb
|
88
|
+
- test/unit/test_indexer.rb
|
89
|
+
- test/unit/test_multi_attribute.rb
|
90
|
+
- test/unit/test_search.rb
|
91
|
+
has_rdoc: true
|
92
|
+
homepage: http://github.com/rafaelss/dnif
|
93
|
+
licenses: []
|
94
|
+
|
95
|
+
post_install_message:
|
96
|
+
rdoc_options:
|
97
|
+
- --charset=UTF-8
|
98
|
+
require_paths:
|
99
|
+
- lib
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
101
|
+
none: false
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
segments:
|
106
|
+
- 0
|
107
|
+
version: "0"
|
108
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
109
|
+
none: false
|
110
|
+
requirements:
|
111
|
+
- - ">"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
segments:
|
114
|
+
- 1
|
115
|
+
- 3
|
116
|
+
- 1
|
117
|
+
version: 1.3.1
|
118
|
+
requirements: []
|
119
|
+
|
120
|
+
rubyforge_project:
|
121
|
+
rubygems_version: 1.3.7
|
122
|
+
signing_key:
|
123
|
+
specification_version: 3
|
124
|
+
summary: dnif is the new find... for sphinx
|
125
|
+
test_files:
|
126
|
+
- test/fixtures/db/schema.rb
|
127
|
+
- test/fixtures/models.rb
|
128
|
+
- test/test_helper.rb
|
129
|
+
- test/unit/test_configuration.rb
|
130
|
+
- test/unit/test_dnif.rb
|
131
|
+
- test/unit/test_index_builder.rb
|
132
|
+
- test/unit/test_indexer.rb
|
133
|
+
- test/unit/test_multi_attribute.rb
|
134
|
+
- test/unit/test_search.rb
|