thinkingtank 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README +67 -0
- data/lib/indextank.rb +150 -0
- data/lib/thinkingtank/init.rb +105 -0
- data/lib/thinkingtank/tasks.rb +52 -0
- data/rails/init.rb +4 -0
- data/tasks/rails.rake +1 -0
- metadata +58 -0
data/README
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
ThinkingTank
|
2
|
+
=============
|
3
|
+
|
4
|
+
ActiveRecord extension that allows to define models that should be indexed
|
5
|
+
in an existing IndexTank index. It supports a very similar syntax to
|
6
|
+
ThinkingSphinx allowing to easily port an existing project.
|
7
|
+
|
8
|
+
Every indexable model should include a define_index block in its class
|
9
|
+
definition, see the example for more details. This block supports the indexes
|
10
|
+
method and receives a field name.
|
11
|
+
|
12
|
+
Model classes now have a search method that receives one or more string
|
13
|
+
arguments with query strings (according to the query specifications) and
|
14
|
+
supports the :conditions argument as a hash from field name to query string.
|
15
|
+
|
16
|
+
In order for this extension to work you need to define a config/indextank.yml
|
17
|
+
in your application with the api_key and index_code (or index_name) settings
|
18
|
+
for each environment (similar to config/database.yml).
|
19
|
+
|
20
|
+
Indexed fields in ActiveRecord are prepended an underscore when sent to
|
21
|
+
IndexTank so if you plan to write query strings that use your field names you
|
22
|
+
will have to prepend the underscore to the field names.
|
23
|
+
|
24
|
+
In order for the ThinkingTank rake tasks to be available you need to add:
|
25
|
+
|
26
|
+
require 'thinkingtank/tasks'
|
27
|
+
|
28
|
+
to your Rakefile. You can use the following task to reindex your entire database:
|
29
|
+
|
30
|
+
rake indextank:reindex
|
31
|
+
|
32
|
+
|
33
|
+
Example
|
34
|
+
=======
|
35
|
+
|
36
|
+
Sample config/indextank.yml file:
|
37
|
+
|
38
|
+
development:
|
39
|
+
api_key: '<YOUR API KEY>'
|
40
|
+
index_code: '<INDEX CODE>'
|
41
|
+
# instead of index_code you can also use an index_name
|
42
|
+
test:
|
43
|
+
api_key: '<YOUR API KEY>'
|
44
|
+
index_code: '<INDEX CODE>'
|
45
|
+
# instead of index_code you can also use an index_name
|
46
|
+
production:
|
47
|
+
api_key: '<YOUR API KEY>'
|
48
|
+
index_code: '<INDEX CODE>'
|
49
|
+
# instead of index_code you can also use an index_name
|
50
|
+
|
51
|
+
Sample model:
|
52
|
+
class Person < ActiveRecord::Base
|
53
|
+
define_index do
|
54
|
+
indexes :name
|
55
|
+
indexes gender
|
56
|
+
indexes age
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
Sample query code:
|
61
|
+
Person.search("john")
|
62
|
+
Person.search("john OR stacey")
|
63
|
+
Person.search("stacey", :conditions => { :age => 25 } )
|
64
|
+
Person.search("james", :conditions => { :age => 25 , :gender => "female" } )
|
65
|
+
|
66
|
+
|
67
|
+
Copyright(c) 2010 Flaptor Inc.
|
data/lib/indextank.rb
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'rubygems'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
BASE_URL = 'http://api.indextank.com/api/v0'
|
7
|
+
class IndexTank
|
8
|
+
def initialize(api_key, options={})
|
9
|
+
@api_key = api_key
|
10
|
+
@def_index_code = options[:index_code]
|
11
|
+
@def_index_name = options[:index_name]
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_index(index_name)
|
15
|
+
return api_call("admin/create", "index_code", :index_name => index_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
# the options argument may contain an :index_code definition to override
|
19
|
+
# this instance's default index_code
|
20
|
+
def delete_index(options={})
|
21
|
+
return api_call("admin/delete", nil, options)
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
def list_indexes()
|
26
|
+
return api_call("admin/list", "results")
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
# the options argument may contain an :index_code definition to override
|
31
|
+
# this instance's default index_code
|
32
|
+
def add(doc_id, content, options={})
|
33
|
+
options.merge!(:document_id => doc_id, :document => content.to_json)
|
34
|
+
options[:boosts] = options[:boosts].to_json if options.key?(:boosts)
|
35
|
+
return api_call("index/add", nil, options)
|
36
|
+
end
|
37
|
+
|
38
|
+
# the options argument may contain an :index_code definition to override
|
39
|
+
# this instance's default index_code
|
40
|
+
def boost(doc_id, boosts, options={})
|
41
|
+
options.merge!(:document_id => doc_id, :boosts => boosts.to_json)
|
42
|
+
return api_call("index/boost", nil, options)
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
# the options argument may contain an :index_code definition to override
|
47
|
+
# this instance's default index_code
|
48
|
+
def promote(doc_id, query, options={})
|
49
|
+
options.merge!(:document_id => doc_id, :query => query)
|
50
|
+
return api_call("index/promote", nil, options)
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
# the options argument may contain an :index_code definition to override
|
55
|
+
# this instance's default index_code
|
56
|
+
def update(doc_id, content, options={})
|
57
|
+
options.merge!(:document_id => doc_id, :document => content.to_json)
|
58
|
+
return api_call("index/update", nil, options)
|
59
|
+
end
|
60
|
+
|
61
|
+
# the options argument may contain an :index_code definition to override
|
62
|
+
# this instance's default index_code
|
63
|
+
def delete(doc_id, options={})
|
64
|
+
options.merge!(:document_id => doc_id)
|
65
|
+
return api_call("index/delete", nil, options)
|
66
|
+
end
|
67
|
+
|
68
|
+
# the options argument may contain an :index_code definition to override
|
69
|
+
# this instance's default index_code
|
70
|
+
# it can also contain any of the following:
|
71
|
+
# :start => an int with the number of results to skip
|
72
|
+
# :len => an int with the number of results to return
|
73
|
+
# :snippet_fields => a comma separated list of field names for which a snippet
|
74
|
+
# should be returned. (requires an index that supports snippets)
|
75
|
+
# :fetch_fields => a comma separated list of field names for which its content
|
76
|
+
# should be returned. (requires an index that supports storage)
|
77
|
+
# :relevance_function => an int with the index of the relevance function to be used
|
78
|
+
# for this query
|
79
|
+
def search(query, options={})
|
80
|
+
options = { :start => 0, :len => 10 }.merge(options)
|
81
|
+
options.merge!(:query => query)
|
82
|
+
return api_call("search/query", "results", options)
|
83
|
+
end
|
84
|
+
|
85
|
+
# the options argument may contain an :index_code definition to override
|
86
|
+
# this instance's default index_code
|
87
|
+
def add_function(function_index, definition, options={})
|
88
|
+
options.merge!( :function_id => function_index, :definition => definition )
|
89
|
+
return api_call("index/add_function", nil, options)
|
90
|
+
end
|
91
|
+
|
92
|
+
# the options argument may contain an :index_code definition to override
|
93
|
+
# this instance's default index_code
|
94
|
+
def del_function(function_index, options={})
|
95
|
+
options.merge!( :function_id => function_index )
|
96
|
+
return api_call("index/remove_function", nil, options)
|
97
|
+
end
|
98
|
+
|
99
|
+
# the options argument may contain an :index_code definition to override
|
100
|
+
# this instance's default index_code
|
101
|
+
def list_functions(options={})
|
102
|
+
return api_call("index/list_functions", "results", options)
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
# the options argument may contain an :index_code definition to override
|
107
|
+
# this instance's default index_code
|
108
|
+
def index_stats(options={})
|
109
|
+
return api_call("index/stats", "results", options)
|
110
|
+
end
|
111
|
+
|
112
|
+
|
113
|
+
# the options argument may contain an :index_code definition to override
|
114
|
+
# this instance's default index_code
|
115
|
+
def search_stats(options={})
|
116
|
+
return api_call("search/stats", "results", options)
|
117
|
+
end
|
118
|
+
|
119
|
+
private
|
120
|
+
|
121
|
+
def base_url
|
122
|
+
return ENV['INDEXTANK_BASE_URL'] || BASE_URL
|
123
|
+
end
|
124
|
+
|
125
|
+
def api_call(method, return_key, params={})
|
126
|
+
params = { "api_key" => @api_key }.merge(params)
|
127
|
+
params = { :index_code => @def_index_code }.merge(params) unless @def_index_code.nil?
|
128
|
+
params = { :index_name => @def_index_name }.merge(params) unless @def_index_name.nil?
|
129
|
+
url = base_url + '/' + method
|
130
|
+
req = Net::HTTP::Post.new(url)
|
131
|
+
req.set_form_data(params, ';')
|
132
|
+
res = Net::HTTP.new(URI.parse(base_url).host).start {|http| http.request(req) }
|
133
|
+
if res.is_a? Net::HTTPOK
|
134
|
+
result = JSON.parse(res.body)
|
135
|
+
ok = result['status'] == 'OK'
|
136
|
+
return ok ? [ok, (return_key.nil? ? nil : result[return_key])] : [ok, result['message']]
|
137
|
+
elsif res.is_a? Net::HTTPForbidden
|
138
|
+
return false, "Access forbidden"
|
139
|
+
elsif res.is_a? Net::HTTPClientError
|
140
|
+
return false, "Unknown client error"
|
141
|
+
elsif res.is_a? Net::HTTPServerError
|
142
|
+
puts res.body
|
143
|
+
return false, "Unknown server error"
|
144
|
+
else
|
145
|
+
puts res.body
|
146
|
+
return false, "Unexpected response"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'indextank'
|
2
|
+
|
3
|
+
module ThinkingTank
|
4
|
+
class Builder
|
5
|
+
def initialize(model, &block)
|
6
|
+
@index_fields = []
|
7
|
+
self.instance_eval &block
|
8
|
+
end
|
9
|
+
def indexes(*args)
|
10
|
+
options = args.extract_options!
|
11
|
+
args.each do |field|
|
12
|
+
@index_fields << field
|
13
|
+
end
|
14
|
+
end
|
15
|
+
def index_fields
|
16
|
+
return @index_fields
|
17
|
+
end
|
18
|
+
def method_missing(method)
|
19
|
+
return method
|
20
|
+
end
|
21
|
+
end
|
22
|
+
class Configuration
|
23
|
+
include Singleton
|
24
|
+
attr_accessor :app_root, :client
|
25
|
+
def initialize
|
26
|
+
self.app_root = RAILS_ROOT if defined?(RAILS_ROOT)
|
27
|
+
self.app_root = Merb.root if defined?(Merb)
|
28
|
+
self.app_root ||= Dir.pwd
|
29
|
+
|
30
|
+
path = "#{app_root}/config/indextank.yml"
|
31
|
+
return unless File.exists?(path)
|
32
|
+
|
33
|
+
conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
|
34
|
+
self.client = IndexTank.new(conf['api_key'], :index_code => conf['index_code'], :index_name => conf['index_name'])
|
35
|
+
end
|
36
|
+
def environment
|
37
|
+
if defined?(Merb)
|
38
|
+
Merb.environment
|
39
|
+
elsif defined?(RAILS_ENV)
|
40
|
+
RAILS_ENV
|
41
|
+
else
|
42
|
+
ENV['RAILS_ENV'] || 'development'
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
module IndexMethods
|
48
|
+
def update_index
|
49
|
+
it = ThinkingTank::Configuration.instance.client
|
50
|
+
docid = self.class.name + ' ' + self.id.to_s
|
51
|
+
data = {}
|
52
|
+
self.class.thinkingtank_builder.index_fields.each do |field|
|
53
|
+
val = self.instance_eval(field.to_s)
|
54
|
+
data["_" + field.to_s] = val.to_s unless val.nil?
|
55
|
+
end
|
56
|
+
data[:text] = data.values.join " . "
|
57
|
+
data[:type] = self.class.name
|
58
|
+
it.add(docid, data)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
|
64
|
+
class << ActiveRecord::Base
|
65
|
+
@indexable = false
|
66
|
+
def search(*args)
|
67
|
+
options = args.extract_options!
|
68
|
+
query = args.join(' ')
|
69
|
+
if options.has_key? :conditions
|
70
|
+
options[:conditions].each do |field,value|
|
71
|
+
field = "_#{field}" # underscore prepended to ActiveRecord fields
|
72
|
+
query += " #{field}:(#{value})"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
# TODO : add relevance functions
|
76
|
+
|
77
|
+
it = IndexTankPlugin::Configuration.instance.client
|
78
|
+
models = []
|
79
|
+
ok, res = it.search("#{query.to_s} type:#{self.name}")
|
80
|
+
if ok
|
81
|
+
res['docs'].each do |doc|
|
82
|
+
type, docid = doc['docid'].split(" ", 2)
|
83
|
+
models << self.find(id=docid)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
return models
|
87
|
+
end
|
88
|
+
|
89
|
+
def define_index(name = nil, &block)
|
90
|
+
include ThinkingTank::IndexMethods
|
91
|
+
@thinkingtank_builder = ThinkingTank::Builder.new self, &block
|
92
|
+
@indexable = true
|
93
|
+
after_save :update_index
|
94
|
+
end
|
95
|
+
|
96
|
+
def is_indexable?
|
97
|
+
return @indexable
|
98
|
+
end
|
99
|
+
|
100
|
+
def thinkingtank_builder
|
101
|
+
return @thinkingtank_builder
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'erb'
|
2
|
+
require 'active_record'
|
3
|
+
|
4
|
+
def load_models
|
5
|
+
app_root = ThinkingTank::Configuration.instance.app_root
|
6
|
+
dirs = ["#{app_root}/app/models/"] + Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
|
7
|
+
|
8
|
+
dirs.each do |base|
|
9
|
+
Dir["#{base}**/*.rb"].each do |file|
|
10
|
+
model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
|
11
|
+
|
12
|
+
next if model_name.nil?
|
13
|
+
next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
|
14
|
+
model.name == model_name
|
15
|
+
}
|
16
|
+
|
17
|
+
begin
|
18
|
+
model_name.camelize.constantize
|
19
|
+
rescue LoadError
|
20
|
+
model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
|
21
|
+
rescue NameError
|
22
|
+
next
|
23
|
+
rescue StandardError
|
24
|
+
STDERR.puts "Warning: Error loading #{file}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def reindex_models
|
31
|
+
Object.subclasses_of(ActiveRecord::Base).each do |klass|
|
32
|
+
reindex klass if klass.is_indexable?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def reindex(klass)
|
37
|
+
klass.find(:all).each do |obj|
|
38
|
+
puts "re-indexing #{obj.class.name}:#{obj.id}"
|
39
|
+
obj.update_index
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
namespace :indextank do
|
44
|
+
task :reindex => :environment do
|
45
|
+
load_models
|
46
|
+
reindex_models
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
namespace :it do
|
51
|
+
task :reindex => "indextank:reindex"
|
52
|
+
end
|
data/rails/init.rb
ADDED
data/tasks/rails.rake
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), '/tasks')
|
metadata
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: thinkingtank
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Flaptor
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-05-19 00:00:00 -03:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: ActiveRecord extension that allows to define models that should be indexed in an existing IndexTank index. It supports a very similar syntax to ThinkingSphinx allowing to easily port an existing project.
|
17
|
+
email: indextank@flaptor.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- rails/init.rb
|
26
|
+
- lib/indextank.rb
|
27
|
+
- lib/thinkingtank/init.rb
|
28
|
+
- README
|
29
|
+
- tasks/rails.rake
|
30
|
+
- lib/thinkingtank/tasks.rb
|
31
|
+
has_rdoc: false
|
32
|
+
homepage: http://indextank.com/
|
33
|
+
post_install_message:
|
34
|
+
rdoc_options: []
|
35
|
+
|
36
|
+
require_paths:
|
37
|
+
- lib
|
38
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: "0"
|
43
|
+
version:
|
44
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: "0"
|
49
|
+
version:
|
50
|
+
requirements: []
|
51
|
+
|
52
|
+
rubyforge_project:
|
53
|
+
rubygems_version: 1.0.1
|
54
|
+
signing_key:
|
55
|
+
specification_version: 2
|
56
|
+
summary: Thinking-Sphinx-like Indextank plugin.
|
57
|
+
test_files: []
|
58
|
+
|