thinkingtank 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +67 -0
- data/lib/indextank.rb +150 -0
- data/lib/thinkingtank/init.rb +105 -0
- data/lib/thinkingtank/tasks.rb +52 -0
- data/rails/init.rb +4 -0
- data/tasks/rails.rake +1 -0
- metadata +58 -0
data/README
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
ThinkingTank
|
2
|
+
=============
|
3
|
+
|
4
|
+
ActiveRecord extension that allows to define models that should be indexed
|
5
|
+
in an existing IndexTank index. It supports a very similar syntax to
|
6
|
+
ThinkingSphinx allowing to easily port an existing project.
|
7
|
+
|
8
|
+
Every indexable model should include a define_index block in its class
|
9
|
+
definition, see the example for more details. This block supports the indexes
|
10
|
+
method and receives a field name.
|
11
|
+
|
12
|
+
Model classes now have a search method that receives one or more string
|
13
|
+
arguments with query strings (according to the query specifications) and
|
14
|
+
supports the :conditions argument as a hash from field name to query string.
|
15
|
+
|
16
|
+
In order for this extension to work you need to define a config/indextank.yml
|
17
|
+
in your application with the api_key and index_code (or index_name) settings
|
18
|
+
for each environment (similar to config/database.yml).
|
19
|
+
|
20
|
+
Indexed fields in ActiveRecord are prepended an underscore when sent to
|
21
|
+
IndexTank so if you plan to write query strings that use your field names you
|
22
|
+
will have to prepend the underscore to the field names.
|
23
|
+
|
24
|
+
In order for the ThinkingTank rake tasks to be available you need to add:
|
25
|
+
|
26
|
+
require 'thinkingtank/tasks'
|
27
|
+
|
28
|
+
to your Rakefile. You can use the following task to reindex your entire database:
|
29
|
+
|
30
|
+
rake indextank:reindex
|
31
|
+
|
32
|
+
|
33
|
+
Example
|
34
|
+
=======
|
35
|
+
|
36
|
+
Sample config/indextank.yml file:
|
37
|
+
|
38
|
+
development:
|
39
|
+
api_key: '<YOUR API KEY>'
|
40
|
+
index_code: '<INDEX CODE>'
|
41
|
+
# instead of index_code you can also use an index_name
|
42
|
+
test:
|
43
|
+
api_key: '<YOUR API KEY>'
|
44
|
+
index_code: '<INDEX CODE>'
|
45
|
+
# instead of index_code you can also use an index_name
|
46
|
+
production:
|
47
|
+
api_key: '<YOUR API KEY>'
|
48
|
+
index_code: '<INDEX CODE>'
|
49
|
+
# instead of index_code you can also use an index_name
|
50
|
+
|
51
|
+
Sample model:
|
52
|
+
class Person < ActiveRecord::Base
|
53
|
+
define_index do
|
54
|
+
indexes :name
|
55
|
+
indexes gender
|
56
|
+
indexes age
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
Sample query code:
|
61
|
+
Person.search("john")
|
62
|
+
Person.search("john OR stacey")
|
63
|
+
Person.search("stacey", :conditions => { :age => 25 } )
|
64
|
+
Person.search("james", :conditions => { :age => 25 , :gender => "female" } )
|
65
|
+
|
66
|
+
|
67
|
+
Copyright(c) 2010 Flaptor Inc.
|
data/lib/indextank.rb
ADDED
@@ -0,0 +1,150 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'rubygems'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
BASE_URL = 'http://api.indextank.com/api/v0'
|
7
|
+
class IndexTank
|
8
|
+
def initialize(api_key, options={})
|
9
|
+
@api_key = api_key
|
10
|
+
@def_index_code = options[:index_code]
|
11
|
+
@def_index_name = options[:index_name]
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_index(index_name)
|
15
|
+
return api_call("admin/create", "index_code", :index_name => index_name)
|
16
|
+
end
|
17
|
+
|
18
|
+
# the options argument may contain an :index_code definition to override
|
19
|
+
# this instance's default index_code
|
20
|
+
def delete_index(options={})
|
21
|
+
return api_call("admin/delete", nil, options)
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
def list_indexes()
|
26
|
+
return api_call("admin/list", "results")
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
# the options argument may contain an :index_code definition to override
|
31
|
+
# this instance's default index_code
|
32
|
+
def add(doc_id, content, options={})
|
33
|
+
options.merge!(:document_id => doc_id, :document => content.to_json)
|
34
|
+
options[:boosts] = options[:boosts].to_json if options.key?(:boosts)
|
35
|
+
return api_call("index/add", nil, options)
|
36
|
+
end
|
37
|
+
|
38
|
+
# the options argument may contain an :index_code definition to override
|
39
|
+
# this instance's default index_code
|
40
|
+
def boost(doc_id, boosts, options={})
|
41
|
+
options.merge!(:document_id => doc_id, :boosts => boosts.to_json)
|
42
|
+
return api_call("index/boost", nil, options)
|
43
|
+
end
|
44
|
+
|
45
|
+
|
46
|
+
# the options argument may contain an :index_code definition to override
|
47
|
+
# this instance's default index_code
|
48
|
+
def promote(doc_id, query, options={})
|
49
|
+
options.merge!(:document_id => doc_id, :query => query)
|
50
|
+
return api_call("index/promote", nil, options)
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
# the options argument may contain an :index_code definition to override
|
55
|
+
# this instance's default index_code
|
56
|
+
def update(doc_id, content, options={})
|
57
|
+
options.merge!(:document_id => doc_id, :document => content.to_json)
|
58
|
+
return api_call("index/update", nil, options)
|
59
|
+
end
|
60
|
+
|
61
|
+
# the options argument may contain an :index_code definition to override
|
62
|
+
# this instance's default index_code
|
63
|
+
def delete(doc_id, options={})
|
64
|
+
options.merge!(:document_id => doc_id)
|
65
|
+
return api_call("index/delete", nil, options)
|
66
|
+
end
|
67
|
+
|
68
|
+
# the options argument may contain an :index_code definition to override
|
69
|
+
# this instance's default index_code
|
70
|
+
# it can also contain any of the following:
|
71
|
+
# :start => an int with the number of results to skip
|
72
|
+
# :len => an int with the number of results to return
|
73
|
+
# :snippet_fields => a comma separated list of field names for which a snippet
|
74
|
+
# should be returned. (requires an index that supports snippets)
|
75
|
+
# :fetch_fields => a comma separated list of field names for which its content
|
76
|
+
# should be returned. (requires an index that supports storage)
|
77
|
+
# :relevance_function => an int with the index of the relevance function to be used
|
78
|
+
# for this query
|
79
|
+
def search(query, options={})
|
80
|
+
options = { :start => 0, :len => 10 }.merge(options)
|
81
|
+
options.merge!(:query => query)
|
82
|
+
return api_call("search/query", "results", options)
|
83
|
+
end
|
84
|
+
|
85
|
+
# the options argument may contain an :index_code definition to override
|
86
|
+
# this instance's default index_code
|
87
|
+
def add_function(function_index, definition, options={})
|
88
|
+
options.merge!( :function_id => function_index, :definition => definition )
|
89
|
+
return api_call("index/add_function", nil, options)
|
90
|
+
end
|
91
|
+
|
92
|
+
# the options argument may contain an :index_code definition to override
|
93
|
+
# this instance's default index_code
|
94
|
+
def del_function(function_index, options={})
|
95
|
+
options.merge!( :function_id => function_index )
|
96
|
+
return api_call("index/remove_function", nil, options)
|
97
|
+
end
|
98
|
+
|
99
|
+
# the options argument may contain an :index_code definition to override
|
100
|
+
# this instance's default index_code
|
101
|
+
def list_functions(options={})
|
102
|
+
return api_call("index/list_functions", "results", options)
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
# the options argument may contain an :index_code definition to override
|
107
|
+
# this instance's default index_code
|
108
|
+
def index_stats(options={})
|
109
|
+
return api_call("index/stats", "results", options)
|
110
|
+
end
|
111
|
+
|
112
|
+
|
113
|
+
# the options argument may contain an :index_code definition to override
|
114
|
+
# this instance's default index_code
|
115
|
+
def search_stats(options={})
|
116
|
+
return api_call("search/stats", "results", options)
|
117
|
+
end
|
118
|
+
|
119
|
+
private
|
120
|
+
|
121
|
+
def base_url
|
122
|
+
return ENV['INDEXTANK_BASE_URL'] || BASE_URL
|
123
|
+
end
|
124
|
+
|
125
|
+
def api_call(method, return_key, params={})
|
126
|
+
params = { "api_key" => @api_key }.merge(params)
|
127
|
+
params = { :index_code => @def_index_code }.merge(params) unless @def_index_code.nil?
|
128
|
+
params = { :index_name => @def_index_name }.merge(params) unless @def_index_name.nil?
|
129
|
+
url = base_url + '/' + method
|
130
|
+
req = Net::HTTP::Post.new(url)
|
131
|
+
req.set_form_data(params, ';')
|
132
|
+
res = Net::HTTP.new(URI.parse(base_url).host).start {|http| http.request(req) }
|
133
|
+
if res.is_a? Net::HTTPOK
|
134
|
+
result = JSON.parse(res.body)
|
135
|
+
ok = result['status'] == 'OK'
|
136
|
+
return ok ? [ok, (return_key.nil? ? nil : result[return_key])] : [ok, result['message']]
|
137
|
+
elsif res.is_a? Net::HTTPForbidden
|
138
|
+
return false, "Access forbidden"
|
139
|
+
elsif res.is_a? Net::HTTPClientError
|
140
|
+
return false, "Unknown client error"
|
141
|
+
elsif res.is_a? Net::HTTPServerError
|
142
|
+
puts res.body
|
143
|
+
return false, "Unknown server error"
|
144
|
+
else
|
145
|
+
puts res.body
|
146
|
+
return false, "Unexpected response"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'indextank'
|
2
|
+
|
3
|
+
module ThinkingTank
|
4
|
+
class Builder
|
5
|
+
def initialize(model, &block)
|
6
|
+
@index_fields = []
|
7
|
+
self.instance_eval &block
|
8
|
+
end
|
9
|
+
def indexes(*args)
|
10
|
+
options = args.extract_options!
|
11
|
+
args.each do |field|
|
12
|
+
@index_fields << field
|
13
|
+
end
|
14
|
+
end
|
15
|
+
def index_fields
|
16
|
+
return @index_fields
|
17
|
+
end
|
18
|
+
def method_missing(method)
|
19
|
+
return method
|
20
|
+
end
|
21
|
+
end
|
22
|
+
class Configuration
|
23
|
+
include Singleton
|
24
|
+
attr_accessor :app_root, :client
|
25
|
+
def initialize
|
26
|
+
self.app_root = RAILS_ROOT if defined?(RAILS_ROOT)
|
27
|
+
self.app_root = Merb.root if defined?(Merb)
|
28
|
+
self.app_root ||= Dir.pwd
|
29
|
+
|
30
|
+
path = "#{app_root}/config/indextank.yml"
|
31
|
+
return unless File.exists?(path)
|
32
|
+
|
33
|
+
conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
|
34
|
+
self.client = IndexTank.new(conf['api_key'], :index_code => conf['index_code'], :index_name => conf['index_name'])
|
35
|
+
end
|
36
|
+
def environment
|
37
|
+
if defined?(Merb)
|
38
|
+
Merb.environment
|
39
|
+
elsif defined?(RAILS_ENV)
|
40
|
+
RAILS_ENV
|
41
|
+
else
|
42
|
+
ENV['RAILS_ENV'] || 'development'
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
module IndexMethods
|
48
|
+
def update_index
|
49
|
+
it = ThinkingTank::Configuration.instance.client
|
50
|
+
docid = self.class.name + ' ' + self.id.to_s
|
51
|
+
data = {}
|
52
|
+
self.class.thinkingtank_builder.index_fields.each do |field|
|
53
|
+
val = self.instance_eval(field.to_s)
|
54
|
+
data["_" + field.to_s] = val.to_s unless val.nil?
|
55
|
+
end
|
56
|
+
data[:text] = data.values.join " . "
|
57
|
+
data[:type] = self.class.name
|
58
|
+
it.add(docid, data)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
|
64
|
+
class << ActiveRecord::Base
|
65
|
+
@indexable = false
|
66
|
+
def search(*args)
|
67
|
+
options = args.extract_options!
|
68
|
+
query = args.join(' ')
|
69
|
+
if options.has_key? :conditions
|
70
|
+
options[:conditions].each do |field,value|
|
71
|
+
field = "_#{field}" # underscore prepended to ActiveRecord fields
|
72
|
+
query += " #{field}:(#{value})"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
# TODO : add relevance functions
|
76
|
+
|
77
|
+
it = IndexTankPlugin::Configuration.instance.client
|
78
|
+
models = []
|
79
|
+
ok, res = it.search("#{query.to_s} type:#{self.name}")
|
80
|
+
if ok
|
81
|
+
res['docs'].each do |doc|
|
82
|
+
type, docid = doc['docid'].split(" ", 2)
|
83
|
+
models << self.find(id=docid)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
return models
|
87
|
+
end
|
88
|
+
|
89
|
+
def define_index(name = nil, &block)
|
90
|
+
include ThinkingTank::IndexMethods
|
91
|
+
@thinkingtank_builder = ThinkingTank::Builder.new self, &block
|
92
|
+
@indexable = true
|
93
|
+
after_save :update_index
|
94
|
+
end
|
95
|
+
|
96
|
+
def is_indexable?
|
97
|
+
return @indexable
|
98
|
+
end
|
99
|
+
|
100
|
+
def thinkingtank_builder
|
101
|
+
return @thinkingtank_builder
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'erb'
|
2
|
+
require 'active_record'
|
3
|
+
|
4
|
+
def load_models
|
5
|
+
app_root = ThinkingTank::Configuration.instance.app_root
|
6
|
+
dirs = ["#{app_root}/app/models/"] + Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
|
7
|
+
|
8
|
+
dirs.each do |base|
|
9
|
+
Dir["#{base}**/*.rb"].each do |file|
|
10
|
+
model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
|
11
|
+
|
12
|
+
next if model_name.nil?
|
13
|
+
next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
|
14
|
+
model.name == model_name
|
15
|
+
}
|
16
|
+
|
17
|
+
begin
|
18
|
+
model_name.camelize.constantize
|
19
|
+
rescue LoadError
|
20
|
+
model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
|
21
|
+
rescue NameError
|
22
|
+
next
|
23
|
+
rescue StandardError
|
24
|
+
STDERR.puts "Warning: Error loading #{file}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def reindex_models
|
31
|
+
Object.subclasses_of(ActiveRecord::Base).each do |klass|
|
32
|
+
reindex klass if klass.is_indexable?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def reindex(klass)
|
37
|
+
klass.find(:all).each do |obj|
|
38
|
+
puts "re-indexing #{obj.class.name}:#{obj.id}"
|
39
|
+
obj.update_index
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
namespace :indextank do
|
44
|
+
task :reindex => :environment do
|
45
|
+
load_models
|
46
|
+
reindex_models
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
namespace :it do
|
51
|
+
task :reindex => "indextank:reindex"
|
52
|
+
end
|
data/rails/init.rb
ADDED
data/tasks/rails.rake
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), '/tasks')
|
metadata
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: thinkingtank
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Flaptor
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-05-19 00:00:00 -03:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: ActiveRecord extension that allows to define models that should be indexed in an existing IndexTank index. It supports a very similar syntax to ThinkingSphinx allowing to easily port an existing project.
|
17
|
+
email: indextank@flaptor.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- rails/init.rb
|
26
|
+
- lib/indextank.rb
|
27
|
+
- lib/thinkingtank/init.rb
|
28
|
+
- README
|
29
|
+
- tasks/rails.rake
|
30
|
+
- lib/thinkingtank/tasks.rb
|
31
|
+
has_rdoc: false
|
32
|
+
homepage: http://indextank.com/
|
33
|
+
post_install_message:
|
34
|
+
rdoc_options: []
|
35
|
+
|
36
|
+
require_paths:
|
37
|
+
- lib
|
38
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: "0"
|
43
|
+
version:
|
44
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: "0"
|
49
|
+
version:
|
50
|
+
requirements: []
|
51
|
+
|
52
|
+
rubyforge_project:
|
53
|
+
rubygems_version: 1.0.1
|
54
|
+
signing_key:
|
55
|
+
specification_version: 2
|
56
|
+
summary: Thinking-Sphinx-like Indextank plugin.
|
57
|
+
test_files: []
|
58
|
+
|