thinkingtank 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/indextank_client.rb +255 -0
- data/lib/thinkingtank/init.rb +45 -26
- data/lib/thinkingtank/tasks.rb +13 -0
- metadata +4 -4
- data/lib/indextank.rb +0 -150
@@ -0,0 +1,255 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'rubygems'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
module IndexTank
|
7
|
+
|
8
|
+
private
|
9
|
+
|
10
|
+
class RestClient
|
11
|
+
def GET(path, params={})
|
12
|
+
path = "#{path}?#{to_query(params)}" if params
|
13
|
+
request = Net::HTTP::Get.new "#{@uri}#{path}"
|
14
|
+
authorize request
|
15
|
+
return execute(request)
|
16
|
+
end
|
17
|
+
|
18
|
+
def PUT(path, body={})
|
19
|
+
request = Net::HTTP::Put.new "#{@uri}#{path}"
|
20
|
+
authorize request
|
21
|
+
request.body = body.to_json if body
|
22
|
+
return execute(request)
|
23
|
+
end
|
24
|
+
|
25
|
+
def DELETE(path, params={})
|
26
|
+
path = "#{path}?#{to_query(params)}" if params
|
27
|
+
request = Net::HTTP::Delete.new "#{@uri}#{path}"
|
28
|
+
authorize request
|
29
|
+
return execute(request)
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def to_query(params)
|
35
|
+
require 'cgi' unless defined?(CGI) && defined?(CGI::escape)
|
36
|
+
r = ''
|
37
|
+
params.each do |k,v|
|
38
|
+
r << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}&"
|
39
|
+
end
|
40
|
+
return r
|
41
|
+
end
|
42
|
+
|
43
|
+
def authorize(req)
|
44
|
+
req.basic_auth(@uri.user, @uri.password)
|
45
|
+
end
|
46
|
+
|
47
|
+
def execute(req)
|
48
|
+
res = Net::HTTP.new(@uri.host).start { |http| http.request(req) }
|
49
|
+
if res.is_a? Net::HTTPSuccess
|
50
|
+
if res.body.nil? or res.body.empty?
|
51
|
+
return res.code, nil
|
52
|
+
else
|
53
|
+
begin
|
54
|
+
return res.code, JSON.parse(res.body)
|
55
|
+
rescue
|
56
|
+
raise "Invalid JSON response: #{res.body}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
elsif res.is_a? Net::HTTPUnauthorized
|
60
|
+
raise SecurityError, "Authorization required"
|
61
|
+
elsif res.is_a? Net::HTTPBadRequest
|
62
|
+
raise ArgumentError, res.body
|
63
|
+
else
|
64
|
+
raise HttpCodeException.new(res.code, res.body)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
public
|
71
|
+
|
72
|
+
class ApiClient < RestClient
|
73
|
+
def initialize(api_url)
|
74
|
+
@uri = URI.parse(api_url)
|
75
|
+
end
|
76
|
+
|
77
|
+
def get_index(name)
|
78
|
+
return IndexClient.new("#{@uri}/v1/indexes/#{name}")
|
79
|
+
end
|
80
|
+
|
81
|
+
def create_index(name)
|
82
|
+
index = get_index(name)
|
83
|
+
index.create_index()
|
84
|
+
return index
|
85
|
+
end
|
86
|
+
|
87
|
+
def delete_index(name)
|
88
|
+
return get_index(name).delete_index()
|
89
|
+
end
|
90
|
+
|
91
|
+
def list_indexes()
|
92
|
+
code, indexes = GET "/v1/indexes"
|
93
|
+
return indexes.map do |name,metadata| IndexClient.new "#{@uri}/v1/indexes/#{name}", metadata end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
class IndexClient < RestClient
|
98
|
+
def initialize(index_url, metadata=nil)
|
99
|
+
@uri = URI.parse(index_url)
|
100
|
+
@metadata = metadata
|
101
|
+
end
|
102
|
+
|
103
|
+
def code
|
104
|
+
return metadata['code']
|
105
|
+
end
|
106
|
+
|
107
|
+
def running?
|
108
|
+
return metadata!['started']
|
109
|
+
end
|
110
|
+
|
111
|
+
def creation_time
|
112
|
+
return metadata['creation_time']
|
113
|
+
end
|
114
|
+
|
115
|
+
def size
|
116
|
+
return metadata['size']
|
117
|
+
end
|
118
|
+
|
119
|
+
def exists?
|
120
|
+
begin
|
121
|
+
metadata!
|
122
|
+
return true
|
123
|
+
rescue HttpCodeException
|
124
|
+
if $!.code == "404"
|
125
|
+
return false
|
126
|
+
end
|
127
|
+
raise
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
# the options argument may contain a :variables key
|
132
|
+
# with a Hash from variable numbers to their float values
|
133
|
+
# this variables can be used in the scoring functions
|
134
|
+
# when sorting a search
|
135
|
+
def add_document(docid, fields, options={})
|
136
|
+
options.merge!( :docid => docid, :fields => fields )
|
137
|
+
code, r = PUT "/docs", options
|
138
|
+
return r
|
139
|
+
end
|
140
|
+
|
141
|
+
def update_variables(docid, variables, options={})
|
142
|
+
options.merge!( :docid => docid, :variables => variables )
|
143
|
+
code, r = PUT "/docs/variables", options
|
144
|
+
return r
|
145
|
+
end
|
146
|
+
|
147
|
+
def delete_document(docid, options={})
|
148
|
+
options.merge!( :docid => docid )
|
149
|
+
code, r = DELETE "/docs", options
|
150
|
+
return r
|
151
|
+
end
|
152
|
+
|
153
|
+
# the options argument may contain an :index_code definition to override
|
154
|
+
# this instance's default index_code
|
155
|
+
def promote(docid, query, options={})
|
156
|
+
options.merge!( :docid => docid, :query => query )
|
157
|
+
code, r = PUT "/promote", options
|
158
|
+
return r
|
159
|
+
end
|
160
|
+
|
161
|
+
|
162
|
+
# the options argument may contain an :index_code definition to override
|
163
|
+
# this instance's default index_code
|
164
|
+
# it can also contain any of the following:
|
165
|
+
# :start => an int with the number of results to skip
|
166
|
+
# :len => an int with the number of results to return
|
167
|
+
# :snippet => a comma separated list of field names for which a snippet
|
168
|
+
# should be returned. (requires an index that supports snippets)
|
169
|
+
# :fetch => a comma separated list of field names for which its content
|
170
|
+
# should be returned. (requires an index that supports storage)
|
171
|
+
# :function => an int with the index of the scoring function to be used
|
172
|
+
# for this query
|
173
|
+
def search(query, options={})
|
174
|
+
options = { :start => 0, :len => 10 }.merge(options)
|
175
|
+
options.merge!( :q => query )
|
176
|
+
begin
|
177
|
+
code, r = GET "/search", options
|
178
|
+
return r
|
179
|
+
rescue HttpCodeException
|
180
|
+
raise
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def add_function(function_index, definition, options={})
|
185
|
+
options.merge!( :definition => definition )
|
186
|
+
code, r = PUT "/functions/#{function_index}", options
|
187
|
+
return r
|
188
|
+
end
|
189
|
+
|
190
|
+
def del_function(function_index, options={})
|
191
|
+
code, r = DELETE "/functions/#{function_index}", options
|
192
|
+
return r
|
193
|
+
end
|
194
|
+
|
195
|
+
def list_functions(options={})
|
196
|
+
code, r = GET "/functions"
|
197
|
+
return r
|
198
|
+
end
|
199
|
+
|
200
|
+
def create_index()
|
201
|
+
begin
|
202
|
+
code, r = PUT ""
|
203
|
+
raise IndexAlreadyExists if code == "204"
|
204
|
+
return r
|
205
|
+
rescue HttpCodeException
|
206
|
+
if $!.code == "409"
|
207
|
+
puts $!.code
|
208
|
+
raise TooManyIndexes
|
209
|
+
end
|
210
|
+
raise
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
def delete_index()
|
215
|
+
code, r = DELETE ""
|
216
|
+
return r
|
217
|
+
end
|
218
|
+
|
219
|
+
def metadata
|
220
|
+
metadata! if @metadata.nil?
|
221
|
+
return @metadata
|
222
|
+
end
|
223
|
+
|
224
|
+
def metadata!
|
225
|
+
code, @metadata = GET ""
|
226
|
+
return @metadata
|
227
|
+
end
|
228
|
+
|
229
|
+
end
|
230
|
+
|
231
|
+
class IndexAlreadyExists < StandardError
|
232
|
+
end
|
233
|
+
class TooManyIndexes < StandardError
|
234
|
+
end
|
235
|
+
|
236
|
+
class HttpCodeException < StandardError
|
237
|
+
def initialize(code, message)
|
238
|
+
@code = code
|
239
|
+
@message = message
|
240
|
+
super("#{code}: #{message}")
|
241
|
+
end
|
242
|
+
|
243
|
+
attr_accessor :code
|
244
|
+
attr_accessor :message
|
245
|
+
end
|
246
|
+
|
247
|
+
class HerokuClient < ApiClient
|
248
|
+
def initialize()
|
249
|
+
super(ENV['HEROKUTANK_API_URL'])
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
|
254
|
+
end
|
255
|
+
|
data/lib/thinkingtank/init.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require '
|
1
|
+
require 'indextank_client'
|
2
2
|
|
3
3
|
module ThinkingTank
|
4
4
|
class Builder
|
@@ -31,7 +31,7 @@ module ThinkingTank
|
|
31
31
|
return unless File.exists?(path)
|
32
32
|
|
33
33
|
conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
|
34
|
-
self.client = IndexTank.new(conf['
|
34
|
+
self.client = IndexTank::ApiClient.new(conf['api_url']).get_index(conf['index_name'])
|
35
35
|
end
|
36
36
|
def environment
|
37
37
|
if defined?(Merb)
|
@@ -51,11 +51,11 @@ module ThinkingTank
|
|
51
51
|
data = {}
|
52
52
|
self.class.thinkingtank_builder.index_fields.each do |field|
|
53
53
|
val = self.instance_eval(field.to_s)
|
54
|
-
data[
|
54
|
+
data[field.to_s] = val.to_s unless val.nil?
|
55
55
|
end
|
56
|
-
data[:
|
57
|
-
data[:
|
58
|
-
it.
|
56
|
+
data[:__any] = data.values.join " . "
|
57
|
+
data[:__type] = self.class.name
|
58
|
+
it.add_document(docid, data)
|
59
59
|
end
|
60
60
|
end
|
61
61
|
|
@@ -64,26 +64,10 @@ end
|
|
64
64
|
class << ActiveRecord::Base
|
65
65
|
@indexable = false
|
66
66
|
def search(*args)
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
field = "_#{field}" # underscore prepended to ActiveRecord fields
|
72
|
-
query += " #{field}:(#{value})"
|
73
|
-
end
|
74
|
-
end
|
75
|
-
# TODO : add relevance functions
|
76
|
-
|
77
|
-
it = ThinkingTank::Configuration.instance.client
|
78
|
-
models = []
|
79
|
-
ok, res = it.search("#{query.to_s} type:#{self.name}")
|
80
|
-
if ok
|
81
|
-
res['docs'].each do |doc|
|
82
|
-
type, docid = doc['docid'].split(" ", 2)
|
83
|
-
models << self.find(id=docid)
|
84
|
-
end
|
85
|
-
end
|
86
|
-
return models
|
67
|
+
return indextank_search(true, *args)
|
68
|
+
end
|
69
|
+
def search_raw(*args)
|
70
|
+
return indextank_search(false, *args)
|
87
71
|
end
|
88
72
|
|
89
73
|
def define_index(name = nil, &block)
|
@@ -100,6 +84,41 @@ class << ActiveRecord::Base
|
|
100
84
|
def thinkingtank_builder
|
101
85
|
return @thinkingtank_builder
|
102
86
|
end
|
87
|
+
|
88
|
+
private
|
89
|
+
|
90
|
+
def indextank_search(models, *args)
|
91
|
+
options = args.extract_options!
|
92
|
+
query = args.join(' ')
|
93
|
+
|
94
|
+
# transform fields in query
|
95
|
+
|
96
|
+
if options.has_key? :conditions
|
97
|
+
options[:conditions].each do |field,value|
|
98
|
+
query += " #{field}:(#{value})"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
options.slice!(:snippet, :fetch, :function)
|
103
|
+
|
104
|
+
it = ThinkingTank::Configuration.instance.client
|
105
|
+
models = []
|
106
|
+
res = it.search("__any:(#{query.to_s}) __type:#{self.name}", options)
|
107
|
+
if models
|
108
|
+
res['results'].each do |doc|
|
109
|
+
type, docid = doc['docid'].split(" ", 2)
|
110
|
+
models << self.find(id=docid)
|
111
|
+
end
|
112
|
+
return models
|
113
|
+
else
|
114
|
+
res['results'].each do |doc|
|
115
|
+
type, docid = doc['docid'].split(" ", 2)
|
116
|
+
doc['model'] = self.find(id=docid)
|
117
|
+
end
|
118
|
+
return res
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
103
122
|
end
|
104
123
|
|
105
124
|
|
data/lib/thinkingtank/tasks.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'erb'
|
2
2
|
require 'active_record'
|
3
|
+
require 'indextank_client'
|
3
4
|
|
4
5
|
def load_models
|
5
6
|
app_root = ThinkingTank::Configuration.instance.app_root
|
@@ -28,6 +29,18 @@ def load_models
|
|
28
29
|
end
|
29
30
|
|
30
31
|
def reindex_models
|
32
|
+
it = ThinkingTank::Configuration.instance.client
|
33
|
+
if it.exists?
|
34
|
+
puts "Deleting existing index"
|
35
|
+
it.delete_index()
|
36
|
+
end
|
37
|
+
puts "Creating a new empty index"
|
38
|
+
it.create_index()
|
39
|
+
puts "Waiting for the index to be ready"
|
40
|
+
while not it.running?
|
41
|
+
sleep 0.5
|
42
|
+
end
|
43
|
+
|
31
44
|
Object.subclasses_of(ActiveRecord::Base).each do |klass|
|
32
45
|
reindex klass if klass.is_indexable?
|
33
46
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 3
|
9
|
+
version: 0.0.3
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Flaptor
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-
|
17
|
+
date: 2010-07-26 00:00:00 -07:00
|
18
18
|
default_executable:
|
19
19
|
dependencies: []
|
20
20
|
|
@@ -30,7 +30,7 @@ extra_rdoc_files: []
|
|
30
30
|
|
31
31
|
files:
|
32
32
|
- rails/init.rb
|
33
|
-
- lib/
|
33
|
+
- lib/indextank_client.rb
|
34
34
|
- lib/thinkingtank/init.rb
|
35
35
|
- README
|
36
36
|
- tasks/rails.rake
|
data/lib/indextank.rb
DELETED
@@ -1,150 +0,0 @@
|
|
1
|
-
require 'net/http'
|
2
|
-
require 'uri'
|
3
|
-
require 'rubygems'
|
4
|
-
require 'json'
|
5
|
-
|
6
|
-
BASE_URL = 'http://api.indextank.com/api/v0'
|
7
|
-
class IndexTank
|
8
|
-
def initialize(api_key, options={})
|
9
|
-
@api_key = api_key
|
10
|
-
@def_index_code = options[:index_code]
|
11
|
-
@def_index_name = options[:index_name]
|
12
|
-
end
|
13
|
-
|
14
|
-
def create_index(index_name)
|
15
|
-
return api_call("admin/create", "index_code", :index_name => index_name)
|
16
|
-
end
|
17
|
-
|
18
|
-
# the options argument may contain an :index_code definition to override
|
19
|
-
# this instance's default index_code
|
20
|
-
def delete_index(options={})
|
21
|
-
return api_call("admin/delete", nil, options)
|
22
|
-
end
|
23
|
-
|
24
|
-
|
25
|
-
def list_indexes()
|
26
|
-
return api_call("admin/list", "results")
|
27
|
-
end
|
28
|
-
|
29
|
-
|
30
|
-
# the options argument may contain an :index_code definition to override
|
31
|
-
# this instance's default index_code
|
32
|
-
def add(doc_id, content, options={})
|
33
|
-
options.merge!(:document_id => doc_id, :document => content.to_json)
|
34
|
-
options[:boosts] = options[:boosts].to_json if options.key?(:boosts)
|
35
|
-
return api_call("index/add", nil, options)
|
36
|
-
end
|
37
|
-
|
38
|
-
# the options argument may contain an :index_code definition to override
|
39
|
-
# this instance's default index_code
|
40
|
-
def boost(doc_id, boosts, options={})
|
41
|
-
options.merge!(:document_id => doc_id, :boosts => boosts.to_json)
|
42
|
-
return api_call("index/boost", nil, options)
|
43
|
-
end
|
44
|
-
|
45
|
-
|
46
|
-
# the options argument may contain an :index_code definition to override
|
47
|
-
# this instance's default index_code
|
48
|
-
def promote(doc_id, query, options={})
|
49
|
-
options.merge!(:document_id => doc_id, :query => query)
|
50
|
-
return api_call("index/promote", nil, options)
|
51
|
-
end
|
52
|
-
|
53
|
-
|
54
|
-
# the options argument may contain an :index_code definition to override
|
55
|
-
# this instance's default index_code
|
56
|
-
def update(doc_id, content, options={})
|
57
|
-
options.merge!(:document_id => doc_id, :document => content.to_json)
|
58
|
-
return api_call("index/update", nil, options)
|
59
|
-
end
|
60
|
-
|
61
|
-
# the options argument may contain an :index_code definition to override
|
62
|
-
# this instance's default index_code
|
63
|
-
def delete(doc_id, options={})
|
64
|
-
options.merge!(:document_id => doc_id)
|
65
|
-
return api_call("index/delete", nil, options)
|
66
|
-
end
|
67
|
-
|
68
|
-
# the options argument may contain an :index_code definition to override
|
69
|
-
# this instance's default index_code
|
70
|
-
# it can also contain any of the following:
|
71
|
-
# :start => an int with the number of results to skip
|
72
|
-
# :len => an int with the number of results to return
|
73
|
-
# :snippet_fields => a comma separated list of field names for which a snippet
|
74
|
-
# should be returned. (requires an index that supports snippets)
|
75
|
-
# :fetch_fields => a comma separated list of field names for which its content
|
76
|
-
# should be returned. (requires an index that supports storage)
|
77
|
-
# :relevance_function => an int with the index of the relevance function to be used
|
78
|
-
# for this query
|
79
|
-
def search(query, options={})
|
80
|
-
options = { :start => 0, :len => 10 }.merge(options)
|
81
|
-
options.merge!(:query => query)
|
82
|
-
return api_call("search/query", "results", options)
|
83
|
-
end
|
84
|
-
|
85
|
-
# the options argument may contain an :index_code definition to override
|
86
|
-
# this instance's default index_code
|
87
|
-
def add_function(function_index, definition, options={})
|
88
|
-
options.merge!( :function_id => function_index, :definition => definition )
|
89
|
-
return api_call("index/add_function", nil, options)
|
90
|
-
end
|
91
|
-
|
92
|
-
# the options argument may contain an :index_code definition to override
|
93
|
-
# this instance's default index_code
|
94
|
-
def del_function(function_index, options={})
|
95
|
-
options.merge!( :function_id => function_index )
|
96
|
-
return api_call("index/remove_function", nil, options)
|
97
|
-
end
|
98
|
-
|
99
|
-
# the options argument may contain an :index_code definition to override
|
100
|
-
# this instance's default index_code
|
101
|
-
def list_functions(options={})
|
102
|
-
return api_call("index/list_functions", "results", options)
|
103
|
-
end
|
104
|
-
|
105
|
-
|
106
|
-
# the options argument may contain an :index_code definition to override
|
107
|
-
# this instance's default index_code
|
108
|
-
def index_stats(options={})
|
109
|
-
return api_call("index/stats", "results", options)
|
110
|
-
end
|
111
|
-
|
112
|
-
|
113
|
-
# the options argument may contain an :index_code definition to override
|
114
|
-
# this instance's default index_code
|
115
|
-
def search_stats(options={})
|
116
|
-
return api_call("search/stats", "results", options)
|
117
|
-
end
|
118
|
-
|
119
|
-
private
|
120
|
-
|
121
|
-
def base_url
|
122
|
-
return ENV['INDEXTANK_BASE_URL'] || BASE_URL
|
123
|
-
end
|
124
|
-
|
125
|
-
def api_call(method, return_key, params={})
|
126
|
-
params = { "api_key" => @api_key }.merge(params)
|
127
|
-
params = { :index_code => @def_index_code }.merge(params) unless @def_index_code.nil?
|
128
|
-
params = { :index_name => @def_index_name }.merge(params) unless @def_index_name.nil?
|
129
|
-
url = base_url + '/' + method
|
130
|
-
req = Net::HTTP::Post.new(url)
|
131
|
-
req.set_form_data(params, ';')
|
132
|
-
res = Net::HTTP.new(URI.parse(base_url).host).start {|http| http.request(req) }
|
133
|
-
if res.is_a? Net::HTTPOK
|
134
|
-
result = JSON.parse(res.body)
|
135
|
-
ok = result['status'] == 'OK'
|
136
|
-
return ok ? [ok, (return_key.nil? ? nil : result[return_key])] : [ok, result['message']]
|
137
|
-
elsif res.is_a? Net::HTTPForbidden
|
138
|
-
return false, "Access forbidden"
|
139
|
-
elsif res.is_a? Net::HTTPClientError
|
140
|
-
return false, "Unknown client error"
|
141
|
-
elsif res.is_a? Net::HTTPServerError
|
142
|
-
puts res.body
|
143
|
-
return false, "Unknown server error"
|
144
|
-
else
|
145
|
-
puts res.body
|
146
|
-
return false, "Unexpected response"
|
147
|
-
end
|
148
|
-
end
|
149
|
-
|
150
|
-
end
|