thinkingtank 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,255 @@
1
+ require 'net/http'
2
+ require 'uri'
3
+ require 'rubygems'
4
+ require 'json'
5
+
6
+ module IndexTank
7
+
8
+ private
9
+
10
+ class RestClient
11
+ def GET(path, params={})
12
+ path = "#{path}?#{to_query(params)}" if params
13
+ request = Net::HTTP::Get.new "#{@uri}#{path}"
14
+ authorize request
15
+ return execute(request)
16
+ end
17
+
18
+ def PUT(path, body={})
19
+ request = Net::HTTP::Put.new "#{@uri}#{path}"
20
+ authorize request
21
+ request.body = body.to_json if body
22
+ return execute(request)
23
+ end
24
+
25
+ def DELETE(path, params={})
26
+ path = "#{path}?#{to_query(params)}" if params
27
+ request = Net::HTTP::Delete.new "#{@uri}#{path}"
28
+ authorize request
29
+ return execute(request)
30
+ end
31
+
32
+ private
33
+
34
+ def to_query(params)
35
+ require 'cgi' unless defined?(CGI) && defined?(CGI::escape)
36
+ r = ''
37
+ params.each do |k,v|
38
+ r << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}&"
39
+ end
40
+ return r
41
+ end
42
+
43
+ def authorize(req)
44
+ req.basic_auth(@uri.user, @uri.password)
45
+ end
46
+
47
+ def execute(req)
48
+ res = Net::HTTP.new(@uri.host).start { |http| http.request(req) }
49
+ if res.is_a? Net::HTTPSuccess
50
+ if res.body.nil? or res.body.empty?
51
+ return res.code, nil
52
+ else
53
+ begin
54
+ return res.code, JSON.parse(res.body)
55
+ rescue
56
+ raise "Invalid JSON response: #{res.body}"
57
+ end
58
+ end
59
+ elsif res.is_a? Net::HTTPUnauthorized
60
+ raise SecurityError, "Authorization required"
61
+ elsif res.is_a? Net::HTTPBadRequest
62
+ raise ArgumentError, res.body
63
+ else
64
+ raise HttpCodeException.new(res.code, res.body)
65
+ end
66
+ end
67
+
68
+ end
69
+
70
+ public
71
+
72
+ class ApiClient < RestClient
73
+ def initialize(api_url)
74
+ @uri = URI.parse(api_url)
75
+ end
76
+
77
+ def get_index(name)
78
+ return IndexClient.new("#{@uri}/v1/indexes/#{name}")
79
+ end
80
+
81
+ def create_index(name)
82
+ index = get_index(name)
83
+ index.create_index()
84
+ return index
85
+ end
86
+
87
+ def delete_index(name)
88
+ return get_index(name).delete_index()
89
+ end
90
+
91
+ def list_indexes()
92
+ code, indexes = GET "/v1/indexes"
93
+ return indexes.map do |name,metadata| IndexClient.new "#{@uri}/v1/indexes/#{name}", metadata end
94
+ end
95
+ end
96
+
97
+ class IndexClient < RestClient
98
+ def initialize(index_url, metadata=nil)
99
+ @uri = URI.parse(index_url)
100
+ @metadata = metadata
101
+ end
102
+
103
+ def code
104
+ return metadata['code']
105
+ end
106
+
107
+ def running?
108
+ return metadata!['started']
109
+ end
110
+
111
+ def creation_time
112
+ return metadata['creation_time']
113
+ end
114
+
115
+ def size
116
+ return metadata['size']
117
+ end
118
+
119
+ def exists?
120
+ begin
121
+ metadata!
122
+ return true
123
+ rescue HttpCodeException
124
+ if $!.code == "404"
125
+ return false
126
+ end
127
+ raise
128
+ end
129
+ end
130
+
131
+ # the options argument may contain a :variables key
132
+ # with a Hash from variable numbers to their float values
133
+ # this variables can be used in the scoring functions
134
+ # when sorting a search
135
+ def add_document(docid, fields, options={})
136
+ options.merge!( :docid => docid, :fields => fields )
137
+ code, r = PUT "/docs", options
138
+ return r
139
+ end
140
+
141
+ def update_variables(docid, variables, options={})
142
+ options.merge!( :docid => docid, :variables => variables )
143
+ code, r = PUT "/docs/variables", options
144
+ return r
145
+ end
146
+
147
+ def delete_document(docid, options={})
148
+ options.merge!( :docid => docid )
149
+ code, r = DELETE "/docs", options
150
+ return r
151
+ end
152
+
153
+ # the options argument may contain an :index_code definition to override
154
+ # this instance's default index_code
155
+ def promote(docid, query, options={})
156
+ options.merge!( :docid => docid, :query => query )
157
+ code, r = PUT "/promote", options
158
+ return r
159
+ end
160
+
161
+
162
+ # the options argument may contain an :index_code definition to override
163
+ # this instance's default index_code
164
+ # it can also contain any of the following:
165
+ # :start => an int with the number of results to skip
166
+ # :len => an int with the number of results to return
167
+ # :snippet => a comma separated list of field names for which a snippet
168
+ # should be returned. (requires an index that supports snippets)
169
+ # :fetch => a comma separated list of field names for which its content
170
+ # should be returned. (requires an index that supports storage)
171
+ # :function => an int with the index of the scoring function to be used
172
+ # for this query
173
+ def search(query, options={})
174
+ options = { :start => 0, :len => 10 }.merge(options)
175
+ options.merge!( :q => query )
176
+ begin
177
+ code, r = GET "/search", options
178
+ return r
179
+ rescue HttpCodeException
180
+ raise
181
+ end
182
+ end
183
+
184
+ def add_function(function_index, definition, options={})
185
+ options.merge!( :definition => definition )
186
+ code, r = PUT "/functions/#{function_index}", options
187
+ return r
188
+ end
189
+
190
+ def del_function(function_index, options={})
191
+ code, r = DELETE "/functions/#{function_index}", options
192
+ return r
193
+ end
194
+
195
+ def list_functions(options={})
196
+ code, r = GET "/functions"
197
+ return r
198
+ end
199
+
200
+ def create_index()
201
+ begin
202
+ code, r = PUT ""
203
+ raise IndexAlreadyExists if code == "204"
204
+ return r
205
+ rescue HttpCodeException
206
+ if $!.code == "409"
207
+ puts $!.code
208
+ raise TooManyIndexes
209
+ end
210
+ raise
211
+ end
212
+ end
213
+
214
+ def delete_index()
215
+ code, r = DELETE ""
216
+ return r
217
+ end
218
+
219
+ def metadata
220
+ metadata! if @metadata.nil?
221
+ return @metadata
222
+ end
223
+
224
+ def metadata!
225
+ code, @metadata = GET ""
226
+ return @metadata
227
+ end
228
+
229
+ end
230
+
231
+ class IndexAlreadyExists < StandardError
232
+ end
233
+ class TooManyIndexes < StandardError
234
+ end
235
+
236
+ class HttpCodeException < StandardError
237
+ def initialize(code, message)
238
+ @code = code
239
+ @message = message
240
+ super("#{code}: #{message}")
241
+ end
242
+
243
+ attr_accessor :code
244
+ attr_accessor :message
245
+ end
246
+
247
+ class HerokuClient < ApiClient
248
+ def initialize()
249
+ super(ENV['HEROKUTANK_API_URL'])
250
+ end
251
+ end
252
+
253
+
254
+ end
255
+
@@ -1,4 +1,4 @@
1
- require 'indextank'
1
+ require 'indextank_client'
2
2
 
3
3
  module ThinkingTank
4
4
  class Builder
@@ -31,7 +31,7 @@ module ThinkingTank
31
31
  return unless File.exists?(path)
32
32
 
33
33
  conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
34
- self.client = IndexTank.new(conf['api_key'], :index_code => conf['index_code'], :index_name => conf['index_name'])
34
+ self.client = IndexTank::ApiClient.new(conf['api_url']).get_index(conf['index_name'])
35
35
  end
36
36
  def environment
37
37
  if defined?(Merb)
@@ -51,11 +51,11 @@ module ThinkingTank
51
51
  data = {}
52
52
  self.class.thinkingtank_builder.index_fields.each do |field|
53
53
  val = self.instance_eval(field.to_s)
54
- data["_" + field.to_s] = val.to_s unless val.nil?
54
+ data[field.to_s] = val.to_s unless val.nil?
55
55
  end
56
- data[:text] = data.values.join " . "
57
- data[:type] = self.class.name
58
- it.add(docid, data)
56
+ data[:__any] = data.values.join " . "
57
+ data[:__type] = self.class.name
58
+ it.add_document(docid, data)
59
59
  end
60
60
  end
61
61
 
@@ -64,26 +64,10 @@ end
64
64
  class << ActiveRecord::Base
65
65
  @indexable = false
66
66
  def search(*args)
67
- options = args.extract_options!
68
- query = args.join(' ')
69
- if options.has_key? :conditions
70
- options[:conditions].each do |field,value|
71
- field = "_#{field}" # underscore prepended to ActiveRecord fields
72
- query += " #{field}:(#{value})"
73
- end
74
- end
75
- # TODO : add relevance functions
76
-
77
- it = ThinkingTank::Configuration.instance.client
78
- models = []
79
- ok, res = it.search("#{query.to_s} type:#{self.name}")
80
- if ok
81
- res['docs'].each do |doc|
82
- type, docid = doc['docid'].split(" ", 2)
83
- models << self.find(id=docid)
84
- end
85
- end
86
- return models
67
+ return indextank_search(true, *args)
68
+ end
69
+ def search_raw(*args)
70
+ return indextank_search(false, *args)
87
71
  end
88
72
 
89
73
  def define_index(name = nil, &block)
@@ -100,6 +84,41 @@ class << ActiveRecord::Base
100
84
  def thinkingtank_builder
101
85
  return @thinkingtank_builder
102
86
  end
87
+
88
+ private
89
+
90
+ def indextank_search(models, *args)
91
+ options = args.extract_options!
92
+ query = args.join(' ')
93
+
94
+ # transform fields in query
95
+
96
+ if options.has_key? :conditions
97
+ options[:conditions].each do |field,value|
98
+ query += " #{field}:(#{value})"
99
+ end
100
+ end
101
+
102
+ options.slice!(:snippet, :fetch, :function)
103
+
104
+ it = ThinkingTank::Configuration.instance.client
105
+ models = []
106
+ res = it.search("__any:(#{query.to_s}) __type:#{self.name}", options)
107
+ if models
108
+ res['results'].each do |doc|
109
+ type, docid = doc['docid'].split(" ", 2)
110
+ models << self.find(id=docid)
111
+ end
112
+ return models
113
+ else
114
+ res['results'].each do |doc|
115
+ type, docid = doc['docid'].split(" ", 2)
116
+ doc['model'] = self.find(id=docid)
117
+ end
118
+ return res
119
+ end
120
+ end
121
+
103
122
  end
104
123
 
105
124
 
@@ -1,5 +1,6 @@
1
1
  require 'erb'
2
2
  require 'active_record'
3
+ require 'indextank_client'
3
4
 
4
5
  def load_models
5
6
  app_root = ThinkingTank::Configuration.instance.app_root
@@ -28,6 +29,18 @@ def load_models
28
29
  end
29
30
 
30
31
  def reindex_models
32
+ it = ThinkingTank::Configuration.instance.client
33
+ if it.exists?
34
+ puts "Deleting existing index"
35
+ it.delete_index()
36
+ end
37
+ puts "Creating a new empty index"
38
+ it.create_index()
39
+ puts "Waiting for the index to be ready"
40
+ while not it.running?
41
+ sleep 0.5
42
+ end
43
+
31
44
  Object.subclasses_of(ActiveRecord::Base).each do |klass|
32
45
  reindex klass if klass.is_indexable?
33
46
  end
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 0
7
7
  - 0
8
- - 2
9
- version: 0.0.2
8
+ - 3
9
+ version: 0.0.3
10
10
  platform: ruby
11
11
  authors:
12
12
  - Flaptor
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-05-19 00:00:00 -03:00
17
+ date: 2010-07-26 00:00:00 -07:00
18
18
  default_executable:
19
19
  dependencies: []
20
20
 
@@ -30,7 +30,7 @@ extra_rdoc_files: []
30
30
 
31
31
  files:
32
32
  - rails/init.rb
33
- - lib/indextank.rb
33
+ - lib/indextank_client.rb
34
34
  - lib/thinkingtank/init.rb
35
35
  - README
36
36
  - tasks/rails.rake
data/lib/indextank.rb DELETED
@@ -1,150 +0,0 @@
1
- require 'net/http'
2
- require 'uri'
3
- require 'rubygems'
4
- require 'json'
5
-
6
- BASE_URL = 'http://api.indextank.com/api/v0'
7
- class IndexTank
8
- def initialize(api_key, options={})
9
- @api_key = api_key
10
- @def_index_code = options[:index_code]
11
- @def_index_name = options[:index_name]
12
- end
13
-
14
- def create_index(index_name)
15
- return api_call("admin/create", "index_code", :index_name => index_name)
16
- end
17
-
18
- # the options argument may contain an :index_code definition to override
19
- # this instance's default index_code
20
- def delete_index(options={})
21
- return api_call("admin/delete", nil, options)
22
- end
23
-
24
-
25
- def list_indexes()
26
- return api_call("admin/list", "results")
27
- end
28
-
29
-
30
- # the options argument may contain an :index_code definition to override
31
- # this instance's default index_code
32
- def add(doc_id, content, options={})
33
- options.merge!(:document_id => doc_id, :document => content.to_json)
34
- options[:boosts] = options[:boosts].to_json if options.key?(:boosts)
35
- return api_call("index/add", nil, options)
36
- end
37
-
38
- # the options argument may contain an :index_code definition to override
39
- # this instance's default index_code
40
- def boost(doc_id, boosts, options={})
41
- options.merge!(:document_id => doc_id, :boosts => boosts.to_json)
42
- return api_call("index/boost", nil, options)
43
- end
44
-
45
-
46
- # the options argument may contain an :index_code definition to override
47
- # this instance's default index_code
48
- def promote(doc_id, query, options={})
49
- options.merge!(:document_id => doc_id, :query => query)
50
- return api_call("index/promote", nil, options)
51
- end
52
-
53
-
54
- # the options argument may contain an :index_code definition to override
55
- # this instance's default index_code
56
- def update(doc_id, content, options={})
57
- options.merge!(:document_id => doc_id, :document => content.to_json)
58
- return api_call("index/update", nil, options)
59
- end
60
-
61
- # the options argument may contain an :index_code definition to override
62
- # this instance's default index_code
63
- def delete(doc_id, options={})
64
- options.merge!(:document_id => doc_id)
65
- return api_call("index/delete", nil, options)
66
- end
67
-
68
- # the options argument may contain an :index_code definition to override
69
- # this instance's default index_code
70
- # it can also contain any of the following:
71
- # :start => an int with the number of results to skip
72
- # :len => an int with the number of results to return
73
- # :snippet_fields => a comma separated list of field names for which a snippet
74
- # should be returned. (requires an index that supports snippets)
75
- # :fetch_fields => a comma separated list of field names for which its content
76
- # should be returned. (requires an index that supports storage)
77
- # :relevance_function => an int with the index of the relevance function to be used
78
- # for this query
79
- def search(query, options={})
80
- options = { :start => 0, :len => 10 }.merge(options)
81
- options.merge!(:query => query)
82
- return api_call("search/query", "results", options)
83
- end
84
-
85
- # the options argument may contain an :index_code definition to override
86
- # this instance's default index_code
87
- def add_function(function_index, definition, options={})
88
- options.merge!( :function_id => function_index, :definition => definition )
89
- return api_call("index/add_function", nil, options)
90
- end
91
-
92
- # the options argument may contain an :index_code definition to override
93
- # this instance's default index_code
94
- def del_function(function_index, options={})
95
- options.merge!( :function_id => function_index )
96
- return api_call("index/remove_function", nil, options)
97
- end
98
-
99
- # the options argument may contain an :index_code definition to override
100
- # this instance's default index_code
101
- def list_functions(options={})
102
- return api_call("index/list_functions", "results", options)
103
- end
104
-
105
-
106
- # the options argument may contain an :index_code definition to override
107
- # this instance's default index_code
108
- def index_stats(options={})
109
- return api_call("index/stats", "results", options)
110
- end
111
-
112
-
113
- # the options argument may contain an :index_code definition to override
114
- # this instance's default index_code
115
- def search_stats(options={})
116
- return api_call("search/stats", "results", options)
117
- end
118
-
119
- private
120
-
121
- def base_url
122
- return ENV['INDEXTANK_BASE_URL'] || BASE_URL
123
- end
124
-
125
- def api_call(method, return_key, params={})
126
- params = { "api_key" => @api_key }.merge(params)
127
- params = { :index_code => @def_index_code }.merge(params) unless @def_index_code.nil?
128
- params = { :index_name => @def_index_name }.merge(params) unless @def_index_name.nil?
129
- url = base_url + '/' + method
130
- req = Net::HTTP::Post.new(url)
131
- req.set_form_data(params, ';')
132
- res = Net::HTTP.new(URI.parse(base_url).host).start {|http| http.request(req) }
133
- if res.is_a? Net::HTTPOK
134
- result = JSON.parse(res.body)
135
- ok = result['status'] == 'OK'
136
- return ok ? [ok, (return_key.nil? ? nil : result[return_key])] : [ok, result['message']]
137
- elsif res.is_a? Net::HTTPForbidden
138
- return false, "Access forbidden"
139
- elsif res.is_a? Net::HTTPClientError
140
- return false, "Unknown client error"
141
- elsif res.is_a? Net::HTTPServerError
142
- puts res.body
143
- return false, "Unknown server error"
144
- else
145
- puts res.body
146
- return false, "Unexpected response"
147
- end
148
- end
149
-
150
- end