supernova 0.3.10 → 0.3.11
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/supernova/solr_indexer.rb +27 -6
- data/spec/integration/solr_spec.rb +53 -5
- data/spec/spec_helper.rb +3 -1
- data/spec/supernova/solr_indexer_spec.rb +95 -14
- data/supernova.gemspec +2 -2
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.11
|
@@ -1,9 +1,12 @@
|
|
1
1
|
require "json"
|
2
|
+
require "fileutils"
|
2
3
|
|
3
4
|
class Supernova::SolrIndexer
|
4
|
-
attr_accessor :options, :db, :ids
|
5
|
+
attr_accessor :options, :db, :ids, :max_rows_to_direct_index, :local_solr
|
5
6
|
attr_writer :index_file_path
|
6
7
|
|
8
|
+
MAX_ROWS_TO_DIRECT_INDEX = 100
|
9
|
+
|
7
10
|
include Supernova::Solr
|
8
11
|
|
9
12
|
class << self
|
@@ -62,6 +65,7 @@ class Supernova::SolrIndexer
|
|
62
65
|
options.each do |key, value|
|
63
66
|
self.send(:"#{key}=", value) if self.respond_to?(:"#{key}=")
|
64
67
|
end
|
68
|
+
self.max_rows_to_direct_index ||= MAX_ROWS_TO_DIRECT_INDEX
|
65
69
|
self.options = options
|
66
70
|
self.ids ||= :all
|
67
71
|
end
|
@@ -143,8 +147,23 @@ class Supernova::SolrIndexer
|
|
143
147
|
end
|
144
148
|
|
145
149
|
def index_query(query)
|
146
|
-
query_db(query)
|
147
|
-
|
150
|
+
rows = query_db(query)
|
151
|
+
if self.max_rows_to_direct_index < rows.count
|
152
|
+
index_with_json_file(rows)
|
153
|
+
else
|
154
|
+
index_directly(rows)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def index_directly(rows)
|
159
|
+
rows.each do |row|
|
160
|
+
Supernova::Solr.connection.add(row)
|
161
|
+
end
|
162
|
+
Supernova::Solr.connection.commit if rows.any?
|
163
|
+
end
|
164
|
+
|
165
|
+
def index_with_json_file(rows)
|
166
|
+
rows.each do |row|
|
148
167
|
write_to_file(row)
|
149
168
|
end
|
150
169
|
finish
|
@@ -187,16 +206,18 @@ class Supernova::SolrIndexer
|
|
187
206
|
end
|
188
207
|
|
189
208
|
def solr_url
|
190
|
-
Supernova::Solr.url
|
209
|
+
Supernova::Solr.url.present? ? Supernova::Solr.url.to_s.gsub(/\/$/, "") : nil
|
191
210
|
end
|
192
211
|
|
193
212
|
def do_index_file(options = {})
|
194
213
|
raise "solr not configured" if solr_url.nil?
|
195
|
-
cmd = if
|
214
|
+
cmd = if self.local_solr
|
196
215
|
%(curl -s '#{solr_url}/update/json?commit=true\\&stream.file=#{index_file_path}')
|
197
216
|
else
|
198
217
|
%(cd #{File.dirname(index_file_path)} && curl -s '#{solr_url}/update/json?commit=true' --data-binary @#{File.basename(index_file_path)} -H 'Content-type:application/json')
|
199
218
|
end
|
200
|
-
Kernel.send(:`, cmd)
|
219
|
+
out = Kernel.send(:`, cmd)
|
220
|
+
FileUtils.rm_f(self.index_file_path) if out.to_s.include?(%(<int name=\"status\">0</int>))
|
221
|
+
out
|
201
222
|
end
|
202
223
|
end
|
@@ -31,6 +31,52 @@ describe "Solr" do
|
|
31
31
|
Offer.search_scope
|
32
32
|
end
|
33
33
|
|
34
|
+
describe "#indexing" do
|
35
|
+
before(:each) do
|
36
|
+
Supernova::Solr.truncate!
|
37
|
+
Supernova::Solr.connection.commit
|
38
|
+
end
|
39
|
+
|
40
|
+
class OfferIndex < Supernova::SolrIndexer
|
41
|
+
has :user_id, :type => :integer
|
42
|
+
has :popularity, :type => :integer
|
43
|
+
|
44
|
+
clazz Offer
|
45
|
+
end
|
46
|
+
|
47
|
+
it "indexes all Offers without file" do
|
48
|
+
offer1 = Offer.create!(:user_id => 1, :popularity => 10)
|
49
|
+
offer2 = Offer.create!(:user_id => 2, :popularity => 20)
|
50
|
+
indexer = OfferIndex.new(:db => ActiveRecord::Base.connection)
|
51
|
+
indexer.index!
|
52
|
+
OfferIndex.search_scope.to_a.total_entries.should == 2
|
53
|
+
OfferIndex.search_scope.order("user_id desc").to_a.should == [offer2, offer1]
|
54
|
+
indexer.instance_variable_get("@index_file_path").should be_nil
|
55
|
+
end
|
56
|
+
|
57
|
+
it "indexes with a file" do
|
58
|
+
offer1 = Offer.create!(:user_id => 1, :popularity => 10)
|
59
|
+
offer2 = Offer.create!(:user_id => 2, :popularity => 20)
|
60
|
+
indexer = OfferIndex.new(:db => ActiveRecord::Base.connection, :max_rows_to_direct_index => 0)
|
61
|
+
indexer.index!
|
62
|
+
indexer.instance_variable_get("@index_file_path").should_not be_nil
|
63
|
+
OfferIndex.search_scope.to_a.total_entries.should == 2
|
64
|
+
OfferIndex.search_scope.order("user_id desc").to_a.should == [offer2, offer1]
|
65
|
+
File.should_not be_exists(indexer.instance_variable_get("@index_file_path"))
|
66
|
+
end
|
67
|
+
|
68
|
+
it "indexes with a local file" do
|
69
|
+
offer1 = Offer.create!(:user_id => 1, :popularity => 10)
|
70
|
+
offer2 = Offer.create!(:user_id => 2, :popularity => 20)
|
71
|
+
indexer = OfferIndex.new(:db => ActiveRecord::Base.connection, :max_rows_to_direct_index => 0, :local_solr => true)
|
72
|
+
indexer.index!
|
73
|
+
indexer.instance_variable_get("@index_file_path").should_not be_nil
|
74
|
+
OfferIndex.search_scope.to_a.total_entries.should == 2
|
75
|
+
OfferIndex.search_scope.order("user_id desc").to_a.should == [offer2, offer1]
|
76
|
+
File.should_not be_exists(indexer.instance_variable_get("@index_file_path"))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
34
80
|
describe "searching" do
|
35
81
|
it "returns the correct current_page when nil" do
|
36
82
|
new_criteria.to_a.current_page.should == 1
|
@@ -60,11 +106,13 @@ describe "Solr" do
|
|
60
106
|
end
|
61
107
|
end
|
62
108
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
}
|
109
|
+
{
|
110
|
+
"id" => "offers/1", "type" => "Offer", "user_id_i" => 1, "enabled_b" => false, "text_t" => "Hans Meyer",
|
111
|
+
"popularity_i" => 10, "location_p" => "47,11"
|
112
|
+
}.each do |key, value|
|
113
|
+
it "sets #{key} to #{value}" do
|
114
|
+
doc = new_criteria.search("text_t:Hans").to_a.first.instance_variable_get("@solr_doc")[key].should == value
|
115
|
+
end
|
68
116
|
end
|
69
117
|
|
70
118
|
describe "nearby search" do
|
data/spec/spec_helper.rb
CHANGED
@@ -18,7 +18,9 @@ end
|
|
18
18
|
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
19
19
|
|
20
20
|
RSpec.configure do |config|
|
21
|
-
|
21
|
+
config.before(:each) do
|
22
|
+
ActiveRecord::Base.connection.execute("TRUNCATE offers")
|
23
|
+
end
|
22
24
|
end
|
23
25
|
|
24
26
|
ActiveRecord::Base.establish_connection(
|
@@ -5,11 +5,11 @@ describe Supernova::SolrIndexer do
|
|
5
5
|
let(:db) { double("db", :query => [to_index]) }
|
6
6
|
let(:to_index) { { :id => 1, :title => "Some Title"} }
|
7
7
|
let(:file_stub) { double("file").as_null_object }
|
8
|
+
let(:solr) { double("solr").as_null_object }
|
8
9
|
|
9
10
|
let(:indexer) do
|
10
11
|
indexer = Supernova::SolrIndexer.new
|
11
12
|
indexer.db = db
|
12
|
-
Supernova::Solr.url = "http://solr.xx:9333/solr"
|
13
13
|
indexer.stub!(:system).and_return true
|
14
14
|
indexer
|
15
15
|
end
|
@@ -17,6 +17,8 @@ describe Supernova::SolrIndexer do
|
|
17
17
|
let(:custom_indexer) { indexer_clazz.new }
|
18
18
|
|
19
19
|
before(:each) do
|
20
|
+
Supernova::Solr.url = "http://solr.xx:9333/solr"
|
21
|
+
Supernova::Solr.stub!(:connection).and_return solr
|
20
22
|
indexer_clazz.has(:title, :type => :text)
|
21
23
|
indexer_clazz.has(:artist_id, :type => :integer)
|
22
24
|
indexer_clazz.has(:description, :type => :text)
|
@@ -47,6 +49,10 @@ describe Supernova::SolrIndexer do
|
|
47
49
|
it "sets ids to all when nil" do
|
48
50
|
Supernova::SolrIndexer.new.ids.should == :all
|
49
51
|
end
|
52
|
+
|
53
|
+
it "sets max_rows_to_direct_index to 100" do
|
54
|
+
Supernova::SolrIndexer.new.max_rows_to_direct_index.should == 100
|
55
|
+
end
|
50
56
|
end
|
51
57
|
|
52
58
|
describe "index!" do
|
@@ -135,24 +141,78 @@ describe Supernova::SolrIndexer do
|
|
135
141
|
describe "#index_query" do
|
136
142
|
let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
|
137
143
|
|
138
|
-
it "
|
139
|
-
indexer.
|
144
|
+
it "calls index_with_json_file when rows > max_rows_to_direct_index" do
|
145
|
+
indexer.max_rows_to_direct_index = 0
|
146
|
+
rows = [to_index]
|
147
|
+
indexer.should_receive(:query_db).with(query).and_return rows
|
148
|
+
indexer.should_receive(:index_with_json_file).with(rows)
|
140
149
|
indexer.index_query(query)
|
141
150
|
end
|
142
151
|
|
143
|
-
it "calls
|
144
|
-
|
145
|
-
|
146
|
-
indexer.should_receive(:
|
147
|
-
indexer.should_receive(:
|
148
|
-
indexer.stub!(:finish)
|
152
|
+
it "calls index_directly with rows when rows = max_rows_to_direct_index" do
|
153
|
+
indexer.max_rows_to_direct_index = 1
|
154
|
+
rows = [to_index]
|
155
|
+
indexer.should_receive(:query_db).with(query).and_return rows
|
156
|
+
indexer.should_receive(:index_directly).with(rows)
|
149
157
|
indexer.index_query(query)
|
150
158
|
end
|
151
159
|
|
152
|
-
|
153
|
-
|
154
|
-
|
160
|
+
describe "with number of rows > max_rows_to_direct_index" do
|
161
|
+
before(:each) do
|
162
|
+
indexer.max_rows_to_direct_index = 0
|
163
|
+
end
|
164
|
+
|
165
|
+
it "calls max_rows_to_direct_index" do
|
166
|
+
indexer.should_receive(:max_rows_to_direct_index).and_return 0
|
167
|
+
indexer.index_query(query)
|
168
|
+
end
|
169
|
+
|
170
|
+
it "executes the query" do
|
171
|
+
indexer.should_receive(:query_db).with(query).and_return [to_index]
|
172
|
+
indexer.index_query(query)
|
173
|
+
end
|
174
|
+
|
175
|
+
it "calls write_to_file on all rows" do
|
176
|
+
rows = [double("1"), double("2")]
|
177
|
+
indexer.stub(:query_db).and_return rows
|
178
|
+
indexer.should_receive(:write_to_file).with(rows.first)
|
179
|
+
indexer.should_receive(:write_to_file).with(rows.at(1))
|
180
|
+
indexer.stub!(:finish)
|
181
|
+
indexer.index_query(query)
|
182
|
+
end
|
183
|
+
|
184
|
+
it "calls finish" do
|
185
|
+
indexer.should_receive(:finish)
|
186
|
+
indexer.index_query(query)
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
describe "#index_directly" do
|
192
|
+
before(:each) do
|
193
|
+
Supernova::Solr.stub!(:connection).and_return solr
|
194
|
+
end
|
195
|
+
|
196
|
+
it "calls the correct add statement" do
|
197
|
+
row1 = double("1")
|
198
|
+
row2 = double("2")
|
199
|
+
rows = [row1, row2]
|
200
|
+
solr.should_receive(:add).with(row1)
|
201
|
+
solr.should_receive(:add).with(row2)
|
202
|
+
indexer.index_directly(rows)
|
203
|
+
end
|
204
|
+
|
205
|
+
it "calls commit" do
|
206
|
+
solr.should_receive(:commit)
|
207
|
+
indexer.index_directly([double("1")])
|
155
208
|
end
|
209
|
+
|
210
|
+
it "does not call commit when rows is empty" do
|
211
|
+
solr.should_not_receive(:commit)
|
212
|
+
indexer.index_directly([])
|
213
|
+
end
|
214
|
+
|
215
|
+
it "calls a block given given"
|
156
216
|
end
|
157
217
|
|
158
218
|
describe "#index_file_path" do
|
@@ -239,8 +299,22 @@ describe Supernova::SolrIndexer do
|
|
239
299
|
end
|
240
300
|
|
241
301
|
it "calls the correct curl command" do
|
242
|
-
indexer.index_file_path
|
243
|
-
Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'")
|
302
|
+
indexer = Supernova::SolrIndexer.new(:index_file_path => "/tmp/some_path.json", :local_solr => true)
|
303
|
+
Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'").and_return "<\"status\">0"
|
304
|
+
indexer.do_index_file(:local => true)
|
305
|
+
end
|
306
|
+
|
307
|
+
it "calls rm on file" do
|
308
|
+
indexer = Supernova::SolrIndexer.new(:index_file_path => "/tmp/some_path.json", :local_solr => true)
|
309
|
+
Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'").and_return %(<int name="status">0</int>)
|
310
|
+
FileUtils.should_receive(:rm_f).with("/tmp/some_path.json")
|
311
|
+
indexer.do_index_file(:local => true)
|
312
|
+
end
|
313
|
+
|
314
|
+
it "does not call rm when not successful" do
|
315
|
+
indexer = Supernova::SolrIndexer.new(:index_file_path => "/tmp/some_path.json", :local_solr => true)
|
316
|
+
Kernel.should_receive(:`).with("curl -s 'http://solr.xx:9333/solr/update/json?commit=true\\&stream.file=/tmp/some_path.json'").and_return %(<int name="status">1</int>)
|
317
|
+
FileUtils.should_not_receive(:rm_f).with("/tmp/some_path.json")
|
244
318
|
indexer.do_index_file(:local => true)
|
245
319
|
end
|
246
320
|
|
@@ -492,4 +566,11 @@ describe Supernova::SolrIndexer do
|
|
492
566
|
Supernova::SolrIndexer.solr_field_for_field_name_and_mapping(:artist, nil).should == "artist"
|
493
567
|
end
|
494
568
|
end
|
569
|
+
|
570
|
+
describe "#solr_url" do
|
571
|
+
it "strips slashes from defined solr url" do
|
572
|
+
Supernova::Solr.url = "http://solr.xx:9333/solr/"
|
573
|
+
indexer.solr_url.should == "http://solr.xx:9333/solr"
|
574
|
+
end
|
575
|
+
end
|
495
576
|
end
|
data/supernova.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{supernova}
|
8
|
-
s.version = "0.3.
|
8
|
+
s.version = "0.3.11"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Tobias Schwab"]
|
12
|
-
s.date = %q{2011-06-
|
12
|
+
s.date = %q{2011-06-17}
|
13
13
|
s.description = %q{Unified search scopes}
|
14
14
|
s.email = %q{tobias.schwab@dynport.de}
|
15
15
|
s.extra_rdoc_files = [
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: supernova
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 5
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 3
|
9
|
-
-
|
10
|
-
version: 0.3.
|
9
|
+
- 11
|
10
|
+
version: 0.3.11
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Tobias Schwab
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-06-
|
18
|
+
date: 2011-06-17 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
requirement: &id001 !ruby/object:Gem::Requirement
|