supernova 0.4.15 → 0.4.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/supernova/solr_indexer.rb +32 -16
- data/spec/spec_helper.rb +3 -1
- data/spec/supernova/solr_indexer_spec.rb +65 -45
- data/supernova.gemspec +2 -2
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.4.
|
1
|
+
0.4.16
|
@@ -77,7 +77,9 @@ class Supernova::SolrIndexer
|
|
77
77
|
response = yield if block_given?
|
78
78
|
end
|
79
79
|
if @debug == true
|
80
|
-
|
80
|
+
message.gsub!("%COUNT%", response.count.to_s) if message.include?("%COUNT%") && response.respond_to?(:count)
|
81
|
+
message.gsub!("%TIME%", "%.3f" % time) if message.include?("%TIME%")
|
82
|
+
puts "%s: %s" % [Time.now.iso8601(3), message]
|
81
83
|
end
|
82
84
|
response
|
83
85
|
end
|
@@ -104,14 +106,18 @@ class Supernova::SolrIndexer
|
|
104
106
|
end
|
105
107
|
|
106
108
|
def map_hash_keys_to_solr(hash)
|
107
|
-
|
109
|
+
@indexed_at ||= Time.now.utc.iso8601.to_s
|
110
|
+
hash["indexed_at_dt"] = @indexed_at
|
108
111
|
hash["id_s"] = [self.class.table_name, hash["id"]].compact.join("/") if hash["id"]
|
109
112
|
self.class.field_definitions.each do |field, options|
|
110
113
|
if hash.has_key?(field.to_s)
|
111
114
|
value = hash.delete(field.to_s)
|
112
|
-
if options[:type] == :date
|
113
|
-
|
114
|
-
|
115
|
+
if options[:type] == :date
|
116
|
+
if value.is_a?(Date)
|
117
|
+
value = "#{value}T00:00:00Z"
|
118
|
+
elsif value.respond_to?(:utc)
|
119
|
+
value = value.utc.iso8601
|
120
|
+
end
|
115
121
|
end
|
116
122
|
hash["#{field}_#{self.class.suffix_from_type(options[:type])}"] = value
|
117
123
|
end
|
@@ -191,19 +197,14 @@ class Supernova::SolrIndexer
|
|
191
197
|
end
|
192
198
|
|
193
199
|
def rows(query = nil)
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
def solr_rows_to_index_for_query(query)
|
198
|
-
query_db(query).map do |row|
|
199
|
-
map_for_solr(row)
|
200
|
+
debug "fetched rows in %TIME%" do
|
201
|
+
query_db(query || query_to_index)
|
200
202
|
end
|
201
203
|
end
|
202
204
|
|
203
|
-
def
|
204
|
-
debug "
|
205
|
-
|
206
|
-
solr_rows_to_index_for_query(query)
|
205
|
+
def index_rows(rows)
|
206
|
+
debug "mapped %COUNT% rows to solr in %TIME%" do
|
207
|
+
rows.map! { |r| map_for_solr(r) }
|
207
208
|
end
|
208
209
|
if self.max_rows_to_direct_index < rows.count
|
209
210
|
debug "indexed #{rows.length} rows with json in %TIME%" do
|
@@ -216,6 +217,17 @@ class Supernova::SolrIndexer
|
|
216
217
|
end
|
217
218
|
end
|
218
219
|
|
220
|
+
def solr_rows_to_index_for_query(query)
|
221
|
+
query_db(query).map do |row|
|
222
|
+
map_for_solr(row)
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
def index_query(query)
|
227
|
+
debug "getting rows for #{query[0,100]}"
|
228
|
+
index_rows(query_db(query))
|
229
|
+
end
|
230
|
+
|
219
231
|
def index_directly(rows)
|
220
232
|
rows.each do |row|
|
221
233
|
row = Supernova::Solr.connection.add(row)
|
@@ -279,8 +291,12 @@ class Supernova::SolrIndexer
|
|
279
291
|
else
|
280
292
|
%(cd #{File.dirname(index_file_path)} && curl -s '#{solr_url}/update/json?commit=true' --data-binary @#{File.basename(index_file_path)} -H 'Content-type:application/json')
|
281
293
|
end
|
294
|
+
debug "run command: #{cmd}"
|
282
295
|
out = Kernel.send(:`, cmd)
|
283
|
-
|
296
|
+
if !out.to_s.include?(%(<int name=\"status\">0</int>))
|
297
|
+
debug "ERROR: #{out}"
|
298
|
+
raise "unable to index #{index_file_path}: #{out}"
|
299
|
+
end
|
284
300
|
FileUtils.rm_f(self.index_file_path)
|
285
301
|
out
|
286
302
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -8,6 +8,7 @@ require "fileutils"
|
|
8
8
|
require "ruby-debug"
|
9
9
|
require "geokit"
|
10
10
|
require "active_record"
|
11
|
+
PROJECT_ROOT = Pathname.new(File.expand_path("..", File.dirname(__FILE__)))
|
11
12
|
|
12
13
|
if defined?(Debugger) && Debugger.respond_to?(:settings)
|
13
14
|
Debugger.settings[:autolist] = 1
|
@@ -24,6 +25,7 @@ RSpec.configure do |config|
|
|
24
25
|
end
|
25
26
|
end
|
26
27
|
|
28
|
+
|
27
29
|
ActiveRecord::Base.establish_connection(
|
28
30
|
:adapter => "mysql2",
|
29
31
|
:host => "localhost",
|
@@ -32,7 +34,7 @@ ActiveRecord::Base.establish_connection(
|
|
32
34
|
:encoding => "utf8"
|
33
35
|
)
|
34
36
|
|
35
|
-
|
37
|
+
|
36
38
|
FileUtils.mkdir_p(PROJECT_ROOT.join("log"))
|
37
39
|
|
38
40
|
ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS offers")
|
@@ -225,6 +225,14 @@ describe Supernova::SolrIndexer do
|
|
225
225
|
112
|
226
226
|
end
|
227
227
|
end
|
228
|
+
|
229
|
+
it "replaces %COUNT% when responding to .count" do
|
230
|
+
index = CustomSolrIndex.new(:debug => true)
|
231
|
+
index.should_receive(:puts).with(/indexed 2/)
|
232
|
+
index.debug "indexed %COUNT%" do
|
233
|
+
[1, 2]
|
234
|
+
end
|
235
|
+
end
|
228
236
|
end
|
229
237
|
|
230
238
|
describe "#map_hash_keys_to_solr" do
|
@@ -331,59 +339,64 @@ describe Supernova::SolrIndexer do
|
|
331
339
|
end
|
332
340
|
end
|
333
341
|
|
334
|
-
describe "#
|
335
|
-
let(:
|
342
|
+
describe "#index_rows" do
|
343
|
+
let(:row1) { double("row1") }
|
344
|
+
let(:row2) { double("row2") }
|
345
|
+
let(:mapped1) { double("mapped 1") }
|
346
|
+
let(:mapped2) { double("mapped 2") }
|
336
347
|
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
indexer.index_query(query)
|
348
|
+
before(:each) do
|
349
|
+
custom_indexer.stub(:map_for_solr).with(row1).and_return(mapped1)
|
350
|
+
custom_indexer.stub(:map_for_solr).with(row2).and_return(mapped2)
|
341
351
|
end
|
342
352
|
|
343
|
-
it "
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
353
|
+
it "is callable" do
|
354
|
+
custom_indexer.index_rows([])
|
355
|
+
end
|
356
|
+
|
357
|
+
it "calls map_for_solr on all rows" do
|
358
|
+
custom_indexer.should_receive(:map_for_solr).with(row1).and_return(mapped1)
|
359
|
+
custom_indexer.should_receive(:map_for_solr).with(row2).and_return(mapped2)
|
360
|
+
custom_indexer.index_rows([row1, row2])
|
361
|
+
end
|
362
|
+
|
363
|
+
it "calls map_directly when number of rows < max_rows_to_direct_index" do
|
364
|
+
custom_indexer.should_receive(:max_rows_to_direct_index).and_return 100
|
365
|
+
custom_indexer.should_receive(:index_directly).with([mapped1, mapped2])
|
366
|
+
custom_indexer.index_rows([row1, row2])
|
367
|
+
end
|
368
|
+
|
369
|
+
it "calls map_directly when number of rows < max_rows_to_direct_index" do
|
370
|
+
custom_indexer.should_receive(:max_rows_to_direct_index).and_return 1
|
371
|
+
custom_indexer.should_receive(:index_with_json_file).with([mapped1, mapped2])
|
372
|
+
custom_indexer.index_rows([row1, row2])
|
349
373
|
end
|
374
|
+
end
|
375
|
+
|
376
|
+
describe "#index_query" do
|
377
|
+
let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
|
350
378
|
|
351
|
-
it "calls
|
352
|
-
indexer.max_rows_to_direct_index = 1
|
379
|
+
it "calls index_rows with result of query" do
|
353
380
|
rows = [to_index]
|
354
381
|
indexer.should_receive(:query_db).with(query).and_return rows
|
355
|
-
indexer.should_receive(:
|
382
|
+
indexer.should_receive(:index_rows).with(rows)
|
356
383
|
indexer.index_query(query)
|
357
384
|
end
|
385
|
+
end
|
386
|
+
|
387
|
+
describe "#index_with_json_file" do
|
388
|
+
let(:rows) { [{ "b" => 2 }, { "a" => 1 }] }
|
358
389
|
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
indexer.should_receive(:max_rows_to_direct_index).and_return 0
|
366
|
-
indexer.index_query(query)
|
367
|
-
end
|
368
|
-
|
369
|
-
it "executes the query" do
|
370
|
-
indexer.should_receive(:query_db).with(query).and_return [to_index]
|
371
|
-
indexer.index_query(query)
|
372
|
-
end
|
373
|
-
|
374
|
-
it "calls write_to_file on all rows" do
|
375
|
-
rows = [{ "b" => 2 }, { "a" => 1 }]
|
376
|
-
indexer.stub(:query_db).and_return rows
|
377
|
-
indexer.should_receive(:write_to_file).with(rows.first)
|
378
|
-
indexer.should_receive(:write_to_file).with(rows.at(1))
|
379
|
-
indexer.stub!(:finish)
|
380
|
-
indexer.index_query(query)
|
381
|
-
end
|
390
|
+
it "calls write_to_file on all rows" do
|
391
|
+
indexer.should_receive(:write_to_file).with(rows.first)
|
392
|
+
indexer.should_receive(:write_to_file).with(rows.at(1))
|
393
|
+
indexer.stub!(:finish)
|
394
|
+
indexer.index_with_json_file(rows)
|
395
|
+
end
|
382
396
|
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
end
|
397
|
+
it "calls finish" do
|
398
|
+
indexer.should_receive(:finish)
|
399
|
+
indexer.index_with_json_file(rows)
|
387
400
|
end
|
388
401
|
end
|
389
402
|
|
@@ -434,18 +447,25 @@ describe Supernova::SolrIndexer do
|
|
434
447
|
|
435
448
|
it "writes the opening brackets and the first line" do
|
436
449
|
file_stub.should_receive(:puts).with("\{")
|
437
|
-
file_stub.should_receive(:print).with
|
450
|
+
file_stub.should_receive(:print).with do |str|
|
451
|
+
str.should include("add")
|
452
|
+
str.should include("\"title\":\"Some Title\"")
|
453
|
+
str.should include("\"id\":1")
|
454
|
+
end
|
438
455
|
indexer.write_to_file(to_index)
|
439
456
|
end
|
440
457
|
|
441
458
|
it "only write fields which are not null" do
|
442
|
-
file_stub.
|
459
|
+
file_stub.stub(:print)
|
460
|
+
file_stub.should_not_receive(:print).with do |str|
|
461
|
+
str.include?("text")
|
462
|
+
end
|
443
463
|
indexer.write_to_file(to_index.merge(:text => nil))
|
444
464
|
end
|
445
465
|
|
446
466
|
it "separates the first and the second line" do
|
447
467
|
file_stub.should_receive(:puts).with("\{")
|
448
|
-
file_stub.should_receive(:print).with("
|
468
|
+
file_stub.should_receive(:print).with(/\"add\":{\"doc\"/)
|
449
469
|
file_stub.should_receive(:print).with(%(,\n"add":{"doc":{"id":2}}))
|
450
470
|
indexer.write_to_file(to_index)
|
451
471
|
indexer.write_to_file({:id => 2})
|
data/supernova.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{supernova}
|
8
|
-
s.version = "0.4.
|
8
|
+
s.version = "0.4.16"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Tobias Schwab"]
|
12
|
-
s.date = %q{2011-
|
12
|
+
s.date = %q{2011-08-02}
|
13
13
|
s.description = %q{Unified search scopes}
|
14
14
|
s.email = %q{tobias.schwab@dynport.de}
|
15
15
|
s.executables = ["start_solr"]
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: supernova
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 47
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 4
|
9
|
-
-
|
10
|
-
version: 0.4.
|
9
|
+
- 16
|
10
|
+
version: 0.4.16
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Tobias Schwab
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-08-02 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
requirement: &id001 !ruby/object:Gem::Requirement
|