supernova 0.4.15 → 0.4.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.4.15
1
+ 0.4.16
@@ -77,7 +77,9 @@ class Supernova::SolrIndexer
77
77
  response = yield if block_given?
78
78
  end
79
79
  if @debug == true
80
- puts "%s: %s" % [Time.now.iso8601(3), message.gsub("%TIME%", "%.3f" % time)]
80
+ message.gsub!("%COUNT%", response.count.to_s) if message.include?("%COUNT%") && response.respond_to?(:count)
81
+ message.gsub!("%TIME%", "%.3f" % time) if message.include?("%TIME%")
82
+ puts "%s: %s" % [Time.now.iso8601(3), message]
81
83
  end
82
84
  response
83
85
  end
@@ -104,14 +106,18 @@ class Supernova::SolrIndexer
104
106
  end
105
107
 
106
108
  def map_hash_keys_to_solr(hash)
107
- hash["indexed_at_dt"] = Time.now.utc.iso8601
109
+ @indexed_at ||= Time.now.utc.iso8601.to_s
110
+ hash["indexed_at_dt"] = @indexed_at
108
111
  hash["id_s"] = [self.class.table_name, hash["id"]].compact.join("/") if hash["id"]
109
112
  self.class.field_definitions.each do |field, options|
110
113
  if hash.has_key?(field.to_s)
111
114
  value = hash.delete(field.to_s)
112
- if options[:type] == :date
113
- value = Time.utc(value.year, value.month, value.day) if value.is_a?(Date)
114
- value = value.utc.iso8601 if value
115
+ if options[:type] == :date
116
+ if value.is_a?(Date)
117
+ value = "#{value}T00:00:00Z"
118
+ elsif value.respond_to?(:utc)
119
+ value = value.utc.iso8601
120
+ end
115
121
  end
116
122
  hash["#{field}_#{self.class.suffix_from_type(options[:type])}"] = value
117
123
  end
@@ -191,19 +197,14 @@ class Supernova::SolrIndexer
191
197
  end
192
198
 
193
199
  def rows(query = nil)
194
- query_db(query || query_to_index)
195
- end
196
-
197
- def solr_rows_to_index_for_query(query)
198
- query_db(query).map do |row|
199
- map_for_solr(row)
200
+ debug "fetched rows in %TIME%" do
201
+ query_db(query || query_to_index)
200
202
  end
201
203
  end
202
204
 
203
- def index_query(query)
204
- debug "getting rows for #{query[0,100]}"
205
- rows = debug "got all rows in %TIME%" do
206
- solr_rows_to_index_for_query(query)
205
+ def index_rows(rows)
206
+ debug "mapped %COUNT% rows to solr in %TIME%" do
207
+ rows.map! { |r| map_for_solr(r) }
207
208
  end
208
209
  if self.max_rows_to_direct_index < rows.count
209
210
  debug "indexed #{rows.length} rows with json in %TIME%" do
@@ -216,6 +217,17 @@ class Supernova::SolrIndexer
216
217
  end
217
218
  end
218
219
 
220
+ def solr_rows_to_index_for_query(query)
221
+ query_db(query).map do |row|
222
+ map_for_solr(row)
223
+ end
224
+ end
225
+
226
+ def index_query(query)
227
+ debug "getting rows for #{query[0,100]}"
228
+ index_rows(query_db(query))
229
+ end
230
+
219
231
  def index_directly(rows)
220
232
  rows.each do |row|
221
233
  row = Supernova::Solr.connection.add(row)
@@ -279,8 +291,12 @@ class Supernova::SolrIndexer
279
291
  else
280
292
  %(cd #{File.dirname(index_file_path)} && curl -s '#{solr_url}/update/json?commit=true' --data-binary @#{File.basename(index_file_path)} -H 'Content-type:application/json')
281
293
  end
294
+ debug "run command: #{cmd}"
282
295
  out = Kernel.send(:`, cmd)
283
- raise "unable to index #{index_file_path}: #{out}" if !out.to_s.include?(%(<int name=\"status\">0</int>))
296
+ if !out.to_s.include?(%(<int name=\"status\">0</int>))
297
+ debug "ERROR: #{out}"
298
+ raise "unable to index #{index_file_path}: #{out}"
299
+ end
284
300
  FileUtils.rm_f(self.index_file_path)
285
301
  out
286
302
  end
@@ -8,6 +8,7 @@ require "fileutils"
8
8
  require "ruby-debug"
9
9
  require "geokit"
10
10
  require "active_record"
11
+ PROJECT_ROOT = Pathname.new(File.expand_path("..", File.dirname(__FILE__)))
11
12
 
12
13
  if defined?(Debugger) && Debugger.respond_to?(:settings)
13
14
  Debugger.settings[:autolist] = 1
@@ -24,6 +25,7 @@ RSpec.configure do |config|
24
25
  end
25
26
  end
26
27
 
28
+
27
29
  ActiveRecord::Base.establish_connection(
28
30
  :adapter => "mysql2",
29
31
  :host => "localhost",
@@ -32,7 +34,7 @@ ActiveRecord::Base.establish_connection(
32
34
  :encoding => "utf8"
33
35
  )
34
36
 
35
- PROJECT_ROOT = Pathname.new(File.expand_path("..", File.dirname(__FILE__)))
37
+
36
38
  FileUtils.mkdir_p(PROJECT_ROOT.join("log"))
37
39
 
38
40
  ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS offers")
@@ -225,6 +225,14 @@ describe Supernova::SolrIndexer do
225
225
  112
226
226
  end
227
227
  end
228
+
229
+ it "replaces %COUNT% when responding to .count" do
230
+ index = CustomSolrIndex.new(:debug => true)
231
+ index.should_receive(:puts).with(/indexed 2/)
232
+ index.debug "indexed %COUNT%" do
233
+ [1, 2]
234
+ end
235
+ end
228
236
  end
229
237
 
230
238
  describe "#map_hash_keys_to_solr" do
@@ -331,59 +339,64 @@ describe Supernova::SolrIndexer do
331
339
  end
332
340
  end
333
341
 
334
- describe "#index_query" do
335
- let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
342
+ describe "#index_rows" do
343
+ let(:row1) { double("row1") }
344
+ let(:row2) { double("row2") }
345
+ let(:mapped1) { double("mapped 1") }
346
+ let(:mapped2) { double("mapped 2") }
336
347
 
337
- it "calls solr_rows_to_index_for_query with query" do
338
- result = []
339
- indexer.should_receive(:solr_rows_to_index_for_query).with(query).and_return(result)
340
- indexer.index_query(query)
348
+ before(:each) do
349
+ custom_indexer.stub(:map_for_solr).with(row1).and_return(mapped1)
350
+ custom_indexer.stub(:map_for_solr).with(row2).and_return(mapped2)
341
351
  end
342
352
 
343
- it "calls index_with_json_file when rows > max_rows_to_direct_index" do
344
- indexer.max_rows_to_direct_index = 0
345
- rows = [to_index]
346
- indexer.should_receive(:query_db).with(query).and_return rows
347
- indexer.should_receive(:index_with_json_file).with(rows)
348
- indexer.index_query(query)
353
+ it "is callable" do
354
+ custom_indexer.index_rows([])
355
+ end
356
+
357
+ it "calls map_for_solr on all rows" do
358
+ custom_indexer.should_receive(:map_for_solr).with(row1).and_return(mapped1)
359
+ custom_indexer.should_receive(:map_for_solr).with(row2).and_return(mapped2)
360
+ custom_indexer.index_rows([row1, row2])
361
+ end
362
+
363
+ it "calls map_directly when number of rows < max_rows_to_direct_index" do
364
+ custom_indexer.should_receive(:max_rows_to_direct_index).and_return 100
365
+ custom_indexer.should_receive(:index_directly).with([mapped1, mapped2])
366
+ custom_indexer.index_rows([row1, row2])
367
+ end
368
+
369
+ it "calls map_directly when number of rows < max_rows_to_direct_index" do
370
+ custom_indexer.should_receive(:max_rows_to_direct_index).and_return 1
371
+ custom_indexer.should_receive(:index_with_json_file).with([mapped1, mapped2])
372
+ custom_indexer.index_rows([row1, row2])
349
373
  end
374
+ end
375
+
376
+ describe "#index_query" do
377
+ let(:query) { %(SELECT CONCAT("user_", id) AS id, title FROM people WHERE type = 'User') }
350
378
 
351
- it "calls index_directly with rows when rows = max_rows_to_direct_index" do
352
- indexer.max_rows_to_direct_index = 1
379
+ it "calls index_rows with result of query" do
353
380
  rows = [to_index]
354
381
  indexer.should_receive(:query_db).with(query).and_return rows
355
- indexer.should_receive(:index_directly).with(rows)
382
+ indexer.should_receive(:index_rows).with(rows)
356
383
  indexer.index_query(query)
357
384
  end
385
+ end
386
+
387
+ describe "#index_with_json_file" do
388
+ let(:rows) { [{ "b" => 2 }, { "a" => 1 }] }
358
389
 
359
- describe "with number of rows > max_rows_to_direct_index" do
360
- before(:each) do
361
- indexer.max_rows_to_direct_index = 0
362
- end
363
-
364
- it "calls max_rows_to_direct_index" do
365
- indexer.should_receive(:max_rows_to_direct_index).and_return 0
366
- indexer.index_query(query)
367
- end
368
-
369
- it "executes the query" do
370
- indexer.should_receive(:query_db).with(query).and_return [to_index]
371
- indexer.index_query(query)
372
- end
373
-
374
- it "calls write_to_file on all rows" do
375
- rows = [{ "b" => 2 }, { "a" => 1 }]
376
- indexer.stub(:query_db).and_return rows
377
- indexer.should_receive(:write_to_file).with(rows.first)
378
- indexer.should_receive(:write_to_file).with(rows.at(1))
379
- indexer.stub!(:finish)
380
- indexer.index_query(query)
381
- end
390
+ it "calls write_to_file on all rows" do
391
+ indexer.should_receive(:write_to_file).with(rows.first)
392
+ indexer.should_receive(:write_to_file).with(rows.at(1))
393
+ indexer.stub!(:finish)
394
+ indexer.index_with_json_file(rows)
395
+ end
382
396
 
383
- it "calls finish" do
384
- indexer.should_receive(:finish)
385
- indexer.index_query(query)
386
- end
397
+ it "calls finish" do
398
+ indexer.should_receive(:finish)
399
+ indexer.index_with_json_file(rows)
387
400
  end
388
401
  end
389
402
 
@@ -434,18 +447,25 @@ describe Supernova::SolrIndexer do
434
447
 
435
448
  it "writes the opening brackets and the first line" do
436
449
  file_stub.should_receive(:puts).with("\{")
437
- file_stub.should_receive(:print).with("\"add\":{\"doc\":{\"title\":\"Some Title\",\"id\":1}}")
450
+ file_stub.should_receive(:print).with do |str|
451
+ str.should include("add")
452
+ str.should include("\"title\":\"Some Title\"")
453
+ str.should include("\"id\":1")
454
+ end
438
455
  indexer.write_to_file(to_index)
439
456
  end
440
457
 
441
458
  it "only write fields which are not null" do
442
- file_stub.should_receive(:print).with("\"add\":{\"doc\":{\"title\":\"Some Title\",\"id\":1}}")
459
+ file_stub.stub(:print)
460
+ file_stub.should_not_receive(:print).with do |str|
461
+ str.include?("text")
462
+ end
443
463
  indexer.write_to_file(to_index.merge(:text => nil))
444
464
  end
445
465
 
446
466
  it "separates the first and the second line" do
447
467
  file_stub.should_receive(:puts).with("\{")
448
- file_stub.should_receive(:print).with("\"add\":{\"doc\":{\"title\":\"Some Title\",\"id\":1}}")
468
+ file_stub.should_receive(:print).with(/\"add\":{\"doc\"/)
449
469
  file_stub.should_receive(:print).with(%(,\n"add":{"doc":{"id":2}}))
450
470
  indexer.write_to_file(to_index)
451
471
  indexer.write_to_file({:id => 2})
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{supernova}
8
- s.version = "0.4.15"
8
+ s.version = "0.4.16"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Tobias Schwab"]
12
- s.date = %q{2011-07-28}
12
+ s.date = %q{2011-08-02}
13
13
  s.description = %q{Unified search scopes}
14
14
  s.email = %q{tobias.schwab@dynport.de}
15
15
  s.executables = ["start_solr"]
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: supernova
3
3
  version: !ruby/object:Gem::Version
4
- hash: 17
4
+ hash: 47
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
8
  - 4
9
- - 15
10
- version: 0.4.15
9
+ - 16
10
+ version: 0.4.16
11
11
  platform: ruby
12
12
  authors:
13
13
  - Tobias Schwab
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2011-07-28 00:00:00 Z
18
+ date: 2011-08-02 00:00:00 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  requirement: &id001 !ruby/object:Gem::Requirement