lucid_works 0.3.9 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +33 -3
- data/config/locales/en.yml +21 -12
- data/lib/lucid_works.rb +10 -0
- data/lib/lucid_works/associations.rb +14 -12
- data/lib/lucid_works/base.rb +13 -13
- data/lib/lucid_works/collection.rb +65 -5
- data/lib/lucid_works/collection/activity.rb +33 -0
- data/lib/lucid_works/collection/activity/history.rb +20 -0
- data/lib/lucid_works/collection/activity/status.rb +14 -0
- data/lib/lucid_works/collection/settings.rb +28 -8
- data/lib/lucid_works/crawler.rb +3 -3
- data/lib/lucid_works/datasource.rb +29 -3
- data/lib/lucid_works/datasource/job.rb +9 -0
- data/lib/lucid_works/datasource/status.rb +6 -11
- data/lib/lucid_works/patch_time.rb +13 -0
- data/lib/lucid_works/schema.rb +36 -8
- data/lib/lucid_works/utils.rb +22 -0
- data/lib/lucid_works/version.rb +1 -1
- data/lucid_works.gemspec +2 -0
- data/spec/lib/lucid_works/associations_spec.rb +12 -1
- data/spec/lib/lucid_works/base_spec.rb +26 -10
- data/spec/lib/lucid_works/collection/activity/history_spec.rb +33 -0
- data/spec/lib/lucid_works/collection/activity/status_spec.rb +20 -0
- data/spec/lib/lucid_works/collection/activity_spec.rb +88 -0
- data/spec/lib/lucid_works/collection/prime_activities_spec.rb +86 -0
- data/spec/lib/lucid_works/collection_spec.rb +140 -1
- data/spec/lib/lucid_works/datasource/history_spec.rb +11 -7
- data/spec/lib/lucid_works/datasource/status_spec.rb +64 -32
- data/spec/lib/lucid_works/datasource_spec.rb +48 -13
- data/spec/lib/lucid_works/schema_spec.rb +56 -4
- data/spec/lib/lucid_works/utils_spec.rb +62 -0
- data/spec/spec_helper.rb +17 -14
- metadata +41 -3
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe LucidWorks::Collection::Activity do
|
4
|
+
before :all do
|
5
|
+
@server = connect_to_live_server
|
6
|
+
@server.reset_collections!
|
7
|
+
@collection = @server.collections!.first
|
8
|
+
end
|
9
|
+
|
10
|
+
def activity_count
|
11
|
+
@collection.activities!.size
|
12
|
+
end
|
13
|
+
|
14
|
+
class LucidWorks::Collection::Activity
|
15
|
+
def history_size
|
16
|
+
self.histories!.size
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe "CRUD" do
|
21
|
+
describe ".create"do
|
22
|
+
context "with invalid parameters" do
|
23
|
+
before do
|
24
|
+
@activity_params = {:type => 'bogus', :start_time => 8600}
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should not create, and should add errors to model" do
|
28
|
+
a = nil
|
29
|
+
lambda {
|
30
|
+
a = LucidWorks::Collection::Activity.create(@activity_params.merge(:parent => @collection))
|
31
|
+
}.should_not change(self, :activity_count)
|
32
|
+
a.should_not be_persisted
|
33
|
+
a.errors[:type].should_not be_blank
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
context "with valid parameters" do
|
38
|
+
before do
|
39
|
+
@activity_params = {:type => 'optimize', :start_time => 8600}
|
40
|
+
end
|
41
|
+
it "should create a new activity" do
|
42
|
+
a = nil
|
43
|
+
lambda {
|
44
|
+
a = LucidWorks::Collection::Activity.create(@activity_params.merge(:parent => @collection))
|
45
|
+
}.should change(self, :activity_count).by(1)
|
46
|
+
a.should be_persisted
|
47
|
+
a.errors[:type].should be_blank
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe ".find" do
|
52
|
+
before do
|
53
|
+
@activity = LucidWorks::Collection::Activity.create(
|
54
|
+
:type => 'optimize',
|
55
|
+
:start_time => 8600,
|
56
|
+
:collection => @collection
|
57
|
+
)
|
58
|
+
@activity.should be_persisted
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should return a valid activity" do
|
62
|
+
a = LucidWorks::Collection::Activity.find(@activity.id, :parent => @collection)
|
63
|
+
a.should be_a(LucidWorks::Collection::Activity)
|
64
|
+
end
|
65
|
+
|
66
|
+
%w{ type period start_time }.each do |attr|
|
67
|
+
it "should have a value for #{attr}" do
|
68
|
+
@activity.send(attr.to_sym).should_not be_nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
describe "#start" do
|
74
|
+
before do
|
75
|
+
@activity = LucidWorks::Collection::Activity.first(:parent => @collection)
|
76
|
+
end
|
77
|
+
|
78
|
+
# can't check #running? b/c w/out a big index, operation is done faster than we can check
|
79
|
+
|
80
|
+
it "should create a new history" do
|
81
|
+
lambda {
|
82
|
+
@collection.activities!.first.start
|
83
|
+
}.should change(@activity, :history_size).by(1)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe "LucidWorks::Collection#prime_activities" do
|
4
|
+
class LucidWorks::Collection
|
5
|
+
def activities_count
|
6
|
+
self.activities!.count
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
before :all do
|
11
|
+
@server = connect_to_live_server
|
12
|
+
@server.reset_collections!
|
13
|
+
end
|
14
|
+
|
15
|
+
context "without pre_existing activities" do
|
16
|
+
describe "the virtual attribute convenience accessors" do
|
17
|
+
before :all do
|
18
|
+
@collection = @server.create_collection(:name => 'collection_with_no_activities_1', :parent => @server)
|
19
|
+
@collection.should be_a LucidWorks::Collection
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should produce activities of the appropriate type" do
|
23
|
+
@collection.optimize_activity.should be_a LucidWorks::Collection::Activity
|
24
|
+
@collection.optimize_activity.type.should == 'optimize'
|
25
|
+
@collection.spelling_activity.should be_a LucidWorks::Collection::Activity
|
26
|
+
@collection.spelling_activity.type.should == 'spelling'
|
27
|
+
@collection.click_activity.should be_a LucidWorks::Collection::Activity
|
28
|
+
@collection.click_activity.type.should == 'click'
|
29
|
+
@collection.autocomplete_activity.should be_a LucidWorks::Collection::Activity
|
30
|
+
@collection.autocomplete_activity.type.should == 'autocomplete'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "#prime_activities" do
|
36
|
+
context "there are already some activities" do
|
37
|
+
before :all do
|
38
|
+
@collection = @server.create_collection(:name => 'collection_with_no_activities_2', :parent => @server)
|
39
|
+
spelling_activity = @collection.create_activity(:type => 'spelling', :start_time => 3600, :active => true)
|
40
|
+
spelling_activity.should be_a LucidWorks::Collection::Activity
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should not create duplicates for existing types" do
|
44
|
+
lambda {
|
45
|
+
@collection.prime_activities
|
46
|
+
}.should change(@collection, :activities_count).by 3
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context "there are no activities before calling" do
|
51
|
+
before :all do
|
52
|
+
@collection = @server.create_collection(:name => 'collection_with_no_activities_3', :parent => @server)
|
53
|
+
@collection.should be_a LucidWorks::Collection
|
54
|
+
@collection.activities_count.should == 0
|
55
|
+
end
|
56
|
+
|
57
|
+
describe "and after calling" do
|
58
|
+
before :all do
|
59
|
+
@collection.prime_activities
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should have a virtual attribute for each activity type" do
|
63
|
+
@collection.activities!.count.should == 4
|
64
|
+
@collection.optimize_activity.should be_a LucidWorks::Collection::Activity
|
65
|
+
@collection.optimize_activity.type.should == 'optimize'
|
66
|
+
@collection.spelling_activity.should be_a LucidWorks::Collection::Activity
|
67
|
+
@collection.spelling_activity.type.should == 'spelling'
|
68
|
+
@collection.click_activity.should be_a LucidWorks::Collection::Activity
|
69
|
+
@collection.click_activity.type.should == 'click'
|
70
|
+
@collection.autocomplete_activity.should be_a LucidWorks::Collection::Activity
|
71
|
+
@collection.autocomplete_activity.type.should == 'autocomplete'
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should have start times at least an hour apart for each activity" do
|
75
|
+
start_times = @collection.activities!.map(&:start_time).map{|t| Time.iso8601 t}.sort
|
76
|
+
last_time = nil
|
77
|
+
start_times.each do |time|
|
78
|
+
(last_time = time) && next unless last_time
|
79
|
+
(time - last_time).should >= 3600
|
80
|
+
last_time = time
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -84,6 +84,37 @@ describe LucidWorks::Collection do
|
|
84
84
|
end
|
85
85
|
end
|
86
86
|
|
87
|
+
describe "#activities" do
|
88
|
+
before do
|
89
|
+
@collection = LucidWorks::Collection.first(:parent => @server)
|
90
|
+
end
|
91
|
+
|
92
|
+
context "for a collection with some activities" do
|
93
|
+
before do
|
94
|
+
activity = LucidWorks::Collection::Activity.create(
|
95
|
+
:type => 'optimize',
|
96
|
+
:start_time => 8600,
|
97
|
+
:collection => @collection
|
98
|
+
)
|
99
|
+
activity.should be_persisted
|
100
|
+
activity = LucidWorks::Collection::Activity.create(
|
101
|
+
:type => 'spelling',
|
102
|
+
:start_time => 8600,
|
103
|
+
:collection => @collection
|
104
|
+
)
|
105
|
+
activity.should be_persisted
|
106
|
+
end
|
107
|
+
|
108
|
+
it "should return a list of activities" do
|
109
|
+
activities = @collection.activities
|
110
|
+
activities.count.should == 2
|
111
|
+
activities.each do |activity|
|
112
|
+
activity.should be_a(LucidWorks::Collection::Activity)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
87
118
|
describe "#info" do
|
88
119
|
before :all do
|
89
120
|
@collection = LucidWorks::Collection.first(:parent => @server)
|
@@ -229,11 +260,119 @@ describe LucidWorks::Collection do
|
|
229
260
|
end
|
230
261
|
end
|
231
262
|
|
263
|
+
describe "#rsolr_connect" do
|
264
|
+
before :each do
|
265
|
+
@collection = LucidWorks::Collection.new(:parent => @server)
|
266
|
+
end
|
267
|
+
|
268
|
+
it "should call RSolr.connect" do
|
269
|
+
RSolr.should_receive(:connect).with(:url => 'fake_url')
|
270
|
+
@collection.rsolr_connect('fake_url')
|
271
|
+
end
|
272
|
+
|
273
|
+
it "should return an RSolr::Client" do
|
274
|
+
@collection.rsolr_connect('fake_url').should be_a(RSolr::Client)
|
275
|
+
end
|
276
|
+
|
277
|
+
it "should not call RSolr.connect again if it is called again" do
|
278
|
+
mock_rsolr_client = double("RSolr::Client")
|
279
|
+
RSolr.should_receive(:connect).once.and_return(mock_rsolr_client)
|
280
|
+
@collection.rsolr_connect('fake_uri')
|
281
|
+
@collection.rsolr_connect('fake_uri')
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
describe "search" do
|
286
|
+
context "using faked out Solr" do
|
287
|
+
before do
|
288
|
+
@collection = LucidWorks::Collection.new(:name => "collection_to_search", :parent => @server)
|
289
|
+
@mock_rsolr_client = double("RSolr::Client")
|
290
|
+
RSolr.stub(:connect) { @mock_rsolr_client }
|
291
|
+
end
|
292
|
+
|
293
|
+
it "should use default search params stored by #rsolr_connect" do
|
294
|
+
@mock_rsolr_client.should_receive(:paginate)
|
295
|
+
.with(1, 10, '/fake_access_path/solr/collection_to_search/select',
|
296
|
+
:params => {:q => 'baz', :foo => :bar})
|
297
|
+
.and_return({})
|
298
|
+
|
299
|
+
@collection.rsolr_connect('http://fake-solr-host.com/fake_access_path', :foo => :bar)
|
300
|
+
@collection.search :q => 'baz'
|
301
|
+
end
|
302
|
+
|
303
|
+
it "should respect pagination variables" do
|
304
|
+
@mock_rsolr_client.should_receive(:paginate)
|
305
|
+
.with(33, 44, '/fake_access_path/solr/collection_to_search/select',
|
306
|
+
:params => {:q => 'fake_query', :wt => :ruby})
|
307
|
+
.and_return({})
|
308
|
+
|
309
|
+
@collection.rsolr_connect('http://fake-solr-host.com/fake_access_path', :wt => :ruby)
|
310
|
+
@collection.search({:q => 'fake_query'}, :page => 33, :per_page => 44)
|
311
|
+
end
|
312
|
+
|
313
|
+
context "when requesting an XML response" do
|
314
|
+
before do
|
315
|
+
@default_search_params = {:wt => :xml}
|
316
|
+
@collection.rsolr_connect('http://fake-solr-host.com/fake_access_path', @default_search_params)
|
317
|
+
@good_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<response>FooBar</response>\n</xml>"
|
318
|
+
end
|
319
|
+
|
320
|
+
it "should call RSolr.get to perform the search" do
|
321
|
+
@mock_rsolr_client.should_receive(:paginate)
|
322
|
+
.with(1, 10, '/fake_access_path/solr/collection_to_search/select',
|
323
|
+
:params => {:q => 'fake_query', :wt => :xml})
|
324
|
+
.and_return(@good_xml)
|
325
|
+
|
326
|
+
@collection.search(:q => 'fake_query')
|
327
|
+
end
|
328
|
+
|
329
|
+
it "should parse the XML results" do
|
330
|
+
@mock_rsolr_client.stub(:paginate) { @good_xml }
|
331
|
+
results = @collection.search(:q => 'fake_query')
|
332
|
+
results.should be_a(Nokogiri::XML::Document)
|
333
|
+
results.root.children.first.text.should == "FooBar"
|
334
|
+
end
|
335
|
+
|
336
|
+
context "when the search returns invalid XML" do
|
337
|
+
before do
|
338
|
+
@mock_rsolr_client.stub(:get) { "this is not xml" }
|
339
|
+
end
|
340
|
+
|
341
|
+
it "should raise an error" do
|
342
|
+
lambda {
|
343
|
+
@collection.search
|
344
|
+
}.should raise_error
|
345
|
+
end
|
346
|
+
end
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
350
|
+
context "when using a live collection" do
|
351
|
+
before do
|
352
|
+
@collection = @server.collections.first
|
353
|
+
end
|
354
|
+
|
355
|
+
describe "when given a bad search" do
|
356
|
+
before do
|
357
|
+
@query_params = { :defType => 'lucene', :q => 'datefield:[FOO' }
|
358
|
+
end
|
359
|
+
|
360
|
+
it "should raise an error (until I decide what the right answer is)" do
|
361
|
+
collection = LucidWorks::Collection.new(:name => "collection1", :parent => @server)
|
362
|
+
collection.rsolr_connect(@server.instance_variable_get(:'@host'))
|
363
|
+
lambda {
|
364
|
+
collection.search @query_params
|
365
|
+
}.should raise_error
|
366
|
+
end
|
367
|
+
end
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
232
371
|
describe "#fields" do
|
233
372
|
before do
|
234
373
|
@collection = @server.collections.first
|
235
374
|
end
|
236
|
-
it "should return a valid LucidWorks::
|
375
|
+
it "should return a valid LucidWorks::Field" do
|
237
376
|
@collection.should respond_to(:field)
|
238
377
|
field = @collection.field('body')
|
239
378
|
field.should be_a(LucidWorks::Field)
|
@@ -20,21 +20,25 @@ describe LucidWorks::Datasource::History do
|
|
20
20
|
end
|
21
21
|
|
22
22
|
describe "#crawl_stopped" do
|
23
|
-
it "should
|
24
|
-
|
25
|
-
|
23
|
+
it "should convert to a Time with the correct value" do
|
24
|
+
history = LucidWorks::Datasource::History.new(:parent => @fake_server, :crawlStopped => "2011-03-25T21:20:52+0000")
|
25
|
+
history.crawl_stopped.should be_a(Time)
|
26
|
+
# Time.iso8601("2011-03-25T21:20:52+0000").to_yaml # => "--- 2011-03-25 21:20:52 Z\n"
|
27
|
+
history.crawl_stopped.should == YAML.load("--- 2011-03-25 21:20:52 Z")
|
26
28
|
end
|
27
29
|
end
|
28
30
|
|
29
31
|
describe "#crawl_started" do
|
30
|
-
it "should
|
31
|
-
|
32
|
-
|
32
|
+
it "should convert to a Time with the correct value" do
|
33
|
+
history = LucidWorks::Datasource::History.new(:parent => @fake_server, :crawlStarted => "2011-03-25T21:20:52+0000")
|
34
|
+
history.crawl_started.should be_a(Time)
|
35
|
+
# Time.iso8601("2011-03-25T21:20:52+0000").to_yaml # => "--- 2011-03-25 21:20:52 Z\n"
|
36
|
+
history.crawl_started.should == YAML.load("--- 2011-03-25 21:20:52 Z")
|
33
37
|
end
|
34
38
|
end
|
35
39
|
|
36
40
|
describe "#duration" do
|
37
|
-
it "should
|
41
|
+
it "should subtract crawlStarted form crawlStopped and return the difference in seconds" do
|
38
42
|
include ActiveSupport::Duration
|
39
43
|
|
40
44
|
history = LucidWorks::Datasource::History.new(:parent => @fake_server)
|
@@ -1,52 +1,84 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe LucidWorks::Datasource::Status do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
# TODO
|
8
|
-
end
|
4
|
+
before do
|
5
|
+
@server = connect_to_live_server
|
6
|
+
@status = LucidWorks::Datasource::Status.new(:parent => @server)
|
9
7
|
end
|
10
8
|
|
11
|
-
describe "
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
describe "state predicate methods" do
|
10
|
+
|
11
|
+
STATES = %w{idle stopped aborted exception finished stopping aborting running}
|
12
|
+
|
13
|
+
STATES.each do |actual_state|
|
14
|
+
describe "when state is #{actual_state}," do
|
15
|
+
before { @status.crawlState = actual_state.upcase }
|
16
|
+
|
17
|
+
(STATES - ['stopped']).each do |testing_state|
|
18
|
+
predicate = "#{testing_state}?"
|
19
|
+
describe predicate do
|
20
|
+
if actual_state == testing_state
|
21
|
+
it "should return true" do
|
22
|
+
@status.send(predicate).should be_true
|
23
|
+
end
|
24
|
+
else
|
25
|
+
it "should return false" do
|
26
|
+
@status.send(predicate).should be_false
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
15
32
|
end
|
16
33
|
end
|
17
34
|
|
18
35
|
describe "#stopped?" do
|
19
|
-
|
20
|
-
pending "write some tests"
|
21
|
-
# TODO
|
22
|
-
end
|
23
|
-
end
|
36
|
+
subject { @status }
|
24
37
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
38
|
+
truth_table = {
|
39
|
+
'IDLE' => true, 'STOPPED' => true, 'ABORTED' => true, 'EXCEPTION' => true,
|
40
|
+
'FINISHED' => true, 'STOPPING' => false, 'ABORTING' => false, 'RUNNING' => false
|
41
|
+
}
|
42
|
+
truth_table.each do |state, expected_result|
|
43
|
+
describe "when state is #{state}" do
|
44
|
+
before { @status.crawlState = state }
|
45
|
+
if expected_result
|
46
|
+
it { should be_stopped }
|
47
|
+
else
|
48
|
+
it { should_not be_stopped }
|
49
|
+
end
|
50
|
+
end
|
29
51
|
end
|
30
52
|
end
|
31
53
|
|
32
|
-
describe "#
|
33
|
-
|
34
|
-
pending "write some tests"
|
35
|
-
# TODO
|
36
|
-
end
|
37
|
-
end
|
54
|
+
describe "#post_processing?" do
|
55
|
+
subject { @status }
|
38
56
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
57
|
+
truth_table = {
|
58
|
+
'IDLE' => false, 'STOPPED' => false, 'ABORTED' => false, 'EXCEPTION' => false,
|
59
|
+
'FINISHED' => false, 'STOPPING' => true, 'ABORTING' => true, 'RUNNING' => false
|
60
|
+
}
|
61
|
+
truth_table.each do |state, expected_result|
|
62
|
+
describe "when state is #{state}" do
|
63
|
+
before { @status.crawlState = state }
|
64
|
+
if expected_result
|
65
|
+
it { should be_post_processing }
|
66
|
+
else
|
67
|
+
it { should_not be_post_processing }
|
68
|
+
end
|
69
|
+
end
|
43
70
|
end
|
44
71
|
end
|
45
72
|
|
46
|
-
describe "#
|
47
|
-
it "should
|
48
|
-
|
49
|
-
|
73
|
+
describe "#doc_count" do
|
74
|
+
it "should sum numUpdated, numNew and numUnchanged" do
|
75
|
+
status = LucidWorks::Datasource::Status.new(
|
76
|
+
:parent => @server,
|
77
|
+
:numUpdated => 11,
|
78
|
+
:numNew => 22,
|
79
|
+
:numUnchanged => 33
|
80
|
+
)
|
81
|
+
status.doc_count.should == 66
|
50
82
|
end
|
51
83
|
end
|
52
84
|
end
|