lucid_works 0.3.9 → 0.4.9
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +33 -3
- data/config/locales/en.yml +21 -12
- data/lib/lucid_works.rb +10 -0
- data/lib/lucid_works/associations.rb +14 -12
- data/lib/lucid_works/base.rb +13 -13
- data/lib/lucid_works/collection.rb +65 -5
- data/lib/lucid_works/collection/activity.rb +33 -0
- data/lib/lucid_works/collection/activity/history.rb +20 -0
- data/lib/lucid_works/collection/activity/status.rb +14 -0
- data/lib/lucid_works/collection/settings.rb +28 -8
- data/lib/lucid_works/crawler.rb +3 -3
- data/lib/lucid_works/datasource.rb +29 -3
- data/lib/lucid_works/datasource/job.rb +9 -0
- data/lib/lucid_works/datasource/status.rb +6 -11
- data/lib/lucid_works/patch_time.rb +13 -0
- data/lib/lucid_works/schema.rb +36 -8
- data/lib/lucid_works/utils.rb +22 -0
- data/lib/lucid_works/version.rb +1 -1
- data/lucid_works.gemspec +2 -0
- data/spec/lib/lucid_works/associations_spec.rb +12 -1
- data/spec/lib/lucid_works/base_spec.rb +26 -10
- data/spec/lib/lucid_works/collection/activity/history_spec.rb +33 -0
- data/spec/lib/lucid_works/collection/activity/status_spec.rb +20 -0
- data/spec/lib/lucid_works/collection/activity_spec.rb +88 -0
- data/spec/lib/lucid_works/collection/prime_activities_spec.rb +86 -0
- data/spec/lib/lucid_works/collection_spec.rb +140 -1
- data/spec/lib/lucid_works/datasource/history_spec.rb +11 -7
- data/spec/lib/lucid_works/datasource/status_spec.rb +64 -32
- data/spec/lib/lucid_works/datasource_spec.rb +48 -13
- data/spec/lib/lucid_works/schema_spec.rb +56 -4
- data/spec/lib/lucid_works/utils_spec.rb +62 -0
- data/spec/spec_helper.rb +17 -14
- metadata +41 -3
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe LucidWorks::Collection::Activity do
|
4
|
+
before :all do
|
5
|
+
@server = connect_to_live_server
|
6
|
+
@server.reset_collections!
|
7
|
+
@collection = @server.collections!.first
|
8
|
+
end
|
9
|
+
|
10
|
+
def activity_count
|
11
|
+
@collection.activities!.size
|
12
|
+
end
|
13
|
+
|
14
|
+
class LucidWorks::Collection::Activity
|
15
|
+
def history_size
|
16
|
+
self.histories!.size
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe "CRUD" do
|
21
|
+
describe ".create"do
|
22
|
+
context "with invalid parameters" do
|
23
|
+
before do
|
24
|
+
@activity_params = {:type => 'bogus', :start_time => 8600}
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should not create, and should add errors to model" do
|
28
|
+
a = nil
|
29
|
+
lambda {
|
30
|
+
a = LucidWorks::Collection::Activity.create(@activity_params.merge(:parent => @collection))
|
31
|
+
}.should_not change(self, :activity_count)
|
32
|
+
a.should_not be_persisted
|
33
|
+
a.errors[:type].should_not be_blank
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
context "with valid parameters" do
|
38
|
+
before do
|
39
|
+
@activity_params = {:type => 'optimize', :start_time => 8600}
|
40
|
+
end
|
41
|
+
it "should create a new activity" do
|
42
|
+
a = nil
|
43
|
+
lambda {
|
44
|
+
a = LucidWorks::Collection::Activity.create(@activity_params.merge(:parent => @collection))
|
45
|
+
}.should change(self, :activity_count).by(1)
|
46
|
+
a.should be_persisted
|
47
|
+
a.errors[:type].should be_blank
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe ".find" do
|
52
|
+
before do
|
53
|
+
@activity = LucidWorks::Collection::Activity.create(
|
54
|
+
:type => 'optimize',
|
55
|
+
:start_time => 8600,
|
56
|
+
:collection => @collection
|
57
|
+
)
|
58
|
+
@activity.should be_persisted
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should return a valid activity" do
|
62
|
+
a = LucidWorks::Collection::Activity.find(@activity.id, :parent => @collection)
|
63
|
+
a.should be_a(LucidWorks::Collection::Activity)
|
64
|
+
end
|
65
|
+
|
66
|
+
%w{ type period start_time }.each do |attr|
|
67
|
+
it "should have a value for #{attr}" do
|
68
|
+
@activity.send(attr.to_sym).should_not be_nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
describe "#start" do
|
74
|
+
before do
|
75
|
+
@activity = LucidWorks::Collection::Activity.first(:parent => @collection)
|
76
|
+
end
|
77
|
+
|
78
|
+
# can't check #running? b/c w/out a big index, operation is done faster than we can check
|
79
|
+
|
80
|
+
it "should create a new history" do
|
81
|
+
lambda {
|
82
|
+
@collection.activities!.first.start
|
83
|
+
}.should change(@activity, :history_size).by(1)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe "LucidWorks::Collection#prime_activities" do
|
4
|
+
class LucidWorks::Collection
|
5
|
+
def activities_count
|
6
|
+
self.activities!.count
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
before :all do
|
11
|
+
@server = connect_to_live_server
|
12
|
+
@server.reset_collections!
|
13
|
+
end
|
14
|
+
|
15
|
+
context "without pre_existing activities" do
|
16
|
+
describe "the virtual attribute convenience accessors" do
|
17
|
+
before :all do
|
18
|
+
@collection = @server.create_collection(:name => 'collection_with_no_activities_1', :parent => @server)
|
19
|
+
@collection.should be_a LucidWorks::Collection
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should produce activities of the appropriate type" do
|
23
|
+
@collection.optimize_activity.should be_a LucidWorks::Collection::Activity
|
24
|
+
@collection.optimize_activity.type.should == 'optimize'
|
25
|
+
@collection.spelling_activity.should be_a LucidWorks::Collection::Activity
|
26
|
+
@collection.spelling_activity.type.should == 'spelling'
|
27
|
+
@collection.click_activity.should be_a LucidWorks::Collection::Activity
|
28
|
+
@collection.click_activity.type.should == 'click'
|
29
|
+
@collection.autocomplete_activity.should be_a LucidWorks::Collection::Activity
|
30
|
+
@collection.autocomplete_activity.type.should == 'autocomplete'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "#prime_activities" do
|
36
|
+
context "there are already some activities" do
|
37
|
+
before :all do
|
38
|
+
@collection = @server.create_collection(:name => 'collection_with_no_activities_2', :parent => @server)
|
39
|
+
spelling_activity = @collection.create_activity(:type => 'spelling', :start_time => 3600, :active => true)
|
40
|
+
spelling_activity.should be_a LucidWorks::Collection::Activity
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should not create duplicates for existing types" do
|
44
|
+
lambda {
|
45
|
+
@collection.prime_activities
|
46
|
+
}.should change(@collection, :activities_count).by 3
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context "there are no activities before calling" do
|
51
|
+
before :all do
|
52
|
+
@collection = @server.create_collection(:name => 'collection_with_no_activities_3', :parent => @server)
|
53
|
+
@collection.should be_a LucidWorks::Collection
|
54
|
+
@collection.activities_count.should == 0
|
55
|
+
end
|
56
|
+
|
57
|
+
describe "and after calling" do
|
58
|
+
before :all do
|
59
|
+
@collection.prime_activities
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should have a virtual attribute for each activity type" do
|
63
|
+
@collection.activities!.count.should == 4
|
64
|
+
@collection.optimize_activity.should be_a LucidWorks::Collection::Activity
|
65
|
+
@collection.optimize_activity.type.should == 'optimize'
|
66
|
+
@collection.spelling_activity.should be_a LucidWorks::Collection::Activity
|
67
|
+
@collection.spelling_activity.type.should == 'spelling'
|
68
|
+
@collection.click_activity.should be_a LucidWorks::Collection::Activity
|
69
|
+
@collection.click_activity.type.should == 'click'
|
70
|
+
@collection.autocomplete_activity.should be_a LucidWorks::Collection::Activity
|
71
|
+
@collection.autocomplete_activity.type.should == 'autocomplete'
|
72
|
+
end
|
73
|
+
|
74
|
+
it "should have start times at least an hour apart for each activity" do
|
75
|
+
start_times = @collection.activities!.map(&:start_time).map{|t| Time.iso8601 t}.sort
|
76
|
+
last_time = nil
|
77
|
+
start_times.each do |time|
|
78
|
+
(last_time = time) && next unless last_time
|
79
|
+
(time - last_time).should >= 3600
|
80
|
+
last_time = time
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -84,6 +84,37 @@ describe LucidWorks::Collection do
|
|
84
84
|
end
|
85
85
|
end
|
86
86
|
|
87
|
+
describe "#activities" do
|
88
|
+
before do
|
89
|
+
@collection = LucidWorks::Collection.first(:parent => @server)
|
90
|
+
end
|
91
|
+
|
92
|
+
context "for a collection with some activities" do
|
93
|
+
before do
|
94
|
+
activity = LucidWorks::Collection::Activity.create(
|
95
|
+
:type => 'optimize',
|
96
|
+
:start_time => 8600,
|
97
|
+
:collection => @collection
|
98
|
+
)
|
99
|
+
activity.should be_persisted
|
100
|
+
activity = LucidWorks::Collection::Activity.create(
|
101
|
+
:type => 'spelling',
|
102
|
+
:start_time => 8600,
|
103
|
+
:collection => @collection
|
104
|
+
)
|
105
|
+
activity.should be_persisted
|
106
|
+
end
|
107
|
+
|
108
|
+
it "should return a list of activities" do
|
109
|
+
activities = @collection.activities
|
110
|
+
activities.count.should == 2
|
111
|
+
activities.each do |activity|
|
112
|
+
activity.should be_a(LucidWorks::Collection::Activity)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
87
118
|
describe "#info" do
|
88
119
|
before :all do
|
89
120
|
@collection = LucidWorks::Collection.first(:parent => @server)
|
@@ -229,11 +260,119 @@ describe LucidWorks::Collection do
|
|
229
260
|
end
|
230
261
|
end
|
231
262
|
|
263
|
+
describe "#rsolr_connect" do
|
264
|
+
before :each do
|
265
|
+
@collection = LucidWorks::Collection.new(:parent => @server)
|
266
|
+
end
|
267
|
+
|
268
|
+
it "should call RSolr.connect" do
|
269
|
+
RSolr.should_receive(:connect).with(:url => 'fake_url')
|
270
|
+
@collection.rsolr_connect('fake_url')
|
271
|
+
end
|
272
|
+
|
273
|
+
it "should return an RSolr::Client" do
|
274
|
+
@collection.rsolr_connect('fake_url').should be_a(RSolr::Client)
|
275
|
+
end
|
276
|
+
|
277
|
+
it "should not call RSolr.connect again if it is called again" do
|
278
|
+
mock_rsolr_client = double("RSolr::Client")
|
279
|
+
RSolr.should_receive(:connect).once.and_return(mock_rsolr_client)
|
280
|
+
@collection.rsolr_connect('fake_uri')
|
281
|
+
@collection.rsolr_connect('fake_uri')
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
describe "search" do
|
286
|
+
context "using faked out Solr" do
|
287
|
+
before do
|
288
|
+
@collection = LucidWorks::Collection.new(:name => "collection_to_search", :parent => @server)
|
289
|
+
@mock_rsolr_client = double("RSolr::Client")
|
290
|
+
RSolr.stub(:connect) { @mock_rsolr_client }
|
291
|
+
end
|
292
|
+
|
293
|
+
it "should use default search params stored by #rsolr_connect" do
|
294
|
+
@mock_rsolr_client.should_receive(:paginate)
|
295
|
+
.with(1, 10, '/fake_access_path/solr/collection_to_search/select',
|
296
|
+
:params => {:q => 'baz', :foo => :bar})
|
297
|
+
.and_return({})
|
298
|
+
|
299
|
+
@collection.rsolr_connect('http://fake-solr-host.com/fake_access_path', :foo => :bar)
|
300
|
+
@collection.search :q => 'baz'
|
301
|
+
end
|
302
|
+
|
303
|
+
it "should respect pagination variables" do
|
304
|
+
@mock_rsolr_client.should_receive(:paginate)
|
305
|
+
.with(33, 44, '/fake_access_path/solr/collection_to_search/select',
|
306
|
+
:params => {:q => 'fake_query', :wt => :ruby})
|
307
|
+
.and_return({})
|
308
|
+
|
309
|
+
@collection.rsolr_connect('http://fake-solr-host.com/fake_access_path', :wt => :ruby)
|
310
|
+
@collection.search({:q => 'fake_query'}, :page => 33, :per_page => 44)
|
311
|
+
end
|
312
|
+
|
313
|
+
context "when requesting an XML response" do
|
314
|
+
before do
|
315
|
+
@default_search_params = {:wt => :xml}
|
316
|
+
@collection.rsolr_connect('http://fake-solr-host.com/fake_access_path', @default_search_params)
|
317
|
+
@good_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<response>FooBar</response>\n</xml>"
|
318
|
+
end
|
319
|
+
|
320
|
+
it "should call RSolr.get to perform the search" do
|
321
|
+
@mock_rsolr_client.should_receive(:paginate)
|
322
|
+
.with(1, 10, '/fake_access_path/solr/collection_to_search/select',
|
323
|
+
:params => {:q => 'fake_query', :wt => :xml})
|
324
|
+
.and_return(@good_xml)
|
325
|
+
|
326
|
+
@collection.search(:q => 'fake_query')
|
327
|
+
end
|
328
|
+
|
329
|
+
it "should parse the XML results" do
|
330
|
+
@mock_rsolr_client.stub(:paginate) { @good_xml }
|
331
|
+
results = @collection.search(:q => 'fake_query')
|
332
|
+
results.should be_a(Nokogiri::XML::Document)
|
333
|
+
results.root.children.first.text.should == "FooBar"
|
334
|
+
end
|
335
|
+
|
336
|
+
context "when the search returns invalid XML" do
|
337
|
+
before do
|
338
|
+
@mock_rsolr_client.stub(:get) { "this is not xml" }
|
339
|
+
end
|
340
|
+
|
341
|
+
it "should raise an error" do
|
342
|
+
lambda {
|
343
|
+
@collection.search
|
344
|
+
}.should raise_error
|
345
|
+
end
|
346
|
+
end
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
350
|
+
context "when using a live collection" do
|
351
|
+
before do
|
352
|
+
@collection = @server.collections.first
|
353
|
+
end
|
354
|
+
|
355
|
+
describe "when given a bad search" do
|
356
|
+
before do
|
357
|
+
@query_params = { :defType => 'lucene', :q => 'datefield:[FOO' }
|
358
|
+
end
|
359
|
+
|
360
|
+
it "should raise an error (until I decide what the right answer is)" do
|
361
|
+
collection = LucidWorks::Collection.new(:name => "collection1", :parent => @server)
|
362
|
+
collection.rsolr_connect(@server.instance_variable_get(:'@host'))
|
363
|
+
lambda {
|
364
|
+
collection.search @query_params
|
365
|
+
}.should raise_error
|
366
|
+
end
|
367
|
+
end
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
232
371
|
describe "#fields" do
|
233
372
|
before do
|
234
373
|
@collection = @server.collections.first
|
235
374
|
end
|
236
|
-
it "should return a valid LucidWorks::
|
375
|
+
it "should return a valid LucidWorks::Field" do
|
237
376
|
@collection.should respond_to(:field)
|
238
377
|
field = @collection.field('body')
|
239
378
|
field.should be_a(LucidWorks::Field)
|
@@ -20,21 +20,25 @@ describe LucidWorks::Datasource::History do
|
|
20
20
|
end
|
21
21
|
|
22
22
|
describe "#crawl_stopped" do
|
23
|
-
it "should
|
24
|
-
|
25
|
-
|
23
|
+
it "should convert to a Time with the correct value" do
|
24
|
+
history = LucidWorks::Datasource::History.new(:parent => @fake_server, :crawlStopped => "2011-03-25T21:20:52+0000")
|
25
|
+
history.crawl_stopped.should be_a(Time)
|
26
|
+
# Time.iso8601("2011-03-25T21:20:52+0000").to_yaml # => "--- 2011-03-25 21:20:52 Z\n"
|
27
|
+
history.crawl_stopped.should == YAML.load("--- 2011-03-25 21:20:52 Z")
|
26
28
|
end
|
27
29
|
end
|
28
30
|
|
29
31
|
describe "#crawl_started" do
|
30
|
-
it "should
|
31
|
-
|
32
|
-
|
32
|
+
it "should convert to a Time with the correct value" do
|
33
|
+
history = LucidWorks::Datasource::History.new(:parent => @fake_server, :crawlStarted => "2011-03-25T21:20:52+0000")
|
34
|
+
history.crawl_started.should be_a(Time)
|
35
|
+
# Time.iso8601("2011-03-25T21:20:52+0000").to_yaml # => "--- 2011-03-25 21:20:52 Z\n"
|
36
|
+
history.crawl_started.should == YAML.load("--- 2011-03-25 21:20:52 Z")
|
33
37
|
end
|
34
38
|
end
|
35
39
|
|
36
40
|
describe "#duration" do
|
37
|
-
it "should
|
41
|
+
it "should subtract crawlStarted form crawlStopped and return the difference in seconds" do
|
38
42
|
include ActiveSupport::Duration
|
39
43
|
|
40
44
|
history = LucidWorks::Datasource::History.new(:parent => @fake_server)
|
@@ -1,52 +1,84 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe LucidWorks::Datasource::Status do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
# TODO
|
8
|
-
end
|
4
|
+
before do
|
5
|
+
@server = connect_to_live_server
|
6
|
+
@status = LucidWorks::Datasource::Status.new(:parent => @server)
|
9
7
|
end
|
10
8
|
|
11
|
-
describe "
|
12
|
-
|
13
|
-
|
14
|
-
|
9
|
+
describe "state predicate methods" do
|
10
|
+
|
11
|
+
STATES = %w{idle stopped aborted exception finished stopping aborting running}
|
12
|
+
|
13
|
+
STATES.each do |actual_state|
|
14
|
+
describe "when state is #{actual_state}," do
|
15
|
+
before { @status.crawlState = actual_state.upcase }
|
16
|
+
|
17
|
+
(STATES - ['stopped']).each do |testing_state|
|
18
|
+
predicate = "#{testing_state}?"
|
19
|
+
describe predicate do
|
20
|
+
if actual_state == testing_state
|
21
|
+
it "should return true" do
|
22
|
+
@status.send(predicate).should be_true
|
23
|
+
end
|
24
|
+
else
|
25
|
+
it "should return false" do
|
26
|
+
@status.send(predicate).should be_false
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
15
32
|
end
|
16
33
|
end
|
17
34
|
|
18
35
|
describe "#stopped?" do
|
19
|
-
|
20
|
-
pending "write some tests"
|
21
|
-
# TODO
|
22
|
-
end
|
23
|
-
end
|
36
|
+
subject { @status }
|
24
37
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
38
|
+
truth_table = {
|
39
|
+
'IDLE' => true, 'STOPPED' => true, 'ABORTED' => true, 'EXCEPTION' => true,
|
40
|
+
'FINISHED' => true, 'STOPPING' => false, 'ABORTING' => false, 'RUNNING' => false
|
41
|
+
}
|
42
|
+
truth_table.each do |state, expected_result|
|
43
|
+
describe "when state is #{state}" do
|
44
|
+
before { @status.crawlState = state }
|
45
|
+
if expected_result
|
46
|
+
it { should be_stopped }
|
47
|
+
else
|
48
|
+
it { should_not be_stopped }
|
49
|
+
end
|
50
|
+
end
|
29
51
|
end
|
30
52
|
end
|
31
53
|
|
32
|
-
describe "#
|
33
|
-
|
34
|
-
pending "write some tests"
|
35
|
-
# TODO
|
36
|
-
end
|
37
|
-
end
|
54
|
+
describe "#post_processing?" do
|
55
|
+
subject { @status }
|
38
56
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
57
|
+
truth_table = {
|
58
|
+
'IDLE' => false, 'STOPPED' => false, 'ABORTED' => false, 'EXCEPTION' => false,
|
59
|
+
'FINISHED' => false, 'STOPPING' => true, 'ABORTING' => true, 'RUNNING' => false
|
60
|
+
}
|
61
|
+
truth_table.each do |state, expected_result|
|
62
|
+
describe "when state is #{state}" do
|
63
|
+
before { @status.crawlState = state }
|
64
|
+
if expected_result
|
65
|
+
it { should be_post_processing }
|
66
|
+
else
|
67
|
+
it { should_not be_post_processing }
|
68
|
+
end
|
69
|
+
end
|
43
70
|
end
|
44
71
|
end
|
45
72
|
|
46
|
-
describe "#
|
47
|
-
it "should
|
48
|
-
|
49
|
-
|
73
|
+
describe "#doc_count" do
|
74
|
+
it "should sum numUpdated, numNew and numUnchanged" do
|
75
|
+
status = LucidWorks::Datasource::Status.new(
|
76
|
+
:parent => @server,
|
77
|
+
:numUpdated => 11,
|
78
|
+
:numNew => 22,
|
79
|
+
:numUnchanged => 33
|
80
|
+
)
|
81
|
+
status.doc_count.should == 66
|
50
82
|
end
|
51
83
|
end
|
52
84
|
end
|