rummageable 0.6.2 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/rummageable/version.rb +1 -1
- data/lib/rummageable.rb +81 -60
- data/test/add_test.rb +107 -0
- data/test/amend_test.rb +15 -0
- data/test/commit_test.rb +14 -0
- data/test/delete_test.rb +52 -0
- metadata +45 -19
- data/lib/rummageable/fake.rb +0 -18
- data/lib/rummageable/implementation.rb +0 -64
- data/test/rummageable_test.rb +0 -227
data/lib/rummageable/version.rb
CHANGED
data/lib/rummageable.rb
CHANGED
@@ -1,76 +1,97 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require "plek"
|
1
|
+
require 'benchmark'
|
2
|
+
require 'json'
|
4
3
|
|
5
|
-
require
|
6
|
-
require
|
4
|
+
require 'multi_json'
|
5
|
+
require 'null_logger'
|
6
|
+
require 'rest_client'
|
7
7
|
|
8
|
-
module Rummageable
|
9
8
|
|
10
|
-
|
11
|
-
|
9
|
+
module Rummageable
|
10
|
+
class Index
|
11
|
+
def initialize(base_url, index_name, options = {})
|
12
|
+
@index_url = [base_url, index_name.sub(%r{^/}, '')].join('/')
|
13
|
+
@logger = options[:logger] || NullLogger.instance
|
14
|
+
@batch_size = options.fetch(:batch_size, 20)
|
15
|
+
@retry_delay = options.fetch(:retry_delay, 2)
|
16
|
+
@attempts = options.fetch(:attempts, 3)
|
17
|
+
end
|
12
18
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
19
|
+
def add(entry)
|
20
|
+
repeatedly do
|
21
|
+
make_request(:post, documents_url, MultiJson.encode([entry]))
|
22
|
+
end
|
23
|
+
end
|
17
24
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
25
|
+
def add_batch(entries)
|
26
|
+
entries.each_slice(@batch_size) do |batch|
|
27
|
+
repeatedly do
|
28
|
+
make_request(:post, documents_url, MultiJson.encode(batch))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
22
32
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
33
|
+
def amend(link, changes)
|
34
|
+
repeatedly do
|
35
|
+
make_request(:post, documents_url(link: link), changes)
|
36
|
+
end
|
37
|
+
end
|
27
38
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
39
|
+
def delete(link)
|
40
|
+
repeatedly do
|
41
|
+
make_request(:delete, documents_url(link: link))
|
42
|
+
end
|
43
|
+
end
|
32
44
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
end
|
45
|
+
def delete_all
|
46
|
+
repeatedly do
|
47
|
+
make_request(:delete, documents_url + '?delete_all=1')
|
48
|
+
end
|
49
|
+
end
|
39
50
|
|
40
|
-
|
41
|
-
|
42
|
-
|
51
|
+
def commit
|
52
|
+
repeatedly do
|
53
|
+
make_request(:post, [@index_url, 'commit'].join('/'), MultiJson.encode({}))
|
54
|
+
end
|
55
|
+
end
|
43
56
|
|
44
|
-
|
45
|
-
|
46
|
-
|
57
|
+
private
|
58
|
+
def repeatedly(&block)
|
59
|
+
@attempts.times do |i|
|
60
|
+
begin
|
61
|
+
return yield
|
62
|
+
rescue RestClient::RequestFailed, RestClient::RequestTimeout, RestClient::ServerBrokeConnection => e
|
63
|
+
@logger.warn e.message
|
64
|
+
raise if @attempts == i + 1
|
65
|
+
@logger.info 'Retrying...'
|
66
|
+
sleep(@retry_delay) if @retry_delay
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
47
70
|
|
48
|
-
|
49
|
-
|
50
|
-
|
71
|
+
def log_request(method, url, payload = nil)
|
72
|
+
@logger.info("Rummageable request: #{method.upcase} #{url}")
|
73
|
+
end
|
51
74
|
|
52
|
-
|
53
|
-
|
54
|
-
|
75
|
+
def log_response(method, url, call_time, response)
|
76
|
+
time = sprintf('%.03f', call_time)
|
77
|
+
result = JSON.parse(response).fetch('result', 'UNKNOWN')
|
78
|
+
@logger.info("Rummageable response: #{method.upcase} #{url} - time: #{time}s, result: #{result}")
|
79
|
+
end
|
55
80
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
%w[boost_phrases],
|
66
|
-
%w[additional_links title],
|
67
|
-
%w[additional_links link],
|
68
|
-
%w[additional_links link_order],
|
69
|
-
]
|
81
|
+
def make_request(method, *args)
|
82
|
+
response = nil
|
83
|
+
log_request(method, *args)
|
84
|
+
call_time = Benchmark.realtime do
|
85
|
+
response = RestClient.send(method, *args, content_type: :json, accept: :json)
|
86
|
+
end
|
87
|
+
log_response(method, args.first, call_time, response)
|
88
|
+
response
|
89
|
+
end
|
70
90
|
|
71
|
-
|
72
|
-
|
91
|
+
def documents_url(options = {})
|
92
|
+
parts = [@index_url, 'documents']
|
93
|
+
parts << CGI.escape(options[:link]) if options[:link]
|
94
|
+
parts.join('/')
|
95
|
+
end
|
73
96
|
end
|
74
|
-
|
75
|
-
extend self
|
76
97
|
end
|
data/test/add_test.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class AddTest < MiniTest::Unit::TestCase
|
4
|
+
def build_document(index)
|
5
|
+
{
|
6
|
+
'title' => "TITLE #{index}",
|
7
|
+
'link' => "/link#{index}"
|
8
|
+
}
|
9
|
+
end
|
10
|
+
|
11
|
+
def one_document
|
12
|
+
build_document(1)
|
13
|
+
end
|
14
|
+
|
15
|
+
def two_documents
|
16
|
+
[one_document] << build_document(2)
|
17
|
+
end
|
18
|
+
|
19
|
+
def stub_successful_request
|
20
|
+
stub_request(:post, documents_url).to_return(status(200))
|
21
|
+
end
|
22
|
+
|
23
|
+
def stub_one_failed_request
|
24
|
+
stub_request(:post, documents_url).
|
25
|
+
to_return(status(502)).times(1).then.to_return(status(200))
|
26
|
+
end
|
27
|
+
|
28
|
+
def stub_repeatedly_failing_requests(failures)
|
29
|
+
stub_request(:post, documents_url).to_return(status(502)).times(failures)
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_add_should_index_a_single_document_by_posting_it_as_json
|
33
|
+
stub_successful_request
|
34
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
35
|
+
index.add(one_document)
|
36
|
+
assert_requested :post, documents_url, times: 1 do |request|
|
37
|
+
request.body == json_for([one_document]) &&
|
38
|
+
request.headers['Content-Type'] == 'application/json' &&
|
39
|
+
request.headers['Accept'] == 'application/json'
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_add_batch_should_index_multiple_documents_in_one_request
|
44
|
+
stub_successful_request
|
45
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
46
|
+
index.add_batch(two_documents)
|
47
|
+
assert_requested :post, documents_url, body: json_for(two_documents)
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_add_batch_should_split_large_batches_into_multiple_requests
|
51
|
+
stub_successful_request
|
52
|
+
documents = (1..3).map { |i| build_document(i) }
|
53
|
+
index = Rummageable::Index.new(rummager_url, index_name, batch_size: 2)
|
54
|
+
index.add_batch(documents)
|
55
|
+
assert_requested :post, documents_url, body: json_for(documents[0, 2])
|
56
|
+
assert_requested :post, documents_url, body: json_for(documents[2, 1])
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_add_should_return_true_when_successful
|
60
|
+
stub_successful_request
|
61
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
62
|
+
assert index.add(one_document), 'should return true on success'
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_add_should_sleep_and_retry_on_bad_gateway_errors
|
66
|
+
stub_one_failed_request
|
67
|
+
Rummageable::Index.any_instance.expects(:sleep).with(1)
|
68
|
+
index = Rummageable::Index.new(rummager_url, index_name, retry_delay: 1)
|
69
|
+
assert index.add(one_document), 'should return true on success'
|
70
|
+
assert_requested :post, documents_url, times: 2
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_add_should_not_sleep_between_attempts_if_retry_delay_nil
|
74
|
+
stub_one_failed_request
|
75
|
+
Rummageable::Index.any_instance.expects(:sleep).never
|
76
|
+
index = Rummageable::Index.new(rummager_url, index_name, retry_delay: nil)
|
77
|
+
index.add(one_document)
|
78
|
+
assert_requested :post, documents_url, times: 2
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_add_should_propogate_exceptions_after_too_many_failed_attempts
|
82
|
+
failures = attempts = 2
|
83
|
+
stub_repeatedly_failing_requests(failures)
|
84
|
+
Rummageable::Index.any_instance.stubs(:sleep)
|
85
|
+
index = Rummageable::Index.new(rummager_url, index_name, attempts: attempts)
|
86
|
+
assert_raises RestClient::BadGateway do
|
87
|
+
index.add(one_document)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_add_should_log_attempts_to_post_to_rummager
|
92
|
+
stub_successful_request
|
93
|
+
logger = stub('logger', debug: true)
|
94
|
+
logger.expects(:info).twice
|
95
|
+
index = Rummageable::Index.new(rummager_url, index_name, logger: logger)
|
96
|
+
index.add(one_document)
|
97
|
+
end
|
98
|
+
|
99
|
+
def test_add_should_log_failures
|
100
|
+
stub_one_failed_request
|
101
|
+
Rummageable::Index.any_instance.stubs(:sleep)
|
102
|
+
logger = stub('logger', debug: true, info: true)
|
103
|
+
logger.expects(:warn).once
|
104
|
+
index = Rummageable::Index.new(rummager_url, index_name, logger: logger)
|
105
|
+
index.add(one_document)
|
106
|
+
end
|
107
|
+
end
|
data/test/amend_test.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class AmendTest < MiniTest::Unit::TestCase
|
4
|
+
def link
|
5
|
+
'/path'
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_should_post_amendments_to_a_document_by_its_link
|
9
|
+
new_document = { 'title' => 'Cheese', 'indexable_content' => 'Blah' }
|
10
|
+
stub_request(:post, link_url).with(body: new_document).to_return(status(200))
|
11
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
12
|
+
index.amend(link, { 'title' => 'Cheese', 'indexable_content' => 'Blah' })
|
13
|
+
assert_requested :post, link_url, body: new_document
|
14
|
+
end
|
15
|
+
end
|
data/test/commit_test.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class CommitTest < MiniTest::Unit::TestCase
|
4
|
+
def commit_url
|
5
|
+
[rummager_url, index_name, 'commit'].join('/')
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_commit_should_post_to_rummager
|
9
|
+
stub_request(:post, commit_url).to_return(status(200))
|
10
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
11
|
+
index.commit
|
12
|
+
assert_requested :post, commit_url, body: json_for({})
|
13
|
+
end
|
14
|
+
end
|
data/test/delete_test.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class DeleteTest < MiniTest::Unit::TestCase
|
4
|
+
def link
|
5
|
+
'http://example.com/foo'
|
6
|
+
end
|
7
|
+
|
8
|
+
def stub_successful_delete_request
|
9
|
+
stub_request(:delete, link_url).to_return(status(200))
|
10
|
+
end
|
11
|
+
|
12
|
+
def stub_one_failed_delete_request
|
13
|
+
stub_request(:delete, link_url).
|
14
|
+
to_return(status(502)).times(1).then.to_return(status(200))
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_should_delete_a_document_by_its_link
|
18
|
+
stub_successful_delete_request
|
19
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
20
|
+
index.delete(link)
|
21
|
+
assert_requested :delete, link_url do |request|
|
22
|
+
request.headers['Content-Type'] == 'application/json' &&
|
23
|
+
request.headers['Accept'] == 'application/json'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_should_be_able_to_delete_all_documents
|
28
|
+
stub_request(:delete, /#{documents_url}/).to_return(status(200))
|
29
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
30
|
+
index.delete_all
|
31
|
+
assert_requested :delete, documents_url, query: { delete_all: 1 } do |request|
|
32
|
+
request.headers['Content-Type'] == 'application/json' &&
|
33
|
+
request.headers['Accept'] == 'application/json'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_delete_should_handle_connection_errors
|
38
|
+
stub_one_failed_delete_request
|
39
|
+
Rummageable::Index.any_instance.expects(:sleep).once
|
40
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
41
|
+
index.delete(link)
|
42
|
+
assert_requested :delete, link_url, times: 2
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_delete_should_log_attempts_to_delete_documents_from_rummager
|
46
|
+
stub_successful_delete_request
|
47
|
+
logger = stub('logger')
|
48
|
+
logger.expects(:info).twice
|
49
|
+
index = Rummageable::Index.new(rummager_url, index_name, logger: logger)
|
50
|
+
index.delete(link)
|
51
|
+
end
|
52
|
+
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: rummageable
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.
|
5
|
+
version: 1.0.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- GovUK Beta Team
|
@@ -10,10 +10,10 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-07-15 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
-
name:
|
16
|
+
name: multi_json
|
17
17
|
requirement: &id001 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
@@ -24,7 +24,7 @@ dependencies:
|
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: *id001
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
|
-
name:
|
27
|
+
name: null_logger
|
28
28
|
requirement: &id002 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
@@ -35,29 +35,29 @@ dependencies:
|
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: *id002
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
38
|
+
name: rest-client
|
39
39
|
requirement: &id003 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ">="
|
43
43
|
- !ruby/object:Gem::Version
|
44
|
-
version: 0
|
44
|
+
version: "0"
|
45
45
|
type: :runtime
|
46
46
|
prerelease: false
|
47
47
|
version_requirements: *id003
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
|
-
name:
|
49
|
+
name: gem_publisher
|
50
50
|
requirement: &id004 !ruby/object:Gem::Requirement
|
51
51
|
none: false
|
52
52
|
requirements:
|
53
|
-
- - "
|
53
|
+
- - "="
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version:
|
55
|
+
version: 1.0.0
|
56
56
|
type: :development
|
57
57
|
prerelease: false
|
58
58
|
version_requirements: *id004
|
59
59
|
- !ruby/object:Gem::Dependency
|
60
|
-
name:
|
60
|
+
name: mocha
|
61
61
|
requirement: &id005 !ruby/object:Gem::Requirement
|
62
62
|
none: false
|
63
63
|
requirements:
|
@@ -68,16 +68,38 @@ dependencies:
|
|
68
68
|
prerelease: false
|
69
69
|
version_requirements: *id005
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
|
-
name:
|
71
|
+
name: rake
|
72
72
|
requirement: &id006 !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
|
-
- - "
|
75
|
+
- - ">="
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version:
|
77
|
+
version: "0"
|
78
78
|
type: :development
|
79
79
|
prerelease: false
|
80
80
|
version_requirements: *id006
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: webmock
|
83
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
89
|
+
type: :development
|
90
|
+
prerelease: false
|
91
|
+
version_requirements: *id007
|
92
|
+
- !ruby/object:Gem::Dependency
|
93
|
+
name: yajl-ruby
|
94
|
+
requirement: &id008 !ruby/object:Gem::Requirement
|
95
|
+
none: false
|
96
|
+
requirements:
|
97
|
+
- - "="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: 1.1.0
|
100
|
+
type: :development
|
101
|
+
prerelease: false
|
102
|
+
version_requirements: *id008
|
81
103
|
description: Mediator for apps that want their content to be in the search index
|
82
104
|
email:
|
83
105
|
executables: []
|
@@ -87,11 +109,12 @@ extensions: []
|
|
87
109
|
extra_rdoc_files: []
|
88
110
|
|
89
111
|
files:
|
90
|
-
- lib/rummageable/implementation.rb
|
91
|
-
- lib/rummageable/fake.rb
|
92
112
|
- lib/rummageable/version.rb
|
93
113
|
- lib/rummageable.rb
|
94
|
-
- test/
|
114
|
+
- test/commit_test.rb
|
115
|
+
- test/delete_test.rb
|
116
|
+
- test/add_test.rb
|
117
|
+
- test/amend_test.rb
|
95
118
|
homepage: https://github.com/alphagov/rummageable
|
96
119
|
licenses: []
|
97
120
|
|
@@ -105,7 +128,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
105
128
|
requirements:
|
106
129
|
- - ">="
|
107
130
|
- !ruby/object:Gem::Version
|
108
|
-
hash:
|
131
|
+
hash: 3448417689166380779
|
109
132
|
segments:
|
110
133
|
- 0
|
111
134
|
version: "0"
|
@@ -114,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
114
137
|
requirements:
|
115
138
|
- - ">="
|
116
139
|
- !ruby/object:Gem::Version
|
117
|
-
hash:
|
140
|
+
hash: 3448417689166380779
|
118
141
|
segments:
|
119
142
|
- 0
|
120
143
|
version: "0"
|
@@ -126,4 +149,7 @@ signing_key:
|
|
126
149
|
specification_version: 3
|
127
150
|
summary: Mediator for apps that want their content to be in the search index
|
128
151
|
test_files:
|
129
|
-
- test/
|
152
|
+
- test/commit_test.rb
|
153
|
+
- test/delete_test.rb
|
154
|
+
- test/add_test.rb
|
155
|
+
- test/amend_test.rb
|
data/lib/rummageable/fake.rb
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
module Rummageable
|
2
|
-
class Fake
|
3
|
-
def index(documents, index_name)
|
4
|
-
end
|
5
|
-
|
6
|
-
def delete(link, index_name)
|
7
|
-
end
|
8
|
-
|
9
|
-
def amend(link, amendments, index_name)
|
10
|
-
end
|
11
|
-
|
12
|
-
def commit(index_name)
|
13
|
-
end
|
14
|
-
|
15
|
-
def validate_structure(hash, parents=[])
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,64 +0,0 @@
|
|
1
|
-
module Rummageable
|
2
|
-
class Implementation
|
3
|
-
def index(documents, index_name)
|
4
|
-
documents = documents.is_a?(Hash) ? [documents] : documents
|
5
|
-
url = Rummageable.rummager_host + index_name + "/documents"
|
6
|
-
documents.each_slice(CHUNK_SIZE).each do |slice|
|
7
|
-
slice.each do |document|
|
8
|
-
validate_structure document
|
9
|
-
end
|
10
|
-
body = JSON.dump(slice)
|
11
|
-
RestClient.post url, body, content_type: :json, accept: :json
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def amend(link, amendments, index_name)
|
16
|
-
validate_structure amendments
|
17
|
-
RestClient.post url_for(link, index_name), amendments
|
18
|
-
end
|
19
|
-
|
20
|
-
def delete(link, index_name)
|
21
|
-
RestClient.delete url_for(link, index_name), content_type: :json, accept: :json
|
22
|
-
end
|
23
|
-
|
24
|
-
def delete_all(index_name)
|
25
|
-
url = Rummageable.rummager_host + index_name + "/documents?delete_all=1"
|
26
|
-
RestClient.delete url, content_type: :json, accept: :json
|
27
|
-
end
|
28
|
-
|
29
|
-
def commit(index_name)
|
30
|
-
url = Rummageable.rummager_host + index_name + "/commit"
|
31
|
-
RestClient.post url, {}
|
32
|
-
end
|
33
|
-
|
34
|
-
def validate_structure(hash, parents=[])
|
35
|
-
hash.each do |key, value|
|
36
|
-
full_key = parents + [key]
|
37
|
-
case value
|
38
|
-
when Array
|
39
|
-
value.each do |el|
|
40
|
-
validate_structure el, full_key
|
41
|
-
end
|
42
|
-
when Hash
|
43
|
-
validate_structure value, full_key
|
44
|
-
else
|
45
|
-
raise InvalidDocument unless VALID_KEYS.include?(full_key)
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
private
|
51
|
-
|
52
|
-
def url_components(index_name)
|
53
|
-
[Rummageable.rummager_host, index_name, "/documents/"]
|
54
|
-
end
|
55
|
-
|
56
|
-
def url_for(link, index_name)
|
57
|
-
(url_components(index_name) << CGI.escape(link)).join
|
58
|
-
end
|
59
|
-
|
60
|
-
def unescaped_url_for(link, index_name)
|
61
|
-
(url_components(index_name) << link).join
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
data/test/rummageable_test.rb
DELETED
@@ -1,227 +0,0 @@
|
|
1
|
-
require "minitest/autorun"
|
2
|
-
require "webmock/minitest"
|
3
|
-
require "rummageable"
|
4
|
-
ENV["RACK_ENV"] = "test"
|
5
|
-
|
6
|
-
class RummageableTest < MiniTest::Unit::TestCase
|
7
|
-
API = Plek.current.find("search")
|
8
|
-
|
9
|
-
def test_should_index_a_single_document_by_posting_it_as_json
|
10
|
-
document = {
|
11
|
-
"title" => "TITLE",
|
12
|
-
"description" => "DESCRIPTION",
|
13
|
-
"format" => "NAME OF FORMAT",
|
14
|
-
"section" => "NAME OF SECTION",
|
15
|
-
"subsection" => "NAME OF SUBSECTION",
|
16
|
-
"subsubsection" => "NAME OF SUBSUBSECTION",
|
17
|
-
"link" => "/link",
|
18
|
-
"indexable_content" => "TEXT",
|
19
|
-
"boost_phrases" => "BOOST",
|
20
|
-
"additional_links" => [
|
21
|
-
{"title" => "LINK1", "link" => "/link1"},
|
22
|
-
{"title" => "LINK2", "link" => "/link2"},
|
23
|
-
]
|
24
|
-
}
|
25
|
-
json = JSON.dump([document])
|
26
|
-
|
27
|
-
stub_request(:post, "#{API}/documents").
|
28
|
-
with(body: json).
|
29
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
30
|
-
|
31
|
-
Rummageable.index(document)
|
32
|
-
end
|
33
|
-
|
34
|
-
def test_should_index_multiple_documents_by_posting_them_as_json
|
35
|
-
documents = [
|
36
|
-
{"title" => "DOC1"},
|
37
|
-
{"title" => "DOC2"}
|
38
|
-
]
|
39
|
-
json = JSON.dump(documents)
|
40
|
-
|
41
|
-
stub_request(:post, "#{API}/documents").
|
42
|
-
with(body: json).
|
43
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
44
|
-
|
45
|
-
Rummageable.index(documents)
|
46
|
-
end
|
47
|
-
|
48
|
-
def test_should_send_documents_in_batches_of_20
|
49
|
-
documents = 21.times.map { |i| {"title" => "DOC#{i}"} }
|
50
|
-
|
51
|
-
stub_request(:post, "#{API}/documents").
|
52
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
53
|
-
|
54
|
-
Rummageable.index(documents)
|
55
|
-
assert_requested :post, "#{API}/documents", body: JSON.dump(documents[0, 20])
|
56
|
-
assert_requested :post, "#{API}/documents", body: JSON.dump(documents[20, 1])
|
57
|
-
end
|
58
|
-
|
59
|
-
def test_should_raise_an_exception_if_a_document_has_symbol_keys
|
60
|
-
document = {title: "TITLE"}
|
61
|
-
assert_raises Rummageable::InvalidDocument do
|
62
|
-
Rummageable.index(document)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def test_should_raise_an_exception_if_a_document_has_unrecognised_keys
|
67
|
-
document = {"foo" => "bar"}
|
68
|
-
assert_raises Rummageable::InvalidDocument do
|
69
|
-
Rummageable.index(document)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def test_should_raise_an_exception_if_a_document_has_unrecognised_nested_keys
|
74
|
-
document = {
|
75
|
-
"additional_links" => [
|
76
|
-
{"foo" => "bar"}
|
77
|
-
]
|
78
|
-
}
|
79
|
-
assert_raises Rummageable::InvalidDocument do
|
80
|
-
Rummageable.index(document)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
def test_allows_indexing_to_an_alternative_index
|
85
|
-
document = {
|
86
|
-
"title" => "TITLE",
|
87
|
-
"description" => "DESCRIPTION",
|
88
|
-
"format" => "NAME OF FORMAT",
|
89
|
-
"section" => "NAME OF SECTION",
|
90
|
-
"subsection" => "NAME OF SUBSECTION",
|
91
|
-
"link" => "/link",
|
92
|
-
"indexable_content" => "TEXT",
|
93
|
-
"boost_phrases" => "BOOST",
|
94
|
-
"additional_links" => [
|
95
|
-
{"title" => "LINK1", "link" => "/link1"},
|
96
|
-
{"title" => "LINK2", "link" => "/link2"},
|
97
|
-
]
|
98
|
-
}
|
99
|
-
json = JSON.dump([document])
|
100
|
-
|
101
|
-
stub_request(:post, "#{API}/alternative/documents").
|
102
|
-
with(body: json).
|
103
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
104
|
-
|
105
|
-
Rummageable.index(document, "/alternative")
|
106
|
-
end
|
107
|
-
|
108
|
-
def test_should_post_to_rummageable_host_determined_by_rummager_service_name
|
109
|
-
document = {"title" => "TITLE"}
|
110
|
-
stub_request(:post, "#{API}/documents")
|
111
|
-
stub_request(:post, "#{Plek.current.find("whitehall-search")}/documents")
|
112
|
-
with_rummager_service_name("whitehall-search") do
|
113
|
-
Rummageable.index(document)
|
114
|
-
end
|
115
|
-
assert_not_requested(:post, "#{API}/documents")
|
116
|
-
assert_requested(:post, "#{Plek.current.find("whitehall-search")}/documents")
|
117
|
-
end
|
118
|
-
|
119
|
-
def test_should_delete_a_document_by_its_link
|
120
|
-
link = "http://example.com/foo"
|
121
|
-
|
122
|
-
stub_request(:delete, "#{API}/documents/http:%2F%2Fexample.com%2Ffoo").
|
123
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
124
|
-
|
125
|
-
Rummageable.delete(link)
|
126
|
-
end
|
127
|
-
|
128
|
-
def test_should_allow_deletion_from_an_alternative_index
|
129
|
-
link = "http://example.com/foo"
|
130
|
-
|
131
|
-
stub_request(:delete, "#{API}/alternative/documents/http:%2F%2Fexample.com%2Ffoo").
|
132
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
133
|
-
|
134
|
-
Rummageable.delete(link, '/alternative')
|
135
|
-
end
|
136
|
-
|
137
|
-
def test_should_allow_delete_all
|
138
|
-
stub_request(:delete, "#{API}/documents?delete_all=1").
|
139
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
140
|
-
|
141
|
-
Rummageable.delete_all
|
142
|
-
end
|
143
|
-
|
144
|
-
def test_should_allow_delete_all_from_an_alternative_index
|
145
|
-
stub_request(:delete, "#{API}/alternative/documents?delete_all=1").
|
146
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
147
|
-
|
148
|
-
Rummageable.delete_all('/alternative')
|
149
|
-
end
|
150
|
-
|
151
|
-
def test_should_post_amendments
|
152
|
-
stub_request(:post, "#{API}/documents/%2Ffoobang").
|
153
|
-
with(body: {"title" => "Cheese", "indexable_content" => "Blah"}).
|
154
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
155
|
-
|
156
|
-
Rummageable.amend("/foobang", {"title" => "Cheese", "indexable_content" => "Blah"})
|
157
|
-
end
|
158
|
-
|
159
|
-
def test_should_reject_unknown_amendments
|
160
|
-
stub_request(:post, "#{API}/documents/%2Ffoobang").
|
161
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
162
|
-
|
163
|
-
assert_raises Rummageable::InvalidDocument do
|
164
|
-
Rummageable.amend("/foobang", {"title" => "Cheese", "face" => "Blah"})
|
165
|
-
end
|
166
|
-
|
167
|
-
assert_not_requested :any, "#{API}/documents/%2Ffoobang"
|
168
|
-
end
|
169
|
-
|
170
|
-
def test_should_fail_amendments_with_symbols
|
171
|
-
stub_request(:post, "#{API}/documents/%2Ffoobang").
|
172
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
173
|
-
|
174
|
-
assert_raises Rummageable::InvalidDocument do
|
175
|
-
Rummageable.amend("/foobang", {title: "Cheese"})
|
176
|
-
end
|
177
|
-
|
178
|
-
assert_not_requested :any, "#{API}/documents/%2Ffoobang"
|
179
|
-
end
|
180
|
-
|
181
|
-
def test_should_delete_to_rummageable_host_determined_by_rummager_service_name
|
182
|
-
link = "http://example.com/foo"
|
183
|
-
stub_request(:delete, "#{API}/documents/http:%2F%2Fexample.com%2Ffoo")
|
184
|
-
stub_request(:delete, "#{Plek.current.find("whitehall-search")}/documents/http:%2F%2Fexample.com%2Ffoo")
|
185
|
-
with_rummager_service_name("whitehall-search") do
|
186
|
-
Rummageable.delete(link)
|
187
|
-
end
|
188
|
-
assert_not_requested(:delete, "#{API}/#{API}/documents/http:%2F%2Fexample.com%2Ffoo")
|
189
|
-
assert_requested(:delete, "#{Plek.current.find("whitehall-search")}/documents/http:%2F%2Fexample.com%2Ffoo")
|
190
|
-
end
|
191
|
-
|
192
|
-
def test_should_defer_to_plek_for_the_location_of_the_rummager_host
|
193
|
-
assert_equal Plek.current.find("search"), Rummageable.rummager_host
|
194
|
-
end
|
195
|
-
|
196
|
-
def test_should_allow_the_rummager_host_to_be_set_manually_so_that_we_can_connect_to_rummager_running_at_arbitrary_locations
|
197
|
-
Rummageable.rummager_host = "http://example.com"
|
198
|
-
assert_equal "http://example.com", Rummageable.rummager_host
|
199
|
-
ensure
|
200
|
-
Rummageable.rummager_host = nil
|
201
|
-
end
|
202
|
-
|
203
|
-
def test_should_commit_to_rummmageable_host
|
204
|
-
stub_request(:post, "#{API}/commit").
|
205
|
-
to_return(status: 200, body: '{"result":"OK"}')
|
206
|
-
|
207
|
-
Rummageable.commit
|
208
|
-
end
|
209
|
-
|
210
|
-
def test_should_allow_committing_an_alternative_index
|
211
|
-
stub_request(:post, "#{API}/alternative/commit").
|
212
|
-
to_return(status: 200, body: '{"result":"OK"}')
|
213
|
-
|
214
|
-
Rummageable.commit '/alternative'
|
215
|
-
end
|
216
|
-
|
217
|
-
private
|
218
|
-
|
219
|
-
def with_rummager_service_name(service_name)
|
220
|
-
original_rummager_service_name = Rummageable.rummager_service_name
|
221
|
-
Rummageable.rummager_service_name = "whitehall-search"
|
222
|
-
yield
|
223
|
-
ensure
|
224
|
-
Rummageable.rummager_service_name = original_rummager_service_name
|
225
|
-
end
|
226
|
-
|
227
|
-
end
|