rummageable 0.6.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/rummageable/version.rb +1 -1
- data/lib/rummageable.rb +81 -60
- data/test/add_test.rb +107 -0
- data/test/amend_test.rb +15 -0
- data/test/commit_test.rb +14 -0
- data/test/delete_test.rb +52 -0
- metadata +45 -19
- data/lib/rummageable/fake.rb +0 -18
- data/lib/rummageable/implementation.rb +0 -64
- data/test/rummageable_test.rb +0 -227
data/lib/rummageable/version.rb
CHANGED
data/lib/rummageable.rb
CHANGED
@@ -1,76 +1,97 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require "plek"
|
1
|
+
require 'benchmark'
|
2
|
+
require 'json'
|
4
3
|
|
5
|
-
require
|
6
|
-
require
|
4
|
+
require 'multi_json'
|
5
|
+
require 'null_logger'
|
6
|
+
require 'rest_client'
|
7
7
|
|
8
|
-
module Rummageable
|
9
8
|
|
10
|
-
|
11
|
-
|
9
|
+
module Rummageable
|
10
|
+
class Index
|
11
|
+
def initialize(base_url, index_name, options = {})
|
12
|
+
@index_url = [base_url, index_name.sub(%r{^/}, '')].join('/')
|
13
|
+
@logger = options[:logger] || NullLogger.instance
|
14
|
+
@batch_size = options.fetch(:batch_size, 20)
|
15
|
+
@retry_delay = options.fetch(:retry_delay, 2)
|
16
|
+
@attempts = options.fetch(:attempts, 3)
|
17
|
+
end
|
12
18
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
19
|
+
def add(entry)
|
20
|
+
repeatedly do
|
21
|
+
make_request(:post, documents_url, MultiJson.encode([entry]))
|
22
|
+
end
|
23
|
+
end
|
17
24
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
25
|
+
def add_batch(entries)
|
26
|
+
entries.each_slice(@batch_size) do |batch|
|
27
|
+
repeatedly do
|
28
|
+
make_request(:post, documents_url, MultiJson.encode(batch))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
22
32
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
33
|
+
def amend(link, changes)
|
34
|
+
repeatedly do
|
35
|
+
make_request(:post, documents_url(link: link), changes)
|
36
|
+
end
|
37
|
+
end
|
27
38
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
39
|
+
def delete(link)
|
40
|
+
repeatedly do
|
41
|
+
make_request(:delete, documents_url(link: link))
|
42
|
+
end
|
43
|
+
end
|
32
44
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
end
|
45
|
+
def delete_all
|
46
|
+
repeatedly do
|
47
|
+
make_request(:delete, documents_url + '?delete_all=1')
|
48
|
+
end
|
49
|
+
end
|
39
50
|
|
40
|
-
|
41
|
-
|
42
|
-
|
51
|
+
def commit
|
52
|
+
repeatedly do
|
53
|
+
make_request(:post, [@index_url, 'commit'].join('/'), MultiJson.encode({}))
|
54
|
+
end
|
55
|
+
end
|
43
56
|
|
44
|
-
|
45
|
-
|
46
|
-
|
57
|
+
private
|
58
|
+
def repeatedly(&block)
|
59
|
+
@attempts.times do |i|
|
60
|
+
begin
|
61
|
+
return yield
|
62
|
+
rescue RestClient::RequestFailed, RestClient::RequestTimeout, RestClient::ServerBrokeConnection => e
|
63
|
+
@logger.warn e.message
|
64
|
+
raise if @attempts == i + 1
|
65
|
+
@logger.info 'Retrying...'
|
66
|
+
sleep(@retry_delay) if @retry_delay
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
47
70
|
|
48
|
-
|
49
|
-
|
50
|
-
|
71
|
+
def log_request(method, url, payload = nil)
|
72
|
+
@logger.info("Rummageable request: #{method.upcase} #{url}")
|
73
|
+
end
|
51
74
|
|
52
|
-
|
53
|
-
|
54
|
-
|
75
|
+
def log_response(method, url, call_time, response)
|
76
|
+
time = sprintf('%.03f', call_time)
|
77
|
+
result = JSON.parse(response).fetch('result', 'UNKNOWN')
|
78
|
+
@logger.info("Rummageable response: #{method.upcase} #{url} - time: #{time}s, result: #{result}")
|
79
|
+
end
|
55
80
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
%w[boost_phrases],
|
66
|
-
%w[additional_links title],
|
67
|
-
%w[additional_links link],
|
68
|
-
%w[additional_links link_order],
|
69
|
-
]
|
81
|
+
def make_request(method, *args)
|
82
|
+
response = nil
|
83
|
+
log_request(method, *args)
|
84
|
+
call_time = Benchmark.realtime do
|
85
|
+
response = RestClient.send(method, *args, content_type: :json, accept: :json)
|
86
|
+
end
|
87
|
+
log_response(method, args.first, call_time, response)
|
88
|
+
response
|
89
|
+
end
|
70
90
|
|
71
|
-
|
72
|
-
|
91
|
+
def documents_url(options = {})
|
92
|
+
parts = [@index_url, 'documents']
|
93
|
+
parts << CGI.escape(options[:link]) if options[:link]
|
94
|
+
parts.join('/')
|
95
|
+
end
|
73
96
|
end
|
74
|
-
|
75
|
-
extend self
|
76
97
|
end
|
data/test/add_test.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class AddTest < MiniTest::Unit::TestCase
|
4
|
+
def build_document(index)
|
5
|
+
{
|
6
|
+
'title' => "TITLE #{index}",
|
7
|
+
'link' => "/link#{index}"
|
8
|
+
}
|
9
|
+
end
|
10
|
+
|
11
|
+
def one_document
|
12
|
+
build_document(1)
|
13
|
+
end
|
14
|
+
|
15
|
+
def two_documents
|
16
|
+
[one_document] << build_document(2)
|
17
|
+
end
|
18
|
+
|
19
|
+
def stub_successful_request
|
20
|
+
stub_request(:post, documents_url).to_return(status(200))
|
21
|
+
end
|
22
|
+
|
23
|
+
def stub_one_failed_request
|
24
|
+
stub_request(:post, documents_url).
|
25
|
+
to_return(status(502)).times(1).then.to_return(status(200))
|
26
|
+
end
|
27
|
+
|
28
|
+
def stub_repeatedly_failing_requests(failures)
|
29
|
+
stub_request(:post, documents_url).to_return(status(502)).times(failures)
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_add_should_index_a_single_document_by_posting_it_as_json
|
33
|
+
stub_successful_request
|
34
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
35
|
+
index.add(one_document)
|
36
|
+
assert_requested :post, documents_url, times: 1 do |request|
|
37
|
+
request.body == json_for([one_document]) &&
|
38
|
+
request.headers['Content-Type'] == 'application/json' &&
|
39
|
+
request.headers['Accept'] == 'application/json'
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_add_batch_should_index_multiple_documents_in_one_request
|
44
|
+
stub_successful_request
|
45
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
46
|
+
index.add_batch(two_documents)
|
47
|
+
assert_requested :post, documents_url, body: json_for(two_documents)
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_add_batch_should_split_large_batches_into_multiple_requests
|
51
|
+
stub_successful_request
|
52
|
+
documents = (1..3).map { |i| build_document(i) }
|
53
|
+
index = Rummageable::Index.new(rummager_url, index_name, batch_size: 2)
|
54
|
+
index.add_batch(documents)
|
55
|
+
assert_requested :post, documents_url, body: json_for(documents[0, 2])
|
56
|
+
assert_requested :post, documents_url, body: json_for(documents[2, 1])
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_add_should_return_true_when_successful
|
60
|
+
stub_successful_request
|
61
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
62
|
+
assert index.add(one_document), 'should return true on success'
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_add_should_sleep_and_retry_on_bad_gateway_errors
|
66
|
+
stub_one_failed_request
|
67
|
+
Rummageable::Index.any_instance.expects(:sleep).with(1)
|
68
|
+
index = Rummageable::Index.new(rummager_url, index_name, retry_delay: 1)
|
69
|
+
assert index.add(one_document), 'should return true on success'
|
70
|
+
assert_requested :post, documents_url, times: 2
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_add_should_not_sleep_between_attempts_if_retry_delay_nil
|
74
|
+
stub_one_failed_request
|
75
|
+
Rummageable::Index.any_instance.expects(:sleep).never
|
76
|
+
index = Rummageable::Index.new(rummager_url, index_name, retry_delay: nil)
|
77
|
+
index.add(one_document)
|
78
|
+
assert_requested :post, documents_url, times: 2
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_add_should_propogate_exceptions_after_too_many_failed_attempts
|
82
|
+
failures = attempts = 2
|
83
|
+
stub_repeatedly_failing_requests(failures)
|
84
|
+
Rummageable::Index.any_instance.stubs(:sleep)
|
85
|
+
index = Rummageable::Index.new(rummager_url, index_name, attempts: attempts)
|
86
|
+
assert_raises RestClient::BadGateway do
|
87
|
+
index.add(one_document)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_add_should_log_attempts_to_post_to_rummager
|
92
|
+
stub_successful_request
|
93
|
+
logger = stub('logger', debug: true)
|
94
|
+
logger.expects(:info).twice
|
95
|
+
index = Rummageable::Index.new(rummager_url, index_name, logger: logger)
|
96
|
+
index.add(one_document)
|
97
|
+
end
|
98
|
+
|
99
|
+
def test_add_should_log_failures
|
100
|
+
stub_one_failed_request
|
101
|
+
Rummageable::Index.any_instance.stubs(:sleep)
|
102
|
+
logger = stub('logger', debug: true, info: true)
|
103
|
+
logger.expects(:warn).once
|
104
|
+
index = Rummageable::Index.new(rummager_url, index_name, logger: logger)
|
105
|
+
index.add(one_document)
|
106
|
+
end
|
107
|
+
end
|
data/test/amend_test.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class AmendTest < MiniTest::Unit::TestCase
|
4
|
+
def link
|
5
|
+
'/path'
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_should_post_amendments_to_a_document_by_its_link
|
9
|
+
new_document = { 'title' => 'Cheese', 'indexable_content' => 'Blah' }
|
10
|
+
stub_request(:post, link_url).with(body: new_document).to_return(status(200))
|
11
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
12
|
+
index.amend(link, { 'title' => 'Cheese', 'indexable_content' => 'Blah' })
|
13
|
+
assert_requested :post, link_url, body: new_document
|
14
|
+
end
|
15
|
+
end
|
data/test/commit_test.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class CommitTest < MiniTest::Unit::TestCase
|
4
|
+
def commit_url
|
5
|
+
[rummager_url, index_name, 'commit'].join('/')
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_commit_should_post_to_rummager
|
9
|
+
stub_request(:post, commit_url).to_return(status(200))
|
10
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
11
|
+
index.commit
|
12
|
+
assert_requested :post, commit_url, body: json_for({})
|
13
|
+
end
|
14
|
+
end
|
data/test/delete_test.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class DeleteTest < MiniTest::Unit::TestCase
|
4
|
+
def link
|
5
|
+
'http://example.com/foo'
|
6
|
+
end
|
7
|
+
|
8
|
+
def stub_successful_delete_request
|
9
|
+
stub_request(:delete, link_url).to_return(status(200))
|
10
|
+
end
|
11
|
+
|
12
|
+
def stub_one_failed_delete_request
|
13
|
+
stub_request(:delete, link_url).
|
14
|
+
to_return(status(502)).times(1).then.to_return(status(200))
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_should_delete_a_document_by_its_link
|
18
|
+
stub_successful_delete_request
|
19
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
20
|
+
index.delete(link)
|
21
|
+
assert_requested :delete, link_url do |request|
|
22
|
+
request.headers['Content-Type'] == 'application/json' &&
|
23
|
+
request.headers['Accept'] == 'application/json'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_should_be_able_to_delete_all_documents
|
28
|
+
stub_request(:delete, /#{documents_url}/).to_return(status(200))
|
29
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
30
|
+
index.delete_all
|
31
|
+
assert_requested :delete, documents_url, query: { delete_all: 1 } do |request|
|
32
|
+
request.headers['Content-Type'] == 'application/json' &&
|
33
|
+
request.headers['Accept'] == 'application/json'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_delete_should_handle_connection_errors
|
38
|
+
stub_one_failed_delete_request
|
39
|
+
Rummageable::Index.any_instance.expects(:sleep).once
|
40
|
+
index = Rummageable::Index.new(rummager_url, index_name)
|
41
|
+
index.delete(link)
|
42
|
+
assert_requested :delete, link_url, times: 2
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_delete_should_log_attempts_to_delete_documents_from_rummager
|
46
|
+
stub_successful_delete_request
|
47
|
+
logger = stub('logger')
|
48
|
+
logger.expects(:info).twice
|
49
|
+
index = Rummageable::Index.new(rummager_url, index_name, logger: logger)
|
50
|
+
index.delete(link)
|
51
|
+
end
|
52
|
+
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: rummageable
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.
|
5
|
+
version: 1.0.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- GovUK Beta Team
|
@@ -10,10 +10,10 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-07-15 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
-
name:
|
16
|
+
name: multi_json
|
17
17
|
requirement: &id001 !ruby/object:Gem::Requirement
|
18
18
|
none: false
|
19
19
|
requirements:
|
@@ -24,7 +24,7 @@ dependencies:
|
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: *id001
|
26
26
|
- !ruby/object:Gem::Dependency
|
27
|
-
name:
|
27
|
+
name: null_logger
|
28
28
|
requirement: &id002 !ruby/object:Gem::Requirement
|
29
29
|
none: false
|
30
30
|
requirements:
|
@@ -35,29 +35,29 @@ dependencies:
|
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: *id002
|
37
37
|
- !ruby/object:Gem::Dependency
|
38
|
-
name:
|
38
|
+
name: rest-client
|
39
39
|
requirement: &id003 !ruby/object:Gem::Requirement
|
40
40
|
none: false
|
41
41
|
requirements:
|
42
42
|
- - ">="
|
43
43
|
- !ruby/object:Gem::Version
|
44
|
-
version: 0
|
44
|
+
version: "0"
|
45
45
|
type: :runtime
|
46
46
|
prerelease: false
|
47
47
|
version_requirements: *id003
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
|
-
name:
|
49
|
+
name: gem_publisher
|
50
50
|
requirement: &id004 !ruby/object:Gem::Requirement
|
51
51
|
none: false
|
52
52
|
requirements:
|
53
|
-
- - "
|
53
|
+
- - "="
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version:
|
55
|
+
version: 1.0.0
|
56
56
|
type: :development
|
57
57
|
prerelease: false
|
58
58
|
version_requirements: *id004
|
59
59
|
- !ruby/object:Gem::Dependency
|
60
|
-
name:
|
60
|
+
name: mocha
|
61
61
|
requirement: &id005 !ruby/object:Gem::Requirement
|
62
62
|
none: false
|
63
63
|
requirements:
|
@@ -68,16 +68,38 @@ dependencies:
|
|
68
68
|
prerelease: false
|
69
69
|
version_requirements: *id005
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
|
-
name:
|
71
|
+
name: rake
|
72
72
|
requirement: &id006 !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
|
-
- - "
|
75
|
+
- - ">="
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version:
|
77
|
+
version: "0"
|
78
78
|
type: :development
|
79
79
|
prerelease: false
|
80
80
|
version_requirements: *id006
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: webmock
|
83
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
89
|
+
type: :development
|
90
|
+
prerelease: false
|
91
|
+
version_requirements: *id007
|
92
|
+
- !ruby/object:Gem::Dependency
|
93
|
+
name: yajl-ruby
|
94
|
+
requirement: &id008 !ruby/object:Gem::Requirement
|
95
|
+
none: false
|
96
|
+
requirements:
|
97
|
+
- - "="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: 1.1.0
|
100
|
+
type: :development
|
101
|
+
prerelease: false
|
102
|
+
version_requirements: *id008
|
81
103
|
description: Mediator for apps that want their content to be in the search index
|
82
104
|
email:
|
83
105
|
executables: []
|
@@ -87,11 +109,12 @@ extensions: []
|
|
87
109
|
extra_rdoc_files: []
|
88
110
|
|
89
111
|
files:
|
90
|
-
- lib/rummageable/implementation.rb
|
91
|
-
- lib/rummageable/fake.rb
|
92
112
|
- lib/rummageable/version.rb
|
93
113
|
- lib/rummageable.rb
|
94
|
-
- test/
|
114
|
+
- test/commit_test.rb
|
115
|
+
- test/delete_test.rb
|
116
|
+
- test/add_test.rb
|
117
|
+
- test/amend_test.rb
|
95
118
|
homepage: https://github.com/alphagov/rummageable
|
96
119
|
licenses: []
|
97
120
|
|
@@ -105,7 +128,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
105
128
|
requirements:
|
106
129
|
- - ">="
|
107
130
|
- !ruby/object:Gem::Version
|
108
|
-
hash:
|
131
|
+
hash: 3448417689166380779
|
109
132
|
segments:
|
110
133
|
- 0
|
111
134
|
version: "0"
|
@@ -114,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
114
137
|
requirements:
|
115
138
|
- - ">="
|
116
139
|
- !ruby/object:Gem::Version
|
117
|
-
hash:
|
140
|
+
hash: 3448417689166380779
|
118
141
|
segments:
|
119
142
|
- 0
|
120
143
|
version: "0"
|
@@ -126,4 +149,7 @@ signing_key:
|
|
126
149
|
specification_version: 3
|
127
150
|
summary: Mediator for apps that want their content to be in the search index
|
128
151
|
test_files:
|
129
|
-
- test/
|
152
|
+
- test/commit_test.rb
|
153
|
+
- test/delete_test.rb
|
154
|
+
- test/add_test.rb
|
155
|
+
- test/amend_test.rb
|
data/lib/rummageable/fake.rb
DELETED
@@ -1,18 +0,0 @@
|
|
1
|
-
module Rummageable
|
2
|
-
class Fake
|
3
|
-
def index(documents, index_name)
|
4
|
-
end
|
5
|
-
|
6
|
-
def delete(link, index_name)
|
7
|
-
end
|
8
|
-
|
9
|
-
def amend(link, amendments, index_name)
|
10
|
-
end
|
11
|
-
|
12
|
-
def commit(index_name)
|
13
|
-
end
|
14
|
-
|
15
|
-
def validate_structure(hash, parents=[])
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,64 +0,0 @@
|
|
1
|
-
module Rummageable
|
2
|
-
class Implementation
|
3
|
-
def index(documents, index_name)
|
4
|
-
documents = documents.is_a?(Hash) ? [documents] : documents
|
5
|
-
url = Rummageable.rummager_host + index_name + "/documents"
|
6
|
-
documents.each_slice(CHUNK_SIZE).each do |slice|
|
7
|
-
slice.each do |document|
|
8
|
-
validate_structure document
|
9
|
-
end
|
10
|
-
body = JSON.dump(slice)
|
11
|
-
RestClient.post url, body, content_type: :json, accept: :json
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def amend(link, amendments, index_name)
|
16
|
-
validate_structure amendments
|
17
|
-
RestClient.post url_for(link, index_name), amendments
|
18
|
-
end
|
19
|
-
|
20
|
-
def delete(link, index_name)
|
21
|
-
RestClient.delete url_for(link, index_name), content_type: :json, accept: :json
|
22
|
-
end
|
23
|
-
|
24
|
-
def delete_all(index_name)
|
25
|
-
url = Rummageable.rummager_host + index_name + "/documents?delete_all=1"
|
26
|
-
RestClient.delete url, content_type: :json, accept: :json
|
27
|
-
end
|
28
|
-
|
29
|
-
def commit(index_name)
|
30
|
-
url = Rummageable.rummager_host + index_name + "/commit"
|
31
|
-
RestClient.post url, {}
|
32
|
-
end
|
33
|
-
|
34
|
-
def validate_structure(hash, parents=[])
|
35
|
-
hash.each do |key, value|
|
36
|
-
full_key = parents + [key]
|
37
|
-
case value
|
38
|
-
when Array
|
39
|
-
value.each do |el|
|
40
|
-
validate_structure el, full_key
|
41
|
-
end
|
42
|
-
when Hash
|
43
|
-
validate_structure value, full_key
|
44
|
-
else
|
45
|
-
raise InvalidDocument unless VALID_KEYS.include?(full_key)
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
private
|
51
|
-
|
52
|
-
def url_components(index_name)
|
53
|
-
[Rummageable.rummager_host, index_name, "/documents/"]
|
54
|
-
end
|
55
|
-
|
56
|
-
def url_for(link, index_name)
|
57
|
-
(url_components(index_name) << CGI.escape(link)).join
|
58
|
-
end
|
59
|
-
|
60
|
-
def unescaped_url_for(link, index_name)
|
61
|
-
(url_components(index_name) << link).join
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
data/test/rummageable_test.rb
DELETED
@@ -1,227 +0,0 @@
|
|
1
|
-
require "minitest/autorun"
|
2
|
-
require "webmock/minitest"
|
3
|
-
require "rummageable"
|
4
|
-
ENV["RACK_ENV"] = "test"
|
5
|
-
|
6
|
-
class RummageableTest < MiniTest::Unit::TestCase
|
7
|
-
API = Plek.current.find("search")
|
8
|
-
|
9
|
-
def test_should_index_a_single_document_by_posting_it_as_json
|
10
|
-
document = {
|
11
|
-
"title" => "TITLE",
|
12
|
-
"description" => "DESCRIPTION",
|
13
|
-
"format" => "NAME OF FORMAT",
|
14
|
-
"section" => "NAME OF SECTION",
|
15
|
-
"subsection" => "NAME OF SUBSECTION",
|
16
|
-
"subsubsection" => "NAME OF SUBSUBSECTION",
|
17
|
-
"link" => "/link",
|
18
|
-
"indexable_content" => "TEXT",
|
19
|
-
"boost_phrases" => "BOOST",
|
20
|
-
"additional_links" => [
|
21
|
-
{"title" => "LINK1", "link" => "/link1"},
|
22
|
-
{"title" => "LINK2", "link" => "/link2"},
|
23
|
-
]
|
24
|
-
}
|
25
|
-
json = JSON.dump([document])
|
26
|
-
|
27
|
-
stub_request(:post, "#{API}/documents").
|
28
|
-
with(body: json).
|
29
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
30
|
-
|
31
|
-
Rummageable.index(document)
|
32
|
-
end
|
33
|
-
|
34
|
-
def test_should_index_multiple_documents_by_posting_them_as_json
|
35
|
-
documents = [
|
36
|
-
{"title" => "DOC1"},
|
37
|
-
{"title" => "DOC2"}
|
38
|
-
]
|
39
|
-
json = JSON.dump(documents)
|
40
|
-
|
41
|
-
stub_request(:post, "#{API}/documents").
|
42
|
-
with(body: json).
|
43
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
44
|
-
|
45
|
-
Rummageable.index(documents)
|
46
|
-
end
|
47
|
-
|
48
|
-
def test_should_send_documents_in_batches_of_20
|
49
|
-
documents = 21.times.map { |i| {"title" => "DOC#{i}"} }
|
50
|
-
|
51
|
-
stub_request(:post, "#{API}/documents").
|
52
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
53
|
-
|
54
|
-
Rummageable.index(documents)
|
55
|
-
assert_requested :post, "#{API}/documents", body: JSON.dump(documents[0, 20])
|
56
|
-
assert_requested :post, "#{API}/documents", body: JSON.dump(documents[20, 1])
|
57
|
-
end
|
58
|
-
|
59
|
-
def test_should_raise_an_exception_if_a_document_has_symbol_keys
|
60
|
-
document = {title: "TITLE"}
|
61
|
-
assert_raises Rummageable::InvalidDocument do
|
62
|
-
Rummageable.index(document)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def test_should_raise_an_exception_if_a_document_has_unrecognised_keys
|
67
|
-
document = {"foo" => "bar"}
|
68
|
-
assert_raises Rummageable::InvalidDocument do
|
69
|
-
Rummageable.index(document)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def test_should_raise_an_exception_if_a_document_has_unrecognised_nested_keys
|
74
|
-
document = {
|
75
|
-
"additional_links" => [
|
76
|
-
{"foo" => "bar"}
|
77
|
-
]
|
78
|
-
}
|
79
|
-
assert_raises Rummageable::InvalidDocument do
|
80
|
-
Rummageable.index(document)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
def test_allows_indexing_to_an_alternative_index
|
85
|
-
document = {
|
86
|
-
"title" => "TITLE",
|
87
|
-
"description" => "DESCRIPTION",
|
88
|
-
"format" => "NAME OF FORMAT",
|
89
|
-
"section" => "NAME OF SECTION",
|
90
|
-
"subsection" => "NAME OF SUBSECTION",
|
91
|
-
"link" => "/link",
|
92
|
-
"indexable_content" => "TEXT",
|
93
|
-
"boost_phrases" => "BOOST",
|
94
|
-
"additional_links" => [
|
95
|
-
{"title" => "LINK1", "link" => "/link1"},
|
96
|
-
{"title" => "LINK2", "link" => "/link2"},
|
97
|
-
]
|
98
|
-
}
|
99
|
-
json = JSON.dump([document])
|
100
|
-
|
101
|
-
stub_request(:post, "#{API}/alternative/documents").
|
102
|
-
with(body: json).
|
103
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
104
|
-
|
105
|
-
Rummageable.index(document, "/alternative")
|
106
|
-
end
|
107
|
-
|
108
|
-
def test_should_post_to_rummageable_host_determined_by_rummager_service_name
|
109
|
-
document = {"title" => "TITLE"}
|
110
|
-
stub_request(:post, "#{API}/documents")
|
111
|
-
stub_request(:post, "#{Plek.current.find("whitehall-search")}/documents")
|
112
|
-
with_rummager_service_name("whitehall-search") do
|
113
|
-
Rummageable.index(document)
|
114
|
-
end
|
115
|
-
assert_not_requested(:post, "#{API}/documents")
|
116
|
-
assert_requested(:post, "#{Plek.current.find("whitehall-search")}/documents")
|
117
|
-
end
|
118
|
-
|
119
|
-
def test_should_delete_a_document_by_its_link
|
120
|
-
link = "http://example.com/foo"
|
121
|
-
|
122
|
-
stub_request(:delete, "#{API}/documents/http:%2F%2Fexample.com%2Ffoo").
|
123
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
124
|
-
|
125
|
-
Rummageable.delete(link)
|
126
|
-
end
|
127
|
-
|
128
|
-
def test_should_allow_deletion_from_an_alternative_index
|
129
|
-
link = "http://example.com/foo"
|
130
|
-
|
131
|
-
stub_request(:delete, "#{API}/alternative/documents/http:%2F%2Fexample.com%2Ffoo").
|
132
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
133
|
-
|
134
|
-
Rummageable.delete(link, '/alternative')
|
135
|
-
end
|
136
|
-
|
137
|
-
def test_should_allow_delete_all
|
138
|
-
stub_request(:delete, "#{API}/documents?delete_all=1").
|
139
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
140
|
-
|
141
|
-
Rummageable.delete_all
|
142
|
-
end
|
143
|
-
|
144
|
-
def test_should_allow_delete_all_from_an_alternative_index
|
145
|
-
stub_request(:delete, "#{API}/alternative/documents?delete_all=1").
|
146
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
147
|
-
|
148
|
-
Rummageable.delete_all('/alternative')
|
149
|
-
end
|
150
|
-
|
151
|
-
def test_should_post_amendments
|
152
|
-
stub_request(:post, "#{API}/documents/%2Ffoobang").
|
153
|
-
with(body: {"title" => "Cheese", "indexable_content" => "Blah"}).
|
154
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
155
|
-
|
156
|
-
Rummageable.amend("/foobang", {"title" => "Cheese", "indexable_content" => "Blah"})
|
157
|
-
end
|
158
|
-
|
159
|
-
def test_should_reject_unknown_amendments
|
160
|
-
stub_request(:post, "#{API}/documents/%2Ffoobang").
|
161
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
162
|
-
|
163
|
-
assert_raises Rummageable::InvalidDocument do
|
164
|
-
Rummageable.amend("/foobang", {"title" => "Cheese", "face" => "Blah"})
|
165
|
-
end
|
166
|
-
|
167
|
-
assert_not_requested :any, "#{API}/documents/%2Ffoobang"
|
168
|
-
end
|
169
|
-
|
170
|
-
def test_should_fail_amendments_with_symbols
|
171
|
-
stub_request(:post, "#{API}/documents/%2Ffoobang").
|
172
|
-
to_return(status: 200, body: '{"status":"OK"}')
|
173
|
-
|
174
|
-
assert_raises Rummageable::InvalidDocument do
|
175
|
-
Rummageable.amend("/foobang", {title: "Cheese"})
|
176
|
-
end
|
177
|
-
|
178
|
-
assert_not_requested :any, "#{API}/documents/%2Ffoobang"
|
179
|
-
end
|
180
|
-
|
181
|
-
def test_should_delete_to_rummageable_host_determined_by_rummager_service_name
|
182
|
-
link = "http://example.com/foo"
|
183
|
-
stub_request(:delete, "#{API}/documents/http:%2F%2Fexample.com%2Ffoo")
|
184
|
-
stub_request(:delete, "#{Plek.current.find("whitehall-search")}/documents/http:%2F%2Fexample.com%2Ffoo")
|
185
|
-
with_rummager_service_name("whitehall-search") do
|
186
|
-
Rummageable.delete(link)
|
187
|
-
end
|
188
|
-
assert_not_requested(:delete, "#{API}/#{API}/documents/http:%2F%2Fexample.com%2Ffoo")
|
189
|
-
assert_requested(:delete, "#{Plek.current.find("whitehall-search")}/documents/http:%2F%2Fexample.com%2Ffoo")
|
190
|
-
end
|
191
|
-
|
192
|
-
def test_should_defer_to_plek_for_the_location_of_the_rummager_host
|
193
|
-
assert_equal Plek.current.find("search"), Rummageable.rummager_host
|
194
|
-
end
|
195
|
-
|
196
|
-
def test_should_allow_the_rummager_host_to_be_set_manually_so_that_we_can_connect_to_rummager_running_at_arbitrary_locations
|
197
|
-
Rummageable.rummager_host = "http://example.com"
|
198
|
-
assert_equal "http://example.com", Rummageable.rummager_host
|
199
|
-
ensure
|
200
|
-
Rummageable.rummager_host = nil
|
201
|
-
end
|
202
|
-
|
203
|
-
def test_should_commit_to_rummmageable_host
|
204
|
-
stub_request(:post, "#{API}/commit").
|
205
|
-
to_return(status: 200, body: '{"result":"OK"}')
|
206
|
-
|
207
|
-
Rummageable.commit
|
208
|
-
end
|
209
|
-
|
210
|
-
def test_should_allow_committing_an_alternative_index
|
211
|
-
stub_request(:post, "#{API}/alternative/commit").
|
212
|
-
to_return(status: 200, body: '{"result":"OK"}')
|
213
|
-
|
214
|
-
Rummageable.commit '/alternative'
|
215
|
-
end
|
216
|
-
|
217
|
-
private
|
218
|
-
|
219
|
-
def with_rummager_service_name(service_name)
|
220
|
-
original_rummager_service_name = Rummageable.rummager_service_name
|
221
|
-
Rummageable.rummager_service_name = "whitehall-search"
|
222
|
-
yield
|
223
|
-
ensure
|
224
|
-
Rummageable.rummager_service_name = original_rummager_service_name
|
225
|
-
end
|
226
|
-
|
227
|
-
end
|