toccatore 0.3.9 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile.lock +31 -18
  3. data/README.md +1 -1
  4. data/lib/toccatore.rb +2 -0
  5. data/lib/toccatore/cli.rb +12 -0
  6. data/lib/toccatore/queue.rb +50 -0
  7. data/lib/toccatore/usage_update.rb +170 -0
  8. data/lib/toccatore/version.rb +1 -1
  9. data/spec/cli_spec.rb +34 -1
  10. data/spec/fixtures/event_data_resp_1 +2 -0
  11. data/spec/fixtures/event_data_resp_2 +4 -0
  12. data/spec/fixtures/usage_event.json +1 -0
  13. data/spec/fixtures/usage_event_fail.json +17 -0
  14. data/spec/fixtures/usage_events.json +63 -0
  15. data/spec/fixtures/usage_update.json +101 -0
  16. data/spec/fixtures/usage_update_1.json +36738 -0
  17. data/spec/fixtures/usage_update_2.json +171 -0
  18. data/spec/fixtures/usage_update_3.json +176 -0
  19. data/spec/fixtures/usage_update_4.json +101 -0
  20. data/spec/fixtures/usage_update_nil.json +6 -0
  21. data/spec/fixtures/vcr_cassettes/Toccatore_CLI/usage_update/no_reports_in_the_queue/should_succeed_with_no_works.yml +150 -0
  22. data/spec/fixtures/vcr_cassettes/Toccatore_CLI/usage_update/should_fail.yml +150 -0
  23. data/spec/fixtures/vcr_cassettes/Toccatore_CLI/usage_update/should_succeed_with_no_works.yml +150 -0
  24. data/spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/get_data/when_there_are_messages/should_return_the_data_for_one_message.yml +52 -0
  25. data/spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/get_data/when_there_is_ONE_message/should_return_the_data_for_one_message.yml +52 -0
  26. data/spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/push_data/should_fail_if_format_of_the_event_is_wrong.yml +199 -0
  27. data/spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/push_data/should_work_with_DataCite_Event_Data.yml +199 -0
  28. data/spec/queque_spec.rb +61 -0
  29. data/spec/spec_helper.rb +14 -0
  30. data/spec/usage_update_spec.rb +156 -0
  31. data/toccatore.gemspec +3 -1
  32. metadata +43 -7
@@ -0,0 +1,61 @@
1
+ # require 'spec_helper'
2
+
3
+ # describe Toccatore::UsageUpdate, vcr: true do
4
+
5
+ # # let(:queue_url){"https://sqs.#{ENV['AWS_REGION']}.amazonaws.com/404017989009/test_usage"}
6
+ # # let(:queue_name){:test_usage}
7
+
8
+ # # before do
9
+ # # # subject.new({:stub_responses => true})
10
+ # # body = File.read(fixture_path + 'usage_event.json')
11
+ # # result = OpenStruct.new(body: JSON.parse(body) )
12
+ # # subject.loopy.stub_responses(:create_queue, queue_url: queue_url)
13
+ # # subject.loopy.send_message({queue_url: queue_url, message_body:result.to_json})
14
+ # # end
15
+
16
+ # context "get_total" do
17
+ # it "when there are messages" do
18
+ # puts subject.queue_url
19
+ # expect(subject.get_total()).to eq(1)
20
+ # end
21
+
22
+ # it "when the queue is empty" do
23
+ # expect(subject.get_total()).to eq(0)
24
+ # end
25
+ # end
26
+
27
+ # context "queue_url" do
28
+ # it "should return always correct queue url" do
29
+ # puts subject.queue_url
30
+ # response = subject.queue_url
31
+ # expect(response).to eq("https://sqs.#{ENV['AWS_REGION']}.amazonaws.com/404017989009/test_usage")
32
+ # end
33
+
34
+ # it "should fail if the queue doesn exist" do
35
+
36
+ # end
37
+ # end
38
+
39
+ # context "get_message" do
40
+ # it "should return one message when there are multiple messages" do
41
+ # expect(subject.get_message.size).to eq(1)
42
+ # end
43
+
44
+ # it "should return no meessage when the queue is empty" do
45
+ # expect(subject.get_message.size).to eq(0)
46
+ # end
47
+ # end
48
+
49
+ # context "delete_message" do
50
+ # it "should delete a message that exist" do
51
+ # msg = subject.get_message
52
+ # response = subject.delete_message msg
53
+ # expect(response.successful?).to eq(true)
54
+ # end
55
+
56
+ # it "should return an error if a message doesnot exist" do
57
+
58
+ # end
59
+ # end
60
+
61
+ # end
data/spec/spec_helper.rb CHANGED
@@ -11,6 +11,8 @@ require 'rack/test'
11
11
  require 'webmock/rspec'
12
12
  require 'nokogiri'
13
13
  require 'vcr'
14
+ require 'aws-sdk-sqs'
15
+
14
16
 
15
17
  RSpec.configure do |config|
16
18
  config.order = :random
@@ -27,6 +29,18 @@ RSpec.configure do |config|
27
29
  config.before do
28
30
  ARGV.replace []
29
31
  end
32
+
33
+ # config.expect_with :rspec do |expectations|
34
+ # expectations.include_chain_clauses_in_custom_matcher_descriptions = true
35
+ # end
36
+
37
+ # config.mock_with :rspec do |mocks|
38
+ # mocks.verify_partial_doubles = true
39
+ # end
40
+
41
+ # config.shared_context_metadata_behavior = :apply_to_host_groups
42
+
43
+ # Aws.config.update(stub_responses: true)
30
44
  end
31
45
 
32
46
  def fixture_path
@@ -0,0 +1,156 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toccatore::UsageUpdate, vcr: true do
4
+
5
+ let(:queue_url){"https://sqs.#{ENV['AWS_REGION']}.amazonaws.com/404017989009/test_usage"}
6
+ let(:queue_name){:test_usage}
7
+ let!(:sqs) {Aws::SQS::Client.new(region: ENV['AWS_REGION'].to_s, stub_responses: true)}
8
+ let!(:body) {File.read(fixture_path + 'usage_event.json')}
9
+ let!(:message){[body: body]}
10
+
11
+ let(:dummy_message){sqs.receive_message(queue_url: queue_url, max_number_of_messages: 1, wait_time_seconds: 1)}
12
+
13
+ # context "queue_jobs" do
14
+ # it "should report if there are no works returned by the Usage Queue" do
15
+ # puts subject.queue_url
16
+ # response = subject.queue_jobs
17
+ # expect(response).to eq(0)
18
+ # end
19
+
20
+ # it "should report if there are works returned by the Usage Queue" do
21
+ # response = subject.queue_jobs
22
+ # expect(response).to eq(3)
23
+ # end
24
+ # end
25
+
26
+ # context "format_event" do
27
+ # it "should report if there are no works returned by the Usage Queue" do
28
+ # body = File.read(fixture_path + 'usage_update_nil.json')
29
+ # result = OpenStruct.new(body: JSON.parse(body) )
30
+ # expect(subject.format_event(result)).to eq([])
31
+ # end
32
+
33
+ # it "should report if there are works returned by the Usage Queue" do
34
+ # body = File.read(fixture_path + 'usage_update.json')
35
+ # result = OpenStruct.new(body: JSON.parse(body) )
36
+ # expect(subject.format_event(result).length).to eq(40)
37
+ # end
38
+ # end
39
+
40
+ describe "get_data" do
41
+ context "when there are messages" do
42
+ it "should return the data for one message" do
43
+ sqs.stub_responses(:receive_message, messages: message)
44
+ sqs.stub_responses(:receive_message, messages: message)
45
+ sqs.stub_responses(:receive_message, messages: message)
46
+ sqs.stub_responses(:receive_message, messages: message)
47
+ response = sqs.receive_message({queue_url: queue_url})
48
+ response = subject.get_data(response)
49
+ expect(response.body["data"]["report"]["report-header"]["report-name"]).to eq("Dataset Master Report")
50
+ end
51
+ end
52
+
53
+ context "when there is ONE message" do
54
+ it "should return the data for one message" do
55
+ sqs.stub_responses(:receive_message, messages: message)
56
+ response = sqs.receive_message({queue_url: queue_url})
57
+ response = subject.get_data(response)
58
+ expect(response.body["data"]["report"]["report-header"]["report-name"]).to eq("Dataset Master Report")
59
+ end
60
+ end
61
+
62
+ context "when there are NOT messages" do
63
+ it "should return empty" do
64
+ sqs.stub_responses(:receive_message, messages: [])
65
+ response = sqs.receive_message({queue_url: queue_url})
66
+ response = subject.get_data(response)
67
+ expect(response.body["errors"]).to eq("Queue is empty")
68
+ end
69
+ end
70
+ end
71
+
72
+ describe "parse_data" do
73
+ context "when the usage event was NOT found" do
74
+ it "should return errors" do
75
+ body = File.read(fixture_path + 'usage_update_nil.json')
76
+ result = OpenStruct.new(body: JSON.parse(body) )
77
+ expect(subject.parse_data(result)).to eq([{"status"=>"404", "title"=>"The resource you are looking for doesn't exist."}])
78
+ end
79
+ end
80
+
81
+ context "when the usage report was NOT found" do
82
+ it "should return errors" do
83
+ body = File.read(fixture_path + 'usage_update_nil.json')
84
+ result = OpenStruct.new(body: JSON.parse(body) )
85
+ expect(subject.parse_data(result)).to eq([{"status"=>"404", "title"=>"The resource you are looking for doesn't exist."}])
86
+ end
87
+ end
88
+
89
+ context "when the report was found" do
90
+ it "should parsed it correctly" do
91
+ body = File.read(fixture_path + 'usage_update.json')
92
+ result = OpenStruct.new(body: JSON.parse(body) )
93
+ response = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
94
+ expect(response.length).to eq(2)
95
+ expect(response.last.except("id")).to eq("subj"=>{"pid"=>"https://metrics.test.datacite.org/reports/2018-3-Dash", "issued"=>"2128-04-09"},"total"=>3,"message-action" => "add", "subj-id"=>"https://metrics.test.datacite.org/reports/2018-3-Dash", "obj-id"=>"https://doi.org/10.7291/d1q94r", "relation-type-id"=>"unique-dataset-investigations-regular", "source-id"=>"datacite-usage", "occurred-at"=>"2128-04-09", "license" => "https://creativecommons.org/publicdomain/zero/1.0/", "source-token" => "28276d12-b320-41ba-9272-bb0adc3466ff")
96
+ end
97
+
98
+ it "should parsed it correctly when it has five metrics and two DOIs" do
99
+ body = File.read(fixture_path + 'usage_update_3.json')
100
+ result = OpenStruct.new(body: JSON.parse(body) )
101
+ response = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
102
+ expect(response.length).to eq(5)
103
+ expect(response.last.except("id")).to eq("message-action"=>"add", "subj-id"=>"https://metrics.test.datacite.org/reports/2018-3-Dash", "subj"=>{"pid"=>"https://metrics.test.datacite.org/reports/2018-3-Dash", "issued"=>"2128-04-09"}, "total"=>208, "obj-id"=>"https://doi.org/10.6071/z7wc73", "relation-type-id"=>"Unique-Dataset-Requests-Machine", "source-id"=>"datacite-usage", "source-token"=>"28276d12-b320-41ba-9272-bb0adc3466ff", "occurred-at"=>"2128-04-09", "license"=>"https://creativecommons.org/publicdomain/zero/1.0/")
104
+ end
105
+
106
+ it "should parsed it correctly when it has two metrics per DOI " do
107
+ body = File.read(fixture_path + 'usage_update_2.json')
108
+ result = OpenStruct.new(body: JSON.parse(body) )
109
+ response = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
110
+ expect(response.length).to eq(4)
111
+ expect(response.last.except("id")).to eq("message-action"=>"add", "subj-id"=>"https://metrics.test.datacite.org/reports/2018-3-Dash", "subj"=>{"pid"=>"https://metrics.test.datacite.org/reports/2018-3-Dash", "issued"=>"2128-04-09"}, "total"=>208, "obj-id"=>"https://doi.org/10.6071/z7wc73", "relation-type-id"=>"Unique-Dataset-Requests-Machine", "source-id"=>"datacite-usage", "source-token"=>"28276d12-b320-41ba-9272-bb0adc3466ff", "occurred-at"=>"2128-04-09", "license"=>"https://creativecommons.org/publicdomain/zero/1.0/")
112
+ end
113
+
114
+ it "should send a warning if there are more than 4 metrics" do
115
+ body = File.read(fixture_path + 'usage_update_1.json')
116
+ result = OpenStruct.new(body: JSON.parse(body) )
117
+ response = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
118
+ expect(response.length).to eq(1)
119
+ expect(response.last.body).to eq({"errors"=>"There are too many instances. There can only be 4"})
120
+ end
121
+ end
122
+ end
123
+
124
+ context "push_data" do
125
+ it "should report if there are no works returned by the Queue" do
126
+ result = []
127
+ expect { subject.push_data(result) }.to output("No works found in the Queue.\n").to_stdout
128
+ end
129
+
130
+ # it "should report if there are works returned by the Queue" do
131
+ # body = File.read(fixture_path + 'usage_update.json')
132
+ # result = OpenStruct.new(body: JSON.parse(body) )
133
+ # result = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
134
+ # options = { push_url: ENV['EVENTDATA_URL'], access_token: ENV['EVENTDATA_TOKEN'] }
135
+ # expect { subject.push_data(result, options) }.to output(/https:\/\/doi.org\/10.15468\/dl.mb4das references https:\/\/doi.org\/10.3897\/phytokeys.12.2849 pushed to Event Data service.\n/).to_stdout
136
+ # end
137
+
138
+ it "should work with DataCite Event Data" do
139
+ body = File.read(fixture_path + 'usage_update.json')
140
+ result = OpenStruct.new(body: JSON.parse(body) )
141
+ expect = File.read(fixture_path + 'event_data_resp_1')
142
+ result = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
143
+ options = { push_url: ENV['LAGOTTINO_URL'], access_token: ENV['LAGOTTO_TOKEN'], jsonapi: true }
144
+ expect { subject.push_data(result, options) }.to output(expect).to_stdout
145
+ end
146
+
147
+ it "should fail if format of the event is wrong" do
148
+ body = File.read(fixture_path + 'usage_events.json')
149
+ expect = File.read(fixture_path + 'event_data_resp_2')
150
+ result = JSON.parse(body)
151
+ options = { push_url: ENV['LAGOTTINO_URL'], access_token: ENV['LAGOTTO_TOKEN'], jsonapi: true }
152
+ expect { subject.push_data(result, options) }.to output(expect).to_stdout
153
+ end
154
+ end
155
+
156
+ end
data/toccatore.gemspec CHANGED
@@ -15,13 +15,15 @@ Gem::Specification.new do |s|
15
15
  s.license = 'MIT'
16
16
 
17
17
  # Declary dependencies here, rather than in the Gemfile
18
+
18
19
  s.add_dependency 'maremma', '~> 3.5'
19
20
  s.add_dependency 'activesupport', '~> 4.2', '>= 4.2.5'
20
21
  s.add_dependency 'dotenv', '~> 2.1', '>= 2.1.1'
21
22
  s.add_dependency 'namae', '~> 0.11.0'
22
23
  s.add_dependency 'gender_detector', '~> 1.0'
23
24
  s.add_dependency 'thor', '~> 0.19'
24
- s.add_dependency 'slack-notifier', '~> 2.1'
25
+ s.add_dependency 'slack-notifier', '= 2.2.2'
26
+ s.add_dependency 'aws-sdk-sqs'
25
27
  s.add_development_dependency 'bundler', '~> 1.0'
26
28
  s.add_development_dependency 'rspec', '~> 3.4'
27
29
  s.add_development_dependency 'rake', '~> 12.0'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: toccatore
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.9
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Martin Fenner
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-07-04 00:00:00.000000000 Z
11
+ date: 2018-05-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: maremma
@@ -110,16 +110,30 @@ dependencies:
110
110
  name: slack-notifier
111
111
  requirement: !ruby/object:Gem::Requirement
112
112
  requirements:
113
- - - "~>"
113
+ - - '='
114
114
  - !ruby/object:Gem::Version
115
- version: '2.1'
115
+ version: 2.2.2
116
116
  type: :runtime
117
117
  prerelease: false
118
118
  version_requirements: !ruby/object:Gem::Requirement
119
119
  requirements:
120
- - - "~>"
120
+ - - '='
121
121
  - !ruby/object:Gem::Version
122
- version: '2.1'
122
+ version: 2.2.2
123
+ - !ruby/object:Gem::Dependency
124
+ name: aws-sdk-sqs
125
+ requirement: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - ">="
128
+ - !ruby/object:Gem::Version
129
+ version: '0'
130
+ type: :runtime
131
+ prerelease: false
132
+ version_requirements: !ruby/object:Gem::Requirement
133
+ requirements:
134
+ - - ">="
135
+ - !ruby/object:Gem::Version
136
+ version: '0'
123
137
  - !ruby/object:Gem::Dependency
124
138
  name: bundler
125
139
  requirement: !ruby/object:Gem::Requirement
@@ -273,6 +287,8 @@ files:
273
287
  - lib/toccatore/datacite_related.rb
274
288
  - lib/toccatore/images/toccatore.png
275
289
  - lib/toccatore/orcid_update.rb
290
+ - lib/toccatore/queue.rb
291
+ - lib/toccatore/usage_update.rb
276
292
  - lib/toccatore/version.rb
277
293
  - spec/base_spec.rb
278
294
  - spec/cli_spec.rb
@@ -280,11 +296,22 @@ files:
280
296
  - spec/fixtures/datacite_related.json
281
297
  - spec/fixtures/datacite_related_is_identical.json
282
298
  - spec/fixtures/datacite_related_nil.json
299
+ - spec/fixtures/event_data_resp_1
300
+ - spec/fixtures/event_data_resp_2
283
301
  - spec/fixtures/orcid_update.json
284
302
  - spec/fixtures/orcid_update_is_identical.json
285
303
  - spec/fixtures/orcid_update_is_part.json
286
304
  - spec/fixtures/orcid_update_is_previous.json
287
305
  - spec/fixtures/orcid_update_nil.json
306
+ - spec/fixtures/usage_event.json
307
+ - spec/fixtures/usage_event_fail.json
308
+ - spec/fixtures/usage_events.json
309
+ - spec/fixtures/usage_update.json
310
+ - spec/fixtures/usage_update_1.json
311
+ - spec/fixtures/usage_update_2.json
312
+ - spec/fixtures/usage_update_3.json
313
+ - spec/fixtures/usage_update_4.json
314
+ - spec/fixtures/usage_update_nil.json
288
315
  - spec/fixtures/vcr_cassettes/Toccatore_Base/get_doi_ra/crossref.yml
289
316
  - spec/fixtures/vcr_cassettes/Toccatore_Base/get_doi_ra/datacite.yml
290
317
  - spec/fixtures/vcr_cassettes/Toccatore_Base/send_notification_to_slack/datacite_related.yml
@@ -301,6 +328,9 @@ files:
301
328
  - spec/fixtures/vcr_cassettes/Toccatore_CLI/orcid_update/should_query_by_ORCID_ID.yml
302
329
  - spec/fixtures/vcr_cassettes/Toccatore_CLI/orcid_update/should_succeed.yml
303
330
  - spec/fixtures/vcr_cassettes/Toccatore_CLI/orcid_update/should_succeed_with_no_works.yml
331
+ - spec/fixtures/vcr_cassettes/Toccatore_CLI/usage_update/no_reports_in_the_queue/should_succeed_with_no_works.yml
332
+ - spec/fixtures/vcr_cassettes/Toccatore_CLI/usage_update/should_fail.yml
333
+ - spec/fixtures/vcr_cassettes/Toccatore_CLI/usage_update/should_succeed_with_no_works.yml
304
334
  - spec/fixtures/vcr_cassettes/Toccatore_DataciteRelated/get_data/should_allow_queries_by_DOI_of_the_Datacite_Metadata_Search_API.yml
305
335
  - spec/fixtures/vcr_cassettes/Toccatore_DataciteRelated/get_data/should_allow_queries_by_related_identifier_of_the_Datacite_Metadata_Search_API.yml
306
336
  - spec/fixtures/vcr_cassettes/Toccatore_DataciteRelated/get_data/should_report_if_there_are_no_works_returned_by_the_Datacite_Metadata_Search_API.yml
@@ -320,8 +350,14 @@ files:
320
350
  - spec/fixtures/vcr_cassettes/Toccatore_OrcidUpdate/push_data/should_report_if_there_are_works_returned_by_the_Datacite_Metadata_Search_API.yml
321
351
  - spec/fixtures/vcr_cassettes/Toccatore_OrcidUpdate/queue_jobs/should_report_if_there_are_no_works_returned_by_the_Datacite_Metadata_Search_API.yml
322
352
  - spec/fixtures/vcr_cassettes/Toccatore_OrcidUpdate/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_Metadata_Search_API.yml
353
+ - spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/get_data/when_there_are_messages/should_return_the_data_for_one_message.yml
354
+ - spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/get_data/when_there_is_ONE_message/should_return_the_data_for_one_message.yml
355
+ - spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/push_data/should_fail_if_format_of_the_event_is_wrong.yml
356
+ - spec/fixtures/vcr_cassettes/Toccatore_UsageUpdate/push_data/should_work_with_DataCite_Event_Data.yml
323
357
  - spec/orcid_update_spec.rb
358
+ - spec/queque_spec.rb
324
359
  - spec/spec_helper.rb
360
+ - spec/usage_update_spec.rb
325
361
  - toccatore.gemspec
326
362
  homepage: https://github.com/datacite/toccatore
327
363
  licenses:
@@ -343,7 +379,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
343
379
  version: '0'
344
380
  requirements: []
345
381
  rubyforge_project:
346
- rubygems_version: 2.4.5
382
+ rubygems_version: 2.7.6
347
383
  signing_key:
348
384
  specification_version: 4
349
385
  summary: Ruby library to find ORCID IDs in the DataCite Solr index