toccatore 0.3.9 → 0.4.15

data/spec/cli_spec.rb

@@ -8,7 +8,7 @@ describe Toccatore::CLI do
 
   describe "version" do
     it 'has version' do
-      expect { subject.__print_version }.to output("0.3.9\n").to_stdout
+      expect { subject.__print_version }.to output("0.4.15\n").to_stdout
     end
   end
 
@@ -77,7 +77,7 @@ describe Toccatore::CLI do
 
     it 'should succeed' do
       subject.options = cli_options
-      expect { subject.datacite_related }.to output(/https:\/\/doi.org\/10.5281\/zenodo.16396 is_supplement_to https:\/\/doi.org\/10.1007\/s11548-015-1180-7 pushed to Event Data service.\n/).to_stdout
+      expect { subject.datacite_related }.to output(/https:\/\/doi.org\/10.15468\/dl.mb4das references https:\/\/doi.org\/10.3897\/phytokeys.12.2849 pushed to Event Data service.\n/).to_stdout
     end
 
     it 'should query by DOI' do
@@ -86,7 +86,7 @@ describe Toccatore::CLI do
     end
 
     it 'should query by related_identifier' do
-      subject.options = cli_options.merge(related_identifier: "10.7554/elife.01567", from_date: "2013-01-01", until_date: "2017-12-31")
+      subject.options = cli_options.merge(related_identifier: "10.7554/elife.01567", from_date: "2013-01-01", until_date: "2018-12-31")
       expect { subject.datacite_related }.to output(/https:\/\/doi.org\/10.5061\/dryad.b835k is_referenced_by https:\/\/doi.org\/10.7554\/elife.01567 pushed to Event Data service.\n/).to_stdout
     end
 
@@ -106,4 +106,37 @@ describe Toccatore::CLI do
       expect { subject.datacite_related }.to output(/An error occured: Access token missing.\n/).to_stdout
     end
   end
+
+  describe "usage_update", vcr: true, :order => :defined do
+    let(:push_url) { ENV['LAGOTTINO_URL'] }
+    let(:access_token) { ENV['LAGOTTINO_TOKEN'] }
+    let(:source_token) { ENV['SOURCE_TOKEN'] }
+    let(:slack_webhook_url) { ENV['SLACK_WEBHOOK_URL'] }
+    let(:cli_options) { { push_url: push_url,
+                          slack_webhook_url: slack_webhook_url,
+                          access_token: access_token,
+                          source_token: source_token } }
+
+
+    context "no reports in the queue" do 
+      it 'should succeed with no works' do
+        subject.options = { push_url: push_url,
+                            slack_webhook_url: slack_webhook_url,
+                            access_token: access_token}
+        expect { subject.usage_update }.to output("0 works processed with 0 errors for Usage Reports Queue https://sqs.eu-west-1.amazonaws.com/404017989009/test_usage\n").to_stdout
+      end
+    end
+
+    context "with reports in the queue" do 
+      # TO test this we need a real queue working 
+      # it 'should succeed' do
+      #   subject.options = cli_options
+      #   expect { subject.usage_update }.to output("0 works processed with 0 errors for Usage Reports Queue https://sqs.eu-west-1.amazonaws.com/404017989009/test_usage\n").to_stdout
+      # end
+      # it 'should fail' do
+      #   subject.options = cli_options.except(:access_token)
+      #   expect { subject.usage_update }.to output(/An error occured: Access token missing.\n/).to_stdout
+      # end
+    end
+  end
 end

data/spec/datacite_related_spec.rb

@@ -29,7 +29,7 @@ describe Toccatore::DataciteRelated, vcr: true do
 
   context "get_total" do
     it "with works" do
-      expect(subject.get_total(query_options)).to eq(650)
+      expect(subject.get_total(query_options)).to eq(640)
     end
 
     it "with no works" do
@@ -57,19 +57,19 @@ describe Toccatore::DataciteRelated, vcr: true do
 
     it "should report if there are works returned by the Datacite Metadata Search API" do
       response = subject.get_data(query_options)
-      expect(response.body["data"]["response"]["numFound"]).to eq(650)
+      expect(response.body["data"]["response"]["numFound"]).to eq(640)
       doc = response.body["data"]["response"]["docs"].first
-      expect(doc["doi"]).to eq("10.7480/KNOB.113.2014.3")
+      expect(doc["doi"]).to eq("10.7480/KNOB.113.2014.3.830")
     end
 
     it "should allow queries by related_identifier of the Datacite Metadata Search API" do
-      response = subject.get_data({ related_identifier: "10.7554/elife.01567", from_date: "2013-01-01", until_date: "2017-12-31", rows: 1000, offset: 0 })
-      expect(response.body["data"]["response"]["numFound"]).to eq(1)
+      response = subject.get_data({ related_identifier: "10.3897/phytokeys.12.2849", from_date: "2013-01-01", until_date: "2017-12-31", rows: 1000, offset: 0 })
+      expect(response.body["data"]["response"]["numFound"]).to eq(14223)
       doc = response.body["data"]["response"]["docs"].first
-      expect(doc["doi"]).to eq("10.5061/DRYAD.B835K")
+      expect(doc["doi"]).to eq("10.15468/DL.MBSMUW")
       related_identifiers = doc["relatedIdentifier"]
-      expect(related_identifiers.count).to eq(6)
-      expect(related_identifiers[4]).to eq("IsReferencedBy:DOI:10.7554/ELIFE.01567")
+      expect(related_identifiers.count).to eq(1)
+      expect(related_identifiers.first).to eq("References:DOI:10.3897/phytokeys.12.2849")
     end
 
     it "should allow queries by DOI of the Datacite Metadata Search API" do
@@ -102,7 +102,7 @@ describe Toccatore::DataciteRelated, vcr: true do
       response = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
 
       expect(response.length).to eq(134)
-      expect(response.last.except("id")).to eq("message_action" => "create", "subj_id"=>"https://doi.org/10.17180/obs.yzeron", "obj_id"=>"https://doi.org/10.1016/j.jhydrol.2013.09.055", "relation_type_id"=>"is_referenced_by", "source_id"=>"datacite","source_token" => "28276d12-b320-41ba-9272-bb0adc3466ff", "occurred_at"=>"2015-04-07T12:22:40Z", "license" => "https://creativecommons.org/publicdomain/zero/1.0/")
+      expect(response.last.except("id")).to eq("message_action" => "create", "subj_id"=>"https://doi.org/10.17180/obs.yzeron", "obj_id"=>"https://doi.org/10.1016/j.jhydrol.2013.09.055", "relation_type_id"=>"is_referenced_by", "source_id"=>"datacite","source_token" => ENV['SOURCE_TOKEN'], "occurred_at"=>"2015-04-07T12:22:40Z", "license" => "https://creativecommons.org/publicdomain/zero/1.0/")
     end
 
     it "should report if there are works ignored because of an IsIdenticalTo relation" do
@@ -136,7 +136,7 @@ describe Toccatore::DataciteRelated, vcr: true do
       body = File.read(fixture_path + 'datacite_related.json')
       result = OpenStruct.new(body: { "data" => JSON.parse(body) })
       result = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
-      options = { push_url: ENV['LAGOTTO_URL'], access_token: ENV['LAGOTTO_TOKEN'], jsonapi: true }
+      options = { push_url: ENV['LAGOTTINO_URL'], access_token: ENV['LAGOTTINO_TOKEN'], jsonapi: true }
       expect { subject.push_data(result, options) }.to output(/https:\/\/doi.org\/10.15468\/dl.mb4das references https:\/\/doi.org\/10.3897\/phytokeys.12.2849 pushed to Event Data service.\n/).to_stdout
     end
   end

data/spec/factories/default.rb

@@ -0,0 +1,22 @@
+require "faker"
+require 'securerandom'
+
+
+FactoryBot.define do
+  factory :event , class: OpenStruct do
+      sequence(:uuid) { |n| "#{SecureRandom.uuid}-#{n}" }
+      message_action "create"
+      sequence(:obj_id) { |n| "#{Faker::Internet.url}#{n}" }
+      sequence(:subj_id) { |n| "#{Faker::Internet.url}#{n}" }
+      total Faker::Number.number(3)
+      subj {{
+        "pid" => "fdfdfd",
+        "issued" => Faker::Time.between(DateTime.now - 2, DateTime.now),
+      }}
+      relation_type_id ["total-dataset-investigations-regular","total-dataset-investigations-machine","unique-dataset-investigations-machine","total-dataset-investigations-machine"].sample
+      source_id "datacite-usage"
+      sequence(:source_token) { |n| "#{SecureRandom.uuid}-#{n}" }
+      occurred_at Faker::Time.between(DateTime.now - 2, DateTime.now)
+      license "https://creativecommons.org/publicdomain/zero/1.0/"
+  end
+end

data/spec/fixtures/event_data_resp_1

@@ -0,0 +1,2 @@
+https://api.test.datacite.org/reports/2018-3-Dash total-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.
+https://api.test.datacite.org/reports/2018-3-Dash unique-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.

data/spec/fixtures/event_data_resp_2

@@ -0,0 +1,4 @@
+https://api.test.datacite.org/reports/2018-3-Dash total-dataset-investigations-regular https://doi.org/10.7291/d1q94r had an error:\nthe server responded with status 500/\n
+https://api.test.datacite.org/reports/2018-3-Dash unique-dataset-investigations-regular https://doi.org/10.7291/d1q94r had an error:\nthe server responded with status 500/\n
+https://api.test.datacite.org/reports/2018-3-Dash Total-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 had an error:\nthe server responded with status 500/\n
+https://api.test.datacite.org/reports/2018-3-Dash Unique-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 had an error:\nthe server responded with status 500/\n

data/spec/fixtures/events_empty.json

@@ -0,0 +1,33 @@
+[
+  {
+    "uuid": "",
+    "message-action": "create",
+    "subj-id": "",
+    "subj": {
+      "pid": "",
+      "issued": ""
+    },
+    "total": 3,
+    "obj-id": "",
+    "relation-type-id": "",
+    "source-id": "datacite",
+    "source-token": "",
+    "occurred-at": "",
+    "license": ""
+  }, {
+    "uuid": "fc6860b8-4230-41bc-8b8e-21d60e9a7d9a",
+    "message-action": "create",
+    "subj-id": "https://api.test.datacite.org/reports/2018-3-Dashs",
+    "subj": {
+      "pid": "https://api.test.datacite.org/reports/2018-3-Dashs",
+      "issued": "2128-04-09"
+    },
+    "total": 3,
+    "obj-id": "https://doi.org/10.7291/d1q94r",
+    "relation-type-id": "unique-dataset-investigations-regular",
+    "source-id": "datacite",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }
+]

data/spec/fixtures/usage_event.json

@@ -0,0 +1 @@
+{"report_id":"https://api.test.datacite.org/reports/5cac6ca0-9391-4e1d-95cf-ba2f475cbfad"}

data/spec/fixtures/usage_event_fail.json

@@ -0,0 +1,17 @@
+[{
+    "id": 12332423432432,
+    "message-action": "create",
+    "subj": {
+      "pid": "https://api.test.datacite.org/reports/2018-3-Dash",
+      "issued": "2128-04-09"
+    },
+    "total": "208",
+    "obj-id": "https://doi.org/10.6071/z7wc73",
+    "relation-type-id": "Unique-Dataset-Requests-Machine",
+    "source-id": "datacite",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }
+]
+

data/spec/fixtures/usage_events.json

@@ -0,0 +1,63 @@
+[
+  {
+    "uuid": "a17e783b-8214-451e-9b0d-077213c2acae",
+    "message-action": "create",
+    "subj-id": "https://api.test.datacite.org/reports/2018-3-Dash",
+    "subj": {
+      "pid": "https://api.test.datacite.org/reports/2018-3-Dash",
+      "issued": "2128-04-09"
+    },
+    "total": 3,
+    "obj-id": "https://doi.org/10.7291/d1q94r",
+    "relation-type-id": "total-dataset-investigations-regular",
+    "source-id": "datacite",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }, {
+    "uuid": "fc6860b8-4230-41bc-8b8e-21d60e9a7d9a",
+    "message-action": "create",
+    "subj-id": "https://api.test.datacite.org/reports/2018-3-Dash",
+    "subj": {
+      "pid": "https://api.test.datacite.org/reports/2018-3-Dash",
+      "issued": "2128-04-09"
+    },
+    "total": 3,
+    "obj-id": "https://doi.org/10.7291/d1q94r",
+    "relation-type-id": "unique-dataset-investigations-regular",
+    "source-id": "datacite-usage",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }, {
+    "uuid": "054da0ad-9821-492e-877e-5c3adbfd5ccf",
+    "message-action": "create",
+    "subj-id": "https://api.test.datacite.org/reports/2018-3-Dash",
+    "subj": {
+      "pid": "https://api.test.datacite.org/reports/2018-3-Dash",
+      "issued": "2128-04-09"
+    },
+    "total": 254,
+    "obj-id": "https://doi.org/10.6071/z7wc73",
+    "relation-type-id": "Total-Dataset-Requests-Machine",
+    "source-id": "datacite-usage",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }, {
+    "uuid": "6d6fd620-2123-42da-91f3-6c02e63f3cec",
+    "message-action": "create",
+    "subj-id": "https://api.test.datacite.org/reports/2018-3-Dash",
+    "subj": {
+      "pid": "https://api.test.datacite.org/reports/2018-3-Dash",
+      "issued": "2128-04-09"
+    },
+    "total": 208,
+    "obj-id": "https://doi.org/10.6071/z7wc73",
+    "relation-type-id": "Unique-Dataset-Requests-Machine",
+    "source-id": "datacite-usage",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }
+]

data/spec/fixtures/usage_update.json

@@ -0,0 +1,101 @@
+{
+  "data":{
+  "report": {
+  "id": "2018-3-Dash",
+  "report-header": {
+     "report-name": "dataset report",
+     "report-id": "DSR",
+     "release": "rd1",
+     "created": "2128-04-09",
+     "created-by": "dash",
+     "report-attributes": [],
+     "report-filters": [
+        {
+         "name": "begin-date",
+         "value": "2018-03-01"
+       },
+       {
+         "name": "end-date",
+         "value": "2018-03-31"
+       }
+     ],
+     "exceptions": [
+       {
+         "code": 3040,
+         "severity": "warning",
+         "message": "partial data returned",
+         "help-url": "string",
+         "data": "usage data has not been processed for the entire reporting period"
+       }
+     ]
+  },
+   "report-datasets": [
+     {
+       "dataset-title": "chemical shift-based methods in nmr structure determination",
+       "dataset-id": [
+         {
+           "type": "doi",
+           "value": "10.7291/d1q94r"
+         }
+       ],
+       "dataset-contributors": [
+         {
+           "type": "name",
+           "value": "santrupti nerli"
+         },
+         {
+           "type": "name",
+           "value": "andrew c. mcshan"
+         },
+         {
+           "type": "name",
+           "value": "nikolaos g. sgourakis"
+         }
+       ],
+       "dataset-dates": [
+         {
+           "type": "pub-date",
+           "value": "2018-05-01"
+         }
+       ],
+       "platform": "dash",
+       "publisher": "uc santa cruz",
+       "publisher-id": [
+         {
+           "type": "orcid",
+           "value": "2059-4343-4343-4343"
+         }
+       ],
+       "data-type": "dataset",
+       "yop": "2018",
+       "uri": "https://dash.library.ucsc.edu/stash/dataset/doi:10.7291/d1q94r",
+       "performance": [
+         {
+           "period": {
+             "begin-date": "2018-03-01",
+             "end-date": "2018-03-31"
+           },
+           "instance": [
+             {
+               "country": "gb",
+               "access-method": "regular",
+               "metric-type": "total-dataset-investigations",
+               "count": 3
+             },
+             {
+               "country": "gb",
+               "access-method": "regular",
+               "metric-type": "unique-dataset-investigations",
+               "count": 3
+             }
+           ]
+         }
+       ]
+     }]
+   
+ }
+}
+}
+ 
+     
+