toccatore 0.4.0 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4f98467d1df1b0ce5226b887a5cd1d5b9e3fb551bfc631e9857fca366ee7d29
-  data.tar.gz: fd8c901be0ee18936711f73e959e19cee8e9ecdf114df8b6372af83e2e8922e5
+  metadata.gz: 4d2393ac345f665ff869026daac1b7b682a9e030a7d789282a0bdd2948e03f16
+  data.tar.gz: 7430af2a9767e27c9db91e16146f02c0f63c2f756610ebfd3b07497851c00f01
 SHA512:
-  metadata.gz: 5e3f4a9db657604571d0be6f45222e8bae2a7b7c6c5e0945cf7aeb6aec0099d0caaa0f210665d2fa7c9f88a8dc74e5c9e4f4e2326b29df0332d93fd3d93e51e7
-  data.tar.gz: f81581346057dbe1e0d0e25c37bdfa2be5cf4ba00c59f5da36cb606a0806085c7a6b47724939e5fb032a93aa153090cf17b6c55d426e8480b61ebb5a5e99bebe
+  metadata.gz: df1fc16c9eae4ef0559d8ce8fc182642eb685204e979da0888b0283d164f820f81b3acdbf2c98f8ae4cd370788cb1a62f0f8687d06b2908dc4f7fef4f3a5f307
+  data.tar.gz: 48b01f06f65370caeae8610041c16df4111af98b1cafc748f901d71053168f4d036f7c3757d48d2a276a2c804586a8d0346bdce87d6b19299a903a75ee79f7b6

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    toccatore (0.4.0)
+    toccatore (0.4.9)
       activesupport (~> 4.2, >= 4.2.5)
       aws-sdk-sqs
       dotenv (~> 2.1, >= 2.1.1)
+      factory_bot
       gender_detector (~> 1.0)
       maremma (~> 3.5)
       namae (~> 0.11.0)
@@ -21,8 +22,10 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.5.1)
       public_suffix (~> 2.0, >= 2.0.2)
-    aws-partitions (1.84.0)
-    aws-sdk-core (3.20.2)
+    aws-eventstream (1.0.0)
+    aws-partitions (1.87.0)
+    aws-sdk-core (3.21.2)
+      aws-eventstream (~> 1.0)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
@@ -40,6 +43,10 @@ GEM
     docile (1.1.5)
     dotenv (2.4.0)
     excon (0.45.4)
+    factory_bot (4.8.2)
+      activesupport (>= 3.0.0)
+    faker (1.8.7)
+      i18n (>= 0.7)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
     faraday-encoding (0.0.4)
@@ -112,6 +119,8 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.0)
   codeclimate-test-reporter (~> 1.0, >= 1.0.0)
+  factory_bot
+  faker
   rack-test (~> 0)
   rake (~> 12.0)
   rspec (~> 3.4)

data/lib/toccatore/cli.rb

@@ -53,7 +53,7 @@ module Toccatore
     method_option :push_url, type: :string
     method_option :slack_webhook_url, type: :string
     method_option :doi, type: :string
-    method_option :jsonapi, :type => :boolean, :force => true
+    method_option :jsonapi, :type => :boolean, default: true
     def usage_update
       usage_update = Toccatore::UsageUpdate.new
       usage_update.queue_jobs(usage_update.unfreeze(options))

data/lib/toccatore/queue.rb

@@ -4,7 +4,10 @@ module Toccatore
   module Queue
 
     def queue options={}
-      Aws::SQS::Client.new(region: ENV['AWS_REGION'].to_s, stub_responses: false)
+      puts "Queue name has not been specified" unless ENV['ENVIRONMENT'].present?
+      puts "AWS_REGION has not been specified" unless ENV['AWS_REGION'].present?
+      region = ENV['AWS_REGION'] ||= 'eu-west-1'
+      Aws::SQS::Client.new(region: region.to_s, stub_responses: false)
     end
 
     def get_total options={}
@@ -28,6 +31,7 @@ module Toccatore
     end
 
     def delete_message options={}
+      return 1 if options.messages.size < 1
       reponse = @sqs.delete_message({
         queue_url: queue_url,
         receipt_handle: options.messages[0][:receipt_handle]    
@@ -43,7 +47,9 @@ module Toccatore
     end
 
     def queue_url options={}
-      queue_name = queue_name ||= "#{ENV['ENVIROMENT']}_usage"
+      options[:queue_name] ||= "#{ENV['ENVIRONMENT']}_usage" 
+      queue_name = options[:queue_name] 
+      # puts "Using  #{@sqs.get_queue_url(queue_name: queue_name).queue_url} queue"
       @sqs.get_queue_url(queue_name: queue_name).queue_url
     end
   end

data/lib/toccatore/usage_update.rb

@@ -19,18 +19,16 @@ module Toccatore
         text = "No works found for in the Usage Reports Queue."
       end
 
-      while total > 0
-        # walk through paginated results
-        total_pages = (total.to_f / job_batch_size).ceil
-        error_total = 0
-
-        (0...total_pages).each do |page|
-          options[:offset] = page * job_batch_size
-          options[:total] = total
-          error_total += process_data(options)
-        end
-        text = "#{total} works processed with #{error_total} errors for Usage Reports Queue"
+      error_total = 0
+      proccessed_messages = 0
+      num_messages = total
+      while num_messages != 0 
+          processed = process_data(options)
+          error_total += processed
+          proccessed_messages += 1 if processed == 0
+          num_messages -= proccessed_messages
       end
+      text = "#{proccessed_messages} works processed with #{error_total} errors for Usage Reports Queue #{queue_url}"
 
       puts text
       # send slack notification
@@ -39,25 +37,32 @@ module Toccatore
       send_notification_to_slack(text, options) if options[:slack_webhook_url].present?
 
       # return number of works queued
-      total
+      proccessed_messages
     end
 
     def process_data(options = {})
+      errors = 0 
       message = get_message
-      data = get_data(message)
-      data = parse_data(data, options)
-
-      return [OpenStruct.new(body: { "data" => [] })] if data.empty?
-
-      push_data(data, options)
-      delete_message message
+      unless message.messages.empty?
+        data = get_data(message)
+        events = parse_data(data, options)
+        errors = push_data(events, options)
+        if errors < 1
+          delete_message message
+        end
+      end
+      errors
     end
 
     def get_data reponse 
-      return OpenStruct.new(body: { "errors" => "Queue is empty" }) if reponse.messages.empty?
-
       body = JSON.parse(reponse.messages[0].body)
-      Maremma.get(body["report_id"])
+      url = body["report_id"]
+      host = URI.parse(body["report_id"]).host.downcase
+      puts url
+      puts host
+      puts body
+      puts "%%%%%%%%%%"
+      Maremma.get(url, timeout: 120, host: host)
     end
 
 
@@ -72,22 +77,24 @@ module Toccatore
       else
         error_total = 0
         Array(items).each do |item|
-          error_total += push_item(item, options)
+          puts item
+          puts "*************"
+          error_total += push_item(item, options) 
         end
         error_total
       end
     end
 
-    def metrics_url
-      ENV['SASHIMI_URL']
-    end
-
     def source_id
       "usage_update"
     end
 
     def format_event type, data, options
-      { "id" => SecureRandom.uuid,
+      fail "Not type given. Report #{data[:report_id]} not proccessed" if type.blank?
+      fail "Access token missing." if options[:source_token].blank?
+      fail "Report_id is missing" if data[:report_id].blank?
+
+      { "uuid" => SecureRandom.uuid,
         "message-action" => "add",
         "subj-id" => data[:report_id],
         "subj"=> {
@@ -104,13 +111,15 @@ module Toccatore
       }
     end
 
-
     def parse_data(result, options={})
+      puts result.status
+      puts "*************"
       return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+      return [{ "errors" => { "title" => "The report is blank" }}] if result.body.blank?
 
       items = result.body.dig("data","report","report-datasets")
       header = result.body.dig("data","report","report-header")
-      report_id = metrics_url + "/" + result.body.dig("data","report","id")
+      report_id = result.url
 
       created = header.fetch("created")
       Array.wrap(items).reduce([]) do |x, item|
@@ -123,7 +132,7 @@ module Toccatore
 
         instances = item.dig("performance", 0, "instance")
 
-        return x += [OpenStruct.new(body: { "errors" => "There are too many instances. There can only be 4" })] if instances.size > 8
+        return x += [OpenStruct.new(body: { "errors" => "There are too many instances in #{data[:doi]} for report #{report_id}. There can only be 4" })] if instances.size > 8
      
         x += Array.wrap(instances).reduce([]) do |ssum, instance|
           data[:count] = instance.dig("count")
@@ -135,31 +144,34 @@ module Toccatore
     end
 
     def push_item(item, options={})
-      return OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] }) if options[:access_token].blank?
-
-      host = options[:push_url].presence || "https://api.test.datacite.org"
-      push_url = host + "/events"
-
-      if options[:jsonapi]
-        data = { "data" => {
-                   "id" => item["id"],
-                   "type" => "events",
-                   "attributes" => item.except("id") }}
-        response = Maremma.post(push_url, data: data.to_json,
-                                          bearer: options[:access_token],
-                                          content_type: 'json',
-                                          host: host)
-      else
-        response = Maremma.post(push_url, data: item.to_json,
-                                          bearer: options[:access_token],
-                                          content_type: 'json',
-                                          host: host)
+      if item["subj-id"].blank?
+        puts OpenStruct.new(body: { "errors" => [{ "title" => "There is no Subject" }] })
+        return 1
+      elsif options[:access_token].blank?
+        puts OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] })
+        return 1
+      elsif item["errors"].present?
+        puts OpenStruct.new(body: { "errors" => [{ "title" => "#{item["errors"]["title"]}" }] })
+        return 1
       end
 
-      # return 0 if successful, 1 if error
-      if response.status == 201
+      host = options[:push_url].presence || "https://api.test.datacite.org"
+      push_url = host + "/events/" + item["uuid"].to_s
+      data = { "data" => {
+                  "id" => item["uuid"],
+                  "type" => "events",
+                  "attributes" => item.except("id") }}
+      response = Maremma.put(push_url, data: data.to_json,
+                                        bearer: options[:access_token],
+                                        content_type: 'application/json',
+                                        host: host)
+                                  
+      if response.status == 201 
         puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} pushed to Event Data service."
         0
+      elsif response.status == 200
+        puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} pushed to Event Data service for update."
+        0
       elsif response.body["errors"].present?
         puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} had an error:"
         puts "#{response.body['errors'].first['title']}"

data/lib/toccatore/version.rb

@@ -1,3 +1,3 @@
 module Toccatore
-  VERSION = "0.4.0"
+  VERSION = "0.4.9"
 end

data/spec/cli_spec.rb

@@ -8,7 +8,7 @@ describe Toccatore::CLI do
 
   describe "version" do
     it 'has version' do
-      expect { subject.__print_version }.to output("0.4.0\n").to_stdout
+      expect { subject.__print_version }.to output("0.4.9\n").to_stdout
     end
   end
 
@@ -123,15 +123,15 @@ describe Toccatore::CLI do
           subject.options = { push_url: push_url,
                               slack_webhook_url: slack_webhook_url,
                               access_token: access_token}
-          expect { subject.usage_update }.to output("No works found for in the Usage Reports Queue.\n").to_stdout
+          expect { subject.usage_update }.to output("0 works processed with 0 errors for Usage Reports Queue https://sqs.eu-west-1.amazonaws.com/404017989009/test_usage\n").to_stdout
         end
       end
 
       context "with reports in the queue" do 
-        ## TO test this we need a real queue working 
+        # TO test this we need a real queue working 
         # it 'should succeed' do
         #   subject.options = cli_options
-        #   expect { subject.usage_update }.to output(/https:\/\/doi.org\/10.5281\/zenodo.16396 is_supplement_to https:\/\/doi.org\/10.1007\/s11548-015-1180-7 pushed to Event Data service.\n/).to_stdout
+        #   expect { subject.usage_update }.to output("0 works processed with 0 errors for Usage Reports Queue https://sqs.eu-west-1.amazonaws.com/404017989009/test_usage\n").to_stdout
         # end
         # it 'should fail' do
         #   subject.options = cli_options.except(:access_token)

data/spec/datacite_related_spec.rb

@@ -102,7 +102,7 @@ describe Toccatore::DataciteRelated, vcr: true do
       response = subject.parse_data(result, source_token: ENV['SOURCE_TOKEN'])
 
       expect(response.length).to eq(134)
-      expect(response.last.except("id")).to eq("message_action" => "create", "subj_id"=>"https://doi.org/10.17180/obs.yzeron", "obj_id"=>"https://doi.org/10.1016/j.jhydrol.2013.09.055", "relation_type_id"=>"is_referenced_by", "source_id"=>"datacite","source_token" => "28276d12-b320-41ba-9272-bb0adc3466ff", "occurred_at"=>"2015-04-07T12:22:40Z", "license" => "https://creativecommons.org/publicdomain/zero/1.0/")
+      expect(response.last.except("id")).to eq("message_action" => "create", "subj_id"=>"https://doi.org/10.17180/obs.yzeron", "obj_id"=>"https://doi.org/10.1016/j.jhydrol.2013.09.055", "relation_type_id"=>"is_referenced_by", "source_id"=>"datacite","source_token" => "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc", "occurred_at"=>"2015-04-07T12:22:40Z", "license" => "https://creativecommons.org/publicdomain/zero/1.0/")
     end
 
     it "should report if there are works ignored because of an IsIdenticalTo relation" do

data/spec/factories/default.rb

@@ -0,0 +1,22 @@
+require "faker"
+require 'securerandom'
+
+
+FactoryBot.define do
+  factory :event , class: OpenStruct do
+      sequence(:uuid) { |n| "#{SecureRandom.uuid}-#{n}" }
+      message_action "add"
+      sequence(:obj_id) { |n| "#{Faker::Internet.url}#{n}" }
+      sequence(:subj_id) { |n| "#{Faker::Internet.url}#{n}" }
+      total Faker::Number.number(3)
+      subj {{
+        "pid" => "fdfdfd",
+        "issued" => Faker::Time.between(DateTime.now - 2, DateTime.now),
+      }}
+      relation_type_id ["total-dataset-investigations-regular","total-dataset-investigations-machine","unique-dataset-investigations-machine","total-dataset-investigations-machine"].sample
+      source_id "datacite-usage"
+      sequence(:source_token) { |n| "#{SecureRandom.uuid}-#{n}" }
+      occurred_at Faker::Time.between(DateTime.now - 2, DateTime.now)
+      license "https://creativecommons.org/publicdomain/zero/1.0/"
+  end
+end

data/spec/fixtures/event_data_resp_2

@@ -1,4 +1,4 @@
-https://metrics.test.datacite.org/reports/2018-3-Dash total-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.
-https://metrics.test.datacite.org/reports/2018-3-Dash unique-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.
-https://metrics.test.datacite.org/reports/2018-3-Dash Total-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 pushed to Event Data service.
-https://metrics.test.datacite.org/reports/2018-3-Dash Unique-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 pushed to Event Data service.
+https://metrics.test.datacite.org/reports/2018-3-Dash total-dataset-investigations-regular https://doi.org/10.7291/d1q94r had an error:\nthe server responded with status 500/\n
+https://metrics.test.datacite.org/reports/2018-3-Dash unique-dataset-investigations-regular https://doi.org/10.7291/d1q94r had an error:\nthe server responded with status 500/\n
+https://metrics.test.datacite.org/reports/2018-3-Dash Total-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 had an error:\nthe server responded with status 500/\n
+https://metrics.test.datacite.org/reports/2018-3-Dash Unique-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 had an error:\nthe server responded with status 500/\n

data/spec/fixtures/events_empty.json

@@ -0,0 +1,33 @@
+[
+  {
+    "uuid": "",
+    "message-action": "add",
+    "subj-id": "",
+    "subj": {
+      "pid": "",
+      "issued": ""
+    },
+    "total": 3,
+    "obj-id": "",
+    "relation-type-id": "",
+    "source-id": "datacite",
+    "source-token": "",
+    "occurred-at": "",
+    "license": ""
+  }, {
+    "uuid": "fc6860b8-4230-41bc-8b8e-21d60e9a7d9a",
+    "message-action": "add",
+    "subj-id": "https://metrics.test.datacite.org/reports/2018-3-Dashs",
+    "subj": {
+      "pid": "https://metrics.test.datacite.org/reports/2018-3-Dashs",
+      "issued": "2128-04-09"
+    },
+    "total": 3,
+    "obj-id": "https://doi.org/10.7291/d1q94r",
+    "relation-type-id": "unique-dataset-investigations-regular",
+    "source-id": "datacite",
+    "source-token": "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }
+]

data/spec/fixtures/usage_event.json

@@ -1 +1 @@
-{"report_id":"https://metrics.test.datacite.org/reports/2018-3-DataONE"}
+{"report_id":"https://metrics.test.datacite.org/reports/5cac6ca0-9391-4e1d-95cf-ba2f475cbfad"}

data/spec/fixtures/usage_event_fail.json

@@ -9,7 +9,7 @@
     "obj-id": "https://doi.org/10.6071/z7wc73",
     "relation-type-id": "Unique-Dataset-Requests-Machine",
     "source-id": "datacite",
-    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "source-token": "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc",
     "occurred-at": "2128-04-09",
     "license": "https://creativecommons.org/publicdomain/zero/1.0/"
   }

data/spec/fixtures/usage_events.json

@@ -1,6 +1,6 @@
 [
   {
-    "id": "a17e783b-8214-451e-9b0d-077213c2acae",
+    "uuid": "a17e783b-8214-451e-9b0d-077213c2acae",
     "message-action": "add",
     "subj-id": "https://metrics.test.datacite.org/reports/2018-3-Dash",
     "subj": {
@@ -11,11 +11,11 @@
     "obj-id": "https://doi.org/10.7291/d1q94r",
     "relation-type-id": "total-dataset-investigations-regular",
     "source-id": "datacite",
-    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "source-token": "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc",
     "occurred-at": "2128-04-09",
     "license": "https://creativecommons.org/publicdomain/zero/1.0/"
   }, {
-    "id": "fc6860b8-4230-41bc-8b8e-21d60e9a7d9a",
+    "uuid": "fc6860b8-4230-41bc-8b8e-21d60e9a7d9a",
     "message-action": "add",
     "subj-id": "https://metrics.test.datacite.org/reports/2018-3-Dash",
     "subj": {
@@ -26,11 +26,11 @@
     "obj-id": "https://doi.org/10.7291/d1q94r",
     "relation-type-id": "unique-dataset-investigations-regular",
     "source-id": "datacite",
-    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "source-token": "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc",
     "occurred-at": "2128-04-09",
     "license": "https://creativecommons.org/publicdomain/zero/1.0/"
   }, {
-    "id": "054da0ad-9821-492e-877e-5c3adbfd5ccf",
+    "uuid": "054da0ad-9821-492e-877e-5c3adbfd5ccf",
     "message-action": "add",
     "subj-id": "https://metrics.test.datacite.org/reports/2018-3-Dash",
     "subj": {
@@ -41,11 +41,11 @@
     "obj-id": "https://doi.org/10.6071/z7wc73",
     "relation-type-id": "Total-Dataset-Requests-Machine",
     "source-id": "datacite",
-    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "source-token": "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc",
     "occurred-at": "2128-04-09",
     "license": "https://creativecommons.org/publicdomain/zero/1.0/"
   }, {
-    "id": "6d6fd620-2123-42da-91f3-6c02e63f3cec",
+    "uuid": "6d6fd620-2123-42da-91f3-6c02e63f3cec",
     "message-action": "add",
     "subj-id": "https://metrics.test.datacite.org/reports/2018-3-Dash",
     "subj": {
@@ -56,7 +56,7 @@
     "obj-id": "https://doi.org/10.6071/z7wc73",
     "relation-type-id": "Unique-Dataset-Requests-Machine",
     "source-id": "datacite",
-    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "source-token": "43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc",
     "occurred-at": "2128-04-09",
     "license": "https://creativecommons.org/publicdomain/zero/1.0/"
   }

data/spec/fixtures/vcr_cassettes/Toccatore_CLI/datacite_related/should_query_by_related_identifier.yml

@@ -191,7 +191,7 @@ http_interactions:
     uri: https://bus-staging.eventdata.crossref.org/events
     body:
       encoding: UTF-8
-      string: '{"id":"6accc7dd-d727-4202-9d33-a314d357fc2f","message_action":"create","subj_id":"https://doi.org/10.5061/dryad.b835k","obj_id":"https://doi.org/10.7554/elife.01567","relation_type_id":"is_referenced_by","source_id":"datacite","source_token":"28276d12-b320-41ba-9272-bb0adc3466ff","occurred_at":"2014-01-23T19:20:48Z"}'
+      string: '{"id":"6accc7dd-d727-4202-9d33-a314d357fc2f","message_action":"create","subj_id":"https://doi.org/10.5061/dryad.b835k","obj_id":"https://doi.org/10.7554/elife.01567","relation_type_id":"is_referenced_by","source_id":"datacite","source_token":"43ba99ae-5cf0-11e8-9c2d-fa7ae01bbebc","occurred_at":"2014-01-23T19:20:48Z"}'
     headers:
       User-Agent:
       - Maremma - https://github.com/datacite/maremma