toccatore 0.3.9 → 0.4.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e3f403a9be0abc0a29474ba5afe14da96d2fac0e
-  data.tar.gz: 7d518eaf6d26315f8291b7c71e92a66c0f2a6cae
+SHA256:
+  metadata.gz: 56e3f7a50bedf097a157e0eab460c4131df5541a8ccb64c05a344ce90464060d
+  data.tar.gz: b01a010b2ac258c47c2cb705a4d122bb27f3a342dd838cdbab9c832dac66a563
 SHA512:
-  metadata.gz: bfa83ed964ebd3fdb4cfb6bbf6ecec63477c1d1f3917cecb1040840584468af911add82a09f8c6d079ab814caa8091e3441489dd22ecfab2b67ef24bb98679db
-  data.tar.gz: b8dd0cdd1975f5a0bd1508e6e8711d4e6e1538a46a8cd4b1fad0ec6a2837bb2c61ba54595f370a7c5d25b1153a5b36b707f7e559dd0b9ebf954db32f77518dc2
+  metadata.gz: 43afaa44061b2488a5e5c0a9f243f8bd951d63f3742854c7c65522d78783b7dae2d2959bb33c818462f6e509d0da532686b9bbb1b807a7d989beeea2bbe558cf
+  data.tar.gz: 4ea332dd543c5b933a31dd3736cc87e54bf5dd967e010059e0b1975c4ad045b38536bfceef217f41a9d7db4d66602258405fae6219fbae83cb0c6eaedf2b21d9

data/Gemfile.lock

@@ -1,11 +1,13 @@
 PATH
   remote: .
   specs:
-    toccatore (0.3.9)
-      activesupport (~> 4.2, >= 4.2.5)
+    toccatore (0.4.15)
+      activesupport (>= 4.2.5, < 6)
+      aws-sdk-sqs
       dotenv (~> 2.1, >= 2.1.1)
+      factory_bot
       gender_detector (~> 1.0)
-      maremma (~> 3.5)
+      maremma (>= 3.5.7, < 5)
       namae (~> 0.11.0)
       slack-notifier (~> 2.1)
       thor (~> 0.19)
@@ -13,82 +15,100 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.9)
-      i18n (~> 0.7)
+    activesupport (5.2.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    addressable (2.5.1)
-      public_suffix (~> 2.0, >= 2.0.2)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    aws-eventstream (1.0.1)
+    aws-partitions (1.97.0)
+    aws-sdk-core (3.24.0)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.4.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.3)
     builder (3.2.3)
     codeclimate-test-reporter (1.0.8)
       simplecov (<= 0.13)
+    concurrent-ruby (1.0.5)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
     docile (1.1.5)
-    dotenv (2.2.1)
-    excon (0.45.4)
-    faraday (0.9.2)
+    dotenv (2.5.0)
+    excon (0.62.0)
+    factory_bot (4.10.0)
+      activesupport (>= 3.0.0)
+    faker (1.9.1)
+      i18n (>= 0.7)
+    faraday (0.15.2)
       multipart-post (>= 1.2, < 3)
     faraday-encoding (0.0.4)
       faraday
-    faraday_middleware (0.10.1)
+    faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
     gender_detector (1.0.0)
-    hashdiff (0.3.4)
-    i18n (0.8.4)
+    hashdiff (0.3.7)
+    i18n (1.1.0)
+      concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
     json (2.1.0)
-    maremma (3.5.8)
-      activesupport (>= 4.2.5)
+    maremma (4.1)
+      activesupport (>= 4.2.5, < 6)
       addressable (>= 2.3.6)
       builder (~> 3.2, >= 3.2.2)
-      excon (~> 0.45.0)
-      faraday (~> 0.9.2)
-      faraday-encoding (~> 0.0.1)
-      faraday_middleware (~> 0.10.0)
+      excon (~> 0.60)
+      faraday (~> 0.14)
+      faraday-encoding (~> 0.0.4)
+      faraday_middleware (~> 0.12.0)
       multi_json (~> 1.12)
-      nokogiri (~> 1.6, >= 1.6.8)
-      oj (~> 2.18, >= 2.18.1)
-    mini_portile2 (2.2.0)
-    minitest (5.10.2)
-    multi_json (1.12.1)
+      nokogiri (~> 1.8.1)
+      oj (>= 2.8.3)
+    mini_portile2 (2.3.0)
+    minitest (5.11.3)
+    multi_json (1.13.1)
     multipart-post (2.0.0)
     namae (0.11.3)
-    nokogiri (1.8.0)
-      mini_portile2 (~> 2.2.0)
-    oj (2.18.5)
-    public_suffix (2.0.5)
-    rack (2.0.3)
-    rack-test (0.6.3)
-      rack (>= 1.0)
-    rake (12.0.0)
-    rspec (3.6.0)
-      rspec-core (~> 3.6.0)
-      rspec-expectations (~> 3.6.0)
-      rspec-mocks (~> 3.6.0)
-    rspec-core (3.6.0)
-      rspec-support (~> 3.6.0)
-    rspec-expectations (3.6.0)
+    nokogiri (1.8.4)
+      mini_portile2 (~> 2.3.0)
+    oj (3.6.5)
+    public_suffix (3.0.2)
+    rack (2.0.5)
+    rack-test (0.8.3)
+      rack (>= 1.0, < 3)
+    rake (12.3.1)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-mocks (3.6.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-support (3.6.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
     safe_yaml (1.0.4)
-    simplecov (0.12.0)
+    simplecov (0.13.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.1)
-    slack-notifier (2.2.2)
-    thor (0.19.4)
+    simplecov-html (0.10.2)
+    slack-notifier (2.3.2)
+    thor (0.20.0)
     thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     vcr (3.0.3)
-    webmock (1.24.6)
+    webmock (3.4.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -98,14 +118,16 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0)
-  codeclimate-test-reporter (~> 1.0, >= 1.0.0)
+  codeclimate-test-reporter (~> 1.0)
+  factory_bot
+  faker
   rack-test (~> 0)
   rake (~> 12.0)
   rspec (~> 3.4)
-  simplecov (~> 0.12.0)
+  simplecov
   toccatore!
   vcr (~> 3.0, >= 3.0.3)
-  webmock (~> 1.22, >= 1.22.3)
+  webmock (~> 3.0, >= 3.0.1)
 
 BUNDLED WITH
-   1.12.5
+   1.16.1

data/README.md

@@ -4,7 +4,7 @@
 [![Code Climate](https://codeclimate.com/github/datacite/toccatore/badges/gpa.svg)](https://codeclimate.com/github/datacite/toccatore)
 [![Test Coverage](https://codeclimate.com/github/datacite/toccatore/badges/coverage.svg)](https://codeclimate.com/github/datacite/toccatore/coverage)
 
-Agent for Event Data service, providing links to ORCID IDs and DOIs not from DataCite as related identifiers in DataCite metadata.
+Agent for Event Data service. Extracts links to ORCID IDs and DOIs not from DataCite from DataCite metadata, and pushes them to the other services.
 
 ## Installation and use
 

data/lib/toccatore/base.rb

@@ -72,7 +72,12 @@ module Toccatore
         (0...total_pages).each do |page|
           options[:offset] = page * job_batch_size
           options[:total] = total
-          error_total += process_data(options)
+          err = process_data(options)
+          if err.is_a?(Integer)
+            error_total += err
+          else
+            puts err.inspect
+          end
         end
         text = "#{total} works processed with #{error_total} errors for date range #{options[:from_date]} - #{options[:until_date]}."
       else
@@ -82,7 +87,13 @@ module Toccatore
       puts text
 
       # send slack notification
-      options[:level] = total > 0 ? "good" : "warning"
+      if total == 0
+        options[:level] = "warning"
+      elsif error_total > 0
+        options[:level] = "danger"
+      else
+        options[:level] = "good"
+      end
       options[:title] = "Report for #{source_id}"
       send_notification_to_slack(text, options) if options[:slack_webhook_url].present?
 
@@ -145,8 +156,8 @@ module Toccatore
       notifier = Slack::Notifier.new options[:slack_webhook_url],
                                      username: "Event Data Agent",
                                      icon_url: ICON_URL
-      response = notifier.ping attachments: [attachment]
-      response.body
+      response = notifier.post attachments: [attachment]
+      response.first
     end
 
     def get_doi_ra(prefix)

data/lib/toccatore/cli.rb

@@ -46,5 +46,17 @@ module Toccatore
       datacite_related = Toccatore::DataciteRelated.new
       datacite_related.queue_jobs(datacite_related.unfreeze(options))
     end
+
+    desc "usage_update", "push DataCite DOIs usage from DLM Hub to Event Data"
+    method_option :access_token, type: :string, required: true
+    method_option :source_token, type: :string, required: true
+    method_option :push_url, type: :string
+    method_option :slack_webhook_url, type: :string
+    method_option :doi, type: :string
+    method_option :jsonapi, :type => :boolean, default: true
+    def usage_update
+      usage_update = Toccatore::UsageUpdate.new
+      usage_update.queue_jobs(usage_update.unfreeze(options))
+    end
   end
 end

data/lib/toccatore/datacite_related.rb

@@ -45,7 +45,7 @@ module Toccatore
                         "relation_type_id" => raw_relation_type.underscore,
                         "source_id" => "datacite",
                         "source_token" => options[:source_token],
-                        "occurred_at" => item.fetch("minted"),
+                        "occurred_at" => item.fetch("updated"),
                         "license" => LICENSE }
             else
               ssum
@@ -58,7 +58,10 @@ module Toccatore
     end
 
     def push_item(item, options={})
-      return OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] }) if options[:access_token].blank?
+      if options[:access_token].blank?
+        puts "Access token missing."
+        return 1
+      end
 
       host = options[:push_url].presence || "https://bus.eventdata.crossref.org"
       push_url = host + "/events"
@@ -67,7 +70,16 @@ module Toccatore
         data = { "data" => {
                    "id" => item["id"],
                    "type" => "events",
-                   "attributes" => item.except("id") }}
+                   "attributes" => {
+                     "message-action" => item["message_action"],
+                     "subj-id" => item["subj_id"],
+                     "obj-id" => item["obj_id"],
+                     "relation-type-id" => item["relation_type_id"],
+                     "source-id" => "datacite-crossref",
+                     "source-token" => item["source_token"],
+                     "occurred-at" => item["occurred_at"],
+                     "license" => item["license"] } }}
+
         response = Maremma.post(push_url, data: data.to_json,
                                           bearer: options[:access_token],
                                           content_type: 'json',
@@ -83,6 +95,9 @@ module Toccatore
       if response.status == 201
         puts "#{item['subj_id']} #{item['relation_type_id']} #{item['obj_id']} pushed to Event Data service."
         0
+      elsif response.status == 409
+        puts "#{item['subj_id']} #{item['relation_type_id']} #{item['obj_id']} already pushed to Event Data service."
+        0
       elsif response.body["errors"].present?
         puts "#{item['subj_id']} #{item['relation_type_id']} #{item['obj_id']} had an error:"
         puts "#{response.body['errors'].first['title']}"

data/lib/toccatore/orcid_update.rb

@@ -44,9 +44,13 @@ module Toccatore
     end
 
     def push_item(item, options={})
-      return OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] }) if options[:access_token].blank?
+      if options[:access_token].blank?
+        puts "Access token missing."
+        return 1
+      end
 
-      push_url = (options[:push_url].presence || "https://profiles.datacite.org/api") + "/claims"
+      host = options[:push_url].presence || "https://profiles.datacite.org/api"
+      push_url = host + "/claims"
 
       response = Maremma.post(push_url, data: { "claim" => item }.to_json,
                                         bearer: options[:access_token],

data/lib/toccatore/queue.rb

@@ -0,0 +1,56 @@
+require 'aws-sdk-sqs'
+
+module Toccatore
+  module Queue
+
+    def queue options={}
+      puts "Queue name has not been specified" unless ENV['ENVIRONMENT'].present?
+      puts "AWS_REGION has not been specified" unless ENV['AWS_REGION'].present?
+      region = ENV['AWS_REGION'] ||= 'eu-west-1'
+      Aws::SQS::Client.new(region: region.to_s, stub_responses: false)
+    end
+
+    def get_total options={}
+      req = @sqs.get_queue_attributes(
+        {
+          queue_url: queue_url, attribute_names: 
+            [
+              'ApproximateNumberOfMessages', 
+              'ApproximateNumberOfMessagesNotVisible'
+            ]
+        }
+      )
+
+      msgs_available = req.attributes['ApproximateNumberOfMessages']
+      msgs_in_flight = req.attributes['ApproximateNumberOfMessagesNotVisible']
+      msgs_available.to_i
+    end
+
+    def get_message options={}
+      @sqs.receive_message(queue_url: queue_url, max_number_of_messages: 1, wait_time_seconds: 1)
+    end
+
+    def delete_message options={}
+      return 1 if options.messages.size < 1
+      reponse = @sqs.delete_message({
+        queue_url: queue_url,
+        receipt_handle: options.messages[0][:receipt_handle]    
+      })
+      if reponse.successful?
+        puts "Message #{options.messages[0][:receipt_handle]} deleted"
+        0
+      else
+        puts "Could NOT delete Message #{options.messages[0][:receipt_handle]}"
+        1
+      end
+
+    end
+
+    def queue_url options={}
+      options[:queue_name] ||= "#{ENV['ENVIRONMENT']}_usage" 
+      queue_name = options[:queue_name] 
+      # puts "Using  #{@sqs.get_queue_url(queue_name: queue_name).queue_url} queue"
+      @sqs.get_queue_url(queue_name: queue_name).queue_url
+    end
+  end
+end 

data/lib/toccatore/usage_update.rb

@@ -0,0 +1,183 @@
+require_relative 'base'
+
+
+module Toccatore
+  class UsageUpdate < Base
+    include Toccatore::Queue
+    LICENSE = "https://creativecommons.org/publicdomain/zero/1.0/"
+
+
+    def initialize options={}
+      @sqs = queue options
+    end
+
+    def queue_jobs(options={})
+
+      total = get_total(options)
+      
+      if total < 1
+        text = "No works found for in the Usage Reports Queue."
+      end
+
+      error_total = 0
+      proccessed_messages = 0
+      num_messages = total
+      while num_messages != 0 
+          processed = process_data(options)
+          error_total += processed
+          proccessed_messages += 1 if processed == 0
+          num_messages -= proccessed_messages
+      end
+      text = "#{proccessed_messages} works processed with #{error_total} errors for Usage Reports Queue #{queue_url}"
+
+      puts text
+      # send slack notification
+      options[:level] = total > 0 ? "good" : "warning"
+      options[:title] = "Report for #{source_id}"
+      send_notification_to_slack(text, options) if options[:slack_webhook_url].present?
+
+      # return number of works queued
+      proccessed_messages
+    end
+
+    def process_data(options = {})
+      errors = 0 
+      message = get_message
+      unless message.messages.empty?
+        data = get_data(message)
+        events = parse_data(data, options)
+        errors = push_data(events, options)
+        if errors < 1
+          delete_message message
+        end
+      end
+      errors
+    end
+
+    def get_data reponse 
+      body = JSON.parse(reponse.messages[0].body)
+      url = body["report_id"]
+      host = URI.parse(body["report_id"]).host.downcase
+      puts url
+      puts host
+      puts body
+      puts "%%%%%%%%%%"
+      Maremma.get(url, timeout: 120, host: host)
+    end
+
+
+    # method returns number of errors
+    def push_data(items, options={})
+      if items.empty?
+        puts "No works found in the Queue."
+        0
+      elsif options[:access_token].blank?
+        puts "An error occured: Access token missing."
+        options[:total]
+      else
+        error_total = 0
+        Array(items).each do |item|
+          puts item
+          puts "*************"
+          error_total += push_item(item, options) 
+        end
+        error_total
+      end
+    end
+
+    def source_id
+      "usage_update"
+    end
+
+    def format_event type, data, options
+      fail "Not type given. Report #{data[:report_id]} not proccessed" if type.blank?
+      fail "Access token missing." if options[:source_token].blank?
+      fail "Report_id is missing" if data[:report_id].blank?
+
+      { "uuid" => SecureRandom.uuid,
+        "message-action" => "create",
+        "subj-id" => data[:report_id],
+        "subj"=> {
+          "pid"=> data[:report_id],
+          "issued"=> data[:created]
+        },
+        "total"=> data[:count],
+        "obj-id" => data[:pid],
+        "relation-type-id" => type,
+        "source-id" => "datacite-usage",
+        "source-token" => options[:source_token],
+        "occurred-at" => data[:created_at],
+        "license" => LICENSE 
+      }
+    end
+
+    def parse_data(result, options={})
+      puts result.status
+      puts "*************"
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+      return [{ "errors" => { "title" => "The report is blank" }}] if result.body.blank?
+
+      items = result.body.dig("data","report","report-datasets")
+      header = result.body.dig("data","report","report-header")
+      report_id = result.url
+
+      created = header.fetch("created")
+      Array.wrap(items).reduce([]) do |x, item|
+        data = {}
+        data[:doi] = item.dig("dataset-id").first.dig("value")
+        data[:pid] = normalize_doi(data[:doi])
+        data[:created] = created
+        data[:report_id] = report_id
+        data[:created_at] = created
+
+        instances = item.dig("performance", 0, "instance")
+
+        return x += [OpenStruct.new(body: { "errors" => "There are too many instances in #{data[:doi]} for report #{report_id}. There can only be 4" })] if instances.size > 8
+     
+        x += Array.wrap(instances).reduce([]) do |ssum, instance|
+          data[:count] = instance.dig("count")
+          event_type = "#{instance.dig("metric-type")}-#{instance.dig("access-method")}"
+          ssum << format_event(event_type, data, options)
+          ssum
+        end
+      end    
+    end
+
+    def push_item(item, options={})
+      if item["subj-id"].blank?
+        puts OpenStruct.new(body: { "errors" => [{ "title" => "There is no Subject" }] })
+        return 1
+      elsif options[:access_token].blank?
+        puts OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] })
+        return 1
+      elsif item["errors"].present?
+        puts OpenStruct.new(body: { "errors" => [{ "title" => "#{item["errors"]["title"]}" }] })
+        return 1
+      end
+
+      host = options[:push_url].presence || "https://api.test.datacite.org"
+      push_url = host + "/events/" + item["uuid"].to_s
+      data = { "data" => {
+                  "id" => item["uuid"],
+                  "type" => "events",
+                  "attributes" => item.except("id") }}
+                  
+      response = Maremma.put(push_url, data: data.to_json,
+                                       bearer: options[:access_token],
+                                       content_type: 'json',
+                                       host: host)
+                                  
+      if response.status == 201 
+        puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} pushed to Event Data service."
+        0
+      elsif response.status == 200
+        puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} pushed to Event Data service for update."
+        0
+      elsif response.body["errors"].present?
+        puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} had an error:"
+        puts "#{response.body['errors'].first['title']}"
+        1
+      end
+    end
+  end
+end

data/lib/toccatore/version.rb

@@ -1,3 +1,3 @@
 module Toccatore
-  VERSION = "0.3.9"
+  VERSION = "0.4.15"
 end

data/lib/toccatore.rb

@@ -1,4 +1,6 @@
 require "toccatore/orcid_update"
 require "toccatore/datacite_related"
+require "toccatore/queue"
+require "toccatore/usage_update"
 require "toccatore/cli"
 require "toccatore/version"

data/spec/base_spec.rb

@@ -35,7 +35,7 @@ describe Toccatore::Base, vcr: true do
     it "datacite_related" do
       text =  "12 works processed for date range 2015-04-07 - 2015-04-08."
       options = { title: "Report for datacite_related", slack_webhook_url: slack_webhook_url }
-      expect(subject.send_notification_to_slack(text, options)).to eq("ok")
+      expect(subject.send_notification_to_slack(text, options)).to be_kind_of(Net::HTTPOK)
     end
 
     it "datacite_related no works" do
@@ -43,13 +43,13 @@ describe Toccatore::Base, vcr: true do
       options = { title: "Report for datacite_related",
                   slack_webhook_url: slack_webhook_url,
                   level: "warning" }
-      expect(subject.send_notification_to_slack(text, options)).to eq("ok")
+      expect(subject.send_notification_to_slack(text, options)).to be_kind_of(Net::HTTPOK)
     end
 
     it "orcid_update" do
       text =  "12 works processed for date range 2015-04-07 - 2015-04-08."
       options = { title: "Report for orcid_update", slack_webhook_url: slack_webhook_url }
-      expect(subject.send_notification_to_slack(text, options)).to eq("ok")
+      expect(subject.send_notification_to_slack(text, options)).to be_kind_of(Net::HTTPOK)
     end
 
     it "orcid_update no works" do
@@ -57,7 +57,7 @@ describe Toccatore::Base, vcr: true do
       options = { title: "Report for orcid_update",
                   slack_webhook_url: slack_webhook_url,
                   level: "warning" }
-      expect(subject.send_notification_to_slack(text, options)).to eq("ok")
+      expect(subject.send_notification_to_slack(text, options)).to be_kind_of(Net::HTTPOK)
     end
   end