toccatore 0.3.9 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e3f403a9be0abc0a29474ba5afe14da96d2fac0e
-  data.tar.gz: 7d518eaf6d26315f8291b7c71e92a66c0f2a6cae
+SHA256:
+  metadata.gz: b4f98467d1df1b0ce5226b887a5cd1d5b9e3fb551bfc631e9857fca366ee7d29
+  data.tar.gz: fd8c901be0ee18936711f73e959e19cee8e9ecdf114df8b6372af83e2e8922e5
 SHA512:
-  metadata.gz: bfa83ed964ebd3fdb4cfb6bbf6ecec63477c1d1f3917cecb1040840584468af911add82a09f8c6d079ab814caa8091e3441489dd22ecfab2b67ef24bb98679db
-  data.tar.gz: b8dd0cdd1975f5a0bd1508e6e8711d4e6e1538a46a8cd4b1fad0ec6a2837bb2c61ba54595f370a7c5d25b1153a5b36b707f7e559dd0b9ebf954db32f77518dc2
+  metadata.gz: 5e3f4a9db657604571d0be6f45222e8bae2a7b7c6c5e0945cf7aeb6aec0099d0caaa0f210665d2fa7c9f88a8dc74e5c9e4f4e2326b29df0332d93fd3d93e51e7
+  data.tar.gz: f81581346057dbe1e0d0e25c37bdfa2be5cf4ba00c59f5da36cb606a0806085c7a6b47724939e5fb032a93aa153090cf17b6c55d426e8480b61ebb5a5e99bebe

data/Gemfile.lock

@@ -1,33 +1,44 @@
 PATH
   remote: .
   specs:
-    toccatore (0.3.9)
+    toccatore (0.4.0)
       activesupport (~> 4.2, >= 4.2.5)
+      aws-sdk-sqs
       dotenv (~> 2.1, >= 2.1.1)
       gender_detector (~> 1.0)
       maremma (~> 3.5)
       namae (~> 0.11.0)
-      slack-notifier (~> 2.1)
+      slack-notifier (= 2.2.2)
       thor (~> 0.19)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.2.9)
+    activesupport (4.2.10)
       i18n (~> 0.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
     addressable (2.5.1)
       public_suffix (~> 2.0, >= 2.0.2)
+    aws-partitions (1.84.0)
+    aws-sdk-core (3.20.2)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.3.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     builder (3.2.3)
     codeclimate-test-reporter (1.0.8)
       simplecov (<= 0.13)
+    concurrent-ruby (1.0.5)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
     docile (1.1.5)
-    dotenv (2.2.1)
+    dotenv (2.4.0)
     excon (0.45.4)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
@@ -37,10 +48,12 @@ GEM
       faraday (>= 0.7.4, < 1.0)
     gender_detector (1.0.0)
     hashdiff (0.3.4)
-    i18n (0.8.4)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
     json (2.1.0)
-    maremma (3.5.8)
-      activesupport (>= 4.2.5)
+    maremma (3.6.2)
+      activesupport (>= 4.2.5, < 6)
       addressable (>= 2.3.6)
       builder (~> 3.2, >= 3.2.2)
       excon (~> 0.45.0)
@@ -48,16 +61,16 @@ GEM
       faraday-encoding (~> 0.0.1)
       faraday_middleware (~> 0.10.0)
       multi_json (~> 1.12)
-      nokogiri (~> 1.6, >= 1.6.8)
-      oj (~> 2.18, >= 2.18.1)
-    mini_portile2 (2.2.0)
-    minitest (5.10.2)
-    multi_json (1.12.1)
+      nokogiri (~> 1.8.1)
+      oj (>= 2.8.3)
+    mini_portile2 (2.3.0)
+    minitest (5.11.3)
+    multi_json (1.13.1)
     multipart-post (2.0.0)
     namae (0.11.3)
-    nokogiri (1.8.0)
-      mini_portile2 (~> 2.2.0)
-    oj (2.18.5)
+    nokogiri (1.8.2)
+      mini_portile2 (~> 2.3.0)
+    oj (3.6.0)
     public_suffix (2.0.5)
     rack (2.0.3)
     rack-test (0.6.3)
@@ -83,9 +96,9 @@ GEM
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.1)
     slack-notifier (2.2.2)
-    thor (0.19.4)
+    thor (0.20.0)
     thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     vcr (3.0.3)
     webmock (1.24.6)
@@ -108,4 +121,4 @@ DEPENDENCIES
   webmock (~> 1.22, >= 1.22.3)
 
 BUNDLED WITH
-   1.12.5
+   1.16.1

data/README.md

@@ -4,7 +4,7 @@
 [![Code Climate](https://codeclimate.com/github/datacite/toccatore/badges/gpa.svg)](https://codeclimate.com/github/datacite/toccatore)
 [![Test Coverage](https://codeclimate.com/github/datacite/toccatore/badges/coverage.svg)](https://codeclimate.com/github/datacite/toccatore/coverage)
 
-Agent for Event Data service, providing links to ORCID IDs and DOIs not from DataCite as related identifiers in DataCite metadata.
+Agent for Event Data service. Extracts links to ORCID IDs and DOIs not from DataCite from DataCite metadata, and pushes them to the other services.
 
 ## Installation and use
 

data/lib/toccatore.rb

@@ -1,4 +1,6 @@
 require "toccatore/orcid_update"
 require "toccatore/datacite_related"
+require "toccatore/queue"
+require "toccatore/usage_update"
 require "toccatore/cli"
 require "toccatore/version"

data/lib/toccatore/cli.rb

@@ -46,5 +46,17 @@ module Toccatore
       datacite_related = Toccatore::DataciteRelated.new
       datacite_related.queue_jobs(datacite_related.unfreeze(options))
     end
+
+    desc "usage_update", "push DataCite DOIs usage from DLM Hub to Event Data"
+    method_option :access_token, type: :string, required: true
+    method_option :source_token, type: :string, required: true
+    method_option :push_url, type: :string
+    method_option :slack_webhook_url, type: :string
+    method_option :doi, type: :string
+    method_option :jsonapi, :type => :boolean, :force => true
+    def usage_update
+      usage_update = Toccatore::UsageUpdate.new
+      usage_update.queue_jobs(usage_update.unfreeze(options))
+    end
   end
 end

data/lib/toccatore/queue.rb

@@ -0,0 +1,50 @@
+require 'aws-sdk-sqs'
+
+module Toccatore
+  module Queue
+
+    def queue options={}
+      Aws::SQS::Client.new(region: ENV['AWS_REGION'].to_s, stub_responses: false)
+    end
+
+    def get_total options={}
+      req = @sqs.get_queue_attributes(
+        {
+          queue_url: queue_url, attribute_names: 
+            [
+              'ApproximateNumberOfMessages', 
+              'ApproximateNumberOfMessagesNotVisible'
+            ]
+        }
+      )
+
+      msgs_available = req.attributes['ApproximateNumberOfMessages']
+      msgs_in_flight = req.attributes['ApproximateNumberOfMessagesNotVisible']
+      msgs_available.to_i
+    end
+
+    def get_message options={}
+      @sqs.receive_message(queue_url: queue_url, max_number_of_messages: 1, wait_time_seconds: 1)
+    end
+
+    def delete_message options={}
+      reponse = @sqs.delete_message({
+        queue_url: queue_url,
+        receipt_handle: options.messages[0][:receipt_handle]    
+      })
+      if reponse.successful?
+        puts "Message #{options.messages[0][:receipt_handle]} deleted"
+        0
+      else
+        puts "Could NOT delete Message #{options.messages[0][:receipt_handle]}"
+        1
+      end
+
+    end
+
+    def queue_url options={}
+      queue_name = queue_name ||= "#{ENV['ENVIROMENT']}_usage"
+      @sqs.get_queue_url(queue_name: queue_name).queue_url
+    end
+  end
+end 

data/lib/toccatore/usage_update.rb

@@ -0,0 +1,170 @@
+require_relative 'base'
+
+
+module Toccatore
+  class UsageUpdate < Base
+    include Toccatore::Queue
+    LICENSE = "https://creativecommons.org/publicdomain/zero/1.0/"
+
+
+    def initialize options={}
+      @sqs = queue options
+    end
+
+    def queue_jobs(options={})
+
+      total = get_total(options)
+      
+      if total < 1
+        text = "No works found for in the Usage Reports Queue."
+      end
+
+      while total > 0
+        # walk through paginated results
+        total_pages = (total.to_f / job_batch_size).ceil
+        error_total = 0
+
+        (0...total_pages).each do |page|
+          options[:offset] = page * job_batch_size
+          options[:total] = total
+          error_total += process_data(options)
+        end
+        text = "#{total} works processed with #{error_total} errors for Usage Reports Queue"
+      end
+
+      puts text
+      # send slack notification
+      options[:level] = total > 0 ? "good" : "warning"
+      options[:title] = "Report for #{source_id}"
+      send_notification_to_slack(text, options) if options[:slack_webhook_url].present?
+
+      # return number of works queued
+      total
+    end
+
+    def process_data(options = {})
+      message = get_message
+      data = get_data(message)
+      data = parse_data(data, options)
+
+      return [OpenStruct.new(body: { "data" => [] })] if data.empty?
+
+      push_data(data, options)
+      delete_message message
+    end
+
+    def get_data reponse 
+      return OpenStruct.new(body: { "errors" => "Queue is empty" }) if reponse.messages.empty?
+
+      body = JSON.parse(reponse.messages[0].body)
+      Maremma.get(body["report_id"])
+    end
+
+
+    # method returns number of errors
+    def push_data(items, options={})
+      if items.empty?
+        puts "No works found in the Queue."
+        0
+      elsif options[:access_token].blank?
+        puts "An error occured: Access token missing."
+        options[:total]
+      else
+        error_total = 0
+        Array(items).each do |item|
+          error_total += push_item(item, options)
+        end
+        error_total
+      end
+    end
+
+    def metrics_url
+      ENV['SASHIMI_URL']
+    end
+
+    def source_id
+      "usage_update"
+    end
+
+    def format_event type, data, options
+      { "id" => SecureRandom.uuid,
+        "message-action" => "add",
+        "subj-id" => data[:report_id],
+        "subj"=> {
+          "pid"=> data[:report_id],
+          "issued"=> data[:created]
+        },
+        "total"=> data[:count],
+        "obj-id" => data[:pid],
+        "relation-type-id" => type,
+        "source-id" => "datacite-usage",
+        "source-token" => options[:source_token],
+        "occurred-at" => data[:created_at],
+        "license" => LICENSE 
+      }
+    end
+
+
+    def parse_data(result, options={})
+      return result.body.fetch("errors") if result.body.fetch("errors", nil).present?
+
+      items = result.body.dig("data","report","report-datasets")
+      header = result.body.dig("data","report","report-header")
+      report_id = metrics_url + "/" + result.body.dig("data","report","id")
+
+      created = header.fetch("created")
+      Array.wrap(items).reduce([]) do |x, item|
+        data = {}
+        data[:doi] = item.dig("dataset-id").first.dig("value")
+        data[:pid] = normalize_doi(data[:doi])
+        data[:created] = created
+        data[:report_id] = report_id
+        data[:created_at] = created
+
+        instances = item.dig("performance", 0, "instance")
+
+        return x += [OpenStruct.new(body: { "errors" => "There are too many instances. There can only be 4" })] if instances.size > 8
+     
+        x += Array.wrap(instances).reduce([]) do |ssum, instance|
+          data[:count] = instance.dig("count")
+          event_type = "#{instance.dig("metric-type")}-#{instance.dig("access-method")}"
+          ssum << format_event(event_type, data, options)
+          ssum
+        end
+      end    
+    end
+
+    def push_item(item, options={})
+      return OpenStruct.new(body: { "errors" => [{ "title" => "Access token missing." }] }) if options[:access_token].blank?
+
+      host = options[:push_url].presence || "https://api.test.datacite.org"
+      push_url = host + "/events"
+
+      if options[:jsonapi]
+        data = { "data" => {
+                   "id" => item["id"],
+                   "type" => "events",
+                   "attributes" => item.except("id") }}
+        response = Maremma.post(push_url, data: data.to_json,
+                                          bearer: options[:access_token],
+                                          content_type: 'json',
+                                          host: host)
+      else
+        response = Maremma.post(push_url, data: item.to_json,
+                                          bearer: options[:access_token],
+                                          content_type: 'json',
+                                          host: host)
+      end
+
+      # return 0 if successful, 1 if error
+      if response.status == 201
+        puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} pushed to Event Data service."
+        0
+      elsif response.body["errors"].present?
+        puts "#{item['subj-id']} #{item['relation-type-id']} #{item['obj-id']} had an error:"
+        puts "#{response.body['errors'].first['title']}"
+        1
+      end
+    end
+  end
+end

data/lib/toccatore/version.rb

@@ -1,3 +1,3 @@
 module Toccatore
-  VERSION = "0.3.9"
+  VERSION = "0.4.0"
 end

data/spec/cli_spec.rb

@@ -8,7 +8,7 @@ describe Toccatore::CLI do
 
   describe "version" do
     it 'has version' do
-      expect { subject.__print_version }.to output("0.3.9\n").to_stdout
+      expect { subject.__print_version }.to output("0.4.0\n").to_stdout
     end
   end
 
@@ -106,4 +106,37 @@ describe Toccatore::CLI do
       expect { subject.datacite_related }.to output(/An error occured: Access token missing.\n/).to_stdout
     end
   end
+
+    describe "usage_update", vcr: true, :order => :defined do
+      let(:push_url) { ENV['LAGOTTINO_URL'] }
+      let(:access_token) { ENV['LAGOTTO_TOKEN'] }
+      let(:source_token) { ENV['SOURCE_TOKEN'] }
+      let(:slack_webhook_url) { ENV['SLACK_WEBHOOK_URL'] }
+      let(:cli_options) { { push_url: push_url,
+                            slack_webhook_url: slack_webhook_url,
+                            access_token: access_token,
+                            source_token: source_token } }
+  
+
+      context "no reports in the queue" do 
+        it 'should succeed with no works' do
+          subject.options = { push_url: push_url,
+                              slack_webhook_url: slack_webhook_url,
+                              access_token: access_token}
+          expect { subject.usage_update }.to output("No works found for in the Usage Reports Queue.\n").to_stdout
+        end
+      end
+
+      context "with reports in the queue" do 
+        ## TO test this we need a real queue working 
+        # it 'should succeed' do
+        #   subject.options = cli_options
+        #   expect { subject.usage_update }.to output(/https:\/\/doi.org\/10.5281\/zenodo.16396 is_supplement_to https:\/\/doi.org\/10.1007\/s11548-015-1180-7 pushed to Event Data service.\n/).to_stdout
+        # end
+        # it 'should fail' do
+        #   subject.options = cli_options.except(:access_token)
+        #   expect { subject.usage_update }.to output(/An error occured: Access token missing.\n/).to_stdout
+        # end
+      end
+    end
 end

data/spec/fixtures/event_data_resp_1

@@ -0,0 +1,2 @@
+https://metrics.test.datacite.org/reports/2018-3-Dash total-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.
+https://metrics.test.datacite.org/reports/2018-3-Dash unique-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.

data/spec/fixtures/event_data_resp_2

@@ -0,0 +1,4 @@
+https://metrics.test.datacite.org/reports/2018-3-Dash total-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.
+https://metrics.test.datacite.org/reports/2018-3-Dash unique-dataset-investigations-regular https://doi.org/10.7291/d1q94r pushed to Event Data service.
+https://metrics.test.datacite.org/reports/2018-3-Dash Total-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 pushed to Event Data service.
+https://metrics.test.datacite.org/reports/2018-3-Dash Unique-Dataset-Requests-Machine https://doi.org/10.6071/z7wc73 pushed to Event Data service.

data/spec/fixtures/usage_event.json

@@ -0,0 +1 @@
+{"report_id":"https://metrics.test.datacite.org/reports/2018-3-DataONE"}

data/spec/fixtures/usage_event_fail.json

@@ -0,0 +1,17 @@
+[{
+    "id": 12332423432432,
+    "message-action": "add",
+    "subj": {
+      "pid": "https://metrics.test.datacite.org/reports/2018-3-Dash",
+      "issued": "2128-04-09"
+    },
+    "total": "208",
+    "obj-id": "https://doi.org/10.6071/z7wc73",
+    "relation-type-id": "Unique-Dataset-Requests-Machine",
+    "source-id": "datacite",
+    "source-token": "28276d12-b320-41ba-9272-bb0adc3466ff",
+    "occurred-at": "2128-04-09",
+    "license": "https://creativecommons.org/publicdomain/zero/1.0/"
+  }
+]
+