maltese 0.9.5 → 0.9.11

data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/send_notification_to_slack/send_info.yml

@@ -0,0 +1,60 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: "<SLACK_WEBHOOK_URL>"
+    body:
+      encoding: US-ASCII
+      string: payload=%7B%22username%22%3A%22Fabrica%22%2C%22icon_url%22%3A%22https%3A%2F%2Fraw.githubusercontent.com%2Fdatacite%2Fhomepage%2Fmaster%2Fsource%2Fimages%2Ffabrica.png%22%2C%22attachments%22%3A%5B%7B%22title%22%3A%22DataCite+Fabrica+Stage%3A+Sitemaps+Updated%22%2C%22color%22%3A%22good%22%2C%22fields%22%3A%5B%7B%22title%22%3A%22URL%22%2C%22value%22%3A%22https%3A%2F%2Fcommons.stage.datacite.org%2Fsitemaps%2Fsitemap.xml.gz%22%7D%2C%7B%22title%22%3A%22Number+of+DOIs%22%2C%22value%22%3A%22271%2C605%22%2C%22short%22%3Atrue%7D%2C%7B%22title%22%3A%22Number+of+Sitemaps%22%2C%22value%22%3A%226%22%2C%22short%22%3Atrue%7D%2C%7B%22title%22%3A%22Time+Taken%22%2C%22value%22%3A%2233+min%22%2C%22short%22%3Atrue%7D%5D%7D%5D%7D
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+      Content-Type:
+      - application/x-www-form-urlencoded
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Sun, 01 Nov 2020 07:44:02 GMT
+      Server:
+      - Apache
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubDomains; preload
+      X-Slack-Backend:
+      - r
+      Access-Control-Allow-Origin:
+      - "*"
+      X-Frame-Options:
+      - SAMEORIGIN
+      Vary:
+      - Accept-Encoding
+      Referrer-Policy:
+      - no-referrer
+      Content-Length:
+      - '22'
+      Content-Type:
+      - text/html
+      X-Envoy-Upstream-Service-Time:
+      - '38'
+      X-Backend:
+      - main_normal main_canary_with_overflow main_control_with_overflow
+      X-Server:
+      - 10.128.214.120:80
+      X-Via:
+      - envoy-www-iad-1mri, haproxy-edge-fra-44lk
+      X-Slack-Shared-Secret-Outcome:
+      - shared-secret
+      Via:
+      - envoy-www-iad-1mri
+    body:
+      encoding: ASCII-8BIT
+      string: ok
+    http_version:
+  recorded_at: Sun, 01 Nov 2020 07:44:02 GMT
+recorded_with: VCR 3.0.3

data/spec/sitemap_spec.rb

@@ -3,32 +3,32 @@ require 'spec_helper'
 describe Maltese::Sitemap, vcr: true do
   subject { Maltese::Sitemap.new(rack_env: "test") }
 
-  let(:doi) { "10.3280/ses2013-001015en" }
+  let(:doi) { "10.25601/2f56-s909" }
 
   context "get_query_url" do
     it "default" do
-      expect(subject.get_query_url).to eq("https://api.test.datacite.org/dois?fields%5Bdois%5D=doi%2Cupdated&page%5Bscroll%5D=7m&page%5Bsize%5D=1000")
+      expect(subject.get_query_url).to eq("https://api.stage.datacite.org/dois?fields%5Bdois%5D=doi%2Cupdated&exclude-registration-agencies=true&page%5Bscroll%5D=7m&page%5Bsize%5D=1000")
     end
 
     it "with page[size] one" do
-      expect(subject.get_query_url(size: 1)).to eq("https://api.test.datacite.org/dois?fields%5Bdois%5D=doi%2Cupdated&page%5Bscroll%5D=7m&page%5Bsize%5D=1")
+      expect(subject.get_query_url(size: 1)).to eq("https://api.stage.datacite.org/dois?fields%5Bdois%5D=doi%2Cupdated&exclude-registration-agencies=true&page%5Bscroll%5D=7m&page%5Bsize%5D=1")
     end
 
     it "with size" do
-      expect(subject.get_query_url(size: 250)).to eq("https://api.test.datacite.org/dois?fields%5Bdois%5D=doi%2Cupdated&page%5Bscroll%5D=7m&page%5Bsize%5D=250")
+      expect(subject.get_query_url(size: 250)).to eq("https://api.stage.datacite.org/dois?fields%5Bdois%5D=doi%2Cupdated&exclude-registration-agencies=true&page%5Bscroll%5D=7m&page%5Bsize%5D=250")
     end
   end
 
   context "get_total" do
     it "with works" do
-      expect(subject.get_total).to eq(846853)
+      expect(subject.get_total).to eq(207479)
     end
   end
 
   context "queue_jobs" do
     it "should report if there are works returned by the Datacite REST API" do
       response = subject.queue_jobs
-      expect(response).to eq(895955)
+      expect(response).to eq(207479)
     end
   end
 
@@ -48,14 +48,20 @@ describe Maltese::Sitemap, vcr: true do
     it "should retry 2 times for bad request errors with the Datacite REST API" do
       stub = stub_request(:get, subject.get_query_url).and_return({ status: [502] }, { status: [502] }, { status: [502] })
       response = subject.process_data(total: 10, url: subject.get_query_url)
-      expect(response).to eq(0)
+      expect(response).to eq(1)
+    end
+
+    it "should handle internal server errors with the Datacite REST API" do
+      stub = stub_request(:get, subject.get_query_url).and_return({ status: [500] }, { status: [200] })
+      response = subject.process_data(total: 10, url: subject.get_query_url)
+      expect(response).to eq(1)
     end
   end
 
   context "get_data" do
     it "should report if there are works returned by the Datacite REST API" do
       response = subject.get_data(subject.get_query_url)
-      expect(response.body.dig("meta", "total")).to eq(846839)
+      expect(response.body.dig("meta", "total")).to eq(207479)
       expect(response.body.fetch("data", []).size).to eq(1000)
       doc = response.body.fetch("data", []).first
       expect(doc.dig("attributes", "doi")).to eq(doi)
@@ -97,4 +103,26 @@ describe Maltese::Sitemap, vcr: true do
       expect { subject.push_data }.to output(/1001 links/).to_stdout
     end
   end
+
+  context "send_notification_to_slack" do
+    it "send info" do
+      fields = [
+        { title: "URL", value: "https://commons.stage.datacite.org/sitemaps/sitemap.xml.gz" },
+        { title: "Number of DOIs", value: 271605.to_s(:delimited), short: true },
+        { title: "Number of Sitemaps", value: 6.to_s(:delimited), short: true },
+        { title: "Time Taken", value: "33 min", short: true }
+      ]
+      expect(subject.send_notification_to_slack(nil, title: subject.slack_title + ": Sitemaps Updated", level: "good", fields: fields)).to eq("ok")
+    end
+
+    it "send error" do
+      fields = [
+        { title: "Error", value: "Error: A bad gateway error occured for URL https://api.datacite.org/dois?page%5Bscroll%5D=7m&page%5Bsize%5D=1000&scroll-id=DnF1ZXJ5VGhlbkZldGNoBQAAAAACJPb5FjI5NUhiM3dqU0JleHRSWlJGTnhYaXcAAAAAAiT2-hYyOTVIYjN3alNCZXh0UlpSRk54WGl3AAAAAAIk9vsWMjk1SGIzd2pTQmV4dFJaUkZOeFhpdwAAAAACJPb8FjI5NUhiM3dqU0JleHRSWlJGTnhYaXcAAAAAAiT2_RYyOTVIYjN3alNCZXh0UlpSRk54WGl3." },
+        { title: "Number of DOIs", value: 141572.to_s(:delimited), short: true },
+        { title: "Number of Sitemaps", value: 3.to_s(:delimited), short: true },
+        { title: "Time Taken", value: "17 min", short: true }
+      ]
+      expect(subject.send_notification_to_slack(nil, title: subject.slack_title + ": Sitemaps Not Updated", level: "danger", fields: fields)).to eq("ok")
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -90,6 +90,8 @@ VCR.configure do |c|
   c.configure_rspec_metadata!
   c.filter_sensitive_data("<AWS_ACCESS_KEY_ID>") { ENV["AWS_ACCESS_KEY_ID"] }
   c.filter_sensitive_data("<AWS_SECRET_ACCESS_KEY>") { ENV["AWS_SECRET_ACCESS_KEY"] }
+  c.filter_sensitive_data("<AWS_REGION>") { ENV["AWS_REGION"] }
   c.filter_sensitive_data("<FOG_DIRECTORY>") { ENV['FOG_DIRECTORY'] }
   c.filter_sensitive_data("<FOG_REGION>") { ENV["FOG_REGION"] }
+  c.filter_sensitive_data("<SLACK_WEBHOOK_URL>") { ENV["SLACK_WEBHOOK_URL"] }
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: maltese
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.11
 platform: ruby
 authors:
 - Martin Fenner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-12 00:00:00.000000000 Z
+date: 2020-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: maremma
@@ -182,14 +182,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -306,7 +306,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-description: Ruby library to generate sitemap for DataCite Search.
+description: Ruby library to generate sitemap for DataCite Commons.
 email: mfenner@datacite.org
 executables:
 - maltese
@@ -327,6 +327,7 @@ files:
 - lib/maltese.rb
 - lib/maltese/.dockerignore
 - lib/maltese/cli.rb
+- lib/maltese/datafile.rb
 - lib/maltese/sitemap.rb
 - lib/maltese/version.rb
 - maltese.gemspec
@@ -336,13 +337,15 @@ files:
 - spec/fixtures/vcr_cassettes/Maltese_CLI/sitemap/should_succeed.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_data/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/get_total/with_works.yml
-- spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_catch_bad_request_errors_with_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_bad_request_errors_with_the_Datacite_REST_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_internal_server_errors_with_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_handle_timeout_errors_with_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_retry_2_times_for_bad_request_errors_with_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_no_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/push_data/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
 - spec/fixtures/vcr_cassettes/Maltese_Sitemap/queue_jobs/should_report_if_there_are_works_returned_by_the_Datacite_REST_API.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/send_notification_to_slack/send_error.yml
+- spec/fixtures/vcr_cassettes/Maltese_Sitemap/send_notification_to_slack/send_info.yml
 - spec/sitemap_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/datacite/maltese
@@ -364,9 +367,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
-summary: Ruby library to generate sitemap for DataCite Search
+summary: Ruby library to generate sitemap for DataCite Commons
 test_files: []

data/spec/fixtures/vcr_cassettes/Maltese_Sitemap/process_data/should_catch_bad_request_errors_with_the_Datacite_REST_API.yml

@@ -1,59 +0,0 @@
----
-http_interactions:
-- request:
-    method: put
-    uri: https://s3.eu-west-1.amazonaws.com/search.test.datacite.org/sitemaps/sitemap.xml.gz
-    body:
-      encoding: ASCII-8BIT
-      string: !binary |-
-        H4sIANy18F0AA5WSTU+EMBCG/wrp1dAP3IOQbvfmaT25Jl5rqdCktMjUhf33dgtr0HjAG8w8z8xLGH6YOpud9QDGuz1imKJMO+Vr45o9ejk95g/oIPjnYEGHLLIOqgnMHrUh9BUh4zji8R77oSEFpYy8Ph2fVas7mRsHQTqlURb5ClLx6JUMadFKBxNiq4c0ZOaALEVCcZn9g7094wlqNMf9167FqUwnG/3DbLxvrMbKd7+9PMGEYXazz6bWfqud4LXt9Ahb5Su7zt35N2M3B59pkn767PfxS2Jn64AFX0+Y2tDZv86DlWVJUhelexLceiWuHEQQtBxUi4OGgGsZpIobksbJFeNWQuh8LQrKypwVOWMnWlbFrtrt7iitKI3cgnDVStfo90F/CGlHeQFOViXeD8YPJlxETM3J9xsnKRSZT118AY2t+lEZAwAA
-    headers:
-      Content-Type:
-      - application/x-gzip
-      Accept-Encoding:
-      - ''
-      User-Agent:
-      - aws-sdk-ruby3/3.85.0 ruby/2.6.3 universal.x86_64-darwin19 aws-sdk-s3/1.59.0
-      X-Amz-Acl:
-      - public-read
-      Cache-Control:
-      - private, max-age=0, no-cache
-      Expect:
-      - 100-continue
-      Content-Md5:
-      - lxwQjxbjrf0xGMXvjTOX9g==
-      X-Amz-Date:
-      - 20191211T092444Z
-      X-Amz-Content-Sha256:
-      - 30bde17d18b8abe11ac80301be4511cf1e773a7c993caf77a08f9d216615e7b7
-      Authorization:
-      - AWS4-HMAC-SHA256 Credential=AKIAJAMMCXAR3IXMNCGQ/20191211/eu-west-1/s3/aws4_request,
-        SignedHeaders=cache-control;content-md5;content-type;expect;host;user-agent;x-amz-acl;x-amz-content-sha256;x-amz-date,
-        Signature=99527ec52cae5635e3928767cf9743f88757fc727cf3be085e99071c3959856b
-      Content-Length:
-      - '333'
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      X-Amz-Id-2:
-      - 9SEmB+KZX4UNyualN/z8RPkrjhNZQOrxEREvMwCzXGB35F6gZty/R+QADhxlFJxORz9cYmp9jcs=
-      X-Amz-Request-Id:
-      - 2ED234E38358C809
-      Date:
-      - Wed, 11 Dec 2019 09:24:48 GMT
-      Etag:
-      - '"971c108f16e3adfd3118c5ef8d3397f6"'
-      Content-Length:
-      - '0'
-      Server:
-      - AmazonS3
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Wed, 11 Dec 2019 09:24:48 GMT
-recorded_with: VCR 3.0.3